/*
 * Copyright (c) 2002, 2021, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 */

package javadoc.tester;

import java.io.ByteArrayOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.annotation.Annotation;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.ref.SoftReference;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.charset.Charset;
import java.nio.charset.UnsupportedCharsetException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.tools.StandardJavaFileManager;


/**
 * Test framework for running javadoc and performing tests on the resulting output.
 *
 * <p>
 * Tests are typically written as subtypes of JavadocTester, with a main
 * method that creates an instance of the test class and calls the runTests()
 * method. The runTests() method calls all the test methods declared in the class,
 * and then calls a method to print a summary, and throw an exception if
 * any of the test methods reported a failure.
 *
 * <p>
 * Test methods are identified with a @Test annotation. They have no parameters.
 * The name of the method is not important, but if you have more than one, it is
 * recommended that the names be meaningful and suggestive of the test case
 * contained therein.
 *
 * <p>
 * Typically, a test method will invoke javadoc, and then perform various
 * checks on the results. The standard checks are:
 *
 * <dl>
 * <dt>checkExitCode
 * <dd>Check the exit code returned from javadoc.
 * <dt>checkOutput
 * <dd>Perform a series of checks on the contents on a file or output stream
 *     generated by javadoc.
 *     The checks can be either that a series of strings are found or are not found.
 * <dt>checkFiles
 * <dd>Perform a series of checks on the files generated by javadoc.
 *     The checks can be that a series of files are found or are not found.
 * </dl>
 *
 * <pre><code>
 *  public class MyTester extends JavadocTester {
 *      public static void main(String... args) throws Exception {
 *          MyTester tester = new MyTester();
 *          tester.runTests();
 *      }
 *
 *      // test methods...
 *      {@literal @}Test
 *      void test() {
 *          javadoc(<i>args</i>);
 *          checkExit(Exit.OK);
 *          checkOutput(<i>file</i>, true,
 *              <i>strings-to-find</i>);
 *          checkOutput(<i>file</i>, false,
 *              <i>strings-to-not-find</i>);
 *      }
 *  }
 * </code></pre>
 *
 * <p>
 * If javadoc is run more than once in a test method, you can compare the
 * results that are generated with the diff method. Since files written by
 * javadoc typically contain a timestamp, you may want to use the {@code -notimestamp}
 * option if you are going to compare the results from two runs of javadoc.
 *
 * <p>
 * If you have many calls of checkOutput that are very similar, you can write
 * your own check... method to reduce the amount of duplication. For example,
 * if you want to check that many files contain the same string, you could
 * write a method that takes a varargs list of files and calls checkOutput
 * on each file in turn with the string to be checked.
 *
 * <p>
 * You can also write your own custom check methods. After any setup or
 * argument checking, the method should call {@code checking(...)},
 * and then eventually call either {@code passed(...)} or {@code failed(...)}
 * to report whether the check succeeded or not.
 * Use {@code readFile} to get the contents of a file generated by javadoc.
 *
 * <p>
 * You can have many separate test methods, each identified with a @Test
 * annotation. However, you should <b>not</b> assume they will be called
 * in the order declared in your source file.  If the order of a series
 * of javadoc invocations is important, do that within a single method.
 * If the invocations are independent, for better clarity, use separate
 * test methods, each with their own set of checks on the results.
 */
public abstract class JavadocTester {

    public static final String FS = System.getProperty("file.separator");
    public static final String PS = System.getProperty("path.separator");
    public static final String NL = System.getProperty("line.separator");
    public static final String thisRelease = System.getProperty("java.specification.version");

    public static final Path currDir = Paths.get(".").toAbsolutePath().normalize();

    public enum Output {
        /** The name of the output stream from javadoc. */
        OUT,
        /** The name for any output written to System.out. */
        STDOUT,
        /** The name for any output written to System.err. */
        STDERR
    }

    /** The output directory used in the most recent call of javadoc. */
    protected Path outputDir;

    /** The output charset used in the most recent call of javadoc. */
    protected Charset charset = Charset.defaultCharset();

    /** The exit code of the most recent call of javadoc. */
    private int exitCode;

    /** The output generated by javadoc to the various writers and streams. */
    private final Map<Output, String> outputMap = new EnumMap<>(Output.class);

    /** A cache of file content, to avoid reading files unnecessarily. */
    private final Map<Path,SoftReference<String>> fileContentCache = new HashMap<>();
    /** The charset used for files in the fileContentCache. */
    private Charset fileContentCacheCharset = null;

    /** Stream used for logging messages. */
    protected final PrintStream out = System.out;

    /** The directory containing the source code for the test. */
    public static final String testSrc = System.getProperty("test.src");

    /**
     * Get the path for a source file in the test source directory.
     *
     * @param path the path of a file or directory in the source directory
     * @return the full path of the specified file
     */
    public static String testSrc(String path) {
        return Path.of(testSrc).resolve(path).toString();
    }

    /**
     * Alternatives for checking the contents of a directory.
     */
    public enum DirectoryCheck {
        /**
         * Check that the directory is empty.
         */
        EMPTY(p -> true),
        /**
         * Check that the directory does not contain any HTML files,
         * such as may have been generated by a prior run of javadoc
         * using this directory.
         * For now, the check is only performed on the top level directory.
         */
        NO_HTML_FILES(p -> p.getFileName().toString().endsWith(".html")),
        /**
         * No check is performed on the directory contents.
         */
        NONE(null) { @Override void check(Path dir) { } };

        /** The filter used to detect that files should <i>not</i> be present. */
        DirectoryStream.Filter<Path> filter;

        DirectoryCheck(DirectoryStream.Filter<Path> f) {
            filter = f;
        }

        void check(Path dir) {
            if (Files.isDirectory(dir)) {
                List<Path> contents = new ArrayList<>();
                try (var ds = Files.newDirectoryStream(dir, filter)) {
                    for (Path p : ds) {
                        contents.add(p);
                    }
                } catch (IOException e) {
                    throw new Error("cannot list directory: " + dir + "; " + e, e);
                }
                if (!contents.isEmpty()) {
                    System.err.println("Found extraneous files in dir:" + dir.toAbsolutePath());
                    for (Path p : contents) {
                        System.err.println(p);
                    }
                    throw new Error("directory has unexpected content: " + dir);
                }
            }
        }
    }

    private DirectoryCheck outputDirectoryCheck = DirectoryCheck.EMPTY;

    private boolean automaticCheckAccessibility = true;
    private boolean automaticCheckLinks = true;
    private boolean automaticCheckUniqueOUT = true;
    private boolean useStandardStreams = false;
    private StandardJavaFileManager fileManager = null;

    /** The current subtest number. Incremented when checking(...) is called. */
    private int numTestsRun = 0;

    /** The number of subtests passed. Incremented when passed(...) is called. */
    private int numTestsPassed = 0;

    /** The current run of javadoc. Incremented when javadoc is called. */
    private int javadocRunNum = 0;

    /** The current subtest number for this run of javadoc. Incremented when checking(...) is called. */
    private int javadocTestNum = 0;

    /** Marker annotation for test methods to be invoked by runTests. */
    @Retention(RetentionPolicy.RUNTIME)
    public @interface Test { }

    /**
     * Run all methods annotated with @Test, followed by printSummary.
     * The methods are invoked in the order found using getDeclaredMethods.
     * The arguments for the invocation are provided {@link #getTestArgs(Method)}.
     *
     * Typically called on a tester object in main().
     *
     * @throws IllegalArgumentException if any test method does not have a recognized signature
     * @throws Exception if any errors occurred
     */
    public void runTests() throws Exception {
        runTests(this::getTestArgs);
    }

    /**
     * Runs all methods annotated with @Test, followed by printSummary.
     * The methods are invoked in the order found using getDeclaredMethods.
     * The arguments for the invocation are provided by a given function.
     *
     * Typically called on a tester object in main()
     *
     * @param f a function which will be used to provide arguments to each
     *          invoked method
     * @throws Exception if any errors occurred while executing a test method
     */
    public void runTests(Function<Method, Object[]> f) throws Exception {
        for (Method m : getClass().getDeclaredMethods()) {
            Annotation a = m.getAnnotation(Test.class);
            if (a != null) {
                runTest(m, f);
                out.println();
            }
        }
        printSummary();
    }

    /**
     * Run the specified methods annotated with @Test, or all methods annotated
     * with @Test if none are specified, followed by printSummary.
     * The methods are invoked in the order given in the methodNames argument,
     * or the order returned by getDeclaredMethods if no names are provided.
     * The arguments for the invocation are provided {@link #getTestArgs(Method)}.
     *
     * Typically called on a tester object in main(String[] args), perhaps using
     * args as the list of method names.
     *
     * @throws IllegalStateException if any methods annotated with @Test are overloaded
     * @throws IllegalArgumentException if any of the method names does not name a suitable method
     * @throws NullPointerException if {@code methodNames} is {@code null}, or if any of the names are {@code null}
     * @throws Exception if any errors occurred while executing a test method
     */
    public void runTests(String... methodNames) throws Exception {
        runTests(this::getTestArgs, methodNames);
    }

    /**
     * Run the specified methods annotated with @Test, or all methods annotated
     * with @Test if non are specified, followed by printSummary.
     * The methods are invoked in the order given in the methodNames argument,
     * or the order returned by getDeclaredMethods if no names are provided.
     * The arguments for the invocation are provided {@link #getTestArgs(Method)}.
     *
     * Typically called on a tester object in main(String[] args), perhaps using
     * args as the list of method names.
     *
     * @throws IllegalStateException if any methods annotated with @Test are overloaded
     * @throws IllegalArgumentException if any of the method names does not name a suitable method
     * @throws NullPointerException if {@code methodNames} is {@code null}, or if any of the names are {@code null}
     * @throws Exception if any errors occurred while executing a test method
     */
    public void runTests(Function<Method, Object[]> f, String... methodNames) throws Exception {
        if (methodNames.length == 0) {
            runTests(f);
        } else {
            Map<String, Method> testMethods = Stream.of(getClass().getDeclaredMethods())
                    .filter(this::isTestMethod)
                    .collect(Collectors.toMap(Method::getName, Function.identity(),
                            (o, n) -> {
                                throw new IllegalStateException("test method " + o.getName() + " is overloaded");
                            }));

            List<Method> list = new ArrayList<>();
            for (String mn : methodNames) {
                Method m = testMethods.get(mn);
                if (m == null) {
                    throw new IllegalArgumentException("test method " + mn + " not found");
                }
                list.add(m);
            }

            for (Method m : list) {
                runTest(m, f);
            }
        }
    }

    protected boolean isTestMethod(Method m) {
        return m.getAnnotation(Test.class) != null;
    }

    protected Object[] getTestArgs(Method m) throws IllegalArgumentException {
        Class<?>[] paramTypes = m.getParameterTypes();
        if (paramTypes.length == 0) {
            return new Object[] {};
        } else if (paramTypes.length == 1 && paramTypes[0] == Path.class) {
            return new Object[] { Path.of(m.getName())};
        } else {
            throw new IllegalArgumentException("unknown signature for method "
                    + m + Stream.of(paramTypes)
                    .map(Class::toString)
                    .collect(Collectors.joining(", ", "(", ")")))  ;
        }
    }

    protected void runTest(Method m, Function<Method, Object[]> f) throws Exception {
        try {
            out.println("Running test " + m.getName());
            m.invoke(this, f.apply(m));
        } catch (InvocationTargetException e) {
            Throwable cause = e.getCause();
            throw (cause instanceof Exception) ? ((Exception) cause) : e;
        }

    }

    /**
     * Runs javadoc.
     * The output directory used by this call and the final exit code
     * will be saved for later use.
     * To aid the reader, it is recommended that calls to this method
     * put each option and the arguments it takes on a separate line.
     *
     * Example:
     * {@snippet :
     *  javadoc("-d", "out",
     *          "-sourcepath", testSrc,
     *          "-notimestamp",
     *          "pkg1", "pkg2", "pkg3/C.java");
     * }
     *
     * @param args the arguments to pass to javadoc
     */
    public void javadoc(String... args) {
        outputMap.clear();
        fileContentCache.clear();

        javadocRunNum++;
        javadocTestNum = 0; // reset counter for this run of javadoc
        if (javadocRunNum == 1) {
            out.println("Running javadoc...");
        } else {
            out.println("Running javadoc (run "+ javadocRunNum + ")...");
        }

        outputDir = Path.of(".");
        String charsetArg = null;
        String docencodingArg = null;
        String encodingArg = null;
        for (int i = 0; i < args.length - 2; i++) {
            switch (args[i]) {
                case "-d" -> outputDir = Path.of(args[++i]);
                case "-charset" -> charsetArg = args[++i];
                case "-docencoding" -> docencodingArg = args[++i];
                case "-encoding" -> encodingArg = args[++i];
            }
        }

        // The following replicates HtmlConfiguration.finishOptionSettings0
        // and sets up the charset used to read files.
        String cs;
        if (docencodingArg == null) {
            if (charsetArg == null) {
                cs = (encodingArg == null) ? "UTF-8" : encodingArg;
            } else {
                cs = charsetArg;
            }
        } else {
            cs = docencodingArg;
        }
        try {
            charset = Charset.forName(cs);
        } catch (UnsupportedCharsetException e) {
            charset = Charset.defaultCharset();
        }

        out.println("args: " + Arrays.toString(args));
//        log.setOutDir(outputDir);

        outputDirectoryCheck.check(outputDir);

        // This is the sole stream normally used by javadoc
        WriterOutput outOut = new WriterOutput();

        // These are to catch output to System.out and System.err,
        // in case these are used instead of the primary streams
        StreamOutput sysOut = new StreamOutput(System.out, System::setOut);
        StreamOutput sysErr = new StreamOutput(System.err, System::setErr);

        try {
            jdk.javadoc.internal.tool.Main main = new jdk.javadoc.internal.tool.Main();
            if (useStandardStreams) {
                // use sysOut, sysErr
            } else {
                // default: use single explicit stream
                main.setStreams(outOut.pw, outOut.pw);
            }
            if (fileManager != null) {
                main.setFileManager(fileManager);
            }
            exitCode = main.run(args).exitCode;
        } finally {
            outputMap.put(Output.STDOUT, sysOut.close());
            outputMap.put(Output.STDERR, sysErr.close());
            outputMap.put(Output.OUT, outOut.close());
        }

        outputMap.forEach((name, text) -> {
            if (!text.isEmpty()) {
                out.println("javadoc " + name + ":");
                out.println(text);
            }
        });

        if (exitCode == Exit.OK.code && Files.exists(outputDir)) {
            if (automaticCheckLinks) {
                checkLinks();
            }
            if (automaticCheckAccessibility) {
                checkAccessibility();
            }
            if (automaticCheckUniqueOUT) {
                checkUnique(Output.OUT, "^[A-Z][a-z]+ing ", true);
            }
        }
    }

    /**
     * Sets the kind of check for the initial contents of the output directory
     * before javadoc is run.
     * The filter should return true for files that should <b>not</b> appear.
     *
     * @param c the kind of check to perform
     */
    public void setOutputDirectoryCheck(DirectoryCheck c) {
        outputDirectoryCheck = c;
    }

    /**
     * Sets whether or not to perform an automatic call of checkAccessibility.
     */
    public void setAutomaticCheckAccessibility(boolean b) {
        automaticCheckAccessibility = b;
    }

    /**
     * Sets whether or not to perform an automatic call of checkLinks.
     */
    public void setAutomaticCheckLinks(boolean b) {
        automaticCheckLinks = b;
    }

    /**
     * Sets whether or not to perform an automatic call of checkUnique(OUT).
     */
    public void setAutomaticCheckUniqueOUT(boolean b) {
        automaticCheckUniqueOUT = b;
    }

    /**
     * Sets whether to use standard output streams (stdout and stderr)
     * instead of a single temporary stream.
     * Tests using standard streams should generally take care to avoid
     * conflicting use of stdout and stderr.
     */
    public void setUseStandardStreams(boolean b) {
        useStandardStreams = b;
    }

    /**
     * Sets the file manager to use for subsequent invocations of javadoc.
     * If {@code null}, a default file manager will be created and used
     * for each invocation.
     */
    public void setFileManager(StandardJavaFileManager fm) {
        fileManager = fm;
    }

    /**
     * The exit codes returned by the javadoc tool.
     * @see jdk.javadoc.internal.tool.Main.Result
     */
    public enum Exit {
        OK(0),        // Javadoc completed with no errors.
        ERROR(1),     // Completed but reported errors.
        CMDERR(2),    // Bad command-line arguments
        SYSERR(3),    // System error or resource exhaustion.
        ABNORMAL(4);  // Javadoc terminated abnormally

        Exit(int code) {
            this.code = code;
        }

        final int code;

        @Override
        public String toString() {
            return name() + '(' + code + ')';
        }
    }

    /**
     * Checks the exit code of the most recent call of javadoc.
     *
     * @param expected the exit code that is required for the test to pass
     */
    public void checkExit(Exit expected) {
        checking("check exit code");
        if (exitCode == expected.code) {
            passed("return code " + exitCode);
        } else {
            failed("return code " + exitCode +"; expected " + expected);
        }
    }

    /**
     * Checks for content in (or not in) the generated output.
     * Within the search strings, the newline character \n
     * will be translated to the platform newline character sequence.
     *
     * @param path          a path within the most recent output directory
     *                      or the name of one of the output buffers, identifying
     *                      where to look for the search strings.
     * @param expectedFound true if all of the search strings are expected
     *                      to be found, or false if the file is not expected to be found
     * @param strings       the strings to be searched for
     */
    public void checkFileAndOutput(String path, boolean expectedFound, String... strings) {
        if (expectedFound) {
            checkOutput(path, true, strings);
        } else {
            checkFiles(false, path);
        }
    }

    /**
     * Checks for content in (or not in) the generated output.
     * Within the search strings, the newline character \n
     * will be translated to the platform newline character sequence.
     *
     * @param path          a path within the most recent output directory, identifying
     *                      where to look for the search strings.
     * @param expectedFound true if all of the search strings are expected
     *                      to be found, or false if all of the strings are expected to be
     *                      not found
     * @param strings       the strings to be searched for
     *
     * @see OutputChecker#check(String...)
     */
    public void checkOutput(String path, boolean expectedFound, String... strings) {
        new OutputChecker(path)
                .setExpectFound(expectedFound)
                .setExpectOrdered(false) // TODO, fix tests (32 failures) and change to true
                .check(strings);
    }

    /**
     * Checks for content in (or not in) the one of the output streams written by
     * javadoc. Within the search strings, the newline character \n
     * will be translated to the platform newline character sequence.
     *
     * @param output        the output stream to check
     * @param expectedFound true if all of the search strings are expected
     *                      to be found, or false if all of the strings are expected to be
     *                      not found
     * @param strings       the strings to be searched for
     *
     * @see OutputChecker#check(String...)
     */
    public void checkOutput(Output output, boolean expectedFound, String... strings) {
        new OutputChecker(output)
                .setExpectFound(expectedFound)
                .setExpectOrdered(false) // TODO, fix tests (6 failures) and change to true
                .check(strings);
    }

    /**
     * Checks that there are no duplicate lines in one of the streams written by javadoc.
     *
     * @param output the output stream to check
     *
     * @see OutputChecker#checkUnique()
     */
    public void checkUnique(Output output) {
        checkUnique(output, ".*", true);
    }

    /**
     * Checks that there are no duplicate lines that either match or don't match a given pattern,
     * in one of the streams written by javadoc.
     *
     * @param output  the output stream to check
     * @param pattern a pattern to filter the lines to be checked
     * @param select  if {@code true}, lines that match the pattern will be checked for uniqueness;
     *                if {@code false}, lines that do not match the pattern will be checked
     *
     * @see OutputChecker#checkUnique(Pattern, boolean)
     */
    public void checkUnique(Output output, String pattern, boolean select) {
        new OutputChecker(output).checkUnique(Pattern.compile(pattern), select);
    }

    /**
     * Ensures that a series of strings appear only once, in the generated output.
     * Note: this test does not exhaustively check for all other possible
     * duplicates once one is found.
     *
     * @param path    the file to check
     * @param strings the strings
     *
     * @see OutputChecker#checkUnique(String...)
     */
    public void checkUnique(String path, String... strings) {
        new OutputChecker(path).checkUnique(strings);
    }

    /**
     * Performs some structural accessibility checks on the files generated by the most
     * recent run of javadoc.
     * The checks can be run automatically by calling {@link #setAutomaticCheckAccessibility}.
     */
    public void checkAccessibility() {
        checking("Check accessibility");
        A11yChecker c = new A11yChecker(out, this::readFile);
        try {
            c.checkDirectory(outputDir);
            c.report();
            int errors = c.getErrorCount();
            if (errors == 0) {
                passed("No accessibility errors found");
            } else {
                failed(errors + " errors found when checking accessibility");
            }
        } catch (IOException e) {
            failed("exception thrown when reading files: " + e);
        }
    }

    /**
     * Checks all the links within the files generated by the most
     * recent run of javadoc.
     * The checks can be run automatically by calling {@link #setAutomaticCheckLinks}.
     */
    public void checkLinks() {
        checking("Check links");
        LinkChecker c = new LinkChecker(out, this::readFile);
        try {
            c.checkDirectory(outputDir);
            c.report();
            int errors = c.getErrorCount();
            if (errors == 0) {
                passed("Links are OK");
            } else {
                failed(errors + " errors found when checking links");
            }
        } catch (IOException e) {
            failed("exception thrown when reading files: " + e);
        }
    }

    /**
     * Shows the heading structure for each of the specified files.
     * The structure is is printed in plain text to the main output stream.
     * No errors are reported (unless there is a problem reading a file)
     * but missing headings are noted within the output.
     *
     * @param paths the files
     */
    public void showHeadings(String... paths) {
        ShowHeadings s = new ShowHeadings(out, this::readFile);
        for (String p : paths) {
            try {
                Path f = outputDir.resolve(p);
                s.checkFiles(List.of(f), false, Collections.emptySet());
            } catch (IOException e) {
                checking("Read file");
                failed("Error reading file: " + e);
            }
        }
    }

    /**
     * Returns the content of one of the output streams written by javadoc.
     *
     * @param output the name of the output stream
     * @return the content of the output stream
     */
    public String getOutput(Output output) {
        return outputMap.get(output);
    }

    /**
     * Returns the content of one of the output streams written by javadoc.
     *
     * @param output the name of the output stream
     * @return the content of the output stream, as a line of lines
     */
    public List<String> getOutputLines(Output output) {
        String text = outputMap.get(output);
        return (text == null) ? Collections.emptyList() : Arrays.asList(text.split(NL));
    }

    /**
     * Checks for files in (or not in) the generated output.
     *
     * @param expectedFound true if all of the files are expected
     *                      to be found, or false if all of the files are expected to be
     *                      not found
     * @param paths         the files to check, within the most recent output directory.
     */
    public void checkFiles(boolean expectedFound, String... paths) {
        checkFiles(expectedFound, Arrays.asList(paths));
    }

    /**
     * Checks for files in (or not in) the generated output.
     *
     * @param expectedFound true if all of the files are expected to be found,
     *                      or false if all of the files are expected to be not found
     * @param paths         the files to check, within the most recent output directory.
     */
    public void checkFiles(boolean expectedFound, Collection<String> paths) {
        for (String path: paths) {
//            log.logCheckFile(path, expectedFound);
            checking("checkFile");
            Path file = outputDir.resolve(path);
            boolean isFound = Files.exists(file);
            if (isFound == expectedFound) {
                passed(file, "file " + (isFound ? "found:" : "not found:") + "\n");
            } else {
                failed(file, "file " + (isFound ? "found:" : "not found:") + "\n");
            }
        }
    }

    /**
     * Checks that a series of strings are found in order in a file in
     * the generated output.
     *
     * @param path    the file to check
     * @param strings the strings whose order to check
     *
     * @see OutputChecker#check(String...)
     */
    public void checkOrder(String path, String... strings) {
        new OutputChecker(path)
                .setExpectOrdered(true) // be explicit
                .check(strings);
    }

    /**
     * Compares a set of files in each of two directories.
     *
     * @param baseDir1 the directory containing the first set of files
     * @param baseDir2 the directory containing the second set of files
     * @param files    the set of files to be compared
     */
    public void diff(String baseDir1, String baseDir2, String... files) {
        Path bd1 = Path.of(baseDir1);
        Path bd2 = Path.of(baseDir2);
        for (String file : files) {
            diff(bd1, bd2, Path.of(file));
        }
    }

    /**
     * Read a file from the output directory.
     *
     * @param fileName  the name of the file to read
     * @return          the file in string format
     */
    public String readOutputFile(String fileName) throws Error {
        return readFile(outputDir, Path.of(fileName));
    }

    protected String readFile(String fileName) throws Error {
        return readFile(outputDir, Path.of(fileName));
    }

    protected String readFile(String baseDir, String fileName) throws Error {
        return readFile(Path.of(baseDir), Path.of(fileName));
    }

    protected String readFile(Path file) {
        Path baseDir;
        if (file.startsWith(outputDir)) {
            baseDir = outputDir;
        } else if (file.startsWith(currDir)) {
            baseDir = currDir;
        } else {
            baseDir = file.getParent();
        }
        Path fileName = baseDir.relativize(file);
        return readFile(baseDir, fileName);
    }

    /**
     * Reads the file and return it as a string.
     *
     * @param baseDir   the directory in which to locate the file
     * @param fileName  the name of the file to read
     * @return          the file in string format
     */
    private String readFile(Path baseDir, Path fileName) throws Error {
        if (!Objects.equals(fileContentCacheCharset, charset)) {
            fileContentCache.clear();
            fileContentCacheCharset = charset;
        }
        try {
            Path file = baseDir.resolve(fileName);
            SoftReference<String> ref = fileContentCache.get(file);
            String content = (ref == null) ? null : ref.get();
            if (content != null)
                return content;

            // charset defaults to a value inferred from latest javadoc run
            content = new String(Files.readAllBytes(file), charset);
            fileContentCache.put(file, new SoftReference<>(content));
            return content;
        } catch (FileNotFoundException | NoSuchFileException e) {
            throw new Error("File not found: " + fileName + ": " + e, e);
        } catch (IOException e) {
            throw new Error("Error reading file: " + fileName + ": " + e, e);
        }
    }

    /**
     * Starts a check.
     *
     * <p>This method should be called before subsequently calling {@code pass(...)}
     * or {@code fail(...)}.
     *
     * @param message a short description of the check
     */
    protected void checking(String message) {
        numTestsRun++;
        javadocTestNum++;
        print("Starting subtest " + javadocRunNum + "." + javadocTestNum, message);
    }

    /**
     * Concludes a check for a file, reporting that the check succeeded.
     *
     * <p>This method should be called after previously calling {@code checking(...)}.
     *
     * @param file the file that was the focus of the check
     * @param message a short description of the outcome
     */
    protected void passed(Path file, String message) {
        passed(file + ": " + message);
    }

    /**
     * Concludes a check, reporting that the check succeeded.
     *
     * <p>This method should be called after previously calling {@code checking(...)}.
     *
     * @param message a short description of the outcome
     */
    protected void passed(String message) {
        numTestsPassed++;
        print("Passed", message);
        out.println();
    }

    /**
     * Concludes a check for a file, reporting that the check failed.
     *
     * <p>This method should be called after previously calling {@code checking(...)}.
     *
     * @param file the file that was the focus of the check
     * @param message a short description of the outcome
     */
    protected void failed(Path file, String message) {
        failed(file + ": " + message);
    }

    /**
     * Concludes a check for a file, reporting that the check failed.
     *
     * <p>This method should be called after previously calling {@code checking(...)}.
     *
     * @param message a short description of the outcome
     */
    protected void failed(String message) {
        print("FAILED", message);
        StackWalker.getInstance().walk(s -> {
            s.dropWhile(f -> f.getMethodName().equals("failed"))
                    .takeWhile(f -> !f.getMethodName().equals("runTests"))
                    .forEach(f -> out.println("        at "
                            + f.getClassName() + "." + f.getMethodName()
                            + "(" + f.getFileName() + ":" + f.getLineNumber() + ")"));
            return null;
        });
        out.println();
    }

    private void print(String prefix, String message) {
        if (message.isEmpty())
            out.println(prefix);
        else {
            out.print(prefix);
            out.print(": ");
            out.print(message.replace("\n", NL));
            if (!(message.endsWith("\n") || message.endsWith(NL))) {
                out.println();
            }
        }
    }

    /**
     * Prints a summary of the test results.
     */
    protected void printSummary() {
        String javadocRuns = (javadocRunNum <= 1) ? ""
                : ", in " + javadocRunNum + " runs of javadoc";

        if (numTestsRun != 0 && numTestsPassed == numTestsRun) {
            // Test passed
            out.println();
            out.println("All " + numTestsPassed + " subtests passed" + javadocRuns);
        } else {
            // Test failed
            throw new Error((numTestsRun - numTestsPassed)
                    + " of " + (numTestsRun)
                    + " subtests failed"
                    + javadocRuns);
        }
    }

    /**
     * Compares the two given files.
     *
     * @param baseDir1 the directory in which to locate the first file
     * @param baseDir2 the directory in which to locate the second file
     * @param file the file to compare in the two base directories
     */
    private void diff(Path baseDir1, Path baseDir2, Path file) {
        String file1Contents = readFile(baseDir1, file);
        String file2Contents = readFile(baseDir2, file);
        checking("diff " + baseDir1.resolve(file) + ", " + baseDir2.resolve(file));
        if (file1Contents.trim().compareTo(file2Contents.trim()) == 0) {
            passed("files are equal");
        } else {
            failed("files differ");
        }
    }

    /**
     * A flexible checker for checking the content of generated files and output streams.
     *
     * Configuration can be done with a series of chained method calls.
     * Checks can be specified as either literal strings or regular expressions.
     */
    public class OutputChecker {
        private final String name;
        private final String content;
        private boolean allowOverlaps = false;
        private boolean expectFound = true;
        private boolean expectOrdered = true;
        private List<Range> matches = new ArrayList<>();
        private Range lastMatch;

        private enum SearchKind {
            TEXT, PATTERN;
            @Override
            public String toString() {
                return name().toLowerCase(Locale.ROOT);
            }
        }

        /** A half-open interval {@code [start, end)} to record the position of a match. */
        record Range(int start, int end) {
            static Range of(int start, int end) {
                return new Range(start, end);
            }
            boolean overlaps(Range other) {
                // Intervals do not overlap if one interval is completely before or completely after the other:
                // that is,    other.end <= start || end <= other.start
                // Invert that for when intervals do overlap, and simplify to the following expression:
                return other.end > start && end > other.start;
            }
            String toIntervalString() {
                return "[" + start + "," + end + ")";
            }
        }

        /**
         * Creates an output checker for a file written by the most recent run of javadoc.
         * If the file cannot be found or there is any other error while reading the file,
         * an error will be reported and all subsequent {@code check...} methods will be skipped
         *
         * @param file the file
         */
        public OutputChecker(String file) {
            String c = null;
            try {
                c = readFile(file);
            } catch (Error e) {
                JavadocTester.this.checking("Read file " + file);
                if (e.getCause() instanceof IOException) {
                    // exception probably thrown (with known message) by readFile
                    failed(e.getMessage());
                } else {
                    failed("Error reading file: " + e);
                }
            }

            if (c == null) {
                name = null;
                content = null;
            } else {
                name = file;
                content = c;
            }
        }

        /**
         * Creates an output checker for an output stream written by the most recent run of javadoc.
         *
         * @param output the output
         */
        public OutputChecker(Output output) {
            name = output.name();
            content = getOutput(output);
        }

        /**
         * Specifies whether matches are expected to be found or not.
         * The default is {@code true}.
         *
         * @param expectFound whether matches are expected to be found
         * @return this object
         */
        public OutputChecker setExpectFound(boolean expectFound) {
            this.expectFound = expectFound;
            return this;
        }

        /**
         * Specifies whether matches are expected to be found in order or not.
         * The default is {@code true}.
         *
         * @param expectOrdered  whether matches should be ordered
         * @return this object
         */
        public OutputChecker setExpectOrdered(boolean expectOrdered) {
            this.expectOrdered = expectOrdered;
            return this;
        }

        /**
         * Specifies whether matches are allowed to overlap.
         * The default is {@code false}.
         *
         * @param allowOverlaps whether matches may overlap
         * @return this object
         */
        public OutputChecker setAllowOverlaps(boolean allowOverlaps) {
            this.allowOverlaps = allowOverlaps;
            return this;
        }

        /**
         * Checks for the presence (or absence) of a series of strings.
         * Within the search strings, the newline character {@code \n}
         * will be translated to the platform newline character sequence.
         *
         * @param strings the strings to be searched for
         */
        public OutputChecker check(String... strings) {
            if (name == null) {
                out.println("Skipping checks for:" + NL
                        + List.of(strings).stream()
                        .map(s -> "    " + toShortString(s))
                        .collect(Collectors.joining(NL)));
                return this;
            }

            for (String stringToFind : strings) {
                check(startPos -> findString(stringToFind, startPos), SearchKind.TEXT, stringToFind);
            }
            return this;
        }

        /**
         * Checks for the presence (or absence) of a series of regular expressions.
         * Unlike {@link #check(String...)}, there is no special handling for
         * newline characters. Use {@code \R} to match the platform newline sequence.
         *
         * @param patterns the regular expressions to be searched for
         */
        public OutputChecker check(Pattern... patterns) {
            if (name == null) {
                out.println("Skipping checks for:" + NL
                        + List.of(patterns).stream()
                        .map(p -> "    " + toShortString(p.pattern()))
                        .collect(Collectors.joining(NL)));
                return this;
            }
            for (Pattern pattern : patterns) {
                check(startPos -> findPattern(pattern, startPos), SearchKind.PATTERN, pattern.pattern());
            }
            return this;
        }

        /**
         * Checks for the presence (or absence) of an item.
         *
         * @param finder a function to find the next occurrence of an item starting at a given position
         * @param kind   the kind of the item ({@code "text"} or {@code "pattern:} to include in messages
         * @param s      a string for the item, to be included in messages
         */
        private void check(Function<Integer, Range> finder, SearchKind kind, String s) {
            checking("checkOutput", kind);
            int start = getStart();
            Range r = finder.apply(start);
            boolean isFound = r != null;
            if (isFound == expectFound) {
                matches.add(lastMatch = r);
                passed(name + ": following " + kind + " " + (isFound ? "found:" : "not found:") + "\n"
                        + s);
            } else {
                // item not found in order, so check if the item is found out of order, to determine the best message
                if (expectFound && expectOrdered && start > 0) {
                    Range r2 = finder.apply(0);
                    if (r2 != null) {
                        failed(name + ": following " + kind + " was found on line "
                                + getLineNumber(r2.start)
                                + ", but not in order as expected, on or after line "
                                + getLineNumber(start)
                                + ":\n"
                                + s);
                        return;
                    }
                }
                failed(name + ": following " + kind + " "
                        + (isFound ? "found:" : "not found:") + "\n"
                        + s + '\n' + "found \n" + content);
            }

        }

        /**
         * Checks that there are no duplicate lines in the content.
         */
        public OutputChecker checkUnique() {
            checkUnique(Pattern.compile(".*"), true);
            return this;
        }

        /**
         * Checks that there are no duplicate lines that either match or don't match a given pattern,
         * in one of the streams written by javadoc.
         *
         * @param pattern a pattern to filter the lines to be checked
         * @param select  if {@code true}, lines that match the pattern will be checked for uniqueness;
         *                if {@code false}, lines that do not match the pattern will be checked
         */
        public OutputChecker checkUnique(Pattern pattern, boolean select ) {
            if (name == null) {
                out.println("Skipping checkUnique");
                return this;
            }

            checking("checkUnique", SearchKind.PATTERN);
            Matcher m = pattern.matcher("");
            Map<String, Integer> linesSofar = new HashMap<>();
            int lineNumber = 0;
            int duplicates = 0;
            for (String line : content.split(NL)) {
                m.reset(line);
                if (m.find() == select) {
                    Integer prev = linesSofar.putIfAbsent(line, ++lineNumber);
                    if (prev != null) {
                        out.println("duplicate line detected on line " + lineNumber
                                + "; first occurrence on line " + prev);
                        out.println("line: " + line);
                        duplicates++;
                    }
                }
            }
            if (duplicates == 0) {
                passed("All lines are unique");
            } else {
                failed(duplicates + " duplicate lines found");
            }
            return this;
        }

        /**
         * Checks that each of a series of strings appears only once in the generated output.
         * Note: this test does not exhaustively check for all other possible duplicates once one is found.
         *
         * @param strings the strings
         */
        public OutputChecker checkUnique(String... strings) {
            return checkUnique(SearchKind.TEXT, List.of(strings), this::findString);
        }

        /**
         * Checks that each of a series of pattern matches appears only once in the generated output.
         * Note: this test does not exhaustively check for all other possible duplicates once one is found.
         *
         * @param patterns the patterns
         */
        public OutputChecker checkUnique(Pattern... patterns) {
            return checkUnique(SearchKind.PATTERN, List.of(patterns), this::findPattern);
        }

        private <T> OutputChecker checkUnique(SearchKind kind, List<T> items, BiFunction<T, Integer, Range> finder) {
            if (name == null) {
                out.println("Skipping checkUnique");
                return this;
            }

            Range latest = null;
            for (T item : items) {
                int start = getStart();
                Range r = finder.apply(item, start);
                checking("checkUnique at index " + start, SearchKind.TEXT);
                if (r == null) {
                    failed(name + ": " + item + " not found.");
                    continue;
                }
                // only update lastMatch for the initial match of each item
                if (lastMatch == null) {
                    lastMatch = r;
                }
                Range next = finder.apply(item, r.end);
                if (next == null) {
                    passed(name + ": " + item + " is unique");
                } else {
                    failed(name + ": " + item + " is not unique, found at " + next.start);
                }
            }
            if (latest != null) {
                lastMatch = latest;
            }
            return this;
        }

        /**
         * Checks that all the output has been matched by preceding checks with this object.
         * It does not matter whether the checks were ordered or not.
         * The results of the matches are sorted and then checked to be adjacent and to
         * cover the entire content.
         *
         * @apiNote This is probably most useful for checking diagnostic output,
         *          in which case care must be taken to allow for platform differences
         *          in the output, such as file separators and newline sequences.
         */
        public OutputChecker checkComplete() {
            if (name == null) {
                out.println("Skipping checkComplete");
                return this;
            }

            JavadocTester.this.checking("checking for complete coverage of output");
            List<Range> uncovered = new ArrayList<>();
            List<Range> list = new ArrayList<>(matches);
            list.sort(Comparator.comparing(Range::start));
            int prev = 0;
            for (Range r : list) {
                if (r.start != prev) {
                    uncovered.add(new Range(prev, r.start));
                }
                prev = r.end;
            }
            if (prev != content.length()) {
                uncovered.add(new Range(prev, content.length()));
            }
            if (uncovered.isEmpty()) {
                passed("All output matched");
            } else {
                failed("The following output was not matched: "
                    + uncovered.stream()
                        .map(Range::toIntervalString)
                        .collect(Collectors.joining(", ")));
            }
            return this;
        }

        /**
         * Checks that no output is present.
         */
        public OutputChecker checkEmpty() {
            if (name == null) {
                out.println("Skipping checkEmpty");
                return this;
            }

            JavadocTester.this.checking("empty");
            if (content == null || content.isEmpty()) {
                passed(name + " is empty, as expected");
            } else {
                failed(name + " is not empty; contains:\n"
                        + content);
            }
            return this;
        }

        /**
         * Checks that at least of a set of alternatives is found.
         */
        public OutputChecker checkAnyOf(String... strings) {
            return checkAnyOf(SearchKind.TEXT, List.of(strings), this::findString);
        }

        /**
         * Checks that at least of a set of alternatives is found.
         */
        public OutputChecker checkAnyOf(Pattern... patterns) {
            return checkAnyOf(SearchKind.PATTERN, List.of(patterns), this::findPattern);
        }

        /**
         * Checks that at least of a set of alternatives is found.
         *
         */
        private <T> OutputChecker checkAnyOf(SearchKind kind, List<T> items, BiFunction<T, Integer, Range> finder) {
            if (name == null) {
                out.println("Skipping checkAnyOf");
                return this;
            }

            checking("checkAnyOf", kind);
            Range earliest = null;
            int start = getStart();
            int count = 0;
            for (T item : items) {
                Range r = finder.apply(item, start);
                if (r != null) {
                    count++;
                    if (earliest == null || rangeComparator.compare(earliest, r) > 0) {
                        earliest = r;
                    }
                }
            }
            if (earliest != null) {
                lastMatch = earliest;
            }
            if (count == 0) {
                failed("no match found for any " + kind);
            } else {
                passed(count + " matches found; earliest is " + earliest.toIntervalString());
            }
            return this;
        }

        Comparator<Range> rangeComparator = Comparator.comparing(Range::start).thenComparing(Range::end);

        private void checking(String name, SearchKind kind) {
            JavadocTester.this.checking(name + " " + kind.name()
                + " allowOverlaps:" + allowOverlaps
                + " expectFound:"   + expectFound
                + " expectOrdered:" + expectOrdered);
        }

        private Range findString(String stringToFind, int start) {
            // javadoc (should) always use the platform newline sequence,
            // but in the strings to find it is more convenient to use the Java
            // newline character. So we translate \n to NL before we search.
            stringToFind = stringToFind.replace("\n", NL);
            int i = content.indexOf(stringToFind, start);
            return i >= 0 ? Range.of(i, i + stringToFind.length()) : null;
        }

        private Range findPattern(Pattern p, int start) {
            Matcher m = p.matcher(content);
            return m.find(start) ? Range.of(m.start(), m.end()) : null;
        }
        private int getStart() {
            if (lastMatch == null || !expectOrdered) {
                return 0;
            }
            return allowOverlaps ? lastMatch.start + 1 : lastMatch.end;
        }

        private int getLineNumber(int pos) {
            Pattern p = Pattern.compile("\\R");
            Matcher m = p.matcher(content);
            int line = 1;
            int start = 0;
            while (m.find(start) && m.start() < pos) {
                line++;
                start = m.start() + 1;
            }
            return line;
        }

        private String toShortString(String s) {
            final int MAX = 64;
            s = s.replaceAll("\\s+", " ");
            if (s.length() > MAX) {
                s = s.substring(0, MAX / 2 - 2) + " ... " + s.substring(s.length() - MAX / 2 - 2);
            }
            return s;
        }
    }

    /**
     * Utility class to simplify the handling of temporarily setting a
     * new stream for System.out or System.err.
     */
    private static class StreamOutput {
        // functional interface to set a stream.
        private interface Initializer {
            void set(PrintStream s);
        }

        private final ByteArrayOutputStream baos = new ByteArrayOutputStream();
        private final PrintStream ps = new PrintStream(baos);
        private final PrintStream prev;
        private final Initializer init;

        StreamOutput(PrintStream s, Initializer init) {
            prev = s;
            init.set(ps);
            this.init = init;
        }

        String close() {
            init.set(prev);
            ps.close();
            return baos.toString();
        }
    }

    /**
     * Utility class to simplify the handling of creating an in-memory PrintWriter.
     */
    private static class WriterOutput {
        private final StringWriter sw = new StringWriter();
        final PrintWriter pw = new PrintWriter(sw);
        String close() {
            pw.close();
            return sw.toString();
        }
    }


//    private final Logger log = new Logger();

    //--------- Logging --------------------------------------------------------
    //
    // This class writes out the details of calls to checkOutput and checkFile
    // in a canonical way, so that the resulting file can be checked against
    // similar files from other versions of JavadocTester using the same logging
    // facilities.

    static class Logger {
        private static final int PREFIX = 40;
        private static final int SUFFIX = 20;
        private static final int MAX = PREFIX + SUFFIX;
        List<String> tests = new ArrayList<>();
        Path outDir;
        Path rootDir = rootDir();

        static Path rootDir() {
            Path f = Path.of(".").toAbsolutePath();
            while (f != null && !Files.exists(f.resolve(".git")))
                f = f.getParent();
            return f;
        }

        void setOutDir(Path outDir) {
            this.outDir = outDir;
        }

        void logCheckFile(String file, boolean positive) {
            // Strip the outdir because that will typically not be the same
            Path p = Path.of(file);
            if (p.startsWith(outDir))
                p = p.relativize(outDir);
            tests.add(p + " " + positive);
        }

        void logCheckOutput(String file, boolean positive, String text) {
            // Compress the string to be displayed in the log file
            String simpleText = text.replaceAll("\\s+", " ").replace(rootDir.toString(), "[ROOT]");
            if (simpleText.length() > MAX)
                simpleText = simpleText.substring(0, PREFIX)
                        + "..." + simpleText.substring(simpleText.length() - SUFFIX);
            // Strip the outdir because that will typically not be the same
            Path p = Path.of(file);
            if (p.startsWith(outDir))
                p = p.relativize(outDir);
            // The use of text.hashCode ensure that all of "text" is taken into account
            tests.add(p + " " + positive + " " + text.hashCode() + " " + simpleText);
        }

        void write() {
            // sort the log entries because the subtests may not be executed in the same order
            tests.sort(Comparator.naturalOrder());
            try (var bw = Files.newBufferedWriter(Path.of("tester.log"))) {
                for (String t: tests) {
                    bw.write(t);
                    bw.newLine();
                }
            } catch (IOException e) {
                throw new Error("problem writing log: " + e);
            }
        }
    }

}