/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.tests.store;

import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.AccessDeniedException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.NoDeletionPolicy;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Lock;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.tests.util.ThrottledIndexOutput;
import org.apache.lucene.util.CollectionUtil;
import org.apache.lucene.util.IOUtils;

/**
 * This is a Directory Wrapper that adds methods intended to be used only by unit tests. It also
 * adds a number of features useful for testing:
 *
 * <ul>
 *   <li>Instances created by {@link LuceneTestCase#newDirectory()} are tracked to ensure they are
 *       closed by the test.
 *   <li>When a MockDirectoryWrapper is closed, it will throw an exception if it has any open files
 *       against it (with a stacktrace indicating where they were opened from).
 *   <li>When a MockDirectoryWrapper is closed, it runs CheckIndex to test if the index was
 *       corrupted.
 *   <li>MockDirectoryWrapper simulates some "features" of Windows, such as refusing to write/delete
 *       to open files.
 * </ul>
 */
public class MockDirectoryWrapper extends BaseDirectoryWrapper {
  long maxSize;

  // Max actual bytes used. This is set by MockRAMOutputStream:
  long maxUsedSize;
  double randomIOExceptionRate;
  double randomIOExceptionRateOnOpen;
  Random randomState;
  boolean assertNoDeleteOpenFile = false;
  boolean trackDiskUsage = false;
  boolean useSlowOpenClosers = LuceneTestCase.TEST_NIGHTLY;
  boolean allowRandomFileNotFoundException = true;
  boolean allowReadingFilesStillOpenForWrite = false;
  private Set<String> unSyncedFiles;
  private Set<String> createdFiles;
  private Set<String> openFilesForWrite = new HashSet<>();
  ConcurrentMap<String, RuntimeException> openLocks = new ConcurrentHashMap<>();
  volatile boolean crashed;
  private ThrottledIndexOutput throttledOutput;
  private Throttling throttling =
      LuceneTestCase.TEST_NIGHTLY ? Throttling.SOMETIMES : Throttling.NEVER;

  // for testing
  boolean alwaysCorrupt;

  final AtomicInteger inputCloneCount = new AtomicInteger();

  // use this for tracking files for crash.
  // additionally: provides debugging information in case you leave one open
  private Map<Closeable, Exception> openFileHandles =
      Collections.synchronizedMap(new IdentityHashMap<Closeable, Exception>());

  // NOTE: we cannot initialize the Map here due to the
  // order in which our constructor actually does this
  // member initialization vs when it calls super.  It seems
  // like super is called, then our members are initialized:
  private Map<String, Integer> openFiles;

  // Only tracked if noDeleteOpenFile is true: if an attempt
  // is made to delete an open file, we enroll it here.
  private Set<String> openFilesDeleted;

  private synchronized void init() {
    if (openFiles == null) {
      openFiles = new HashMap<>();
      openFilesDeleted = new HashSet<>();
    }

    if (createdFiles == null) createdFiles = new HashSet<>();
    if (unSyncedFiles == null) unSyncedFiles = new HashSet<>();
  }

  public MockDirectoryWrapper(Random random, Directory delegate) {
    super(delegate);
    // must make a private random since our methods are
    // called from different threads; else test failures may
    // not be reproducible from the original seed
    this.randomState = new Random(random.nextInt());
    this.throttledOutput =
        new ThrottledIndexOutput(
            ThrottledIndexOutput.mBitsToBytes(40 + randomState.nextInt(10)),
            1 + randomState.nextInt(5),
            null);
    init();
  }

  public int getInputCloneCount() {
    return inputCloneCount.get();
  }

  boolean verboseClone;

  /**
   * If set to true, we print a fake exception with filename and stacktrace on every indexinput
   * clone()
   */
  public void setVerboseClone(boolean v) {
    verboseClone = v;
  }

  public void setTrackDiskUsage(boolean v) {
    trackDiskUsage = v;
  }

  /**
   * If set to true (the default), when we throw random IOException on openInput or createOutput, we
   * may sometimes throw FileNotFoundException or NoSuchFileException.
   */
  public void setAllowRandomFileNotFoundException(boolean value) {
    allowRandomFileNotFoundException = value;
  }

  /** If set to true, you can open an inputstream on a file that is still open for writes. */
  public void setAllowReadingFilesStillOpenForWrite(boolean value) {
    allowReadingFilesStillOpenForWrite = value;
  }

  /**
   * Enum for controlling hard disk throttling. Set via {@link MockDirectoryWrapper
   * #setThrottling(Throttling)}
   *
   * <p>WARNING: can make tests very slow.
   */
  public enum Throttling {
    /** always emulate a slow hard disk. could be very slow! */
    ALWAYS,
    /** sometimes (0.5% of the time) emulate a slow hard disk. */
    SOMETIMES,
    /** never throttle output */
    NEVER
  }

  public void setThrottling(Throttling throttling) {
    this.throttling = throttling;
  }

  /**
   * Add a rare small sleep to catch race conditions in open/close
   *
   * <p>You can enable this if you need it.
   */
  public void setUseSlowOpenClosers(boolean v) {
    useSlowOpenClosers = v;
  }

  @Override
  public synchronized void sync(Collection<String> names) throws IOException {
    maybeYield();
    maybeThrowDeterministicException();
    if (crashed) {
      throw new IOException("cannot sync after crash");
    }
    // always pass thru fsync, directories rely on this.
    // 90% of time, we use DisableFsyncFS which omits the real calls.
    for (String name : names) {
      // randomly fail with IOE on any file
      maybeThrowIOException(name);
      in.sync(Collections.singleton(name));
      unSyncedFiles.remove(name);
    }
  }

  @Override
  public synchronized void rename(String source, String dest) throws IOException {
    maybeYield();
    maybeThrowDeterministicException();

    if (crashed) {
      throw new IOException("cannot rename after crash");
    }

    if (openFiles.containsKey(source) && assertNoDeleteOpenFile) {
      throw fillOpenTrace(
          new AssertionError(
              "MockDirectoryWrapper: source file \"" + source + "\" is still open: cannot rename"),
          source,
          true);
    }

    if (openFiles.containsKey(dest) && assertNoDeleteOpenFile) {
      throw fillOpenTrace(
          new AssertionError(
              "MockDirectoryWrapper: dest file \"" + dest + "\" is still open: cannot rename"),
          dest,
          true);
    }

    boolean success = false;
    try {
      in.rename(source, dest);
      success = true;
    } finally {
      if (success) {
        // we don't do this stuff with lucene's commit, but it's just for completeness
        if (unSyncedFiles.contains(source)) {
          unSyncedFiles.remove(source);
          unSyncedFiles.add(dest);
        }
        openFilesDeleted.remove(source);
        createdFiles.remove(source);
        createdFiles.add(dest);
      }
    }
  }

  @Override
  public synchronized void syncMetaData() throws IOException {
    maybeYield();
    maybeThrowDeterministicException();
    if (crashed) {
      throw new IOException("cannot sync metadata after crash");
    }
    in.syncMetaData();
  }

  public final synchronized long sizeInBytes() throws IOException {
    long size = 0;
    for (String file : in.listAll()) {
      // hack 2: see TODO in ExtrasFS (ideally it would always return 0 byte
      // size for extras it creates, even though the size of non-regular files is not defined)
      if (!file.startsWith("extra")) {
        size += in.fileLength(file);
      }
    }
    return size;
  }

  public synchronized void corruptUnknownFiles() throws IOException {
    if (LuceneTestCase.VERBOSE) {
      System.out.println("MDW: corrupt unknown files");
    }
    Set<String> knownFiles = new HashSet<>();
    for (String fileName : listAll()) {
      if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
        if (LuceneTestCase.VERBOSE) {
          System.out.println("MDW: read " + fileName + " to gather files it references");
        }
        SegmentInfos infos;
        try {
          infos = SegmentInfos.readCommit(this, fileName);
        } catch (IOException ioe) {
          if (LuceneTestCase.VERBOSE) {
            System.out.println(
                "MDW: exception reading segment infos "
                    + fileName
                    + "; files: "
                    + Arrays.toString(listAll()));
          }
          throw ioe;
        }
        knownFiles.addAll(infos.files(true));
      }
    }

    Set<String> toCorrupt = new HashSet<>();
    Matcher m = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
    for (String fileName : listAll()) {
      m.reset(fileName);
      if (knownFiles.contains(fileName) == false
          && fileName.endsWith("write.lock") == false
          && (m.matches() || fileName.startsWith(IndexFileNames.PENDING_SEGMENTS))) {
        toCorrupt.add(fileName);
      }
    }

    corruptFiles(toCorrupt);
  }

  public synchronized void corruptFiles(Collection<String> files) throws IOException {
    boolean disabled = TestUtil.disableVirusChecker(in);
    try {
      _corruptFiles(files);
    } finally {
      if (disabled) {
        TestUtil.enableVirusChecker(in);
      }
    }
  }

  private synchronized void _corruptFiles(Collection<String> files) throws IOException {

    // TODO: we should also mess with any recent file renames, file deletions, if
    // syncMetaData was not called!!

    // Must make a copy because we change the incoming unsyncedFiles
    // when we create temp files, delete, etc., below:
    final List<String> filesToCorrupt = new ArrayList<>(files);
    // sort the files otherwise we have reproducibility issues
    // across JVMs if the incoming collection is a hashSet etc.
    CollectionUtil.timSort(filesToCorrupt);
    for (String name : filesToCorrupt) {
      int damage = randomState.nextInt(6);
      if (alwaysCorrupt && damage == 3) {
        damage = 4;
      }
      String action = null;

      switch (damage) {
        case 0:
          action = "deleted";
          deleteFile(name);
          break;

        case 1:
          action = "zeroed";
          // Zero out file entirely
          long length;
          try {
            length = fileLength(name);
          } catch (IOException ioe) {
            throw new RuntimeException(
                "hit unexpected IOException while trying to corrupt file " + name, ioe);
          }

          // Delete original and write zeros back:
          deleteFile(name);

          byte[] zeroes = new byte[256];
          long upto = 0;
          try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) {
            while (upto < length) {
              final int limit = (int) Math.min(length - upto, zeroes.length);
              out.writeBytes(zeroes, 0, limit);
              upto += limit;
            }
          } catch (IOException ioe) {
            throw new RuntimeException(
                "hit unexpected IOException while trying to corrupt file " + name, ioe);
          }
          break;

        case 2:
          {
            action = "partially truncated";
            // Partially Truncate the file:

            // First, make temp file and copy only half this
            // file over:
            String tempFileName = null;
            try (IndexOutput tempOut =
                    in.createTempOutput(
                        "name", "mdw_corrupt", LuceneTestCase.newIOContext(randomState));
                IndexInput ii = in.openInput(name, LuceneTestCase.newIOContext(randomState))) {
              tempFileName = tempOut.getName();
              tempOut.copyBytes(ii, ii.length() / 2);
            } catch (IOException ioe) {
              throw new RuntimeException(
                  "hit unexpected IOException while trying to corrupt file " + name, ioe);
            }

            // Delete original and copy bytes back:
            deleteFile(name);

            try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState));
                IndexInput ii =
                    in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState))) {
              out.copyBytes(ii, ii.length());
            } catch (IOException ioe) {
              throw new RuntimeException(
                  "hit unexpected IOException while trying to corrupt file " + name, ioe);
            }
            deleteFile(tempFileName);
          }
          break;

        case 3:
          // The file survived intact:
          action = "didn't change";
          break;

        case 4:
          // Corrupt one bit randomly in the file:

          {
            String tempFileName = null;
            try (IndexOutput tempOut =
                    in.createTempOutput(
                        "name", "mdw_corrupt", LuceneTestCase.newIOContext(randomState));
                IndexInput ii = in.openInput(name, LuceneTestCase.newIOContext(randomState))) {
              tempFileName = tempOut.getName();
              if (ii.length() > 0) {
                // Copy first part unchanged:
                long byteToCorrupt = (long) (randomState.nextDouble() * ii.length());
                if (byteToCorrupt > 0) {
                  tempOut.copyBytes(ii, byteToCorrupt);
                }

                // Randomly flip one bit from this byte:
                byte b = ii.readByte();
                int bitToFlip = randomState.nextInt(8);
                b = (byte) (b ^ (1 << bitToFlip));
                tempOut.writeByte(b);

                action =
                    "flip bit "
                        + bitToFlip
                        + " of byte "
                        + byteToCorrupt
                        + " out of "
                        + ii.length()
                        + " bytes";

                // Copy last part unchanged:
                long bytesLeft = ii.length() - byteToCorrupt - 1;
                if (bytesLeft > 0) {
                  tempOut.copyBytes(ii, bytesLeft);
                }
              } else {
                action = "didn't change";
              }
            } catch (IOException ioe) {
              throw new RuntimeException(
                  "hit unexpected IOException while trying to corrupt file " + name, ioe);
            }

            // Delete original and copy bytes back:
            deleteFile(name);

            try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState));
                IndexInput ii =
                    in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState))) {
              out.copyBytes(ii, ii.length());
            } catch (IOException ioe) {
              throw new RuntimeException(
                  "hit unexpected IOException while trying to corrupt file " + name, ioe);
            }

            deleteFile(tempFileName);
          }
          break;

        case 5:
          action = "fully truncated";
          // Totally truncate the file to zero bytes
          deleteFile(name);

          try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) {
            out.getFilePointer(); // just fake access to prevent compiler warning
          } catch (IOException ioe) {
            throw new RuntimeException(
                "hit unexpected IOException while trying to corrupt file " + name, ioe);
          }
          break;

        default:
          throw new AssertionError();
      }

      if (LuceneTestCase.VERBOSE) {
        System.out.println("MockDirectoryWrapper: " + action + " unsynced file: " + name);
      }
    }
  }

  /** Simulates a crash of OS or machine by overwriting unsynced files. */
  public synchronized void crash() throws IOException {
    openFiles = new HashMap<>();
    openFilesForWrite = new HashSet<>();
    openFilesDeleted = new HashSet<>();
    // first force-close all files, so we can corrupt on windows etc.
    // clone the file map, as these guys want to remove themselves on close.
    Map<Closeable, Exception> m = new IdentityHashMap<>(openFileHandles);
    for (Closeable f : m.keySet()) {
      try {
        f.close();
      } catch (
          @SuppressWarnings("unused")
          Exception ignored) {
      }
    }
    corruptFiles(unSyncedFiles);
    crashed = true;
    unSyncedFiles = new HashSet<>();
  }

  public synchronized void clearCrash() {
    crashed = false;
    openLocks.clear();
  }

  public void setMaxSizeInBytes(long maxSize) {
    this.maxSize = maxSize;
  }

  public long getMaxSizeInBytes() {
    return this.maxSize;
  }

  /** Returns the peek actual storage used (bytes) in this directory. */
  public long getMaxUsedSizeInBytes() {
    return this.maxUsedSize;
  }

  public void resetMaxUsedSizeInBytes() throws IOException {
    this.maxUsedSize = sizeInBytes();
  }

  /** Trip a test assert if there is an attempt to delete an open file. */
  public void setAssertNoDeleteOpenFile(boolean value) {
    this.assertNoDeleteOpenFile = value;
  }

  public boolean getAssertNoDeleteOpenFile() {
    return assertNoDeleteOpenFile;
  }

  /**
   * If 0.0, no exceptions will be thrown. Else this should be a double 0.0 - 1.0. We will randomly
   * throw an IOException on the first write to an OutputStream based on this probability.
   */
  public void setRandomIOExceptionRate(double rate) {
    randomIOExceptionRate = rate;
  }

  public double getRandomIOExceptionRate() {
    return randomIOExceptionRate;
  }

  /**
   * If 0.0, no exceptions will be thrown during openInput and createOutput. Else this should be a
   * double 0.0 - 1.0 and we will randomly throw an IOException in openInput and createOutput with
   * this probability.
   */
  public void setRandomIOExceptionRateOnOpen(double rate) {
    randomIOExceptionRateOnOpen = rate;
  }

  public double getRandomIOExceptionRateOnOpen() {
    return randomIOExceptionRateOnOpen;
  }

  void maybeThrowIOException(String message) throws IOException {
    if (randomState.nextDouble() < randomIOExceptionRate) {
      IOException ioe =
          new IOException("a random IOException" + (message == null ? "" : " (" + message + ")"));
      if (LuceneTestCase.VERBOSE) {
        System.out.println(
            Thread.currentThread().getName()
                + ": MockDirectoryWrapper: now throw random exception"
                + (message == null ? "" : " (" + message + ")"));
        ioe.printStackTrace(System.out);
      }
      throw ioe;
    }
  }

  void maybeThrowIOExceptionOnOpen(String name) throws IOException {
    if (randomState.nextDouble() < randomIOExceptionRateOnOpen) {
      if (LuceneTestCase.VERBOSE) {
        System.out.println(
            Thread.currentThread().getName()
                + ": MockDirectoryWrapper: now throw random exception during open file="
                + name);
        new Throwable().printStackTrace(System.out);
      }
      if (allowRandomFileNotFoundException == false || randomState.nextBoolean()) {
        throw new IOException("a random IOException (" + name + ")");
      } else {
        throw randomState.nextBoolean()
            ? new FileNotFoundException("a random IOException (" + name + ")")
            : new NoSuchFileException("a random IOException (" + name + ")");
      }
    }
  }

  /** returns current open file handle count */
  public synchronized long getFileHandleCount() {
    return openFileHandles.size();
  }

  @Override
  public synchronized void deleteFile(String name) throws IOException {
    maybeYield();

    maybeThrowDeterministicException();

    if (crashed) {
      throw new IOException("cannot delete after crash");
    }

    if (openFiles.containsKey(name)) {
      openFilesDeleted.add(name);
      if (assertNoDeleteOpenFile) {
        throw fillOpenTrace(
            new IOException(
                "MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"),
            name,
            true);
      }
    } else {
      openFilesDeleted.remove(name);
    }

    unSyncedFiles.remove(name);
    in.deleteFile(name);
    createdFiles.remove(name);
  }

  // sets the cause of the incoming ioe to be the stack
  // trace when the offending file name was opened
  private synchronized <T extends Throwable> T fillOpenTrace(T t, String name, boolean input) {
    for (Map.Entry<Closeable, Exception> ent : openFileHandles.entrySet()) {
      if (input
          && ent.getKey() instanceof MockIndexInputWrapper
          && ((MockIndexInputWrapper) ent.getKey()).name.equals(name)) {
        t.initCause(ent.getValue());
        break;
      } else if (!input
          && ent.getKey() instanceof MockIndexOutputWrapper
          && ((MockIndexOutputWrapper) ent.getKey()).name.equals(name)) {
        t.initCause(ent.getValue());
        break;
      }
    }
    return t;
  }

  private void maybeYield() {
    if (randomState.nextBoolean()) {
      Thread.yield();
    }
  }

  public synchronized Set<String> getOpenDeletedFiles() {
    return new HashSet<>(openFilesDeleted);
  }

  private boolean failOnCreateOutput = true;

  public void setFailOnCreateOutput(boolean v) {
    failOnCreateOutput = v;
  }

  @Override
  public synchronized IndexOutput createOutput(String name, IOContext context) throws IOException {
    maybeThrowDeterministicException();
    maybeThrowIOExceptionOnOpen(name);
    maybeYield();
    if (failOnCreateOutput) {
      maybeThrowDeterministicException();
    }
    if (crashed) {
      throw new IOException("cannot createOutput after crash");
    }
    init();

    if (createdFiles.contains(name)) {
      throw new FileAlreadyExistsException("File \"" + name + "\" was already written to.");
    }

    if (assertNoDeleteOpenFile && openFiles.containsKey(name)) {
      throw new AssertionError(
          "MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite");
    }

    unSyncedFiles.add(name);
    createdFiles.add(name);

    // System.out.println(Thread.currentThread().getName() + ": MDW: create " + name);
    IndexOutput delegateOutput =
        in.createOutput(name, LuceneTestCase.newIOContext(randomState, context));
    final IndexOutput io = new MockIndexOutputWrapper(this, delegateOutput, name);
    addFileHandle(io, name, Handle.Output);
    openFilesForWrite.add(name);
    return maybeThrottle(name, io);
  }

  private IndexOutput maybeThrottle(String name, IndexOutput output) {
    // throttling REALLY slows down tests, so don't do it very often for SOMETIMES.
    if (throttling == Throttling.ALWAYS
        || (throttling == Throttling.SOMETIMES && randomState.nextInt(200) == 0)) {
      if (LuceneTestCase.VERBOSE) {
        System.out.println("MockDirectoryWrapper: throttling indexOutput (" + name + ")");
      }
      return throttledOutput.newFromDelegate(output);
    } else {
      return output;
    }
  }

  @Override
  public synchronized IndexOutput createTempOutput(String prefix, String suffix, IOContext context)
      throws IOException {
    maybeThrowDeterministicException();
    maybeThrowIOExceptionOnOpen("temp: prefix=" + prefix + " suffix=" + suffix);
    maybeYield();
    if (failOnCreateOutput) {
      maybeThrowDeterministicException();
    }
    if (crashed) {
      throw new IOException("cannot createTempOutput after crash");
    }
    init();

    IndexOutput delegateOutput =
        in.createTempOutput(prefix, suffix, LuceneTestCase.newIOContext(randomState, context));
    String name = delegateOutput.getName();
    if (name.toLowerCase(Locale.ROOT).endsWith(".tmp") == false) {
      throw new IllegalStateException(
          "wrapped directory failed to use .tmp extension: got: " + name);
    }

    unSyncedFiles.add(name);
    createdFiles.add(name);
    final IndexOutput io = new MockIndexOutputWrapper(this, delegateOutput, name);
    addFileHandle(io, name, Handle.Output);
    openFilesForWrite.add(name);

    return maybeThrottle(name, io);
  }

  private enum Handle {
    Input,
    Output,
    Slice
  }

  synchronized void addFileHandle(Closeable c, String name, Handle handle) {
    Integer v = openFiles.get(name);
    if (v != null) {
      v = Integer.valueOf(v.intValue() + 1);
      openFiles.put(name, v);
    } else {
      openFiles.put(name, Integer.valueOf(1));
    }

    openFileHandles.put(c, new RuntimeException("unclosed Index" + handle.name() + ": " + name));
  }

  private boolean failOnOpenInput = true;

  public void setFailOnOpenInput(boolean v) {
    failOnOpenInput = v;
  }

  @Override
  public synchronized IndexInput openInput(String name, IOContext context) throws IOException {
    maybeThrowDeterministicException();
    maybeThrowIOExceptionOnOpen(name);
    maybeYield();
    if (failOnOpenInput) {
      maybeThrowDeterministicException();
    }
    if (!LuceneTestCase.slowFileExists(in, name)) {
      throw randomState.nextBoolean()
          ? new FileNotFoundException(name + " in dir=" + in)
          : new NoSuchFileException(name + " in dir=" + in);
    }

    // cannot open a file for input if it's still open for output.
    if (!allowReadingFilesStillOpenForWrite && openFilesForWrite.contains(name)) {
      throw fillOpenTrace(
          new AccessDeniedException(
              "MockDirectoryWrapper: file \"" + name + "\" is still open for writing"),
          name,
          false);
    }

    IndexInput delegateInput =
        in.openInput(name, LuceneTestCase.newIOContext(randomState, context));

    final IndexInput ii;
    int randomInt = randomState.nextInt(500);
    if (useSlowOpenClosers && randomInt == 0) {
      if (LuceneTestCase.VERBOSE) {
        System.out.println(
            "MockDirectoryWrapper: using SlowClosingMockIndexInputWrapper for file " + name);
      }
      ii = new SlowClosingMockIndexInputWrapper(this, name, delegateInput);
    } else if (useSlowOpenClosers && randomInt == 1) {
      if (LuceneTestCase.VERBOSE) {
        System.out.println(
            "MockDirectoryWrapper: using SlowOpeningMockIndexInputWrapper for file " + name);
      }
      ii = new SlowOpeningMockIndexInputWrapper(this, name, delegateInput);
    } else {
      ii = new MockIndexInputWrapper(this, name, delegateInput, null);
    }
    addFileHandle(ii, name, Handle.Input);
    return ii;
  }

  // NOTE: This is off by default; see LUCENE-5574
  private volatile boolean assertNoUnreferencedFilesOnClose;

  public void setAssertNoUnrefencedFilesOnClose(boolean v) {
    assertNoUnreferencedFilesOnClose = v;
  }

  @Override
  public synchronized void close() throws IOException {
    if (isOpen) {
      isOpen = false;
    } else {
      in.close(); // but call it again on our wrapped dir
      return;
    }

    boolean success = false;
    try {
      // files that we tried to delete, but couldn't because readers were open.
      // all that matters is that we tried! (they will eventually go away)
      //   still open when we tried to delete
      maybeYield();
      if (openFiles == null) {
        openFiles = new HashMap<>();
        openFilesDeleted = new HashSet<>();
      }
      if (openFiles.size() > 0) {
        // print the first one as it's very verbose otherwise
        Exception cause = null;
        Iterator<Exception> stacktraces = openFileHandles.values().iterator();
        if (stacktraces.hasNext()) {
          cause = stacktraces.next();
        }
        // RuntimeException instead of IOException because
        // super() does not throw IOException currently:
        throw new RuntimeException(
            "MockDirectoryWrapper: cannot close: there are still "
                + openFiles.size()
                + " open files: "
                + openFiles,
            cause);
      }
      if (openLocks.size() > 0) {
        Exception cause = null;
        Iterator<RuntimeException> stacktraces = openLocks.values().iterator();
        if (stacktraces.hasNext()) {
          cause = stacktraces.next();
        }
        throw new RuntimeException(
            "MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks, cause);
      }
      randomIOExceptionRate = 0.0;
      randomIOExceptionRateOnOpen = 0.0;

      if ((getCheckIndexOnClose() || assertNoUnreferencedFilesOnClose)
          && DirectoryReader.indexExists(this)) {
        if (getCheckIndexOnClose()) {

          if (LuceneTestCase.VERBOSE) {
            System.out.println("\nNOTE: MockDirectoryWrapper: now crush");
          }
          crash(); // corrupt any unsynced-files
          if (LuceneTestCase.VERBOSE) {
            System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
          }

          // Methods in MockDirectoryWrapper hold locks on this, which will cause deadlock when
          // TestUtil#checkIndex checks segment concurrently using another thread, but making
          // call back to synchronized methods such as MockDirectoryWrapper#fileLength.
          // Hence passing concurrent = false to this method to turn off concurrent checks.
          TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose(), true, false, null);
        }

        // TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles
        if (assertNoUnreferencedFilesOnClose) {
          if (LuceneTestCase.VERBOSE) {
            System.out.println("MDW: now assert no unref'd files at close");
          }
          // now look for unreferenced files: discount ones that we tried to delete but could not
          Set<String> allFiles = new HashSet<>(Arrays.asList(listAll()));
          String[] startFiles = allFiles.toArray(new String[0]);
          IndexWriterConfig iwc = new IndexWriterConfig(null);
          iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);

          // We must do this before opening writer otherwise writer will be angry if there are
          // pending deletions:
          TestUtil.disableVirusChecker(in);

          new IndexWriter(in, iwc).rollback();
          String[] endFiles = in.listAll();

          Set<String> startSet = new TreeSet<>(Arrays.asList(startFiles));
          Set<String> endSet = new TreeSet<>(Arrays.asList(endFiles));

          startFiles = startSet.toArray(new String[0]);
          endFiles = endSet.toArray(new String[0]);

          if (!Arrays.equals(startFiles, endFiles)) {
            List<String> removed = new ArrayList<>();
            for (String fileName : startFiles) {
              if (!endSet.contains(fileName)) {
                removed.add(fileName);
              }
            }

            List<String> added = new ArrayList<>();
            for (String fileName : endFiles) {
              if (!startSet.contains(fileName)) {
                added.add(fileName);
              }
            }

            String extras;
            if (removed.size() != 0) {
              extras = "\n\nThese files were removed: " + removed;
            } else {
              extras = "";
            }

            if (added.size() != 0) {
              extras += "\n\nThese files were added (waaaaaaaaaat!): " + added;
            }

            throw new RuntimeException(
                "unreferenced files: before delete:\n    "
                    + Arrays.toString(startFiles)
                    + "\n  after delete:\n    "
                    + Arrays.toString(endFiles)
                    + extras);
          }

          DirectoryReader ir1 = DirectoryReader.open(this);
          int numDocs1 = ir1.numDocs();
          ir1.close();
          new IndexWriter(this, new IndexWriterConfig(null)).close();
          DirectoryReader ir2 = DirectoryReader.open(this);
          int numDocs2 = ir2.numDocs();
          ir2.close();
          assert numDocs1 == numDocs2
              : "numDocs changed after opening/closing IW: before="
                  + numDocs1
                  + " after="
                  + numDocs2;
        }
      }
      success = true;
    } finally {
      if (success) {
        IOUtils.close(in);
      } else {
        IOUtils.closeWhileHandlingException(in);
      }
    }
  }

  synchronized void removeOpenFile(Closeable c, String name) {
    Integer v = openFiles.get(name);
    // Could be null when crash() was called
    if (v != null) {
      if (v.intValue() == 1) {
        openFiles.remove(name);
      } else {
        v = Integer.valueOf(v.intValue() - 1);
        openFiles.put(name, v);
      }
    }

    openFileHandles.remove(c);
  }

  public synchronized void removeIndexOutput(IndexOutput out, String name) {
    openFilesForWrite.remove(name);
    removeOpenFile(out, name);
  }

  public synchronized void removeIndexInput(IndexInput in, String name) {
    removeOpenFile(in, name);
  }

  /**
   * Objects that represent fail-able conditions. Objects of a derived class are created and
   * registered with the mock directory. After register, each object will be invoked once for each
   * first write of a file, giving the object a chance to throw an IOException.
   */
  public static class Failure {
    /** eval is called on the first write of every new file. */
    public void eval(MockDirectoryWrapper dir) throws IOException {}

    /**
     * reset should set the state of the failure to its default (freshly constructed) state. Reset
     * is convenient for tests that want to create one failure object and then reuse it in multiple
     * cases. This, combined with the fact that Failure subclasses are often anonymous classes makes
     * reset difficult to do otherwise.
     *
     * <p>A typical example of use is Failure failure = new Failure() { ... }; ...
     * mock.failOn(failure.reset())
     */
    public Failure reset() {
      return this;
    }

    protected boolean doFail;

    public void setDoFail() {
      doFail = true;
    }

    public void clearDoFail() {
      doFail = false;
    }
  }

  ArrayList<Failure> failures;

  /**
   * add a Failure object to the list of objects to be evaluated at every potential failure point
   */
  public synchronized void failOn(Failure fail) {
    if (failures == null) {
      failures = new ArrayList<>();
    }
    failures.add(fail);
  }

  /** Iterate through the failures list, giving each object a chance to throw an IOE */
  synchronized void maybeThrowDeterministicException() throws IOException {
    if (failures != null) {
      for (int i = 0; i < failures.size(); i++) {
        try {
          failures.get(i).eval(this);
        } catch (Throwable t) {
          if (LuceneTestCase.VERBOSE) {
            System.out.println("MockDirectoryWrapper: throw exc");
            t.printStackTrace(System.out);
          }
          throw IOUtils.rethrowAlways(t);
        }
      }
    }
  }

  @Override
  public synchronized String[] listAll() throws IOException {
    maybeYield();
    return in.listAll();
  }

  @Override
  public synchronized long fileLength(String name) throws IOException {
    maybeYield();
    return in.fileLength(name);
  }

  @Override
  public synchronized Lock obtainLock(String name) throws IOException {
    maybeYield();
    return super.obtainLock(name);
    // TODO: consider mocking locks, but not all the time, can hide bugs
  }

  /**
   * Use this when throwing fake {@code IOException}, e.g. from {@link
   * MockDirectoryWrapper.Failure}.
   */
  public static class FakeIOException extends IOException {}

  @Override
  public String toString() {
    if (maxSize != 0) {
      return "MockDirectoryWrapper(" + in + ", current=" + maxUsedSize + ",max=" + maxSize + ")";
    } else {
      return super.toString();
    }
  }

  // don't override optional methods like copyFrom: we need the default impl for things like disk
  // full checks. we randomly exercise "raw" directories anyway. We ensure default impls are used:

  @Override
  public final ChecksumIndexInput openChecksumInput(String name, IOContext context)
      throws IOException {
    return super.openChecksumInput(name, context);
  }

  @Override
  public final void copyFrom(Directory from, String src, String dest, IOContext context)
      throws IOException {
    super.copyFrom(from, src, dest, context);
  }

  @Override
  protected final void ensureOpen() throws AlreadyClosedException {
    super.ensureOpen();
  }
}
