/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.tests.store;

import com.carrotsearch.randomizedtesting.RandomizedTest;
import com.carrotsearch.randomizedtesting.generators.RandomBytes;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.AccessDeniedException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.zip.CRC32;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexNotFoundException;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.tests.mockfile.ExtrasFS;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.IOUtils;
import org.junit.Assert;

/** Base class for {@link Directory} implementations. */
public abstract class BaseDirectoryTestCase extends LuceneTestCase {

  /**
   * A subclass returns the Directory to be tested; if it's an FS-based directory it should point to
   * the specified path, else it can ignore it.
   */
  protected abstract Directory getDirectory(Path path) throws IOException;

  public void testCopyFrom() throws Exception {
    try (Directory source = getDirectory(createTempDir("testCopy"));
        Directory dest = newDirectory()) {
      runCopyFrom(source, dest);
    }

    try (Directory source = newDirectory();
        Directory dest = getDirectory(createTempDir("testCopyDestination"))) {
      runCopyFrom(source, dest);
    }
  }

  private void runCopyFrom(Directory source, Directory dest) throws IOException {
    IndexOutput output = source.createOutput("foobar", newIOContext(random()));

    byte[] bytes = RandomBytes.randomBytesOfLength(random(), 20000);
    output.writeBytes(bytes, bytes.length);
    output.close();

    dest.copyFrom(source, "foobar", "foobaz", newIOContext(random()));
    assertTrue(slowFileExists(dest, "foobaz"));

    IndexInput input = dest.openInput("foobaz", newIOContext(random()));
    byte[] bytes2 = new byte[bytes.length];
    input.readBytes(bytes2, 0, bytes2.length);
    input.close();

    assertArrayEquals(bytes, bytes2);
  }

  public void testRename() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testRename"))) {
      IndexOutput output = dir.createOutput("foobar", newIOContext(random()));
      int numBytes = random().nextInt(20000);
      byte[] bytes = new byte[numBytes];
      random().nextBytes(bytes);
      output.writeBytes(bytes, bytes.length);
      output.close();

      dir.rename("foobar", "foobaz");

      IndexInput input = dir.openInput("foobaz", newIOContext(random()));
      byte[] bytes2 = new byte[numBytes];
      input.readBytes(bytes2, 0, bytes2.length);
      assertEquals(input.length(), numBytes);
      input.close();

      assertArrayEquals(bytes, bytes2);
    }
  }

  public void testDeleteFile() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testDeleteFile"))) {
      String file = "foo.txt";
      Assert.assertFalse(Arrays.asList(dir.listAll()).contains(file));

      dir.createOutput("foo.txt", IOContext.DEFAULT).close();
      Assert.assertTrue(Arrays.asList(dir.listAll()).contains(file));

      dir.deleteFile("foo.txt");
      Assert.assertFalse(Arrays.asList(dir.listAll()).contains(file));

      expectThrowsAnyOf(
          Arrays.asList(NoSuchFileException.class, FileNotFoundException.class),
          () -> {
            dir.deleteFile("foo.txt");
          });
    }
  }

  public void testByte() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testByte"))) {
      IndexOutput output = dir.createOutput("byte", newIOContext(random()));
      output.writeByte((byte) 128);
      output.close();

      IndexInput input = dir.openInput("byte", newIOContext(random()));
      assertEquals(1, input.length());
      assertEquals((byte) 128, input.readByte());
      input.close();
    }
  }

  public void testShort() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testShort"))) {
      IndexOutput output = dir.createOutput("short", newIOContext(random()));
      output.writeShort((short) -20);
      output.close();

      IndexInput input = dir.openInput("short", newIOContext(random()));
      assertEquals(2, input.length());
      assertEquals((short) -20, input.readShort());
      input.close();
    }
  }

  public void testInt() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testInt"))) {
      IndexOutput output = dir.createOutput("int", newIOContext(random()));
      output.writeInt(-500);
      output.close();

      IndexInput input = dir.openInput("int", newIOContext(random()));
      assertEquals(4, input.length());
      assertEquals(-500, input.readInt());
      input.close();
    }
  }

  public void testLong() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testLong"))) {
      IndexOutput output = dir.createOutput("long", newIOContext(random()));
      output.writeLong(-5000);
      output.close();

      IndexInput input = dir.openInput("long", newIOContext(random()));
      assertEquals(8, input.length());
      assertEquals(-5000L, input.readLong());
      input.close();
    }
  }

  public void testAlignedLittleEndianLongs() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testAlignedLittleEndianLongs"))) {
      try (IndexOutput out = dir.createOutput("littleEndianLongs", newIOContext(random()))) {
        out.writeLong(3L);
        out.writeLong(Long.MAX_VALUE);
        out.writeLong(-3L);
      }
      try (IndexInput input = dir.openInput("littleEndianLongs", newIOContext(random()))) {
        assertEquals(24, input.length());
        long[] l = new long[4];
        input.readLongs(l, 1, 3);
        assertArrayEquals(new long[] {0L, 3L, Long.MAX_VALUE, -3L}, l);
        assertEquals(24, input.getFilePointer());
      }
    }
  }

  public void testUnalignedLittleEndianLongs() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testUnalignedLittleEndianLongs"))) {
      try (IndexOutput out = dir.createOutput("littleEndianLongs", newIOContext(random()))) {
        out.writeByte((byte) 2);
        out.writeLong(3L);
        out.writeLong(Long.MAX_VALUE);
        out.writeLong(-3L);
      }
      try (IndexInput input = dir.openInput("littleEndianLongs", newIOContext(random()))) {
        assertEquals(25, input.length());
        assertEquals(2, input.readByte());
        long[] l = new long[4];
        input.readLongs(l, 1, 3);
        assertArrayEquals(new long[] {0L, 3L, Long.MAX_VALUE, -3L}, l);
        assertEquals(25, input.getFilePointer());
      }
    }
  }

  public void testLittleEndianLongsUnderflow() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testLittleEndianLongsUnderflow"))) {
      final int offset = random().nextInt(8);
      final int length = TestUtil.nextInt(random(), 1, 16);
      try (IndexOutput out = dir.createOutput("littleEndianLongs", newIOContext(random()))) {
        byte[] b =
            new byte[offset + length * Long.BYTES - TestUtil.nextInt(random(), 1, Long.BYTES)];
        random().nextBytes(b);
        out.writeBytes(b, b.length);
      }
      try (IndexInput input = dir.openInput("littleEndianLongs", newIOContext(random()))) {
        input.seek(offset);
        expectThrows(EOFException.class, () -> input.readLongs(new long[length], 0, length));
      }
    }
  }

  public void testAlignedInts() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testAlignedInts"))) {
      try (IndexOutput out = dir.createOutput("Ints", newIOContext(random()))) {
        out.writeInt(3);
        out.writeInt(Integer.MAX_VALUE);
        out.writeInt(-3);
      }
      try (IndexInput input = dir.openInput("Ints", newIOContext(random()))) {
        assertEquals(12, input.length());
        int[] i = new int[4];
        input.readInts(i, 1, 3);
        assertArrayEquals(new int[] {0, 3, Integer.MAX_VALUE, -3}, i);
        assertEquals(12, input.getFilePointer());
      }
    }
  }

  public void testUnalignedInts() throws Exception {
    int padding = random().nextInt(3) + 1;
    try (Directory dir = getDirectory(createTempDir("testUnalignedInts"))) {
      try (IndexOutput out = dir.createOutput("Ints", newIOContext(random()))) {
        for (int i = 0; i < padding; i++) {
          out.writeByte((byte) 2);
        }
        out.writeInt(3);
        out.writeInt(Integer.MAX_VALUE);
        out.writeInt(-3);
      }
      try (IndexInput input = dir.openInput("Ints", newIOContext(random()))) {
        assertEquals(12 + padding, input.length());
        for (int i = 0; i < padding; i++) {
          assertEquals(2, input.readByte());
        }
        int[] i = new int[4];
        input.readInts(i, 1, 3);
        assertArrayEquals(new int[] {0, 3, Integer.MAX_VALUE, -3}, i);
        assertEquals(12 + padding, input.getFilePointer());
      }
    }
  }

  public void testIntsUnderflow() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testIntsUnderflow"))) {
      final int offset = random().nextInt(4);
      final int length = TestUtil.nextInt(random(), 1, 16);
      try (IndexOutput out = dir.createOutput("Ints", newIOContext(random()))) {
        byte[] b =
            new byte
                [offset + length * Integer.BYTES - TestUtil.nextInt(random(), 1, Integer.BYTES)];
        random().nextBytes(b);
        out.writeBytes(b, b.length);
      }
      try (IndexInput input = dir.openInput("Ints", newIOContext(random()))) {
        input.seek(offset);
        expectThrows(EOFException.class, () -> input.readInts(new int[length], 0, length));
      }
    }
  }

  public void testAlignedFloats() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testAlignedFloats"))) {
      try (IndexOutput out = dir.createOutput("Floats", newIOContext(random()))) {
        out.writeInt(Float.floatToIntBits(3f));
        out.writeInt(Float.floatToIntBits(Float.MAX_VALUE));
        out.writeInt(Float.floatToIntBits(-3f));
      }
      try (IndexInput input = dir.openInput("Floats", newIOContext(random()))) {
        assertEquals(12, input.length());
        float[] ff = new float[4];
        input.readFloats(ff, 1, 3);
        assertArrayEquals(new float[] {0, 3f, Float.MAX_VALUE, -3f}, ff, 0);
        assertEquals(12, input.getFilePointer());
      }
    }
  }

  public void testUnalignedFloats() throws Exception {
    int padding = random().nextInt(3) + 1;
    try (Directory dir = getDirectory(createTempDir("testUnalignedFloats"))) {
      try (IndexOutput out = dir.createOutput("Floats", newIOContext(random()))) {
        for (int i = 0; i < padding; i++) {
          out.writeByte((byte) 2);
        }
        out.writeInt(Float.floatToIntBits(3f));
        out.writeInt(Float.floatToIntBits(Float.MAX_VALUE));
        out.writeInt(Float.floatToIntBits(-3f));
      }
      try (IndexInput input = dir.openInput("Floats", newIOContext(random()))) {
        assertEquals(12 + padding, input.length());
        for (int i = 0; i < padding; i++) {
          assertEquals(2, input.readByte());
        }
        float[] ff = new float[4];
        input.readFloats(ff, 1, 3);
        assertArrayEquals(new float[] {0, 3f, Float.MAX_VALUE, -3f}, ff, 0);
        assertEquals(12 + padding, input.getFilePointer());
      }
    }
  }

  public void testFloatsUnderflow() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testFloatsUnderflow"))) {
      final int offset = random().nextInt(4);
      final int length = TestUtil.nextInt(random(), 1, 16);
      try (IndexOutput out = dir.createOutput("Floats", newIOContext(random()))) {
        byte[] b =
            new byte[offset + length * Float.BYTES - TestUtil.nextInt(random(), 1, Float.BYTES)];
        random().nextBytes(b);
        out.writeBytes(b, b.length);
      }
      try (IndexInput input = dir.openInput("Floats", newIOContext(random()))) {
        input.seek(offset);
        expectThrows(EOFException.class, () -> input.readFloats(new float[length], 0, length));
      }
    }
  }

  public void testString() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testString"))) {
      IndexOutput output = dir.createOutput("string", newIOContext(random()));
      output.writeString("hello!");
      output.close();

      IndexInput input = dir.openInput("string", newIOContext(random()));
      assertEquals("hello!", input.readString());
      assertEquals(7, input.length());
      input.close();
    }
  }

  public void testVInt() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testVInt"))) {
      IndexOutput output = dir.createOutput("vint", newIOContext(random()));
      output.writeVInt(500);
      output.close();

      IndexInput input = dir.openInput("vint", newIOContext(random()));
      assertEquals(2, input.length());
      assertEquals(500, input.readVInt());
      input.close();
    }
  }

  public void testVLong() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testVLong"))) {
      IndexOutput output = dir.createOutput("vlong", newIOContext(random()));
      output.writeVLong(Long.MAX_VALUE);
      output.close();

      IndexInput input = dir.openInput("vlong", newIOContext(random()));
      assertEquals(9, input.length());
      assertEquals(Long.MAX_VALUE, input.readVLong());
      input.close();
    }
  }

  public void testZInt() throws Exception {
    final int[] ints = new int[random().nextInt(10)];
    for (int i = 0; i < ints.length; ++i) {
      switch (random().nextInt(3)) {
        case 0:
          ints[i] = random().nextInt();
          break;
        case 1:
          ints[i] = random().nextBoolean() ? Integer.MIN_VALUE : Integer.MAX_VALUE;
          break;
        case 2:
          ints[i] = (random().nextBoolean() ? -1 : 1) * random().nextInt(1024);
          break;
        default:
          throw new AssertionError();
      }
    }

    try (Directory dir = getDirectory(createTempDir("testZInt"))) {
      IndexOutput output = dir.createOutput("zint", newIOContext(random()));
      for (int i : ints) {
        output.writeZInt(i);
      }
      output.close();

      IndexInput input = dir.openInput("zint", newIOContext(random()));
      for (int i : ints) {
        assertEquals(i, input.readZInt());
      }
      assertEquals(input.length(), input.getFilePointer());
      input.close();
    }
  }

  public void testZLong() throws Exception {
    final long[] longs = new long[random().nextInt(10)];
    for (int i = 0; i < longs.length; ++i) {
      switch (random().nextInt(3)) {
        case 0:
          longs[i] = random().nextLong();
          break;
        case 1:
          longs[i] = random().nextBoolean() ? Long.MIN_VALUE : Long.MAX_VALUE;
          break;
        case 2:
          longs[i] = (random().nextBoolean() ? -1 : 1) * (long) random().nextInt(1024);
          break;
        default:
          throw new AssertionError();
      }
    }

    try (Directory dir = getDirectory(createTempDir("testZLong"))) {
      IndexOutput output = dir.createOutput("zlong", newIOContext(random()));
      for (long l : longs) {
        output.writeZLong(l);
      }
      output.close();

      IndexInput input = dir.openInput("zlong", newIOContext(random()));
      for (long l : longs) {
        assertEquals(l, input.readZLong());
      }
      assertEquals(input.length(), input.getFilePointer());
      input.close();
    }
  }

  public void testSetOfStrings() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testSetOfStrings"))) {

      IndexOutput output = dir.createOutput("stringset", newIOContext(random()));
      output.writeSetOfStrings(asSet("test1", "test2"));
      output.writeSetOfStrings(Collections.emptySet());
      output.writeSetOfStrings(asSet("test3"));
      output.close();

      IndexInput input = dir.openInput("stringset", newIOContext(random()));
      Set<String> set1 = input.readSetOfStrings();
      assertEquals(asSet("test1", "test2"), set1);
      // set should be immutable
      expectThrows(
          UnsupportedOperationException.class,
          () -> {
            set1.add("bogus");
          });

      Set<String> set2 = input.readSetOfStrings();
      assertEquals(Collections.emptySet(), set2);
      // set should be immutable
      expectThrows(
          UnsupportedOperationException.class,
          () -> {
            set2.add("bogus");
          });

      Set<String> set3 = input.readSetOfStrings();
      assertEquals(Collections.singleton("test3"), set3);
      // set should be immutable
      expectThrows(
          UnsupportedOperationException.class,
          () -> {
            set3.add("bogus");
          });

      assertEquals(input.length(), input.getFilePointer());
      input.close();
    }
  }

  public void testMapOfStrings() throws Exception {
    Map<String, String> m = new HashMap<>();
    m.put("test1", "value1");
    m.put("test2", "value2");

    try (Directory dir = getDirectory(createTempDir("testMapOfStrings"))) {
      IndexOutput output = dir.createOutput("stringmap", newIOContext(random()));
      output.writeMapOfStrings(m);
      output.writeMapOfStrings(Collections.emptyMap());
      output.writeMapOfStrings(Collections.singletonMap("key", "value"));
      output.close();

      IndexInput input = dir.openInput("stringmap", newIOContext(random()));
      Map<String, String> map1 = input.readMapOfStrings();
      assertEquals(m, map1);
      // map should be immutable
      expectThrows(
          UnsupportedOperationException.class,
          () -> {
            map1.put("bogus1", "bogus2");
          });

      Map<String, String> map2 = input.readMapOfStrings();
      assertEquals(Collections.emptyMap(), map2);
      // map should be immutable
      expectThrows(
          UnsupportedOperationException.class,
          () -> {
            map2.put("bogus1", "bogus2");
          });

      Map<String, String> map3 = input.readMapOfStrings();
      assertEquals(Collections.singletonMap("key", "value"), map3);
      // map should be immutable
      expectThrows(
          UnsupportedOperationException.class,
          () -> {
            map3.put("bogus1", "bogus2");
          });

      assertEquals(input.length(), input.getFilePointer());
      input.close();
    }
  }

  // TODO: fold in some of the testing of o.a.l.index.TestIndexInput in here!
  public void testChecksum() throws Exception {
    CRC32 expected = new CRC32();
    int numBytes = random().nextInt(20000);
    byte[] bytes = new byte[numBytes];
    random().nextBytes(bytes);
    expected.update(bytes);

    try (Directory dir = getDirectory(createTempDir("testChecksum"))) {
      IndexOutput output = dir.createOutput("checksum", newIOContext(random()));
      output.writeBytes(bytes, 0, bytes.length);
      output.close();

      ChecksumIndexInput input = dir.openChecksumInput("checksum", newIOContext(random()));
      input.skipBytes(numBytes);

      assertEquals(expected.getValue(), input.getChecksum());
      input.close();
    }
  }

  /** Make sure directory throws AlreadyClosedException if you try to createOutput after closing. */
  public void testDetectClose() throws Throwable {
    Directory dir = getDirectory(createTempDir("testDetectClose"));
    dir.close();

    expectThrows(
        AlreadyClosedException.class,
        () -> {
          dir.createOutput("test", newIOContext(random()));
        });
  }

  public void testThreadSafetyInListAll() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testThreadSafety"))) {
      if (dir instanceof BaseDirectoryWrapper) {
        // we are not making a real index, just writing, reading files.
        ((BaseDirectoryWrapper) dir).setCheckIndexOnClose(false);
      }
      if (dir instanceof MockDirectoryWrapper) {
        // makes this test really slow
        ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
      }

      AtomicBoolean stop = new AtomicBoolean();
      Thread writer =
          new Thread(
              () -> {
                try {
                  for (int i = 0, max = RandomizedTest.randomIntBetween(500, 1000); i < max; i++) {
                    String fileName = "file-" + i;
                    try (IndexOutput output = dir.createOutput(fileName, newIOContext(random()))) {
                      assert output != null;
                      // Add some lags so that the other thread can read the content of the
                      // directory.
                      Thread.yield();
                    }
                    assertTrue(slowFileExists(dir, fileName));
                  }
                } catch (IOException e) {
                  throw new UncheckedIOException(e);
                } finally {
                  stop.set(true);
                }
              });

      Thread reader =
          new Thread(
              () -> {
                try {
                  Random rnd = new Random(RandomizedTest.randomLong());
                  while (!stop.get()) {
                    String[] files =
                        Arrays.stream(dir.listAll())
                            .filter(
                                name -> !ExtrasFS.isExtra(name)) // Ignore anything from ExtraFS.
                            .toArray(String[]::new);

                    if (files.length > 0) {
                      do {
                        String file = RandomPicks.randomFrom(rnd, files);
                        try (IndexInput input = dir.openInput(file, newIOContext(random()))) {
                          // Just open, nothing else.
                          assert input != null;
                        } catch (
                            @SuppressWarnings("unused")
                            AccessDeniedException e) {
                          // Access denied is allowed for files for which the output is still open
                          // (MockDirectoryWriter enforces
                          // this, for example). Since we don't synchronize with the writer thread,
                          // just ignore it.
                        } catch (IOException e) {
                          throw new UncheckedIOException(
                              "Something went wrong when opening: " + file, e);
                        }
                      } while (rnd.nextInt(3) != 0); // Sometimes break and list files again.
                    }
                  }
                } catch (IOException e) {
                  throw new UncheckedIOException(e);
                }
              });

      reader.start();
      writer.start();

      writer.join();
      reader.join();
    }
  }

  /** LUCENE-1468: once we create an output, we should see it in the dir listing. */
  public void testFileExistsInListAfterCreated() throws IOException {
    try (Directory dir = getDirectory(createTempDir("testFileExistsInListAfterCreated"))) {
      String name = "file";
      dir.createOutput(name, newIOContext(random())).close();
      assertTrue(slowFileExists(dir, name));
      assertTrue(Arrays.asList(dir.listAll()).contains(name));
    }
  }

  // LUCENE-2852
  public void testSeekToEOFThenBack() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testSeekToEOFThenBack"))) {
      int bufferLength = 1024;
      IndexOutput o = dir.createOutput("out", newIOContext(random()));
      byte[] bytes = new byte[3 * bufferLength];
      o.writeBytes(bytes, 0, bytes.length);
      o.close();

      IndexInput i = dir.openInput("out", newIOContext(random()));
      i.seek(2L * bufferLength - 1);
      i.seek(3L * bufferLength);
      i.seek(bufferLength);
      i.readBytes(bytes, 0, 2 * bufferLength);
      i.close();
    }
  }

  // LUCENE-1196
  public void testIllegalEOF() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testIllegalEOF"))) {
      IndexOutput o = dir.createOutput("out", newIOContext(random()));
      byte[] b = new byte[1024];
      o.writeBytes(b, 0, 1024);
      o.close();
      IndexInput i = dir.openInput("out", newIOContext(random()));
      i.seek(1024);
      i.close();
    }
  }

  public void testSeekPastEOF() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testSeekPastEOF"))) {
      IndexOutput o = dir.createOutput("out", newIOContext(random()));
      final int len = random().nextInt(2048);
      byte[] b = new byte[len];
      o.writeBytes(b, 0, len);
      o.close();
      IndexInput i = dir.openInput("out", newIOContext(random()));

      // Seeking past EOF should always throw EOFException
      expectThrows(
          EOFException.class, () -> i.seek(len + RandomizedTest.randomIntBetween(1, 2048)));

      // Seeking exactly to EOF should never throw any exception.
      i.seek(len);

      // But any read following the seek(len) should throw an EOFException.
      expectThrows(EOFException.class, i::readByte);
      expectThrows(
          EOFException.class,
          () -> {
            i.readBytes(new byte[1], 0, 1);
          });

      i.close();
    }
  }

  public void testSliceOutOfBounds() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testSliceOutOfBounds"))) {
      IndexOutput o = dir.createOutput("out", newIOContext(random()));
      final int len = random().nextInt(2040) + 8;
      byte[] b = new byte[len];
      o.writeBytes(b, 0, len);
      o.close();
      IndexInput i = dir.openInput("out", newIOContext(random()));
      expectThrows(
          IllegalArgumentException.class,
          () -> {
            i.slice("slice1", 0, len + 1);
          });

      expectThrows(
          IllegalArgumentException.class,
          () -> {
            i.slice("slice2", -1, len);
          });

      IndexInput slice = i.slice("slice3", 4, len / 2);
      expectThrows(
          IllegalArgumentException.class,
          () -> {
            slice.slice("slice3sub", 1, len / 2);
          });

      i.close();
    }
  }

  // LUCENE-3382 -- make sure we get exception if the directory really does not exist.
  public void testNoDir() throws Throwable {
    Path tempDir = createTempDir("doesnotexist");
    IOUtils.rm(tempDir);
    try (Directory dir = getDirectory(tempDir)) {
      expectThrowsAnyOf(
          Arrays.asList(NoSuchFileException.class, IndexNotFoundException.class),
          () -> {
            DirectoryReader.open(dir);
          });
    }
  }

  public void testCopyBytes() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testCopyBytes"))) {
      IndexOutput out = dir.createOutput("test", newIOContext(random()));
      byte[] bytes = new byte[TestUtil.nextInt(random(), 1, 77777)];
      final int size = TestUtil.nextInt(random(), 1, 1777777);
      int upto = 0;
      int byteUpto = 0;
      while (upto < size) {
        bytes[byteUpto++] = value(upto);
        upto++;
        if (byteUpto == bytes.length) {
          out.writeBytes(bytes, 0, bytes.length);
          byteUpto = 0;
        }
      }

      out.writeBytes(bytes, 0, byteUpto);
      assertEquals(size, out.getFilePointer());
      out.close();
      assertEquals(size, dir.fileLength("test"));

      // copy from test -> test2
      final IndexInput in = dir.openInput("test", newIOContext(random()));

      out = dir.createOutput("test2", newIOContext(random()));

      upto = 0;
      while (upto < size) {
        if (random().nextBoolean()) {
          out.writeByte(in.readByte());
          upto++;
        } else {
          final int chunk = Math.min(TestUtil.nextInt(random(), 1, bytes.length), size - upto);
          out.copyBytes(in, chunk);
          upto += chunk;
        }
      }
      assertEquals(size, upto);
      out.close();
      in.close();

      // verify
      IndexInput in2 = dir.openInput("test2", newIOContext(random()));
      upto = 0;
      while (upto < size) {
        if (random().nextBoolean()) {
          final byte v = in2.readByte();
          assertEquals(value(upto), v);
          upto++;
        } else {
          final int limit = Math.min(TestUtil.nextInt(random(), 1, bytes.length), size - upto);
          in2.readBytes(bytes, 0, limit);
          for (int byteIdx = 0; byteIdx < limit; byteIdx++) {
            assertEquals(value(upto), bytes[byteIdx]);
            upto++;
          }
        }
      }
      in2.close();

      dir.deleteFile("test");
      dir.deleteFile("test2");
    }
  }

  private static byte value(int idx) {
    return (byte) ((idx % 256) * (1 + (idx / 256)));
  }

  // LUCENE-3541
  public void testCopyBytesWithThreads() throws Exception {
    try (Directory d = getDirectory(createTempDir("testCopyBytesWithThreads"))) {
      int headerLen = 100;
      byte[] data = RandomBytes.randomBytesOfLengthBetween(random(), headerLen + 1, 10000);

      IndexOutput output = d.createOutput("data", IOContext.DEFAULT);
      output.writeBytes(data, 0, data.length);
      output.close();

      IndexInput input = d.openInput("data", IOContext.DEFAULT);
      IndexOutput outputHeader = d.createOutput("header", IOContext.DEFAULT);
      // copy our header
      outputHeader.copyBytes(input, headerLen);
      outputHeader.close();

      // now make N copies of the remaining bytes
      int threads = 10;
      CyclicBarrier start = new CyclicBarrier(threads);
      Thread[] copies =
          IntStream.range(0, threads)
              .mapToObj(
                  (i) -> {
                    IndexInput src = input.clone();
                    Thread t =
                        new Thread(
                            () -> {
                              try {
                                start.await();
                                IndexOutput dst = d.createOutput("copy" + i, IOContext.DEFAULT);
                                dst.copyBytes(src, src.length() - headerLen);
                                dst.close();
                              } catch (Exception e) {
                                throw new RuntimeException(e);
                              }
                            });
                    t.start();
                    return t;
                  })
              .toArray(Thread[]::new);

      for (Thread t : copies) {
        t.join();
      }

      for (int i = 0; i < threads; i++) {
        try (IndexInput copiedData = d.openInput("copy" + i, IOContext.DEFAULT)) {
          byte[] dataCopy = new byte[data.length];
          System.arraycopy(data, 0, dataCopy, 0, headerLen);
          copiedData.readBytes(dataCopy, headerLen, data.length - headerLen);
          assertArrayEquals(data, dataCopy);
        }
      }
      input.close();
    }
  }

  // this test backdoors the directory via the filesystem. so it must actually use the filesystem
  // TODO: somehow change this test to
  public void testFsyncDoesntCreateNewFiles() throws Exception {
    Path path = createTempDir("nocreate");
    try (Directory fsdir = getDirectory(path)) {
      // this test backdoors the directory via the filesystem. so it must be an FSDir (for now)
      // TODO: figure a way to test this better/clean it up. E.g. we should be testing for
      // FileSwitchDir,
      // if it's using two FSdirs and so on
      if (fsdir instanceof FSDirectory == false) {
        assumeTrue("test only works for FSDirectory subclasses", false);
        return;
      }

      // create a file
      IndexOutput out = fsdir.createOutput("afile", newIOContext(random()));
      out.writeString("boo");
      out.close();

      // delete it in the file system.
      Files.delete(path.resolve("afile"));

      int fileCount = fsdir.listAll().length;

      // fsync it
      expectThrowsAnyOf(
          Arrays.asList(FileNotFoundException.class, NoSuchFileException.class),
          () -> {
            fsdir.sync(Collections.singleton("afile"));
          });

      // no new files created
      assertEquals(fileCount, fsdir.listAll().length);
    }
  }

  // random access APIs
  public void testRandomLong() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testLongs"))) {
      IndexOutput output = dir.createOutput("longs", newIOContext(random()));
      int num = TestUtil.nextInt(random(), 50, 3000);
      long[] longs = new long[num];
      for (int i = 0; i < longs.length; i++) {
        longs[i] = TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE);
        output.writeLong(longs[i]);
      }
      output.close();

      // slice
      IndexInput input = dir.openInput("longs", newIOContext(random()));
      RandomAccessInput slice = input.randomAccessSlice(0, input.length());
      for (int i = 0; i < longs.length; i++) {
        assertEquals(longs[i], slice.readLong(i * 8L));
      }

      // subslices
      for (int i = 1; i < longs.length; i++) {
        long offset = i * 8L;
        RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset);
        for (int j = i; j < longs.length; j++) {
          assertEquals(longs[j], subslice.readLong((j - i) * 8L));
        }
      }

      // with padding
      for (int i = 0; i < 7; i++) {
        String name = "longs-" + i;
        IndexOutput o = dir.createOutput(name, newIOContext(random()));
        byte[] junk = new byte[i];
        random().nextBytes(junk);
        o.writeBytes(junk, junk.length);
        input.seek(0);
        o.copyBytes(input, input.length());
        o.close();
        IndexInput padded = dir.openInput(name, newIOContext(random()));
        RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i);
        for (int j = 0; j < longs.length; j++) {
          assertEquals(longs[j], whole.readLong(j * 8L));
        }
        padded.close();
      }

      input.close();
    }
  }

  public void testRandomInt() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testInts"))) {
      IndexOutput output = dir.createOutput("ints", newIOContext(random()));
      int num = TestUtil.nextInt(random(), 50, 3000);
      int[] ints = new int[num];
      for (int i = 0; i < ints.length; i++) {
        ints[i] = random().nextInt();
        output.writeInt(ints[i]);
      }
      output.close();

      // slice
      IndexInput input = dir.openInput("ints", newIOContext(random()));
      RandomAccessInput slice = input.randomAccessSlice(0, input.length());
      for (int i = 0; i < ints.length; i++) {
        assertEquals(ints[i], slice.readInt(i * 4L));
      }

      // subslices
      for (int i = 1; i < ints.length; i++) {
        long offset = i * 4L;
        RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset);
        for (int j = i; j < ints.length; j++) {
          assertEquals(ints[j], subslice.readInt((j - i) * 4L));
        }
      }

      // with padding
      for (int i = 0; i < 7; i++) {
        String name = "ints-" + i;
        IndexOutput o = dir.createOutput(name, newIOContext(random()));
        byte[] junk = new byte[i];
        random().nextBytes(junk);
        o.writeBytes(junk, junk.length);
        input.seek(0);
        o.copyBytes(input, input.length());
        o.close();
        IndexInput padded = dir.openInput(name, newIOContext(random()));
        RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i);
        for (int j = 0; j < ints.length; j++) {
          assertEquals(ints[j], whole.readInt(j * 4L));
        }
        padded.close();
      }
      input.close();
    }
  }

  public void testRandomShort() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testShorts"))) {
      IndexOutput output = dir.createOutput("shorts", newIOContext(random()));
      int num = TestUtil.nextInt(random(), 50, 3000);
      short[] shorts = new short[num];
      for (int i = 0; i < shorts.length; i++) {
        shorts[i] = (short) random().nextInt();
        output.writeShort(shorts[i]);
      }
      output.close();

      // slice
      IndexInput input = dir.openInput("shorts", newIOContext(random()));
      RandomAccessInput slice = input.randomAccessSlice(0, input.length());
      for (int i = 0; i < shorts.length; i++) {
        assertEquals(shorts[i], slice.readShort(i * 2L));
      }

      // subslices
      for (int i = 1; i < shorts.length; i++) {
        long offset = i * 2L;
        RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset);
        for (int j = i; j < shorts.length; j++) {
          assertEquals(shorts[j], subslice.readShort((j - i) * 2L));
        }
      }

      // with padding
      for (int i = 0; i < 7; i++) {
        String name = "shorts-" + i;
        IndexOutput o = dir.createOutput(name, newIOContext(random()));
        byte[] junk = new byte[i];
        random().nextBytes(junk);
        o.writeBytes(junk, junk.length);
        input.seek(0);
        o.copyBytes(input, input.length());
        o.close();
        IndexInput padded = dir.openInput(name, newIOContext(random()));
        RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i);
        for (int j = 0; j < shorts.length; j++) {
          assertEquals(shorts[j], whole.readShort(j * 2L));
        }
        padded.close();
      }
      input.close();
    }
  }

  public void testRandomByte() throws Exception {
    try (Directory dir = getDirectory(createTempDir("testBytes"))) {
      IndexOutput output = dir.createOutput("bytes", newIOContext(random()));
      int num = TestUtil.nextInt(random(), 50, 3000);
      byte[] bytes = new byte[num];
      random().nextBytes(bytes);
      for (int i = 0; i < bytes.length; i++) {
        output.writeByte(bytes[i]);
      }
      output.close();

      // slice
      IndexInput input = dir.openInput("bytes", newIOContext(random()));
      RandomAccessInput slice = input.randomAccessSlice(0, input.length());
      for (int i = 0; i < bytes.length; i++) {
        assertEquals(bytes[i], slice.readByte(i));
      }

      // subslices
      for (int i = 1; i < bytes.length; i++) {
        long offset = i;
        RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset);
        for (int j = i; j < bytes.length; j++) {
          assertEquals(bytes[j], subslice.readByte(j - i));
        }
      }

      // with padding
      for (int i = 0; i < 7; i++) {
        String name = "bytes-" + i;
        IndexOutput o = dir.createOutput(name, newIOContext(random()));
        byte[] junk = new byte[i];
        random().nextBytes(junk);
        o.writeBytes(junk, junk.length);
        input.seek(0);
        o.copyBytes(input, input.length());
        o.close();
        IndexInput padded = dir.openInput(name, newIOContext(random()));
        RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i);
        for (int j = 0; j < bytes.length; j++) {
          assertEquals(bytes[j], whole.readByte(j));
        }
        padded.close();
      }
      input.close();
    }
  }

  /** try to stress slices of slices */
  public void testSliceOfSlice() throws Exception {
    try (Directory dir = getDirectory(createTempDir("sliceOfSlice"))) {
      IndexOutput output = dir.createOutput("bytes", newIOContext(random()));
      final int num;
      if (TEST_NIGHTLY) {
        num = TestUtil.nextInt(random(), 250, 2500);
      } else {
        num = TestUtil.nextInt(random(), 50, 250);
      }
      byte[] bytes = new byte[num];
      random().nextBytes(bytes);
      for (int i = 0; i < bytes.length; i++) {
        output.writeByte(bytes[i]);
      }
      output.close();

      IndexInput input = dir.openInput("bytes", newIOContext(random()));
      // seek to a random spot should not impact slicing.
      input.seek(TestUtil.nextLong(random(), 0, input.length()));
      for (int i = 0; i < num; i += 16) {
        IndexInput slice1 = input.slice("slice1", i, num - i);
        assertEquals(0, slice1.getFilePointer());
        assertEquals(num - i, slice1.length());

        // seek to a random spot shouldnt impact slicing.
        slice1.seek(TestUtil.nextLong(random(), 0, slice1.length()));
        for (int j = 0; j < slice1.length(); j += 16) {
          IndexInput slice2 = slice1.slice("slice2", j, num - i - j);
          assertEquals(0, slice2.getFilePointer());
          assertEquals(num - i - j, slice2.length());
          byte[] data = new byte[num];
          System.arraycopy(bytes, 0, data, 0, i + j);
          if (random().nextBoolean()) {
            // read the bytes for this slice-of-slice
            slice2.readBytes(data, i + j, num - i - j);
          } else {
            // seek to a random spot in between, read some, seek back and read the rest
            long seek = TestUtil.nextLong(random(), 0, slice2.length());
            slice2.seek(seek);
            slice2.readBytes(data, (int) (i + j + seek), (int) (num - i - j - seek));
            slice2.seek(0);
            slice2.readBytes(data, i + j, (int) seek);
          }
          assertArrayEquals(bytes, data);
        }
      }

      input.close();
    }
  }

  /**
   * This test that writes larger than the size of the buffer output will correctly increment the
   * file pointer.
   */
  public void testLargeWrites() throws IOException {
    try (Directory dir = getDirectory(createTempDir("largeWrites"))) {
      IndexOutput os = dir.createOutput("testBufferStart.txt", newIOContext(random()));

      byte[] largeBuf = new byte[2048];
      random().nextBytes(largeBuf);

      long currentPos = os.getFilePointer();
      os.writeBytes(largeBuf, largeBuf.length);

      try {
        assertEquals(currentPos + largeBuf.length, os.getFilePointer());
      } finally {
        os.close();
      }
    }
  }

  // LUCENE-6084
  public void testIndexOutputToString() throws Throwable {
    try (Directory dir = getDirectory(createTempDir())) {
      IndexOutput out = dir.createOutput("camelCase.txt", newIOContext(random()));
      assertTrue(out.toString(), out.toString().contains("camelCase.txt"));
      out.close();
    }
  }

  public void testDoubleCloseOutput() throws Throwable {
    try (Directory dir = getDirectory(createTempDir())) {
      IndexOutput out = dir.createOutput("foobar", newIOContext(random()));
      out.writeString("testing");
      out.close();
      out.close(); // close again
    }
  }

  public void testDoubleCloseInput() throws Throwable {
    try (Directory dir = getDirectory(createTempDir())) {
      IndexOutput out = dir.createOutput("foobar", newIOContext(random()));
      out.writeString("testing");
      out.close();
      IndexInput in = dir.openInput("foobar", newIOContext(random()));
      assertEquals("testing", in.readString());
      in.close();
      in.close(); // close again
    }
  }

  public void testCreateTempOutput() throws Throwable {
    try (Directory dir = getDirectory(createTempDir())) {
      List<String> names = new ArrayList<>();
      int iters = atLeast(50);
      for (int iter = 0; iter < iters; iter++) {
        IndexOutput out = dir.createTempOutput("foo", "bar", newIOContext(random()));
        names.add(out.getName());
        out.writeVInt(iter);
        out.close();
      }
      for (int iter = 0; iter < iters; iter++) {
        IndexInput in = dir.openInput(names.get(iter), newIOContext(random()));
        assertEquals(iter, in.readVInt());
        in.close();
      }

      Set<String> files =
          Arrays.stream(dir.listAll())
              .filter(file -> !ExtrasFS.isExtra(file)) // remove any ExtrasFS stuff.
              .collect(Collectors.toSet());

      assertEquals(new HashSet<String>(names), files);
    }
  }

  public void testCreateOutputForExistingFile() throws IOException {
    try (Directory dir = getDirectory(createTempDir())) {
      String name = "file";
      try (IndexOutput out = dir.createOutput(name, IOContext.DEFAULT)) {
        assert out != null;
      }

      // Try to create an existing file should fail.
      expectThrows(
          FileAlreadyExistsException.class,
          () -> {
            try (IndexOutput out = dir.createOutput(name, IOContext.DEFAULT)) {
              assert out != null;
            }
          });

      // Delete file and try to recreate it.
      dir.deleteFile(name);
      dir.createOutput(name, IOContext.DEFAULT).close();
    }
  }

  public void testSeekToEndOfFile() throws IOException {
    try (Directory dir = getDirectory(createTempDir())) {
      try (IndexOutput out = dir.createOutput("a", IOContext.DEFAULT)) {
        for (int i = 0; i < 1024; ++i) {
          out.writeByte((byte) 0);
        }
      }
      try (IndexInput in = dir.openInput("a", IOContext.DEFAULT)) {
        in.seek(100);
        assertEquals(100, in.getFilePointer());
        in.seek(1024);
        assertEquals(1024, in.getFilePointer());
      }
    }
  }

  public void testSeekBeyondEndOfFile() throws IOException {
    try (Directory dir = getDirectory(createTempDir())) {
      try (IndexOutput out = dir.createOutput("a", IOContext.DEFAULT)) {
        for (int i = 0; i < 1024; ++i) {
          out.writeByte((byte) 0);
        }
      }
      try (IndexInput in = dir.openInput("a", IOContext.DEFAULT)) {
        in.seek(100);
        assertEquals(100, in.getFilePointer());
        expectThrows(
            EOFException.class,
            () -> {
              in.seek(1025);
            });
      }
    }
  }

  // Make sure the FSDirectory impl properly "emulates" deletions on filesystems (Windows) with
  // buggy deleteFile:
  public void testPendingDeletions() throws IOException {
    try (Directory dir = getDirectory(addVirusChecker(createTempDir()))) {
      assumeTrue(
          "we can only install VirusCheckingFS on an FSDirectory", dir instanceof FSDirectory);
      FSDirectory fsDir = (FSDirectory) dir;

      // Keep trying until virus checker refuses to delete:
      final String fileName;
      while (true) {
        // create a random filename (segment file name style), so it cannot hit windows problem with
        // special filenames ("con", "com1",...):
        String candidate =
            IndexFileNames.segmentFileName(
                TestUtil.randomSimpleString(random(), 1, 6),
                TestUtil.randomSimpleString(random()),
                "test");
        try (IndexOutput out = dir.createOutput(candidate, IOContext.DEFAULT)) {
          out.getFilePointer(); // just fake access to prevent compiler warning
        }
        fsDir.deleteFile(candidate);
        if (fsDir.getPendingDeletions().size() > 0) {
          // good: virus checker struck and prevented deletion of fileName
          fileName = candidate;
          break;
        }
      }

      // Make sure listAll does NOT include the file:
      assertFalse(Arrays.asList(fsDir.listAll()).contains(fileName));

      // Make sure fileLength claims it's deleted:
      expectThrows(
          NoSuchFileException.class,
          () -> {
            fsDir.fileLength(fileName);
          });

      // Make sure rename fails:
      expectThrows(
          NoSuchFileException.class,
          () -> {
            fsDir.rename(fileName, "file2");
          });

      // Make sure delete fails:
      expectThrows(
          NoSuchFileException.class,
          () -> {
            fsDir.deleteFile(fileName);
          });

      // Make sure we cannot open it for reading:
      expectThrowsAnyOf(
          Arrays.asList(NoSuchFileException.class, FileNotFoundException.class),
          () -> {
            fsDir.openInput(fileName, IOContext.DEFAULT);
          });
    }
  }

  public void testListAllIsSorted() throws IOException {
    try (Directory dir = getDirectory(createTempDir())) {
      int count = atLeast(20);
      Set<String> names = new HashSet<>();
      while (names.size() < count) {
        // create a random filename (segment file name style), so it cannot hit windows problem with
        // special filenames ("con", "com1",...):
        String name =
            IndexFileNames.segmentFileName(
                TestUtil.randomSimpleString(random(), 1, 6),
                TestUtil.randomSimpleString(random()),
                "test");
        if (random().nextInt(5) == 1) {
          IndexOutput out = dir.createTempOutput(name, "foo", IOContext.DEFAULT);
          names.add(out.getName());
          out.close();
        } else if (names.contains(name) == false) {
          IndexOutput out = dir.createOutput(name, IOContext.DEFAULT);
          names.add(out.getName());
          out.close();
        }
      }
      String[] actual = dir.listAll();
      String[] expected = actual.clone();
      Arrays.sort(expected);
      assertArrayEquals(expected, actual);
    }
  }
}
