/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.index;

import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.DocHelper;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.store.BaseDirectoryWrapper;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
import org.junit.Assume;

@LuceneTestCase.SuppressCodecs("SimpleText")
public class TestDirectoryReader extends LuceneTestCase {

  public void testDocument() throws IOException {
    SegmentReader[] readers = new SegmentReader[2];
    Directory dir = newDirectory();
    Document doc1 = new Document();
    Document doc2 = new Document();
    DocHelper.setupDoc(doc1);
    DocHelper.setupDoc(doc2);
    DocHelper.writeDoc(random(), dir, doc1);
    DocHelper.writeDoc(random(), dir, doc2);
    DirectoryReader reader = DirectoryReader.open(dir);
    assertTrue(reader != null);
    assertTrue(reader instanceof StandardDirectoryReader);
    StoredFields storedFields = reader.storedFields();

    Document newDoc1 = storedFields.document(0);
    assertTrue(newDoc1 != null);
    assertTrue(
        DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - DocHelper.unstored.size());
    Document newDoc2 = storedFields.document(1);
    assertTrue(newDoc2 != null);
    assertTrue(
        DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - DocHelper.unstored.size());
    Terms vector = reader.termVectors().get(0).terms(DocHelper.TEXT_FIELD_2_KEY);
    assertNotNull(vector);

    reader.close();
    if (readers[0] != null) readers[0].close();
    if (readers[1] != null) readers[1].close();
    dir.close();
  }

  public void testMultiTermDocs() throws IOException {
    Directory ramDir1 = newDirectory();
    addDoc(random(), ramDir1, "test foo", true);
    Directory ramDir2 = newDirectory();
    addDoc(random(), ramDir2, "test blah", true);
    Directory ramDir3 = newDirectory();
    addDoc(random(), ramDir3, "test wow", true);

    IndexReader[] readers1 =
        new IndexReader[] {DirectoryReader.open(ramDir1), DirectoryReader.open(ramDir3)};
    IndexReader[] readers2 =
        new IndexReader[] {
          DirectoryReader.open(ramDir1),
          DirectoryReader.open(ramDir2),
          DirectoryReader.open(ramDir3)
        };
    MultiReader mr2 = new MultiReader(readers1);
    MultiReader mr3 = new MultiReader(readers2);

    // test mixing up TermDocs and TermEnums from different readers.
    TermsEnum te2 = MultiTerms.getTerms(mr2, "body").iterator();
    te2.seekCeil(new BytesRef("wow"));
    PostingsEnum td = TestUtil.docs(random(), mr2, "body", te2.term(), null, 0);

    TermsEnum te3 = MultiTerms.getTerms(mr3, "body").iterator();
    te3.seekCeil(new BytesRef("wow"));
    td = TestUtil.docs(random(), te3, td, 0);

    int ret = 0;

    // This should blow up if we forget to check that the TermEnum is from the same
    // reader as the TermDocs.
    while (td.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) ret += td.docID();

    // really a dummy assert to ensure that we got some docs and to ensure that
    // nothing is eliminated by hotspot
    assertTrue(ret > 0);
    readers1[0].close();
    readers1[1].close();
    readers2[0].close();
    readers2[1].close();
    readers2[2].close();
    ramDir1.close();
    ramDir2.close();
    ramDir3.close();
  }

  private void addDoc(Random random, Directory ramDir1, String s, boolean create)
      throws IOException {
    IndexWriter iw =
        new IndexWriter(
            ramDir1,
            newIndexWriterConfig(new MockAnalyzer(random))
                .setOpenMode(create ? OpenMode.CREATE : OpenMode.APPEND));
    Document doc = new Document();
    doc.add(newTextField("body", s, Field.Store.NO));
    iw.addDocument(doc);
    iw.close();
  }

  public void testIsCurrent() throws Exception {
    Directory d = newDirectory();
    IndexWriter writer = new IndexWriter(d, newIndexWriterConfig(new MockAnalyzer(random())));
    addDocumentWithFields(writer);
    writer.close();
    // set up reader:
    DirectoryReader reader = DirectoryReader.open(d);
    assertTrue(reader.isCurrent());
    // modify index by adding another document:
    writer =
        new IndexWriter(
            d, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));
    addDocumentWithFields(writer);
    writer.close();
    assertFalse(reader.isCurrent());
    // re-create index:
    writer =
        new IndexWriter(
            d, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE));
    addDocumentWithFields(writer);
    writer.close();
    assertFalse(reader.isCurrent());
    reader.close();
    d.close();
  }

  /**
   * Tests the IndexReader.getFieldNames implementation
   *
   * @throws Exception on error
   */
  public void testGetFieldNames() throws Exception {
    Directory d = newDirectory();
    // set up writer
    IndexWriter writer = new IndexWriter(d, newIndexWriterConfig(new MockAnalyzer(random())));

    Document doc = new Document();

    FieldType customType3 = new FieldType();
    customType3.setStored(true);

    doc.add(new StringField("keyword", "test1", Field.Store.YES));
    doc.add(new TextField("text", "test1", Field.Store.YES));
    doc.add(new Field("unindexed", "test1", customType3));
    doc.add(new TextField("unstored", "test1", Field.Store.NO));
    writer.addDocument(doc);

    writer.close();
    // set up reader
    DirectoryReader reader = DirectoryReader.open(d);
    FieldInfos fieldInfos = FieldInfos.getMergedFieldInfos(reader);
    assertNotNull(fieldInfos.fieldInfo("keyword"));
    assertNotNull(fieldInfos.fieldInfo("text"));
    assertNotNull(fieldInfos.fieldInfo("unindexed"));
    assertNotNull(fieldInfos.fieldInfo("unstored"));
    reader.close();
    // add more documents
    writer =
        new IndexWriter(
            d,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setOpenMode(OpenMode.APPEND)
                .setMergePolicy(newLogMergePolicy()));
    // want to get some more segments here
    int mergeFactor = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor();
    for (int i = 0; i < 5 * mergeFactor; i++) {
      doc = new Document();
      doc.add(new StringField("keyword", "test1", Field.Store.YES));
      doc.add(new TextField("text", "test1", Field.Store.YES));
      doc.add(new Field("unindexed", "test1", customType3));
      doc.add(new TextField("unstored", "test1", Field.Store.NO));
      writer.addDocument(doc);
    }
    // new fields are in some different segments (we hope)
    for (int i = 0; i < 5 * mergeFactor; i++) {
      doc = new Document();
      doc.add(new StringField("keyword2", "test1", Field.Store.YES));
      doc.add(new TextField("text2", "test1", Field.Store.YES));
      doc.add(new Field("unindexed2", "test1", customType3));
      doc.add(new TextField("unstored2", "test1", Field.Store.NO));
      writer.addDocument(doc);
    }
    // new termvector fields

    FieldType customType5 = new FieldType(TextField.TYPE_STORED);
    customType5.setStoreTermVectors(true);
    FieldType customType6 = new FieldType(TextField.TYPE_STORED);
    customType6.setStoreTermVectors(true);
    customType6.setStoreTermVectorOffsets(true);
    FieldType customType7 = new FieldType(TextField.TYPE_STORED);
    customType7.setStoreTermVectors(true);
    customType7.setStoreTermVectorPositions(true);
    FieldType customType8 = new FieldType(TextField.TYPE_STORED);
    customType8.setStoreTermVectors(true);
    customType8.setStoreTermVectorOffsets(true);
    customType8.setStoreTermVectorPositions(true);

    for (int i = 0; i < 5 * mergeFactor; i++) {
      doc = new Document();
      doc.add(new TextField("tvnot", "tvnot", Field.Store.YES));
      doc.add(new Field("termvector", "termvector", customType5));
      doc.add(new Field("tvoffset", "tvoffset", customType6));
      doc.add(new Field("tvposition", "tvposition", customType7));
      doc.add(new Field("tvpositionoffset", "tvpositionoffset", customType8));
      writer.addDocument(doc);
    }

    writer.close();

    // verify fields again
    reader = DirectoryReader.open(d);
    fieldInfos = FieldInfos.getMergedFieldInfos(reader);

    Collection<String> allFieldNames = new HashSet<>();
    Collection<String> indexedFieldNames = new HashSet<>();
    Collection<String> notIndexedFieldNames = new HashSet<>();
    Collection<String> tvFieldNames = new HashSet<>();

    for (FieldInfo fieldInfo : fieldInfos) {
      final String name = fieldInfo.name;
      allFieldNames.add(name);
      if (fieldInfo.getIndexOptions() != IndexOptions.NONE) {
        indexedFieldNames.add(name);
      } else {
        notIndexedFieldNames.add(name);
      }
      if (fieldInfo.hasVectors()) {
        tvFieldNames.add(name);
      }
    }

    assertTrue(allFieldNames.contains("keyword"));
    assertTrue(allFieldNames.contains("text"));
    assertTrue(allFieldNames.contains("unindexed"));
    assertTrue(allFieldNames.contains("unstored"));
    assertTrue(allFieldNames.contains("keyword2"));
    assertTrue(allFieldNames.contains("text2"));
    assertTrue(allFieldNames.contains("unindexed2"));
    assertTrue(allFieldNames.contains("unstored2"));
    assertTrue(allFieldNames.contains("tvnot"));
    assertTrue(allFieldNames.contains("termvector"));
    assertTrue(allFieldNames.contains("tvposition"));
    assertTrue(allFieldNames.contains("tvoffset"));
    assertTrue(allFieldNames.contains("tvpositionoffset"));

    // verify that only indexed fields were returned
    assertEquals(11, indexedFieldNames.size()); // 6 original + the 5 termvector fields
    assertTrue(indexedFieldNames.contains("keyword"));
    assertTrue(indexedFieldNames.contains("text"));
    assertTrue(indexedFieldNames.contains("unstored"));
    assertTrue(indexedFieldNames.contains("keyword2"));
    assertTrue(indexedFieldNames.contains("text2"));
    assertTrue(indexedFieldNames.contains("unstored2"));
    assertTrue(indexedFieldNames.contains("tvnot"));
    assertTrue(indexedFieldNames.contains("termvector"));
    assertTrue(indexedFieldNames.contains("tvposition"));
    assertTrue(indexedFieldNames.contains("tvoffset"));
    assertTrue(indexedFieldNames.contains("tvpositionoffset"));

    // verify that only unindexed fields were returned
    assertEquals(2, notIndexedFieldNames.size()); // the following fields
    assertTrue(notIndexedFieldNames.contains("unindexed"));
    assertTrue(notIndexedFieldNames.contains("unindexed2"));

    // verify index term vector fields
    assertEquals(tvFieldNames.toString(), 4, tvFieldNames.size()); // 4 field has term vector only
    assertTrue(tvFieldNames.contains("termvector"));

    reader.close();
    d.close();
  }

  public void testTermVectors() throws Exception {
    Directory d = newDirectory();
    // set up writer
    IndexWriter writer =
        new IndexWriter(
            d,
            newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
    // want to get some more segments here
    // new termvector fields
    int mergeFactor = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor();
    FieldType customType5 = new FieldType(TextField.TYPE_STORED);
    customType5.setStoreTermVectors(true);
    FieldType customType6 = new FieldType(TextField.TYPE_STORED);
    customType6.setStoreTermVectors(true);
    customType6.setStoreTermVectorOffsets(true);
    FieldType customType7 = new FieldType(TextField.TYPE_STORED);
    customType7.setStoreTermVectors(true);
    customType7.setStoreTermVectorPositions(true);
    FieldType customType8 = new FieldType(TextField.TYPE_STORED);
    customType8.setStoreTermVectors(true);
    customType8.setStoreTermVectorOffsets(true);
    customType8.setStoreTermVectorPositions(true);
    for (int i = 0; i < 5 * mergeFactor; i++) {
      Document doc = new Document();
      doc.add(new TextField("tvnot", "one two two three three three", Field.Store.YES));
      doc.add(new Field("termvector", "one two two three three three", customType5));
      doc.add(new Field("tvoffset", "one two two three three three", customType6));
      doc.add(new Field("tvposition", "one two two three three three", customType7));
      doc.add(new Field("tvpositionoffset", "one two two three three three", customType8));

      writer.addDocument(doc);
    }
    writer.close();
    d.close();
  }

  void assertTermDocsCount(String msg, IndexReader reader, Term term, int expected)
      throws IOException {
    PostingsEnum tdocs =
        TestUtil.docs(random(), reader, term.field(), new BytesRef(term.text()), null, 0);
    int count = 0;
    if (tdocs != null) {
      while (tdocs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
        count++;
      }
    }
    assertEquals(msg + ", count mismatch", expected, count);
  }

  public void testBinaryFields() throws IOException {
    Directory dir = newDirectory();
    byte[] bin = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};

    IndexWriter writer =
        new IndexWriter(
            dir,
            newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));

    for (int i = 0; i < 10; i++) {
      addDoc(writer, "document number " + (i + 1));
      addDocumentWithFields(writer);
      addDocumentWithDifferentFields(writer);
      addDocumentWithTermVectorFields(writer);
    }
    writer.close();
    writer =
        new IndexWriter(
            dir,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setOpenMode(OpenMode.APPEND)
                .setMergePolicy(newLogMergePolicy()));
    Document doc = new Document();
    doc.add(new StoredField("bin1", bin));
    doc.add(new TextField("junk", "junk text", Field.Store.NO));
    writer.addDocument(doc);
    writer.close();
    DirectoryReader reader = DirectoryReader.open(dir);
    Document doc2 = reader.storedFields().document(reader.maxDoc() - 1);
    IndexableField[] fields = doc2.getFields("bin1");
    assertNotNull(fields);
    assertEquals(1, fields.length);
    IndexableField b1 = fields[0];
    assertTrue(b1.binaryValue() != null);
    BytesRef bytesRef = b1.binaryValue();
    assertEquals(bin.length, bytesRef.length);
    for (int i = 0; i < bin.length; i++) {
      assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);
    }
    reader.close();
    // force merge

    writer =
        new IndexWriter(
            dir,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setOpenMode(OpenMode.APPEND)
                .setMergePolicy(newLogMergePolicy()));
    writer.forceMerge(1);
    writer.close();
    reader = DirectoryReader.open(dir);
    doc2 = reader.storedFields().document(reader.maxDoc() - 1);
    fields = doc2.getFields("bin1");
    assertNotNull(fields);
    assertEquals(1, fields.length);
    b1 = fields[0];
    assertTrue(b1.binaryValue() != null);
    bytesRef = b1.binaryValue();
    assertEquals(bin.length, bytesRef.length);
    for (int i = 0; i < bin.length; i++) {
      assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);
    }
    reader.close();
    dir.close();
  }

  /* ??? public void testOpenEmptyDirectory() throws IOException{
    String dirName = "test.empty";
    File fileDirName = new File(dirName);
    if (!fileDirName.exists()) {
      fileDirName.mkdir();
    }
    try {
      DirectoryReader.open(fileDirName);
      fail("opening DirectoryReader on empty directory failed to produce FileNotFoundException/NoSuchFileException");
    } catch (FileNotFoundException | NoSuchFileException e) {
      // GOOD
    }
    rmDir(fileDirName);
  }*/

  public void testFilesOpenClose() throws IOException {
    // Create initial data set
    Path dirFile = createTempDir("TestIndexReader.testFilesOpenClose");
    Directory dir = newFSDirectory(dirFile);

    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    addDoc(writer, "test");
    writer.close();
    dir.close();

    // Try to erase the data - this ensures that the writer closed all files
    IOUtils.rm(dirFile);
    dir = newFSDirectory(dirFile);

    // Now create the data set again, just as before
    writer =
        new IndexWriter(
            dir, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE));
    addDoc(writer, "test");
    writer.close();
    dir.close();

    // Now open existing directory and test that reader closes all files
    dir = newFSDirectory(dirFile);
    DirectoryReader reader1 = DirectoryReader.open(dir);
    reader1.close();
    dir.close();

    // The following will fail if reader did not close
    // all files
    IOUtils.rm(dirFile);
  }

  public void testOpenReaderAfterDelete() throws IOException {
    Path dirFile = createTempDir("deletetest");
    Directory dir = newFSDirectory(dirFile);
    if (dir instanceof BaseDirectoryWrapper) {
      ((BaseDirectoryWrapper) dir)
          .setCheckIndexOnClose(false); // we will hit NoSuchFileException in MDW since we nuked it!
    }
    expectThrowsAnyOf(
        Arrays.asList(FileNotFoundException.class, NoSuchFileException.class),
        () -> DirectoryReader.open(dir));

    IOUtils.rm(dirFile);

    // Make sure we still get a CorruptIndexException (not NPE):
    expectThrowsAnyOf(
        Arrays.asList(FileNotFoundException.class, NoSuchFileException.class),
        () -> DirectoryReader.open(dir));

    dir.close();
  }

  static void addDocumentWithFields(IndexWriter writer) throws IOException {
    Document doc = new Document();

    FieldType customType3 = new FieldType();
    customType3.setStored(true);
    doc.add(newStringField("keyword", "test1", Field.Store.YES));
    doc.add(newTextField("text", "test1", Field.Store.YES));
    doc.add(newField("unindexed", "test1", customType3));
    doc.add(new TextField("unstored", "test1", Field.Store.NO));
    writer.addDocument(doc);
  }

  static void addDocumentWithDifferentFields(IndexWriter writer) throws IOException {
    Document doc = new Document();

    FieldType customType3 = new FieldType();
    customType3.setStored(true);
    doc.add(newStringField("keyword2", "test1", Field.Store.YES));
    doc.add(newTextField("text2", "test1", Field.Store.YES));
    doc.add(newField("unindexed2", "test1", customType3));
    doc.add(new TextField("unstored2", "test1", Field.Store.NO));
    writer.addDocument(doc);
  }

  static void addDocumentWithTermVectorFields(IndexWriter writer) throws IOException {
    Document doc = new Document();
    FieldType customType5 = new FieldType(TextField.TYPE_STORED);
    customType5.setStoreTermVectors(true);
    FieldType customType6 = new FieldType(TextField.TYPE_STORED);
    customType6.setStoreTermVectors(true);
    customType6.setStoreTermVectorOffsets(true);
    FieldType customType7 = new FieldType(TextField.TYPE_STORED);
    customType7.setStoreTermVectors(true);
    customType7.setStoreTermVectorPositions(true);
    FieldType customType8 = new FieldType(TextField.TYPE_STORED);
    customType8.setStoreTermVectors(true);
    customType8.setStoreTermVectorOffsets(true);
    customType8.setStoreTermVectorPositions(true);
    doc.add(newTextField("tvnot", "tvnot", Field.Store.YES));
    doc.add(newField("termvector", "termvector", customType5));
    doc.add(newField("tvoffset", "tvoffset", customType6));
    doc.add(newField("tvposition", "tvposition", customType7));
    doc.add(newField("tvpositionoffset", "tvpositionoffset", customType8));

    writer.addDocument(doc);
  }

  static void addDoc(IndexWriter writer, String value) throws IOException {
    Document doc = new Document();
    doc.add(newTextField("content", value, Field.Store.NO));
    writer.addDocument(doc);
  }

  // TODO: maybe this can reuse the logic of test dueling codecs?
  public static void assertIndexEquals(DirectoryReader index1, DirectoryReader index2)
      throws IOException {
    assertEquals(
        "IndexReaders have different values for numDocs.", index1.numDocs(), index2.numDocs());
    assertEquals(
        "IndexReaders have different values for maxDoc.", index1.maxDoc(), index2.maxDoc());
    assertEquals(
        "Only one IndexReader has deletions.", index1.hasDeletions(), index2.hasDeletions());
    assertEquals(
        "Single segment test differs.", index1.leaves().size() == 1, index2.leaves().size() == 1);

    // check field names
    FieldInfos fieldInfos1 = FieldInfos.getMergedFieldInfos(index1);
    FieldInfos fieldInfos2 = FieldInfos.getMergedFieldInfos(index2);
    assertEquals(
        "IndexReaders have different numbers of fields.", fieldInfos1.size(), fieldInfos2.size());
    final int numFields = fieldInfos1.size();
    for (int fieldID = 0; fieldID < numFields; fieldID++) {
      final FieldInfo fieldInfo1 = fieldInfos1.fieldInfo(fieldID);
      final FieldInfo fieldInfo2 = fieldInfos2.fieldInfo(fieldID);
      assertEquals("Different field names.", fieldInfo1.name, fieldInfo2.name);
    }

    // check norms
    for (FieldInfo fieldInfo : fieldInfos1) {
      String curField = fieldInfo.name;
      NumericDocValues norms1 = MultiDocValues.getNormValues(index1, curField);
      NumericDocValues norms2 = MultiDocValues.getNormValues(index2, curField);
      if (norms1 != null && norms2 != null) {
        // todo: generalize this (like TestDuelingCodecs assert)
        while (true) {
          int docID = norms1.nextDoc();
          assertEquals(docID, norms2.nextDoc());
          if (docID == NO_MORE_DOCS) {
            break;
          }
          assertEquals(
              "Norm different for doc " + docID + " and field '" + curField + "'.",
              norms1.longValue(),
              norms2.longValue());
        }
      } else {
        assertNull(norms1);
        assertNull(norms2);
      }
    }

    // check deletions
    final Bits liveDocs1 = MultiBits.getLiveDocs(index1);
    final Bits liveDocs2 = MultiBits.getLiveDocs(index2);
    for (int i = 0; i < index1.maxDoc(); i++) {
      assertEquals(
          "Doc " + i + " only deleted in one index.",
          liveDocs1 == null || !liveDocs1.get(i),
          liveDocs2 == null || !liveDocs2.get(i));
    }

    // check stored fields
    StoredFields storedFields1 = index1.storedFields();
    StoredFields storedFields2 = index2.storedFields();
    for (int i = 0; i < index1.maxDoc(); i++) {
      if (liveDocs1 == null || liveDocs1.get(i)) {
        Document doc1 = storedFields1.document(i);
        Document doc2 = storedFields2.document(i);
        List<IndexableField> field1 = doc1.getFields();
        List<IndexableField> field2 = doc2.getFields();
        assertEquals(
            "Different numbers of fields for doc " + i + ".", field1.size(), field2.size());
        Iterator<IndexableField> itField1 = field1.iterator();
        Iterator<IndexableField> itField2 = field2.iterator();
        while (itField1.hasNext()) {
          Field curField1 = (Field) itField1.next();
          Field curField2 = (Field) itField2.next();
          assertEquals(
              "Different fields names for doc " + i + ".", curField1.name(), curField2.name());
          assertEquals(
              "Different field values for doc " + i + ".",
              curField1.stringValue(),
              curField2.stringValue());
        }
      }
    }

    // check dictionary and posting lists
    TreeSet<String> fields1 = new TreeSet<>(FieldInfos.getIndexedFields(index1));
    TreeSet<String> fields2 = new TreeSet<>(FieldInfos.getIndexedFields(index2));
    Iterator<String> fenum2 = fields2.iterator();
    for (String field1 : fields1) {
      assertEquals("Different fields", field1, fenum2.next());
      Terms terms1 = MultiTerms.getTerms(index1, field1);
      if (terms1 == null) {
        assertNull(MultiTerms.getTerms(index2, field1));
        continue;
      }
      TermsEnum enum1 = terms1.iterator();

      Terms terms2 = MultiTerms.getTerms(index2, field1);
      assertNotNull(terms2);
      TermsEnum enum2 = terms2.iterator();

      while (enum1.next() != null) {
        assertEquals("Different terms", enum1.term(), enum2.next());
        PostingsEnum tp1 = enum1.postings(null, PostingsEnum.ALL);
        PostingsEnum tp2 = enum2.postings(null, PostingsEnum.ALL);

        while (tp1.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
          assertTrue(tp2.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
          assertEquals(
              "Different doc id in postinglist of term " + enum1.term() + ".",
              tp1.docID(),
              tp2.docID());
          assertEquals(
              "Different term frequence in postinglist of term " + enum1.term() + ".",
              tp1.freq(),
              tp2.freq());
          for (int i = 0; i < tp1.freq(); i++) {
            assertEquals(
                "Different positions in postinglist of term " + enum1.term() + ".",
                tp1.nextPosition(),
                tp2.nextPosition());
          }
        }
      }
    }
    assertFalse(fenum2.hasNext());
  }

  public void testGetIndexCommit() throws IOException {

    Directory d = newDirectory();

    // set up writer
    IndexWriter writer =
        new IndexWriter(
            d,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setMaxBufferedDocs(2)
                .setMergePolicy(newLogMergePolicy(10)));
    for (int i = 0; i < 27; i++) addDocumentWithFields(writer);
    writer.close();

    SegmentInfos sis = SegmentInfos.readLatestCommit(d);
    DirectoryReader r = DirectoryReader.open(d);
    IndexCommit c = r.getIndexCommit();

    assertEquals(sis.getSegmentsFileName(), c.getSegmentsFileName());

    assertTrue(c.equals(r.getIndexCommit()));

    // Change the index
    writer =
        new IndexWriter(
            d,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setOpenMode(OpenMode.APPEND)
                .setMaxBufferedDocs(2)
                .setMergePolicy(newLogMergePolicy(10)));
    for (int i = 0; i < 7; i++) addDocumentWithFields(writer);
    writer.close();

    DirectoryReader r2 = DirectoryReader.openIfChanged(r);
    assertNotNull(r2);
    assertFalse(c.equals(r2.getIndexCommit()));
    assertFalse(r2.getIndexCommit().getSegmentCount() == 1);
    r2.close();

    writer =
        new IndexWriter(
            d, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));
    writer.forceMerge(1);
    writer.close();

    r2 = DirectoryReader.openIfChanged(r);
    assertNotNull(r2);
    assertNull(DirectoryReader.openIfChanged(r2));
    assertEquals(1, r2.getIndexCommit().getSegmentCount());

    r.close();
    r2.close();
    d.close();
  }

  static Document createDocument(String id) {
    Document doc = new Document();
    FieldType customType = new FieldType(TextField.TYPE_STORED);
    customType.setTokenized(false);
    customType.setOmitNorms(true);

    doc.add(newField("id", id, customType));
    return doc;
  }

  // LUCENE-1468 -- make sure on attempting to open an
  // DirectoryReader on a non-existent directory, you get a
  // good exception
  public void testNoDir() throws Throwable {
    Path tempDir = createTempDir("doesnotexist");
    Directory dir = newFSDirectory(tempDir);
    expectThrows(
        IndexNotFoundException.class,
        () -> {
          DirectoryReader.open(dir);
        });
    dir.close();
  }

  // LUCENE-1509
  public void testNoDupCommitFileNames() throws Throwable {

    Directory dir = newDirectory();

    IndexWriter writer =
        new IndexWriter(
            dir, newIndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(2));
    writer.addDocument(createDocument("a"));
    writer.addDocument(createDocument("a"));
    writer.addDocument(createDocument("a"));
    writer.close();

    Collection<IndexCommit> commits = DirectoryReader.listCommits(dir);
    for (final IndexCommit commit : commits) {
      Collection<String> files = commit.getFileNames();
      HashSet<String> seen = new HashSet<>();
      for (final String fileName : files) {
        assertTrue("file " + fileName + " was duplicated", !seen.contains(fileName));
        seen.add(fileName);
      }
    }

    dir.close();
  }

  // LUCENE-1586: getUniqueTermCount
  public void testUniqueTermCount() throws Exception {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    Document doc = new Document();
    doc.add(
        newTextField(
            "field", "a b c d e f g h i j k l m n o p q r s t u v w x y z", Field.Store.NO));
    doc.add(newTextField("number", "0 1 2 3 4 5 6 7 8 9", Field.Store.NO));
    writer.addDocument(doc);
    writer.addDocument(doc);
    writer.commit();

    DirectoryReader r = DirectoryReader.open(dir);
    LeafReader r1 = getOnlyLeafReader(r);
    assertEquals(26, r1.terms("field").size());
    assertEquals(10, r1.terms("number").size());
    writer.addDocument(doc);
    writer.commit();
    DirectoryReader r2 = DirectoryReader.openIfChanged(r);
    assertNotNull(r2);
    r.close();

    for (LeafReaderContext s : r2.leaves()) {
      assertEquals(26, s.reader().terms("field").size());
      assertEquals(10, s.reader().terms("number").size());
    }
    r2.close();
    writer.close();
    dir.close();
  }

  // LUCENE-2046
  public void testPrepareCommitIsCurrent() throws Throwable {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    writer.commit();
    Document doc = new Document();
    writer.addDocument(doc);
    DirectoryReader r = DirectoryReader.open(dir);
    assertTrue(r.isCurrent());
    writer.addDocument(doc);
    writer.prepareCommit();
    assertTrue(r.isCurrent());
    DirectoryReader r2 = DirectoryReader.openIfChanged(r);
    assertNull(r2);
    writer.commit();
    assertFalse(r.isCurrent());
    writer.close();
    r.close();
    dir.close();
  }

  // LUCENE-2753
  public void testListCommits() throws Exception {
    Directory dir = newDirectory();
    IndexWriter writer =
        new IndexWriter(
            dir,
            newIndexWriterConfig(null)
                .setIndexDeletionPolicy(
                    new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy())));
    SnapshotDeletionPolicy sdp =
        (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
    writer.addDocument(new Document());
    writer.commit();
    sdp.snapshot();
    writer.addDocument(new Document());
    writer.commit();
    sdp.snapshot();
    writer.addDocument(new Document());
    writer.commit();
    sdp.snapshot();
    writer.close();
    long currentGen = 0;
    for (IndexCommit ic : DirectoryReader.listCommits(dir)) {
      assertTrue(
          "currentGen=" + currentGen + " commitGen=" + ic.getGeneration(),
          currentGen < ic.getGeneration());
      currentGen = ic.getGeneration();
    }
    dir.close();
  }

  // Make sure totalTermFreq works correctly in the terms
  // dict cache
  public void testTotalTermFreqCached() throws Exception {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    Document d = new Document();
    d.add(newTextField("f", "a a b", Field.Store.NO));
    writer.addDocument(d);
    DirectoryReader r = DirectoryReader.open(writer);
    writer.close();
    try {
      // Make sure codec impls totalTermFreq (eg PreFlex doesn't)
      Assume.assumeTrue(r.totalTermFreq(new Term("f", new BytesRef("b"))) != -1);
      assertEquals(1, r.totalTermFreq(new Term("f", new BytesRef("b"))));
      assertEquals(2, r.totalTermFreq(new Term("f", new BytesRef("a"))));
      assertEquals(1, r.totalTermFreq(new Term("f", new BytesRef("b"))));
    } finally {
      r.close();
      dir.close();
    }
  }

  public void testGetSumDocFreq() throws Exception {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    Document d = new Document();
    d.add(newTextField("f", "a", Field.Store.NO));
    writer.addDocument(d);
    d = new Document();
    d.add(newTextField("f", "b", Field.Store.NO));
    writer.addDocument(d);
    DirectoryReader r = DirectoryReader.open(writer);
    writer.close();
    try {
      // Make sure codec impls getSumDocFreq (eg PreFlex doesn't)
      Assume.assumeTrue(r.getSumDocFreq("f") != -1);
      assertEquals(2, r.getSumDocFreq("f"));
    } finally {
      r.close();
      dir.close();
    }
  }

  public void testGetDocCount() throws Exception {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    Document d = new Document();
    d.add(newTextField("f", "a", Field.Store.NO));
    writer.addDocument(d);
    d = new Document();
    d.add(newTextField("f", "a", Field.Store.NO));
    writer.addDocument(d);
    DirectoryReader r = DirectoryReader.open(writer);
    writer.close();
    try {
      // Make sure codec impls getSumDocFreq (eg PreFlex doesn't)
      Assume.assumeTrue(r.getDocCount("f") != -1);
      assertEquals(2, r.getDocCount("f"));
    } finally {
      r.close();
      dir.close();
    }
  }

  public void testGetSumTotalTermFreq() throws Exception {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    Document d = new Document();
    d.add(newTextField("f", "a b b", Field.Store.NO));
    writer.addDocument(d);
    d = new Document();
    d.add(newTextField("f", "a a b", Field.Store.NO));
    writer.addDocument(d);
    DirectoryReader r = DirectoryReader.open(writer);
    writer.close();
    try {
      // Make sure codec impls getSumDocFreq (eg PreFlex doesn't)
      Assume.assumeTrue(r.getSumTotalTermFreq("f") != -1);
      assertEquals(6, r.getSumTotalTermFreq("f"));
    } finally {
      r.close();
      dir.close();
    }
  }

  // LUCENE-2474
  public void testReaderFinishedListener() throws Exception {
    Directory dir = newDirectory();
    IndexWriter writer =
        new IndexWriter(
            dir,
            newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
    ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(3);
    writer.addDocument(new Document());
    writer.commit();
    writer.addDocument(new Document());
    writer.commit();
    final DirectoryReader reader = DirectoryReader.open(writer);
    final int[] closeCount = new int[1];
    final IndexReader.ClosedListener listener =
        new IndexReader.ClosedListener() {
          @Override
          public void onClose(IndexReader.CacheKey key) {
            closeCount[0]++;
          }
        };

    reader.getReaderCacheHelper().addClosedListener(listener);

    reader.close();

    // Close the top reader, it's the only one that should be closed
    assertEquals(1, closeCount[0]);
    writer.close();

    DirectoryReader reader2 = DirectoryReader.open(dir);
    reader2.getReaderCacheHelper().addClosedListener(listener);

    closeCount[0] = 0;
    reader2.close();
    assertEquals(1, closeCount[0]);
    dir.close();
  }

  public void testOOBDocID() throws Exception {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    writer.addDocument(new Document());
    DirectoryReader r = DirectoryReader.open(writer);
    writer.close();
    r.storedFields().document(0);
    expectThrows(
        IllegalArgumentException.class,
        () -> {
          r.storedFields().document(1);
        });
    r.close();
    dir.close();
  }

  public void testTryIncRef() throws IOException {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    writer.addDocument(new Document());
    writer.commit();
    DirectoryReader r = DirectoryReader.open(dir);
    assertTrue(r.tryIncRef());
    r.decRef();
    r.close();
    assertFalse(r.tryIncRef());
    writer.close();
    dir.close();
  }

  public void testStressTryIncRef() throws IOException, InterruptedException {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    writer.addDocument(new Document());
    writer.commit();
    DirectoryReader r = DirectoryReader.open(dir);
    int numThreads = atLeast(2);

    IncThread[] threads = new IncThread[numThreads];
    for (int i = 0; i < threads.length; i++) {
      threads[i] = new IncThread(r, random());
      threads[i].start();
    }
    Thread.sleep(100);

    assertTrue(r.tryIncRef());
    r.decRef();
    r.close();

    for (int i = 0; i < threads.length; i++) {
      threads[i].join();
      assertNull(threads[i].failed);
    }
    assertFalse(r.tryIncRef());
    writer.close();
    dir.close();
  }

  static class IncThread extends Thread {
    final IndexReader toInc;
    final Random random;
    Throwable failed;

    IncThread(IndexReader toInc, Random random) {
      this.toInc = toInc;
      this.random = random;
    }

    @Override
    public void run() {
      try {
        while (toInc.tryIncRef()) {
          assertFalse(toInc.hasDeletions());
          toInc.decRef();
        }
        assertFalse(toInc.tryIncRef());
      } catch (Throwable e) {
        failed = e;
      }
    }
  }

  public void testLoadCertainFields() throws Exception {
    Directory dir = newDirectory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
    Document doc = new Document();
    doc.add(newStringField("field1", "foobar", Field.Store.YES));
    doc.add(newStringField("field2", "foobaz", Field.Store.YES));
    writer.addDocument(doc);
    DirectoryReader r = writer.getReader();
    writer.close();
    Set<String> fieldsToLoad = new HashSet<>();
    assertEquals(0, r.storedFields().document(0, fieldsToLoad).getFields().size());
    fieldsToLoad.add("field1");
    Document doc2 = r.storedFields().document(0, fieldsToLoad);
    assertEquals(1, doc2.getFields().size());
    assertEquals("foobar", doc2.get("field1"));
    r.close();
    dir.close();
  }

  public void testIndexExistsOnNonExistentDirectory() throws Exception {
    Path tempDir = createTempDir("testIndexExistsOnNonExistentDirectory");
    Directory dir = newFSDirectory(tempDir);
    assertFalse(DirectoryReader.indexExists(dir));
    dir.close();
  }

  public void testOpenWithInvalidMinCompatVersion() throws IOException {
    try (Directory dir = newDirectory();
        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig())) {
      Document doc = new Document();
      doc.add(newStringField("field1", "foobar", Field.Store.YES));
      doc.add(newStringField("field2", "foobaz", Field.Store.YES));
      writer.addDocument(doc);
      writer.commit();
      IndexCommit commit = DirectoryReader.listCommits(dir).get(0);
      expectThrows(IllegalArgumentException.class, () -> DirectoryReader.open(commit, -1, null));
      DirectoryReader.open(commit, random().nextInt(Version.LATEST.major + 1), null).close();
    }
  }
}
