/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.store;

import java.io.FileNotFoundException;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.file.DirectoryStream;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;

/**
 * Base class for Directory implementations that store index files in the file system. <a
 * id="subclasses"></a> There are currently three core subclasses:
 *
 * <ul>
 *   <li>{@link NIOFSDirectory} uses java.nio's FileChannel's positional io when reading to avoid
 *       synchronization when reading from the same file. Unfortunately, due to a Windows-only <a
 *       href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6265734">Sun JRE bug</a> this is a
 *       poor choice for Windows, but on all other platforms this is the preferred choice.
 *       Applications using {@link Thread#interrupt()} or {@link Future#cancel(boolean)} should use
 *       {@code RAFDirectory} instead. See {@link NIOFSDirectory} java doc for details.
 *   <li>{@link MMapDirectory} uses memory-mapped IO when reading. This is a good choice if you have
 *       plenty of virtual memory relative to your index size, eg if you are running on a 64 bit
 *       JRE, or you are running on a 32 bit JRE but your index sizes are small enough to fit into
 *       the virtual memory space. Java has currently the limitation of not being able to unmap
 *       files from user code. The files are unmapped, when GC releases the byte buffers. Due to <a
 *       href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4724038">this bug</a> in Sun's
 *       JRE, MMapDirectory's {@link IndexInput#close} is unable to close the underlying OS file
 *       handle. Only when GC finally collects the underlying objects, which could be quite some
 *       time later, will the file handle be closed. This will consume additional transient disk
 *       usage: on Windows, attempts to delete or overwrite the files will result in an exception;
 *       on other platforms, which typically have a &quot;delete on last close&quot; semantics,
 *       while such operations will succeed, the bytes are still consuming space on disk. For many
 *       applications this limitation is not a problem (e.g. if you have plenty of disk space, and
 *       you don't rely on overwriting files on Windows) but it's still an important limitation to
 *       be aware of. This class supplies a (possibly dangerous) workaround mentioned in the bug
 *       report, which may fail on non-Sun JVMs.
 * </ul>
 *
 * <p>Unfortunately, because of system peculiarities, there is no single overall best
 * implementation. Therefore, we've added the {@link #open} method, to allow Lucene to choose the
 * best FSDirectory implementation given your environment, and the known limitations of each
 * implementation. For users who have no reason to prefer a specific implementation, it's best to
 * simply use {@link #open}. For all others, you should instantiate the desired implementation
 * directly.
 *
 * <p><b>NOTE:</b> Accessing one of the above subclasses either directly or indirectly from a thread
 * while it's interrupted can close the underlying channel immediately if at the same time the
 * thread is blocked on IO. The channel will remain closed and subsequent access to the index will
 * throw a {@link ClosedChannelException}. Applications using {@link Thread#interrupt()} or {@link
 * Future#cancel(boolean)} should use the slower legacy {@code RAFDirectory} from the {@code misc}
 * Lucene module instead.
 *
 * <p>The locking implementation is by default {@link NativeFSLockFactory}, but can be changed by
 * passing in a custom {@link LockFactory} instance.
 *
 * @see Directory
 */
public abstract class FSDirectory extends BaseDirectory {

  protected final Path directory; // The underlying filesystem directory

  /**
   * Maps files that we are trying to delete (or we tried already but failed) before attempting to
   * delete that key.
   */
  private final Set<String> pendingDeletes =
      Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());

  private final AtomicInteger opsSinceLastDelete = new AtomicInteger();

  /** Used to generate temp file names in {@link #createTempOutput}. */
  private final AtomicLong nextTempFileCounter = new AtomicLong();

  /**
   * Create a new FSDirectory for the named location (ctor for subclasses). The directory is created
   * at the named location if it does not yet exist.
   *
   * <p>{@code FSDirectory} resolves the given Path to a canonical / real path to ensure it can
   * correctly lock the index directory and no other process can interfere with changing possible
   * symlinks to the index directory inbetween. If you want to use symlinks and change them
   * dynamically, close all {@code IndexWriters} and create a new {@code FSDirectory} instance.
   *
   * @param path the path of the directory
   * @param lockFactory the lock factory to use, or null for the default ({@link
   *     NativeFSLockFactory});
   * @throws IOException if there is a low-level I/O error
   */
  protected FSDirectory(Path path, LockFactory lockFactory) throws IOException {
    super(lockFactory);
    // If only read access is permitted, createDirectories fails even if the directory already
    // exists.
    if (!Files.isDirectory(path)) {
      Files.createDirectories(path); // create directory, if it doesn't exist
    }
    directory = path.toRealPath();
  }

  /**
   * Creates an FSDirectory instance, trying to pick the best implementation given the current
   * environment. The directory returned uses the {@link NativeFSLockFactory}. The directory is
   * created at the named location if it does not yet exist.
   *
   * <p>{@code FSDirectory} resolves the given Path when calling this method to a canonical / real
   * path to ensure it can correctly lock the index directory and no other process can interfere
   * with changing possible symlinks to the index directory inbetween. If you want to use symlinks
   * and change them dynamically, close all {@code IndexWriters} and create a new {@code
   * FSDirectory} instance.
   *
   * <p>Currently this returns {@link MMapDirectory} for Linux, MacOSX, Solaris, and Windows 64-bit
   * JREs, and {@link NIOFSDirectory} for other JREs. It is highly recommended that you consult the
   * implementation's documentation for your platform before using this method.
   *
   * <p><b>NOTE</b>: this method may suddenly change which implementation is returned from release
   * to release, in the event that higher performance defaults become possible; if the precise
   * implementation is important to your application, please instantiate it directly, instead. For
   * optimal performance you should consider using {@link MMapDirectory} on 64 bit JVMs.
   *
   * <p>See <a href="#subclasses">above</a>
   */
  public static FSDirectory open(Path path) throws IOException {
    return open(path, FSLockFactory.getDefault());
  }

  /** Just like {@link #open(Path)}, but allows you to also specify a custom {@link LockFactory}. */
  public static FSDirectory open(Path path, LockFactory lockFactory) throws IOException {
    if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
      return new MMapDirectory(path, lockFactory);
    } else {
      return new NIOFSDirectory(path, lockFactory);
    }
  }

  /**
   * Lists all files (including subdirectories) in the directory.
   *
   * @throws IOException if there was an I/O error during listing
   */
  public static String[] listAll(Path dir) throws IOException {
    return listAll(dir, null);
  }

  private static String[] listAll(Path dir, Set<String> skipNames) throws IOException {
    List<String> entries = new ArrayList<>();

    try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
      for (Path path : stream) {
        String name = path.getFileName().toString();
        if (skipNames == null || skipNames.contains(name) == false) {
          entries.add(name);
        }
      }
    }

    String[] array = entries.toArray(new String[entries.size()]);
    // Directory.listAll javadocs state that we sort the results here, so we don't let filesystem
    // specifics leak out of this abstraction:
    Arrays.sort(array);
    return array;
  }

  @Override
  public String[] listAll() throws IOException {
    ensureOpen();
    return listAll(directory, pendingDeletes);
  }

  @Override
  public long fileLength(String name) throws IOException {
    ensureOpen();
    if (pendingDeletes.contains(name)) {
      throw new NoSuchFileException("file \"" + name + "\" is pending delete");
    }
    return Files.size(directory.resolve(name));
  }

  @Override
  public IndexOutput createOutput(String name, IOContext context) throws IOException {
    ensureOpen();
    maybeDeletePendingFiles();
    // If this file was pending delete, we are now bringing it back to life:
    if (pendingDeletes.remove(name)) {
      privateDeleteFile(name, true); // try again to delete it - this is best effort
      pendingDeletes.remove(name); // watch out - if the delete fails it put
    }
    return new FSIndexOutput(name);
  }

  @Override
  public IndexOutput createTempOutput(String prefix, String suffix, IOContext context)
      throws IOException {
    ensureOpen();
    maybeDeletePendingFiles();
    while (true) {
      try {
        String name = getTempFileName(prefix, suffix, nextTempFileCounter.getAndIncrement());
        if (pendingDeletes.contains(name)) {
          continue;
        }
        return new FSIndexOutput(name, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
      } catch (
          @SuppressWarnings("unused")
          FileAlreadyExistsException faee) {
        // Retry with next incremented name
      }
    }
  }

  protected void ensureCanRead(String name) throws IOException {
    if (pendingDeletes.contains(name)) {
      throw new NoSuchFileException(
          "file \"" + name + "\" is pending delete and cannot be opened for read");
    }
  }

  @Override
  public void sync(Collection<String> names) throws IOException {
    ensureOpen();

    for (String name : names) {
      fsync(name);
    }
    maybeDeletePendingFiles();
  }

  @Override
  public void rename(String source, String dest) throws IOException {
    ensureOpen();
    if (pendingDeletes.contains(source)) {
      throw new NoSuchFileException(
          "file \"" + source + "\" is pending delete and cannot be moved");
    }
    maybeDeletePendingFiles();
    if (pendingDeletes.remove(dest)) {
      privateDeleteFile(dest, true); // try again to delete it - this is best effort
      pendingDeletes.remove(dest); // watch out if the delete fails it's back in here.
    }
    Files.move(directory.resolve(source), directory.resolve(dest), StandardCopyOption.ATOMIC_MOVE);
  }

  @Override
  public void syncMetaData() throws IOException {
    // TODO: to improve listCommits(), IndexFileDeleter could call this after deleting segments_Ns
    ensureOpen();
    IOUtils.fsync(directory, true);
    maybeDeletePendingFiles();
  }

  @Override
  public synchronized void close() throws IOException {
    isOpen = false;
    deletePendingFiles();
  }

  /**
   * @return the underlying filesystem directory
   */
  public Path getDirectory() {
    ensureOpen();
    return directory;
  }

  @Override
  public String toString() {
    return this.getClass().getSimpleName() + "@" + directory + " lockFactory=" + lockFactory;
  }

  protected void fsync(String name) throws IOException {
    IOUtils.fsync(directory.resolve(name), false);
  }

  @Override
  public void deleteFile(String name) throws IOException {
    if (pendingDeletes.contains(name)) {
      throw new NoSuchFileException("file \"" + name + "\" is already pending delete");
    }
    privateDeleteFile(name, false);
    maybeDeletePendingFiles();
  }

  /**
   * Try to delete any pending files that we had previously tried to delete but failed because we
   * are on Windows and the files were still held open.
   */
  public synchronized void deletePendingFiles() throws IOException {
    if (pendingDeletes.isEmpty() == false) {

      // TODO: we could fix IndexInputs from FSDirectory subclasses to call this when they are
      // closed?

      // Clone the set since we mutate it in privateDeleteFile:
      for (String name : new HashSet<>(pendingDeletes)) {
        privateDeleteFile(name, true);
      }
    }
  }

  private void maybeDeletePendingFiles() throws IOException {
    if (pendingDeletes.isEmpty() == false) {
      // This is a silly heuristic to try to avoid O(N^2), where N = number of files pending
      // deletion, behaviour on Windows:
      int count = opsSinceLastDelete.incrementAndGet();
      if (count >= pendingDeletes.size()) {
        opsSinceLastDelete.addAndGet(-count);
        deletePendingFiles();
      }
    }
  }

  private void privateDeleteFile(String name, boolean isPendingDelete) throws IOException {
    try {
      Files.delete(directory.resolve(name));
      pendingDeletes.remove(name);
    } catch (NoSuchFileException | FileNotFoundException e) {
      // We were asked to delete a non-existent file:
      pendingDeletes.remove(name);
      if (isPendingDelete && Constants.WINDOWS) {
        // TODO: can we remove this OS-specific hacky logic?  If windows deleteFile is buggy, we
        // should instead contain this workaround in
        // a WindowsFSDirectory ...
        // LUCENE-6684: we suppress this check for Windows, since a file could be in a confusing
        // "pending delete" state, failing the first
        // delete attempt with access denied and then apparently falsely failing here when we try ot
        // delete it again, with NSFE/FNFE
      } else {
        throw e;
      }
    } catch (
        @SuppressWarnings("unused")
        IOException ioe) {
      // On windows, a file delete can fail because there's still an open
      // file handle against it.  We record this in pendingDeletes and
      // try again later.

      // TODO: this is hacky/lenient (we don't know which IOException this is), and
      // it should only happen on filesystems that can do this, so really we should
      // move this logic to WindowsDirectory or something

      // TODO: can/should we do if (Constants.WINDOWS) here, else throw the exc?
      // but what about a Linux box with a CIFS mount?
      pendingDeletes.add(name);
    }
  }

  final class FSIndexOutput extends OutputStreamIndexOutput {
    /**
     * The maximum chunk size is 8192 bytes, because file channel mallocs a native buffer outside of
     * stack if the write buffer size is larger.
     */
    static final int CHUNK_SIZE = 8192;

    public FSIndexOutput(String name) throws IOException {
      this(name, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
    }

    FSIndexOutput(String name, OpenOption... options) throws IOException {
      super(
          "FSIndexOutput(path=\"" + directory.resolve(name) + "\")",
          name,
          new FilterOutputStream(Files.newOutputStream(directory.resolve(name), options)) {
            // This implementation ensures, that we never write more than CHUNK_SIZE bytes:
            @Override
            public void write(byte[] b, int offset, int length) throws IOException {
              while (length > 0) {
                final int chunk = Math.min(length, CHUNK_SIZE);
                out.write(b, offset, chunk);
                length -= chunk;
                offset += chunk;
              }
            }
          },
          CHUNK_SIZE);
    }
  }

  @Override
  public synchronized Set<String> getPendingDeletions() throws IOException {
    deletePendingFiles();
    if (pendingDeletes.isEmpty()) {
      return Collections.emptySet();
    } else {
      return Set.copyOf(pendingDeletes);
    }
  }
}
