/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.analysis;

import java.io.Closeable;
import java.io.IOException;
import java.lang.reflect.Modifier;
import org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.AttributeSource;

/**
 * A <code>TokenStream</code> enumerates the sequence of tokens, either from {@link Field}s of a
 * {@link Document} or from query text.
 *
 * <p>This is an abstract class; concrete subclasses are:
 *
 * <ul>
 *   <li>{@link Tokenizer}, a <code>TokenStream</code> whose input is a Reader; and
 *   <li>{@link TokenFilter}, a <code>TokenStream</code> whose input is another <code>TokenStream
 *       </code>.
 * </ul>
 *
 * <code>TokenStream</code> extends {@link AttributeSource}, which provides access to all of the
 * token {@link Attribute}s for the <code>TokenStream</code>. Note that only one instance per {@link
 * AttributeImpl} is created and reused for every token. This approach reduces object creation and
 * allows local caching of references to the {@link AttributeImpl}s. See {@link #incrementToken()}
 * for further details.
 *
 * <p><b>The workflow of the new <code>TokenStream</code> API is as follows:</b>
 *
 * <ol>
 *   <li>Instantiation of <code>TokenStream</code>/{@link TokenFilter}s which add/get attributes
 *       to/from the {@link AttributeSource}.
 *   <li>The consumer calls {@link TokenStream#reset()}.
 *   <li>The consumer retrieves attributes from the stream and stores local references to all
 *       attributes it wants to access.
 *   <li>The consumer calls {@link #incrementToken()} until it returns false consuming the
 *       attributes after each call.
 *   <li>The consumer calls {@link #end()} so that any end-of-stream operations can be performed.
 *   <li>The consumer calls {@link #close()} to release any resource when finished using the <code>
 *       TokenStream</code>.
 * </ol>
 *
 * To make sure that filters and consumers know which attributes are available, the attributes must
 * be added during instantiation. Filters and consumers are not required to check for availability
 * of attributes in {@link #incrementToken()}.
 *
 * <p>You can find some example code for the new API in the analysis package level Javadoc.
 *
 * <p>Sometimes it is desirable to capture a current state of a <code>TokenStream</code>, e.g., for
 * buffering purposes (see {@link CachingTokenFilter}, TeeSinkTokenFilter). For this usecase {@link
 * AttributeSource#captureState} and {@link AttributeSource#restoreState} can be used.
 *
 * <p>The {@code TokenStream}-API in Lucene is based on the decorator pattern. Therefore all
 * non-abstract subclasses must be final or have at least a final implementation of {@link
 * #incrementToken}! This is checked when Java assertions are enabled.
 */
public abstract class TokenStream extends AttributeSource implements Closeable {

  /** Default {@link AttributeFactory} instance that should be used for TokenStreams. */
  public static final AttributeFactory DEFAULT_TOKEN_ATTRIBUTE_FACTORY =
      AttributeFactory.getStaticImplementation(
          AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, PackedTokenAttributeImpl.class);

  /** A TokenStream using the default attribute factory. */
  protected TokenStream() {
    super(DEFAULT_TOKEN_ATTRIBUTE_FACTORY);
    assert assertFinal();
  }

  /** A TokenStream that uses the same attributes as the supplied one. */
  protected TokenStream(AttributeSource input) {
    super(input);
    assert assertFinal();
  }

  /**
   * A TokenStream using the supplied AttributeFactory for creating new {@link Attribute} instances.
   */
  protected TokenStream(AttributeFactory factory) {
    super(factory);
    assert assertFinal();
  }

  private boolean assertFinal() {
    try {
      final Class<?> clazz = getClass();
      if (!clazz.desiredAssertionStatus()) return true;
      assert clazz.isAnonymousClass()
              || (clazz.getModifiers() & (Modifier.FINAL | Modifier.PRIVATE)) != 0
              || Modifier.isFinal(clazz.getMethod("incrementToken").getModifiers())
          : "TokenStream implementation classes or at least their incrementToken() implementation must be final";
      return true;
    } catch (
        @SuppressWarnings("unused")
        NoSuchMethodException nsme) {
      return false;
    }
  }

  /**
   * Consumers (i.e., {@link IndexWriter}) use this method to advance the stream to the next token.
   * Implementing classes must implement this method and update the appropriate {@link
   * AttributeImpl}s with the attributes of the next token.
   *
   * <p>The producer must make no assumptions about the attributes after the method has been
   * returned: the caller may arbitrarily change it. If the producer needs to preserve the state for
   * subsequent calls, it can use {@link #captureState} to create a copy of the current attribute
   * state.
   *
   * <p>This method is called for every token of a document, so an efficient implementation is
   * crucial for good performance. To avoid calls to {@link #addAttribute(Class)} and {@link
   * #getAttribute(Class)}, references to all {@link AttributeImpl}s that this stream uses should be
   * retrieved during instantiation.
   *
   * <p>To ensure that filters and consumers know which attributes are available, the attributes
   * must be added during instantiation. Filters and consumers are not required to check for
   * availability of attributes in {@link #incrementToken()}.
   *
   * @return false for end of stream; true otherwise
   */
  public abstract boolean incrementToken() throws IOException;

  /**
   * This method is called by the consumer after the last token has been consumed, after {@link
   * #incrementToken()} returned <code>false</code> (using the new <code>TokenStream</code> API).
   * Streams implementing the old API should upgrade to use this feature.
   *
   * <p>This method can be used to perform any end-of-stream operations, such as setting the final
   * offset of a stream. The final offset of a stream might differ from the offset of the last token
   * eg in case one or more whitespaces followed after the last token, but a WhitespaceTokenizer was
   * used.
   *
   * <p>Additionally any skipped positions (such as those removed by a stopfilter) can be applied to
   * the position increment, or any adjustment of other attributes where the end-of-stream value may
   * be important.
   *
   * <p>If you override this method, always call {@code super.end()}.
   *
   * @throws IOException If an I/O error occurs
   */
  public void end() throws IOException {
    endAttributes(); // LUCENE-3849: don't consume dirty atts
  }

  /**
   * This method is called by a consumer before it begins consumption using {@link
   * #incrementToken()}.
   *
   * <p>Resets this stream to a clean state. Stateful implementations must implement this method so
   * that they can be reused, just as if they had been created fresh.
   *
   * <p>If you override this method, always call {@code super.reset()}, otherwise some internal
   * state will not be correctly reset (e.g., {@link Tokenizer} will throw {@link
   * IllegalStateException} on further usage).
   */
  public void reset() throws IOException {}

  /**
   * Releases resources associated with this stream.
   *
   * <p>If you override this method, always call {@code super.close()}, otherwise some internal
   * state will not be correctly reset (e.g., {@link Tokenizer} will throw {@link
   * IllegalStateException} on reuse).
   */
  @Override
  public void close() throws IOException {}
}
