Examples of org.apache.lucene.analysis.TokenStreamToAutomaton

org.apache.lucene.analysis.TokenStreamToAutomaton
Consumes a TokenStream and creates an {@link Automaton}where the transition labels are UTF8 bytes from the {@link TermToBytesRefAttribute}. Between tokens we insert POS_SEP and for holes we insert HOLE.

      }


      if (VERBOSE) {
        System.out.println("  analyzed: " + analyzedKey);
      }
      TokenStreamToAutomaton tokenStreamToAutomaton = suggester.getTokenStreamToAutomaton();


      // NOTE: not great that we ask the suggester to give
      // us the "answer key" (ie maybe we have a bug in
      // suggester.toLevA ...) ... but testRandom2() fixes
      // this:

View Full Code Here

    }
  }


  @Override
  TokenStreamToAutomaton getTokenStreamToAutomaton() {
    final TokenStreamToAutomaton tsta = super.getTokenStreamToAutomaton();
    tsta.setUnicodeArcs(unicodeAware);
    return tsta;
  }

View Full Code Here

  protected Automaton convertAutomaton(Automaton a) {
    return a;
  }
  
  TokenStreamToAutomaton getTokenStreamToAutomaton() {
    final TokenStreamToAutomaton tsta = new TokenStreamToAutomaton();
    tsta.setPreservePositionIncrements(preservePositionIncrements);
    return tsta;
  }

View Full Code Here


    Sort.ByteSequencesWriter writer = new Sort.ByteSequencesWriter(tempInput);
    Sort.ByteSequencesReader reader = null;
    BytesRef scratch = new BytesRef();


    TokenStreamToAutomaton ts2a = getTokenStreamToAutomaton();


    boolean success = false;
    byte buffer[] = new byte[8];
    try {
      ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);

View Full Code Here

  protected Automaton convertAutomaton(Automaton a) {
    return a;
  }
  
  TokenStreamToAutomaton getTokenStreamToAutomaton() {
    final TokenStreamToAutomaton tsta = new TokenStreamToAutomaton();
    tsta.setPreservePositionIncrements(preservePositionIncrements);
    return tsta;
  }

View Full Code Here


    Sort.ByteSequencesWriter writer = new Sort.ByteSequencesWriter(tempInput);
    Sort.ByteSequencesReader reader = null;
    BytesRef scratch = new BytesRef();


    TokenStreamToAutomaton ts2a = getTokenStreamToAutomaton();


    boolean success = false;
    count = 0;
    byte buffer[] = new byte[8];
    try {

View Full Code Here

      }


      if (VERBOSE) {
        System.out.println("  analyzed: " + analyzedKey);
      }
      TokenStreamToAutomaton tokenStreamToAutomaton = suggester.getTokenStreamToAutomaton();


      // NOTE: not great that we ask the suggester to give
      // us the "answer key" (ie maybe we have a bug in
      // suggester.toLevA ...) ... but testRandom2() fixes
      // this:

View Full Code Here

    if (preserveSep) {
      return new EscapingTokenStreamToAutomaton();
    } else {
      // When we're not preserving sep, we don't steal 0xff
      // byte, so we don't need to do any escaping:
      return new TokenStreamToAutomaton();
    }
  }

View Full Code Here

    
    Sort.ByteSequencesWriter writer = new Sort.ByteSequencesWriter(tempInput);
    Sort.ByteSequencesReader reader = null;
    BytesRef scratch = new BytesRef();


    TokenStreamToAutomaton ts2a = getTokenStreamToAutomaton();


    // analyzed sequence + 0(byte) + weight(int) + surface + analyzedLength(short) 
    boolean success = false;
    byte buffer[] = new byte[8];
    try {

View Full Code Here

  protected Automaton convertAutomaton(Automaton a) {
    return a;
  }
  
  TokenStreamToAutomaton getTokenStreamToAutomaton() {
    final TokenStreamToAutomaton tsta = new TokenStreamToAutomaton();
    tsta.setPreservePositionIncrements(preservePositionIncrements);
    return tsta;
  }

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.lucene.analysis.TokenStreamToAutomaton

org.apache.lucene.analysis.tokenattributes.OffsetAttribute

org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute

org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute

org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute

org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester

org.apache.lucene.search.suggest.analyzing.FuzzySuggester

org.apache.lucene.search.suggest.analyzing.FuzzySuggesterTest

org.apache.lucene.search.suggest.analyzing.XAnalyzingSuggester

org.apache.lucene.search.suggest.analyzing.XFuzzySuggester

org.apache.lucene.util.automaton.Automaton

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.