Examples of org.apache.lucene.analysis.TokenStreamToAutomaton$Position

org.apache.lucene.util.Vint8.Position
Consumes a TokenStream and creates an {@link Automaton}where the transition labels are UTF8 bytes from the {@link TermToBytesRefAttribute}. Between tokens we insert POS_SEP and for holes we insert HOLE.


    Sort.ByteSequencesWriter writer = new Sort.ByteSequencesWriter(tempInput);
    Sort.ByteSequencesReader reader = null;
    BytesRef scratch = new BytesRef();


    TokenStreamToAutomaton ts2a = getTokenStreamToAutomaton();


    boolean success = false;
    byte buffer[] = new byte[8];
    try {
      ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);

View Full Code Here

      }


      if (VERBOSE) {
        System.out.println("  analyzed: " + analyzedKey);
      }
      TokenStreamToAutomaton tokenStreamToAutomaton = suggester.getTokenStreamToAutomaton();


      // NOTE: not great that we ask the suggester to give
      // us the "answer key" (ie maybe we have a bug in
      // suggester.toLevA ...) ... but testRandom2() fixes
      // this:

View Full Code Here

    }
  }


  @Override
  TokenStreamToAutomaton getTokenStreamToAutomaton() {
    final TokenStreamToAutomaton tsta = super.getTokenStreamToAutomaton();
    tsta.setUnicodeArcs(unicodeAware);
    return tsta;
  }

View Full Code Here

  protected Automaton convertAutomaton(Automaton a) {
    return a;
  }
  
  TokenStreamToAutomaton getTokenStreamToAutomaton() {
    final TokenStreamToAutomaton tsta = new TokenStreamToAutomaton();
    tsta.setPreservePositionIncrements(preservePositionIncrements);
    return tsta;
  }

View Full Code Here


    OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(tempInput);
    OfflineSorter.ByteSequencesReader reader = null;
    BytesRef scratch = new BytesRef();


    TokenStreamToAutomaton ts2a = getTokenStreamToAutomaton();


    boolean success = false;
    count = 0;
    byte buffer[] = new byte[8];
    try {

View Full Code Here

      }
    }


    @Override
    public TokenStreamToAutomaton getTokenStreamToAutomaton() {
      final TokenStreamToAutomaton tsta = super.getTokenStreamToAutomaton();
      tsta.setUnicodeArcs(unicodeAware);
      return tsta;
    }

View Full Code Here

      return spare.get();
    }
  }


  public TokenStreamToAutomaton getTokenStreamToAutomaton() {
    final TokenStreamToAutomaton tsta;
    if (preserveSep) {
      tsta = new EscapingTokenStreamToAutomaton((char) sepLabel);
    } else {
      // When we're not preserving sep, we don't steal 0xff
      // byte, so we don't need to do any escaping:
      tsta = new TokenStreamToAutomaton();
    }
    tsta.setPreservePositionIncrements(preservePositionIncrements);
    return tsta;
  }

View Full Code Here


    OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(tempInput);
    OfflineSorter.ByteSequencesReader reader = null;
    BytesRefBuilder scratch = new BytesRefBuilder();


    TokenStreamToAutomaton ts2a = getTokenStreamToAutomaton();


    boolean success = false;
    count = 0;
    byte buffer[] = new byte[8];
    try {

View Full Code Here


  public Object extractCategoryTokenData(byte[] buffer, int offset, int length) {
    if (length == 0) {
      return null;
    }
    Integer i = Integer.valueOf(Vint8.decode(buffer, new Position(offset)));
    return i;
  }

View Full Code Here

    if (!super.setdoc(docId)) {
      return false;
    }


    // read header - number of enhancements and their lengths
    Position position = new Position();
    nEnhancements = Vint8.decode(buffer, position);
    for (int i = 0; i < nEnhancements; i++) {
      enhancementLength[i] = Vint8.decode(buffer, position);
    }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.lucene.analysis.TokenStreamToAutomaton$Position

ca.uhn.fhir.model.primitive.DecimalDt

com.google.gdt.eclipse.designer.uibinder.model.widgets.WidgetInfo

com.sun.tools.classfile.TypeAnnotation.Position.TypePathEntry

org.apache.lucene.analysis.tokenattributes.OffsetAttribute

org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute

org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute

org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute

org.apache.lucene.facet.enhancements.association.AssociationEnhancement

org.apache.lucene.facet.enhancements.EnhancementsPayloadIterator

org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.