Examples of org.apache.lucene.analysis.Token

org.apache.lucene.analysis.Token
s.apache.org/jira/browse/LUCENE-969">LUCENE-969 for details.

Typical Token reuse patterns:
- Copying text from a string (type is reset to {@link #DEFAULT_TYPE} if not specified):
```
 return reusableToken.reinit(string, startOffset, endOffset[, type]); 
```
- Copying some text from a string (type is reset to {@link #DEFAULT_TYPE} if not specified):
```
 return reusableToken.reinit(string, 0, string.length(), startOffset, endOffset[, type]); 
```
- Copying text from char[] buffer (type is reset to {@link #DEFAULT_TYPE} if not specified):
```
 return reusableToken.reinit(buffer, 0, buffer.length, startOffset, endOffset[, type]); 
```
- Copying some text from a char[] buffer (type is reset to {@link #DEFAULT_TYPE} if not specified):
```
 return reusableToken.reinit(buffer, start, end - start, startOffset, endOffset[, type]); 
```
- Copying from one one Token to another (type is reset to {@link #DEFAULT_TYPE} if not specified):
```
 return reusableToken.reinit(source.termBuffer(), 0, source.termLength(), source.startOffset(), source.endOffset()[, source.type()]); 
```
A few things to note:
- clear() initializes all of the fields to default values. This was changed in contrast to Lucene 2.4, but should affect no one.
- Because TokenStreams can be chained, one cannot assume that the Token's current type is correct.
- The startOffset and endOffset represent the start and offset in the source text, so be careful in adjusting them.
- When caching a reusable token, clone it. When injecting a cached token into a stream that can be reset, clone it again.
@see org.apache.lucene.index.Payload

    BooleanQuery allFieldsQuery = new BooleanQuery();
    String value = String.valueOf(condition.getValue());


    LowercaseWhitespaceAnalyzer analyzer = new LowercaseWhitespaceAnalyzer();
    TokenStream tokenStream = analyzer.tokenStream(String.valueOf(IEntity.ALL_FIELDS), new StringReader(value));
    Token token = null;
    while ((token = tokenStream.next()) != null) {
      String termText = token.termText();


      /* Contained in Title */
      WildcardQuery titleQuery = new WildcardQuery(new Term(String.valueOf(INews.TITLE), termText));
      allFieldsQuery.add(new BooleanClause(titleQuery, Occur.SHOULD));

View Full Code Here

      case SIMILIAR_TO: {
        BooleanQuery similarityQuery = new BooleanQuery();


        LowercaseWhitespaceAnalyzer analyzer = new LowercaseWhitespaceAnalyzer();
        TokenStream tokenStream = analyzer.tokenStream(String.valueOf(IEntity.ALL_FIELDS), new StringReader(value));
        Token token = null;
        while ((token = tokenStream.next()) != null) {
          Term term = new Term(fieldname, token.termText());
          similarityQuery.add(new BooleanClause(new FuzzyQuery(term), Occur.MUST));
        }


        return similarityQuery;
      }

View Full Code Here

      case SIMILIAR_TO: {
        BooleanQuery similarityQuery = new BooleanQuery();


        LowercaseWhitespaceAnalyzer similarAnalyzer = new LowercaseWhitespaceAnalyzer();
        TokenStream tokenStream = similarAnalyzer.tokenStream(String.valueOf(IEntity.ALL_FIELDS), new StringReader(value));
        Token token = null;
        while ((token = tokenStream.next()) != null) {
          String termText = new String(token.termBuffer(), 0, token.termLength());
          Term term = new Term(fieldname, termText);
          similarityQuery.add(new BooleanClause(new FuzzyQuery(term), Occur.MUST));
        }


        return similarityQuery;

View Full Code Here

    return stream;            
  }
  
  private List getTokens(TokenStream stream) throws IOException {
    ArrayList tokens = new ArrayList();
    Token token;
    while ((token = stream.next()) != null) {
      tokens.add(token);
    }
    return tokens;
  }

View Full Code Here

  private void assertEquals(List tokens1, List tokens2) {
    int size = Math.min(tokens1.size(), tokens2.size());
    int i=0;
    try {
      for (; i < size; i++) {
        Token t1 = (Token) tokens1.get(i);
        Token t2 = (Token) tokens2.get(i);
        if (!(t1.termText().equals(t2.termText()))) throw new IllegalStateException("termText");
        if (t1.startOffset() != t2.startOffset()) throw new IllegalStateException("startOffset");
        if (t1.endOffset() != t2.endOffset()) throw new IllegalStateException("endOffset");
        if (!(t1.type().equals(t2.type()))) throw new IllegalStateException("type");
      }
      if (tokens1.size() != tokens2.size())   throw new IllegalStateException("size1=" + tokens1.size() + ", size2=" + tokens2.size());
    }


    catch (IllegalStateException e) {

View Full Code Here

  
  private String toString(List tokens) {
    if (tokens == null) return "null";
    String str = "[";
    for (int i=0; i < tokens.size(); i++) {
      Token t1 = (Token) tokens.get(i);
      str = str + "'" + t1.termText() + "', ";
    }
    return str + "]";
  }

View Full Code Here


            TokenStream includeStream =
                    new StandardAnalyzer().tokenStream(null, new StringReader(escape(terms).toLowerCase()));


            while (true) {
                Token t = includeStream.next();
                if (t == null) break;
                query.add( new Term(fieldName, t.termText()) );
            }


            return query.getTerms().length > 0 ? query : null;
        } catch (Exception ex) {
            throw new RuntimeException(ex);

View Full Code Here


    long b2 = System.currentTimeMillis();// (b2 - b1)Ϊװ�شʵ�ʱ��
    Reader r = new StringReader(testString);
    XTokenizer ts = (XTokenizer) analyzer.tokenStream("", r);


    Token t;
    while ((t = ts.next()) != null) {
       //System.out.println(t);
    }


    long b3 = System.currentTimeMillis();

View Full Code Here

    
    long b2 = System.currentTimeMillis();// (b2 - b1)Ϊװ�شʵ�ʱ��
    Reader r = new StringReader(testString);
    XTokenizer ts = (XTokenizer) analyzer.tokenStream("", r);


    Token t;
    while ((t = ts.next()) != null) {
       //System.out.println(t);
    }


    long b3 = System.currentTimeMillis();

View Full Code Here

    this.tokens = new LinkedList<Token>();
    return iter;
  }


  public void collect(String word, int offset, int end) {
    Token c = candidate != null ? candidate : last;
    if (c == null) {
      candidate = new Token(word, offset, end);
    } else if (offset == c.startOffset()) {
      if (end > c.endOffset()) {
        candidate = new Token(word, offset, end);
      }
    } else if (offset > c.startOffset()) {
      if (candidate != null) {
        select(candidate);
      }
      if (end > c.endOffset()) {
        candidate = new Token(word, offset, end);
      } else {
        candidate = null;
      }
    } else if (end >= c.endOffset()) {
      if (last != null && last.startOffset() >= offset
          && last.endOffset() <= end) {
        for (Iterator iter = tokens.iterator(); iter.hasNext();) {
          last = (Token) iter.next();
          if (last.startOffset() >= offset && last.endOffset() <= end) {
            iter.remove();
          }
        }
      }
      last = null;
      candidate = new Token(word, offset, end);
    }
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.analysis.Token

com.chenlb.mmseg4j.analysis.CutLetterDigitFilter

com.flaptor.hounder.classifier.util.TupleTokenizer

com.stimulus.archiva.search.EmailFilter

com.stimulus.archiva.search.FileNameFilter

net.nutch.analysis.CommonGrams

net.nutch.analysis.CommonGrams$Filter

net.nutch.analysis.NutchDocumentTokenizer

org.apache.cocoon.bean.query.SimpleLuceneCriterionBean

org.apache.jackrabbit.core.query.lucene.AbstractExcerpt

org.apache.jackrabbit.core.query.lucene.SearchIndex

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.