Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.copyBuffer()

1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
  private static Token createToken
    (String term, int start, int offset, int positionIncrement)
  {
    Token token = new Token(start, offset);
    token.copyBuffer(term.toCharArray(), 0, term.length());
    token.setPositionIncrement(positionIncrement);
    return token;
  }
 
  /** blast some random strings through the analyzer */
 
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
    try {
      CharTermAttribute term = stream.addAttribute( CharTermAttribute.class );
      stream.reset();
      while ( stream.incrementToken() ) {
        Token token = new Token();
        token.copyBuffer( term.buffer(), 0, term.length() );
        tokenList.add( token );
      }
      stream.end();
    }
    finally {
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
    int pos1 = tok1!=null ? tok1.getPositionIncrement() : 0;
    int pos2 = tok2!=null ? tok2.getPositionIncrement() : 0;
    while(tok1!=null || tok2!=null) {
      while (tok1 != null && (pos1 <= pos2 || tok2==null)) {
        Token tok = new Token(tok1.startOffset(), tok1.endOffset(), tok1.type());
        tok.copyBuffer(tok1.buffer(), 0, tok1.length());
        tok.setPositionIncrement(pos1-pos);
        result.add(tok);
        pos=pos1;
        tok1 = iter1.hasNext() ? iter1.next() : null;
        pos1 += tok1!=null ? tok1.getPositionIncrement() : 0;
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
        tok1 = iter1.hasNext() ? iter1.next() : null;
        pos1 += tok1!=null ? tok1.getPositionIncrement() : 0;
      }
      while (tok2 != null && (pos2 <= pos1 || tok1==null)) {
        Token tok = new Token(tok2.startOffset(), tok2.endOffset(), tok2.type());
        tok.copyBuffer(tok2.buffer(), 0, tok2.length());
        tok.setPositionIncrement(pos2-pos);
        result.add(tok);
        pos=pos2;
        tok2 = iter2.hasNext() ? iter2.next() : null;
        pos2 += tok2!=null ? tok2.getPositionIncrement() : 0;
View Full Code Here
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
  private static Token createToken
    (String term, int start, int offset, int positionIncrement)
  {
    Token token = new Token(start, offset);
    token.copyBuffer(term.toCharArray(), 0, term.length());
    token.setPositionIncrement(positionIncrement);
    return token;
  }
 
  /** blast some random strings through the analyzer */
 
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
    TokenStream stream = analyzer.tokenStream( field, new StringReader( text ) );
    CharTermAttribute term = stream.addAttribute( CharTermAttribute.class );
    List<Token> tokenList = new ArrayList<Token>();
    while ( stream.incrementToken() ) {
      Token token = new Token();
      token.copyBuffer( term.buffer(), 0, term.length() );
      tokenList.add( token );
    }

    return tokenList.toArray( new Token[tokenList.size()] );
  }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
    try {
      CharTermAttribute term = stream.addAttribute( CharTermAttribute.class );
      stream.reset();
      while ( stream.incrementToken() ) {
        Token token = new Token();
        token.copyBuffer( term.buffer(), 0, term.length() );
        tokenList.add( token );
      }
      stream.end();
    }
    finally {
View Full Code Here
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
  private static Token createToken
    (String term, int start, int offset, int positionIncrement)
  {
    Token token = new Token(start, offset);
    token.copyBuffer(term.toCharArray(), 0, term.length());
    token.setPositionIncrement(positionIncrement);
    return token;
  }
}
View Full Code Here
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
  private static Token createToken
    (String term, int start, int offset, int positionIncrement)
  {
    Token token = new Token(start, offset);
    token.copyBuffer(term.toCharArray(), 0, term.length());
    token.setPositionIncrement(positionIncrement);
    return token;
  }
 
  /** blast some random strings through the analyzer */
 
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.