Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.clone()

1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
      if (expansions == null) {
        return nextRealToken;
      }
      st = new StringTokenizer(expansions, ",");
      if (st.hasMoreTokens()) {
        currentRealToken = (Token) nextRealToken.clone();
      }
      return currentRealToken;
    } else {
      reusableToken.reinit(st.nextToken(),
                           currentRealToken.startOffset(),
View Full Code Here
179
180
181
182
183
184
185
186
187
188
189
                  if (!it.hasNext()) {
                    return null;
                  }
                  // Resettable token streams need to return clones.
                  Token nextToken = (Token) it.next();
                  return (Token) nextToken.clone();
                }

                public void reset() throws IOException {
                  it = tokens.iterator();
                }
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
        Token nextToken = input.next(reusableToken);
        if (nextToken == null) {
          prevToken = null;
          return null;
        }
        prevToken = (Token) nextToken.clone();
        String text = nextToken.term();
        if (text.equals("triplemulti")) {
          multiToken = 2;
          return nextToken;
        } else if (text.equals("multi")) {
View Full Code Here
514
515
516
517
518
519
520
521
522
523
524
      }
      if (!iterator.hasNext()) {
        return null;
      }
      Token nextToken = (Token) iterator.next();
      return (Token) nextToken.clone();
    }


    public void reset() throws IOException {
      iterator = null;
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
    if (UnicodeBlock.of(text.charAt(0)) != UnicodeBlock.THAI) {
      nextToken.setTermBuffer(text.toLowerCase());
      return nextToken;
    }

    thaiToken = (Token) nextToken.clone();
    breaker.setText(text);
    int end = breaker.next();
    if (end != BreakIterator.DONE) {
      nextToken.setTermBuffer(text, 0, end);
      nextToken.setEndOffset(nextToken.startOffset() + end);
View Full Code Here
262
263
264
265
266
267
268
269
270
271
272
    /*
     * Push new tokens to the output buffer.
     */
    if (!shingleBuf.isEmpty()) {
      Token firstShingle = (Token) shingleBuf.get(0);
      shingle = (Token) firstShingle.clone();
      shingle.setType(tokenType);
    }
    for (int j = 1; j < shingleBuf.size(); j++) {
      shingle.setEndOffset(endOffsets[j]);
      StringBuffer buf = shingles[j];
View Full Code Here
241
242
243
244
245
246
247
248
249
250
251
          TokenStream tokenStream = new TokenFilter(child.tokenStream(fieldName, reader)) {

            public Token next(final Token reusableToken) throws IOException {
              assert reusableToken != null;
              Token nextToken = input.next(reusableToken); // from filter super class
              if (nextToken != null) tokens2.add(nextToken.clone());
              return nextToken;
            }
          };
         
          cache.put(fieldName, tokens2);
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
    if (nextToken == null) return null; // EOS; iterator exhausted
   
    stack = synonyms.getSynonyms(nextToken.term()); // push onto stack
    if (stack.length > maxSynonyms) randomize(stack);
    index = 0;
    current = (Token) nextToken.clone();
    todo = maxSynonyms;
    return nextToken;
  }
 
  /**
 
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
      Token token = stream.next( reusableToken );
      if ( token == null ) {
        break;
      }

      tokenList.add( ( Token ) token.clone() );
    }

    return tokenList.toArray( new Token[tokenList.size()] );
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.