Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.clone()

60
61
62
63
64
65
66
67
68
69
70
        value = encoder.encode(value).toString();
      }
      catch (Exception ignored) {} // just use the direct text
      //Token m = new Token(value, t.startOffset(), t.endOffset(), name );
      if( inject ) {
        save = (Token) t.clone();
        save.setPositionIncrement(0);
        save.setTermBuffer(value.toCharArray(), 0, value.length());
      } else {
        t.setTermBuffer(value.toCharArray(), 0, value.length());
      }
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
    public TupleTokenizer(TokenStream ts, int maxTuples ) throws IOException {

        MAX_INCREMENT= maxTuples;
        Token tk = new Token();
        while ((tk = ts.next(tk)) != null) {
            tokens.add((Token)tk.clone());
        }       
    }

    private Token mergeTokens(Token t1, Token t2){
        Token res = new Token();
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
            try {
                Token token = new Token();
                while(true) {
                    token = tokenizer.next(token);
                    if (null == token) { break; }
                    tokens.add(TokenUtil.termText((Token)token.clone()));
                }

                // for every word, suggest something
                for (int i = 0; i < tokens.size(); i++) {
                    StringBuffer sb = new StringBuffer();
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
    Token reusableToken = new Token();
    Token token = null;

    try {
      while ((token = tokenStream.next(reusableToken)) != null) {
        tokens.add((Token) token.clone());
      }
    } catch (IOException ioe) {
      throw new RuntimeException("Error occured while iterating over tokenstream", ioe);
    }
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
      Token token = stream.next( reusableToken );
      if ( token == null ) {
        break;
      }

      tokenList.add( ( Token ) token.clone() );
    }

    return tokenList.toArray( new Token[tokenList.size()] );
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
    Token target = new Token();  // test with token reuse
    while(true) {
      Token t = sf.next(target);
      if (t==null) return lst;
      lst.add((Token)t.clone());
    }
  }


  public void testMatching() throws IOException {
View Full Code Here
262
263
264
265
266
267
268
269
270
271
272
    /*
     * Push new tokens to the output buffer.
     */
    if (!shingleBuf.isEmpty()) {
      Token firstShingle = (Token) shingleBuf.get(0);
      shingle = (Token) firstShingle.clone();
      shingle.setType(tokenType);
    }
    for (int j = 1; j < shingleBuf.size(); j++) {
      shingle.setEndOffset(endOffsets[j]);
      StringBuffer buf = shingles[j];
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
    if (UnicodeBlock.of(text.charAt(0)) != UnicodeBlock.THAI) {
      nextToken.setTermBuffer(text.toLowerCase());
      return nextToken;
    }

    thaiToken = (Token) nextToken.clone();
    breaker.setText(text);
    int end = breaker.next();
    if (end != BreakIterator.DONE) {
      nextToken.setTermBuffer(text, 0, end);
      nextToken.setEndOffset(nextToken.startOffset() + end);
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
      Token token = stream.next( reusableToken );
      if ( token == null ) {
        break;
      }

      tokenList.add( ( Token ) token.clone() );
    }

    return tokenList.toArray( new Token[tokenList.size()] );
  }
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
              nextToken.setPayload(new Payload(new byte[]{100}));
              first = false;
            }

            // index a "synonym" for every token
            buffered = (Token)nextToken.clone();
            buffered.setPayload(null);
            buffered.setPositionIncrement(0);
            buffered.setTermBuffer(new char[]{'b'}, 0, 1);

            return nextToken;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.