Examples of termLength()

Examples of org.apache.lucene.analysis.Token.termLength()

211
212
213
214
215
216
217
218
219
220
221
          // shorter
          // to avoid problems with genitive 's characters and other binding
          // characters
          if (this.onlyLongestMatch) {
            if (longestMatchToken != null) {
              if (longestMatchToken.termLength() < partLength - 1) {
                longestMatchToken = createToken(start, partLength - 1, token);
              }
            } else {
              longestMatchToken = createToken(start, partLength - 1, token);
            }
View Full Code Here

Examples of org.apache.lucene.analysis.Token.termLength()

1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
                        {
                           Fieldable field = fields[k];
                           // assume properties fields use
                           // SingleTokenStream
                           t = field.tokenStreamValue().next(t);
                           String value = new String(t.termBuffer(), 0, t.termLength());
                           if (value.startsWith(namePrefix))
                           {
                              // extract value
                              value = value.substring(namePrefix.length());
                              // create new named value
View Full Code Here

Examples of org.apache.lucene.analysis.Token.termLength()

1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
                        {
                           Fieldable field = fields[k];
                           // assume properties fields use
                           // SingleTokenStream
                           t = field.tokenStreamValue().next(t);
                           String value = new String(t.termBuffer(), 0, t.termLength());
                           if (value.startsWith(namePrefix))
                           {
                              // extract value
                              value = value.substring(namePrefix.length());
                              // create new named value
View Full Code Here

Examples of org.apache.lucene.analysis.Token.termLength()

1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
                        {
                           Fieldable field = fields[k];
                           // assume properties fields use
                           // SingleTokenStream
                           t = field.tokenStreamValue().next(t);
                           String value = new String(t.termBuffer(), 0, t.termLength());
                           if (value.startsWith(namePrefix))
                           {
                              // extract value
                              value = value.substring(namePrefix.length());
                              // create new named value
View Full Code Here

Examples of org.apache.lucene.analysis.Token.termLength()

371
372
373
374
375
376
377
378
379
380
381
    do {
      token = produceNextToken(reusableToken);
    } while (token == request_next_token);
    if (token == null) return false;
   
    termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
    posIncrAtt.setPositionIncrement(token.getPositionIncrement());
    flagsAtt.setFlags(token.getFlags());
    offsetAtt.setOffset(token.startOffset(), token.endOffset());
    typeAtt.setType(token.type());
    payloadAtt.setPayload(token.getPayload());
View Full Code Here

Examples of org.apache.lucene.analysis.Token.termLength()

436
437
438
439
440
441
442
443
444
445
446
        List shingle = new ArrayList();

        for (int i = 0; i < currentShingleLength; i++) {
          Token shingleToken = (Token) currentPermuationTokens.get(i + currentPermutationTokensStartOffset);
          termLength += shingleToken.termLength();
          shingle.add(shingleToken);
        }
        if (spacerCharacter != null) {
          termLength += currentShingleLength - 1;
        }
View Full Code Here

Examples of org.apache.lucene.analysis.Token.termLength()

455
456
457
458
459
460
461
462
463
464
465
        for (Iterator iterator = shingle.iterator(); iterator.hasNext();) {
          Token shingleToken = (Token) iterator.next();
          if (spacerCharacter != null && sb.length() > 0) {
            sb.append(spacerCharacter);
          }
          sb.append(shingleToken.termBuffer(), 0, shingleToken.termLength());
        }
        reusableToken.setTermBuffer(sb.toString());
        updateToken(reusableToken, shingle, currentPermutationTokensStartOffset, currentPermutationRows, currentPermuationTokens);

        return reusableToken;
View Full Code Here

Examples of org.apache.lucene.analysis.Token.termLength()

51
52
53
54
55
56
57
58
59
60
61
    @Override
    public final boolean incrementToken() throws IOException {
      clearAttributes();
      if (index < testToken.length) {
        Token t = testToken[index++];
        termAtt.setTermBuffer(t.termBuffer(), 0, t.termLength());
        offsetAtt.setOffset(t.startOffset(), t.endOffset());
        posIncrAtt.setPositionIncrement(t.getPositionIncrement());
        typeAtt.setType(TypeAttributeImpl.DEFAULT_TYPE);
        return true;
      } else {
View Full Code Here

Examples of org.apache.lucene.analysis.Token.termLength()

35
36
37
38
39
40
41
42
43
44
45
{
  public static String tsToString(TokenStream in) throws IOException {
    StringBuilder out = new StringBuilder();
    Token t = in.next();
    if (null != t)
      out.append(new String(t.termBuffer(), 0, t.termLength()));
   
    for (t = in.next(); null != t; t = in.next()) {
      out.append(" ").append(new String(t.termBuffer(), 0, t.termLength()));
    }
    in.close();
View Full Code Here

Examples of org.apache.lucene.analysis.Token.termLength()

38
39
40
41
42
43
44
45
46
47
48
    Token t = in.next();
    if (null != t)
      out.append(new String(t.termBuffer(), 0, t.termLength()));
   
    for (t = in.next(); null != t; t = in.next()) {
      out.append(" ").append(new String(t.termBuffer(), 0, t.termLength()));
    }
    in.close();
    return out.toString();
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.