Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.type()

112
113
114
115
116
117
118
119
120
121
122
       * "the_sound"|"sound", "sound_of" For case 1 we consume the gram from the
       * input stream and output it rather than the current token This means that
       * the call to super.next() which reads a token from input and passes it on
       * to this process method will always get a token of type word
       */
    if ( next != null && next.type() == "gram" ) {
      // consume "next" token from list and output it
      token = read();
      // use this to clone the token because clone requires all these args but
      // won't take the token.type
      // see
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
  }

  public Token next(final Token reusableToken) throws IOException {
    assert reusableToken != null;
    Token nextToken = input.next(reusableToken);
    if (nextToken != null && nextToken.type().equals(typeMatch)){
      nextToken.setPayload(thePayload);
    }
    return nextToken;
  }
}
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50

  public Token next(final Token reusableToken) throws IOException {
    assert reusableToken != null;
    Token nextToken = input.next(reusableToken);
    if (nextToken != null && nextToken.type() != null && nextToken.type().equals("") == false){
      nextToken.setPayload(new Payload(nextToken.type().getBytes("UTF-8")));
    }
    return nextToken;
  }
}
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50

  public Token next(final Token reusableToken) throws IOException {
    assert reusableToken != null;
    Token nextToken = input.next(reusableToken);
    if (nextToken != null && nextToken.type() != null && nextToken.type().equals("") == false){
      nextToken.setPayload(new Payload(nextToken.type().getBytes("UTF-8")));
    }
    return nextToken;
  }
}
View Full Code Here
41
42
43
44
45
46
47
48
49
50
  public Token next(final Token reusableToken) throws IOException {
    assert reusableToken != null;
    Token nextToken = input.next(reusableToken);
    if (nextToken != null && nextToken.type() != null && nextToken.type().equals("") == false){
      nextToken.setPayload(new Payload(nextToken.type().getBytes("UTF-8")));
    }
    return nextToken;
  }
}
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
    if (nextToken == null)
      return null;

    char[] buffer = nextToken.termBuffer();
    final int bufferLength = nextToken.termLength();
    final String type = nextToken.type();

    if (type == APOSTROPHE_TYPE &&      // remove 's
  bufferLength >= 2 &&
        buffer[bufferLength-2] == '\'' &&
        (buffer[bufferLength-1] == 's' || buffer[bufferLength-1] == 'S')) {
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
        Token token = tokens[index++];
        termAtt.setEmpty().append(token);
        offsetAtt.setOffset(token.startOffset(), token.endOffset());
        posIncAtt.setPositionIncrement(token.getPositionIncrement());
        flagsAtt.setFlags(token.getFlags());
        typeAtt.setType(token.type());
        payloadAtt.setPayload(token.getPayload());
        return true;
      }
    }
  }
View Full Code Here
321
322
323
324
325
326
327
328
329
330
331
            final List<Token> expected = actualTokens.get(term).get(docIDToID.get(doc));
            //System.out.println("      doc=" + docIDToID.get(doc) + " " + expected.size() + " freq");
            assertNotNull(expected);
            assertEquals(expected.size(), docsAndPositions.freq());
            for(Token token : expected) {
              int pos = Integer.parseInt(token.type());
              //System.out.println("        pos=" + pos);
              assertEquals(pos, docsAndPositions.nextPosition());
            }
          }
View Full Code Here
336
337
338
339
340
341
342
343
344
345
346
            final List<Token> expected = actualTokens.get(term).get(docIDToID.get(doc));
            //System.out.println("      doc=" + docIDToID.get(doc) + " " + expected.size() + " freq");
            assertNotNull(expected);
            assertEquals(expected.size(), docsAndPositionsAndOffsets.freq());
            for(Token token : expected) {
              int pos = Integer.parseInt(token.type());
              //System.out.println("        pos=" + pos);
              assertEquals(pos, docsAndPositionsAndOffsets.nextPosition());
              assertEquals(token.startOffset(), docsAndPositionsAndOffsets.startOffset());
              assertEquals(token.endOffset(), docsAndPositionsAndOffsets.endOffset());
            }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
    nextToken = reusableToken.reinit(termAtt.termBuffer(), 0, termAtt.termLength(), offsetAtt.startOffset(), offsetAtt.endOffset(), typeAtt.type());*/
   
    nextToken = TokenUtils.nextToken(input, reusableToken);
   
    if(nextToken != null &&
        (Word.TYPE_LETTER_OR_DIGIT.equalsIgnoreCase(nextToken.type())
          || Word.TYPE_DIGIT_OR_LETTER.equalsIgnoreCase(nextToken.type()))
        ) {
      final char[] buffer = nextToken.buffer();
      final int length = nextToken.length();
      byte lastType = (byte) Character.getType(buffer[0]);  //与上次的字符是否同类
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.