Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.type()

82
83
84
85
86
87
88
89
90
91
92
    final Token t = next();
    if (t != null) {
      termAtt.setTermBuffer(t.termBuffer(), 0, t.termLength());
      offsetAtt.setOffset(t.startOffset(), t.endOffset());
      posIncrAtt.setPositionIncrement(t.getPositionIncrement());
      typeAtt.setType(t.type());
      return true;
    } else {
      return false;
    }
  }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
      Token t = next();
      if (t != null) {
        termAtt.setTermBuffer(t.termBuffer(), 0, t.termLength());
        offsetAtt.setOffset(t.startOffset(), t.endOffset());
        posIncrAtt.setPositionIncrement(t.getPositionIncrement());
        typeAtt.setType(t.type());
      }    
      return t != null;
    }

    private Token inputNext() throws IOException {
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
    clearAttributes();
    Token token = nextToken(reusableToken);
    if(token != null) {
      termAtt.copyBuffer(token.buffer(), 0, token.length());
      offsetAtt.setOffset(token.startOffset(), token.endOffset());
      typeAtt.setType(token.type());
      return true;
    } else {
      end();
      return false;
    }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
    nextToken = reusableToken.reinit(termAtt.termBuffer(), 0, termAtt.termLength(), offsetAtt.startOffset(), offsetAtt.endOffset(), typeAtt.type());*/

    nextToken = TokenUtils.nextToken(input, reusableToken);

    if(nextToken != null &&
        (Word.TYPE_LETTER_OR_DIGIT.equalsIgnoreCase(nextToken.type())
          || Word.TYPE_DIGIT_OR_LETTER.equalsIgnoreCase(nextToken.type()))
        ) {
      final char[] buffer = nextToken.buffer();
      final int length = nextToken.length();
      byte lastType = (byte) Character.getType(buffer[0]);  //与上次的字符是否同类
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
    nextToken = TokenUtils.nextToken(input, reusableToken);

    if(nextToken != null &&
        (Word.TYPE_LETTER_OR_DIGIT.equalsIgnoreCase(nextToken.type())
          || Word.TYPE_DIGIT_OR_LETTER.equalsIgnoreCase(nextToken.type()))
        ) {
      final char[] buffer = nextToken.buffer();
      final int length = nextToken.length();
      byte lastType = (byte) Character.getType(buffer[0]);  //与上次的字符是否同类
      int termBufferOffset = 0;
View Full Code Here
321
322
323
324
325
326
327
328
329
330
331
            final List<Token> expected = actualTokens.get(term).get(docIDToID.get(doc));
            //System.out.println("      doc=" + docIDToID.get(doc) + " " + expected.size() + " freq");
            assertNotNull(expected);
            assertEquals(expected.size(), docsAndPositions.freq());
            for(Token token : expected) {
              int pos = Integer.parseInt(token.type());
              //System.out.println("        pos=" + pos);
              assertEquals(pos, docsAndPositions.nextPosition());
            }
          }
View Full Code Here
336
337
338
339
340
341
342
343
344
345
346
            final List<Token> expected = actualTokens.get(term).get(docIDToID.get(doc));
            //System.out.println("      doc=" + docIDToID.get(doc) + " " + expected.size() + " freq");
            assertNotNull(expected);
            assertEquals(expected.size(), docsAndPositionsAndOffsets.freq());
            for(Token token : expected) {
              int pos = Integer.parseInt(token.type());
              //System.out.println("        pos=" + pos);
              assertEquals(pos, docsAndPositionsAndOffsets.nextPosition());
              assertEquals(token.startOffset(), docsAndPositionsAndOffsets.startOffset());
              assertEquals(token.endOffset(), docsAndPositionsAndOffsets.endOffset());
            }
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
        }
        */
                     
        // read tokens until maxLen
        while ((token=tokens.next())!=null) {       
          if (token.type().equals("<WORD>")) {
            if (firstToken==null) {
              firstToken=token;
            }
            else if (token.endOffset()-firstToken.startOffset()<maxLen) {         
              lastToken=token;                         
View Full Code Here
375
376
377
378
379
380
381
382
383
384
385
   
    termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
    posIncrAtt.setPositionIncrement(token.getPositionIncrement());
    flagsAtt.setFlags(token.getFlags());
    offsetAtt.setOffset(token.startOffset(), token.endOffset());
    typeAtt.setType(token.type());
    payloadAtt.setPayload(token.getPayload());
    return true;
  }
 
  private Token getNextInputToken(Token token) throws IOException {
View Full Code Here
409
410
411
412
413
414
415
416
417
418
419
        Token token = tokens[index++];
        termAtt.setEmpty().append(token);
        offsetAtt.setOffset(token.startOffset(), token.endOffset());
        posIncAtt.setPositionIncrement(token.getPositionIncrement());
        flagsAtt.setFlags(token.getFlags());
        typeAtt.setType(token.type());
        payloadAtt.setPayload(token.getPayload());
        return true;
      }
    }
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.