Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.type()

63
64
65
66
67
68
69
70
71
72
73
   
    nextToken = TokenUtils.nextToken(input, reusableToken);
   
    if(nextToken != null &&
        (Word.TYPE_LETTER_OR_DIGIT.equalsIgnoreCase(nextToken.type())
          || Word.TYPE_DIGIT_OR_LETTER.equalsIgnoreCase(nextToken.type()))
        ) {
      final char[] buffer = nextToken.buffer();
      final int length = nextToken.length();
      byte lastType = (byte) Character.getType(buffer[0]);  //与上次的字符是否同类
      int termBufferOffset = 0;
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
    clearAttributes();
    Token token = nextToken(reusableToken);
    if(token != null) {
      termAtt.copyBuffer(token.buffer(), 0, token.length());
      offsetAtt.setOffset(token.startOffset(), token.endOffset());
      typeAtt.setType(token.type());
      return true;
    } else {
      end();
      return false;
    }
View Full Code Here
336
337
338
339
340
341
342
343
344
345
346
            final List<Token> expected = actualTokens.get(term).get(docIDToID.get(doc));
            //System.out.println("      doc=" + docIDToID.get(doc) + " " + expected.size() + " freq");
            assertNotNull(expected);
            assertEquals(expected.size(), docsAndPositionsAndOffsets.freq());
            for(Token token : expected) {
              int pos = Integer.parseInt(token.type());
              //System.out.println("        pos=" + pos);
              assertEquals(pos, docsAndPositionsAndOffsets.nextPosition());
              assertEquals(token.startOffset(), docsAndPositionsAndOffsets.startOffset());
              assertEquals(token.endOffset(), docsAndPositionsAndOffsets.endOffset());
            }
View Full Code Here
321
322
323
324
325
326
327
328
329
330
331
            final List<Token> expected = actualTokens.get(term).get(docIDToID.get(doc));
            //System.out.println("      doc=" + docIDToID.get(doc) + " " + expected.size() + " freq");
            assertNotNull(expected);
            assertEquals(expected.size(), docsAndPositions.freq());
            for(Token token : expected) {
              int pos = Integer.parseInt(token.type());
              //System.out.println("        pos=" + pos);
              assertEquals(pos, docsAndPositions.nextPosition());
            }
          }
View Full Code Here
245
246
247
248
249
250
251
252
253
254
255
    final Token reusableToken = new Token();
    Token nextToken = tf.next(reusableToken);
    assertTrue("nextToken is null and it shouldn't be", nextToken != null);
    assertTrue(nextToken.term() + " is not equal to " + "http://lucene.apache.org/java/docs/index.html#news",
            nextToken.term().equals("http://lucene.apache.org/java/docs/index.html#news") == true);
    assertTrue(nextToken.type() + " is not equal to " + WikipediaTokenizer.EXTERNAL_LINK_URL, nextToken.type().equals(WikipediaTokenizer.EXTERNAL_LINK_URL) == true);
    tf.next(reusableToken);//skip here
    nextToken = tf.next(reusableToken);
    assertTrue("nextToken is null and it shouldn't be", nextToken != null);
    assertTrue(nextToken.term() + " is not equal to " + "http://lucene.apache.org/java/docs/index.html?b=c",
            nextToken.term().equals("http://lucene.apache.org/java/docs/index.html?b=c") == true);
View Full Code Here
245
246
247
248
249
250
251
252
253
254
255
    final Token reusableToken = new Token();
    Token nextToken = tf.next(reusableToken);
    assertTrue("nextToken is null and it shouldn't be", nextToken != null);
    assertTrue(nextToken.term() + " is not equal to " + "http://lucene.apache.org/java/docs/index.html#news",
            nextToken.term().equals("http://lucene.apache.org/java/docs/index.html#news") == true);
    assertTrue(nextToken.type() + " is not equal to " + WikipediaTokenizer.EXTERNAL_LINK_URL, nextToken.type().equals(WikipediaTokenizer.EXTERNAL_LINK_URL) == true);
    tf.next(reusableToken);//skip here
    nextToken = tf.next(reusableToken);
    assertTrue("nextToken is null and it shouldn't be", nextToken != null);
    assertTrue(nextToken.term() + " is not equal to " + "http://lucene.apache.org/java/docs/index.html?b=c",
            nextToken.term().equals("http://lucene.apache.org/java/docs/index.html?b=c") == true);
View Full Code Here
251
252
253
254
255
256
257
258
259
260
261
    tf.next(reusableToken);//skip here
    nextToken = tf.next(reusableToken);
    assertTrue("nextToken is null and it shouldn't be", nextToken != null);
    assertTrue(nextToken.term() + " is not equal to " + "http://lucene.apache.org/java/docs/index.html?b=c",
            nextToken.term().equals("http://lucene.apache.org/java/docs/index.html?b=c") == true);
    assertTrue(nextToken.type() + " is not equal to " + WikipediaTokenizer.EXTERNAL_LINK_URL, nextToken.type().equals(WikipediaTokenizer.EXTERNAL_LINK_URL) == true);
    tf.next(reusableToken);//skip here
    nextToken = tf.next(reusableToken);
    assertTrue("nextToken is null and it shouldn't be", nextToken != null);
    assertTrue(nextToken.term() + " is not equal to " + "https://lucene.apache.org/java/docs/index.html?b=c",
            nextToken.term().equals("https://lucene.apache.org/java/docs/index.html?b=c") == true);
View Full Code Here
251
252
253
254
255
256
257
258
259
260
261
    tf.next(reusableToken);//skip here
    nextToken = tf.next(reusableToken);
    assertTrue("nextToken is null and it shouldn't be", nextToken != null);
    assertTrue(nextToken.term() + " is not equal to " + "http://lucene.apache.org/java/docs/index.html?b=c",
            nextToken.term().equals("http://lucene.apache.org/java/docs/index.html?b=c") == true);
    assertTrue(nextToken.type() + " is not equal to " + WikipediaTokenizer.EXTERNAL_LINK_URL, nextToken.type().equals(WikipediaTokenizer.EXTERNAL_LINK_URL) == true);
    tf.next(reusableToken);//skip here
    nextToken = tf.next(reusableToken);
    assertTrue("nextToken is null and it shouldn't be", nextToken != null);
    assertTrue(nextToken.term() + " is not equal to " + "https://lucene.apache.org/java/docs/index.html?b=c",
            nextToken.term().equals("https://lucene.apache.org/java/docs/index.html?b=c") == true);
View Full Code Here
257
258
259
260
261
262
263
264
265
266
267
    tf.next(reusableToken);//skip here
    nextToken = tf.next(reusableToken);
    assertTrue("nextToken is null and it shouldn't be", nextToken != null);
    assertTrue(nextToken.term() + " is not equal to " + "https://lucene.apache.org/java/docs/index.html?b=c",
            nextToken.term().equals("https://lucene.apache.org/java/docs/index.html?b=c") == true);
    assertTrue(nextToken.type() + " is not equal to " + WikipediaTokenizer.EXTERNAL_LINK_URL, nextToken.type().equals(WikipediaTokenizer.EXTERNAL_LINK_URL) == true);
    nextToken = tf.next(reusableToken);
    assertTrue("nextToken is null and it shouldn't be", nextToken != null);

    nextToken = tf.next(reusableToken);
    assertTrue("nextToken is not null and it should be", nextToken == null);
View Full Code Here
257
258
259
260
261
262
263
264
265
266
267
    tf.next(reusableToken);//skip here
    nextToken = tf.next(reusableToken);
    assertTrue("nextToken is null and it shouldn't be", nextToken != null);
    assertTrue(nextToken.term() + " is not equal to " + "https://lucene.apache.org/java/docs/index.html?b=c",
            nextToken.term().equals("https://lucene.apache.org/java/docs/index.html?b=c") == true);
    assertTrue(nextToken.type() + " is not equal to " + WikipediaTokenizer.EXTERNAL_LINK_URL, nextToken.type().equals(WikipediaTokenizer.EXTERNAL_LINK_URL) == true);
    nextToken = tf.next(reusableToken);
    assertTrue("nextToken is null and it shouldn't be", nextToken != null);

    nextToken = tf.next(reusableToken);
    assertTrue("nextToken is not null and it should be", nextToken == null);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.