Examples of incrementToken()

Examples of org.apache.lucene.analysis.CachingTokenFilter.incrementToken()

615
616
617
618
619
620
621
622
623
624
625
          int position = -1;
          for (int i = 0; i < numTokens; i++) {
            String term = null;
            int positionIncrement = 1;
            try {
              boolean hasNext = buffer.incrementToken();
              assert hasNext == true;
              term = termAtt.term();
              if (posIncrAtt != null) {
                positionIncrement = posIncrAtt.getPositionIncrement();
              }
View Full Code Here

Examples of org.apache.lucene.analysis.CachingTokenFilter.incrementToken()

655
656
657
658
659
660
661
662
663
664
665
        for (int i = 0; i < numTokens; i++) {
          String term = null;
          int positionIncrement = 1;

          try {
            boolean hasNext = buffer.incrementToken();
            assert hasNext == true;
            term = termAtt.term();
            if (posIncrAtt != null) {
              positionIncrement = posIncrAtt.getPositionIncrement();
            }
View Full Code Here

Examples of org.apache.lucene.analysis.LowerCaseFilter.incrementToken()

62
63
64
65
66
67
68
69
70
71
72
    CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
    tokenStream.clearAttributes();
    String tokenized = "";
    try {
      while (tokenStream.incrementToken()) {
        String token = termAtt.toString();
        if ( vocab != null && vocab.get(token) <= 0) {
          continue;
        }
        tokenized += ( token + " " );
View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer.incrementToken()

114
115
116
117
118
119
120
121
122
123
124
    stream = new PatternKeywordMarkerFilter(stream, Pattern.compile(".*"));
    stream = new BeiderMorseFilter(stream, new PhoneticEngine(NameType.GENERIC, RuleType.EXACT, true));
    KeywordAttribute keyAtt = stream.addAttribute(KeywordAttribute.class);
    stream.reset();
    int i = 0;
    while(stream.incrementToken()) {
      assertTrue(keyAtt.isKeyword());
      i++;
    }
    assertEquals(12, i);
    stream.end();
View Full Code Here

Examples of org.apache.lucene.analysis.NumericTokenStream.incrementToken()

47
48
49
50
51
52
53
54
55
56
57
        final PositionIncrementAttribute posInc1 = ts1.addAttribute(PositionIncrementAttribute.class);
        final PositionIncrementAttribute posInc2 = ts1.addAttribute(PositionIncrementAttribute.class);
        ts1.reset();
        ts2.reset();
        while (ts1.incrementToken()) {
            assertThat(ts2.incrementToken(), is(true));
            assertThat(posInc1, equalTo(posInc2));
            // can't use equalTo directly on the numeric attribute cause it doesn't implement equals (LUCENE-5070)
            assertThat(numTerm1.getRawValue(), equalTo(numTerm2.getRawValue()));
            assertThat(numTerm2.getShift(), equalTo(numTerm2.getShift()));
        }
View Full Code Here

Examples of org.apache.lucene.analysis.PorterStemFilter.incrementToken()

31
32
33
34
35
36
37
38
39
40
41
   
    TermAttribute termAtt = (TermAttribute) result
        .addAttribute(TermAttribute.class);
    StringBuilder buf = new StringBuilder();
    try {
      while (result.incrementToken()) {
        String word = new String(termAtt.termBuffer(), 0, termAtt
            .termLength());
        buf.append(filter.encode(word)).append(" ");
       
      }
View Full Code Here

Examples of org.apache.lucene.analysis.TeeSinkTokenFilter.incrementToken()

45
46
47
48
49
50
51
52
53
54
55
    boolean seenDogs = false;

    TermAttribute termAtt = ttf.addAttribute(TermAttribute.class);
    TypeAttribute typeAtt = ttf.addAttribute(TypeAttribute.class);
    ttf.reset();
    while (ttf.incrementToken()) {
      if (termAtt.term().equals("dogs")) {
        seenDogs = true;
        assertTrue(typeAtt.type() + " is not equal to " + "D", typeAtt.type().equals("D") == true);
      } else {
        assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().equals("word"));
View Full Code Here

Examples of org.apache.lucene.analysis.TeeSinkTokenFilter.SinkTokenStream.incrementToken()

57
58
59
60
61
62
63
64
65
66
67
    }
    assertTrue(seenDogs + " does not equal: " + true, seenDogs == true);
   
    int sinkCount = 0;
    sink.reset();
    while (sink.incrementToken()) {
      sinkCount++;
    }

    assertTrue("sink Size: " + sinkCount + " is not: " + 1, sinkCount == 1);
  }
View Full Code Here

Examples of org.apache.lucene.analysis.TokenFilter.incrementToken()

72
73
74
75
76
77
78
79
80
81
82
    t.setReader(new StringReader(uri));
    t.reset();

    final TokenFilter filter = new MailtoFilter(t);
    for (int i = 0; i < expectedStems.length; i++) {
        assertTrue("token " + i + " exists", filter.incrementToken());
        assertEquals(expectedStems[i], termAtt.toString());
        if (expectedTypes == null)
          assertEquals(uritype, typeAtt.type());
        else
          assertEquals(expectedTypes[i], typeAtt.type());
View Full Code Here

Examples of org.apache.lucene.analysis.TokenStream.incrementToken()

718
719
720
721
722
723
724
725
726
727
728
          TokenStream stream = nestedAnalyzer.tokenStream(fieldName,
                  new StringReader(asString));
          TermAttribute termAtt = (TermAttribute) stream.addAttribute(TermAttribute.class);

          System.out.println("Tokens for '" + asString + "':");
          while (stream.incrementToken()) {
            System.out.println(" '" + termAtt.term() + "'");
          }
          stream.reset();
          return stream;
          // Do the call a second time and return the result this time
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.