Examples of incrementToken()

Examples of org.apache.lucene.analysis.core.KeywordTokenizer.incrementToken()

287
288
289
290
291
292
293
294
295
      assertEquals(0, offsetAtt.startOffset());
      assertEquals(s.length(), offsetAtt.endOffset());
      final int end = Character.offsetByCodePoints(s, 0, i);
      assertEquals(s.substring(0, end), termAtt.toString());
    }
    assertFalse(tk.incrementToken());
  }

}
View Full Code Here

Examples of org.apache.lucene.analysis.icu.segmentation.ICUTokenizer.incrementToken()

211
212
213
214
215
216
217
218
219
220
221
        int lastOffset = 0;
        boolean foundWildcard = false;
        //Lucene tokenizer are really low level ...
        try {
          tokenizer.reset(); //starting with Solr4 reset MUST BE called before using
            while(tokenizer.incrementToken()){
                //only interested in the start/end indexes of tokens
                OffsetAttribute offset = tokenizer.addAttribute(OffsetAttribute.class);
                if(lastAdded < 0){ //rest with this token
                    lastAdded = offset.startOffset();
                }
View Full Code Here

Examples of org.apache.lucene.analysis.payloads.DelimitedPayloadTokenFilter.incrementToken()

40
41
42
43
44
45
46
47
48
49
50
    ResourceLoader loader = new SolrResourceLoader(null, null);
    factory.inform(loader);

    TokenStream input = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("the|0.1 quick|0.1 red|0.1"));
    DelimitedPayloadTokenFilter tf = factory.create(input);
    while (tf.incrementToken()){
      PayloadAttribute payAttr = tf.getAttribute(PayloadAttribute.class);
      assertTrue("payAttr is null and it shouldn't be", payAttr != null);
      byte[] payData = payAttr.getPayload().getData();
      assertTrue("payData is null and it shouldn't be", payData != null);
      assertTrue("payData is null and it shouldn't be", payData != null);
View Full Code Here

Examples of org.apache.lucene.analysis.shingle.ShingleFilter.incrementToken()

127
128
129
130
131
132
133
134
135
136
137
        count++;
        ngrams.adjustOrPutValue(term, 1, 1);
      } else if (emitUnigrams && term.length() > 0) { // unigram
        unigrams.adjustOrPutValue(term, 1, 1);
      }
    } while (sf.incrementToken());
   
    try {
      final GramKey gramKey = new GramKey();
     
      ngrams.forEachPair(new ObjectIntProcedure<String>() {
View Full Code Here

Examples of org.apache.lucene.analysis.standard.StandardFilter.incrementToken()

124
125
126
127
128
129
130
131
132
133
134
      tokenizer = new StandardTokenizer(Version.LUCENE_35, new StringReader(finalTokenized.toString().trim()));
      tokenStream = new ArabicStemFilter(new ArabicNormalizationFilter(tokenizer));
      CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
      tokenStream.clearAttributes();
      try {
        while (tokenStream.incrementToken()) {
          String curToken = termAtt.toString();
          if ( vocab != null && vocab.get(curToken) <= 0) {
            countOOV++;
          }
          countAll++;
View Full Code Here

Examples of org.apache.lucene.analysis.standard.StandardTokenizer.incrementToken()

34
35
36
37
38
39
40
41
42
43
44
    result = new StopFilter(true, result, StandardAnalyzer.STOP_WORDS_SET);
   
    TermAttribute termAtt = (TermAttribute) result.addAttribute(TermAttribute.class);
    StringBuilder buf = new StringBuilder();
    try {
      while (result.incrementToken()) {
        if (termAtt.termLength() < 3) continue;
        String word = new String(termAtt.termBuffer(), 0, termAtt.termLength());
        Matcher m = alphabets.matcher(word);
       
        if (m.matches()) {
View Full Code Here

Examples of org.apache.lucene.analysis.wikipedia.WikipediaTokenizer.incrementToken()

165
166
167
168
169
170
171
172
173
174
175
    int expectedFlags[] = new int[] { UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0,
        0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0 };
    FlagsAttribute flagsAtt = tf.addAttribute(FlagsAttribute.class);
    tf.reset();
    for (int i = 0; i < expectedFlags.length; i++) {
      assertTrue(tf.incrementToken());
      assertEquals("flags " + i, expectedFlags[i], flagsAtt.getFlags());
    }
    assertFalse(tf.incrementToken());
    tf.close();
  }
View Full Code Here

Examples of org.apache.lucene.facet.index.streaming.CategoryAttributesStream.incrementToken()

181
182
183
184
185
186
187
188
189
190
191
  public void testCategoryAttributesStream() throws IOException {
    CategoryAttributesStream stream = new CategoryAttributesStream(
        categoryContainer);
    // count the number of tokens
    int nTokens;
    for (nTokens = 0; stream.incrementToken(); nTokens++) {
    }
    assertEquals("Wrong number of tokens", 3, nTokens);
  }

  /**
 
View Full Code Here

Examples of org.apache.lucene.facet.index.streaming.CategoryParentsStream.incrementToken()

56
57
58
59
60
61
62
63
64
65
66
        new CategoryAttributesStream(categoryContainer),
        taxonomyWriter, new DefaultFacetIndexingParams());

    // count the number of tokens
    int nTokens;
    for (nTokens = 0; stream.incrementToken(); nTokens++) {
    }
    // should be 6 - all categories and parents
    assertEquals("Wrong number of tokens", 6, nTokens);

    taxonomyWriter.close();
View Full Code Here

Examples of org.apache.lucene.facet.index.streaming.CategoryTokenizer.incrementToken()

60
61
62
63
64
65
66
67
68
69
70
      categoryTerms.add(initialCatgeories[i]
          .toString(indexingParams.getFacetDelimChar()));
    }

    int nTokens;
    for (nTokens = 0; tokenizer.incrementToken(); nTokens++) {
      if (!categoryTerms.remove(tokenizer.termAttribute.toString())) {
        fail("Unexpected term: " + tokenizer.termAttribute.toString());
      }
    }
    assertTrue("all category terms should have been found", categoryTerms
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.