Examples of reset()

Examples of org.apache.lucene.analysis.icu.segmentation.ICUTokenizer.reset()

210
211
212
213
214
215
216
217
218
219
220
        int lastAdded = -1;
        int lastOffset = 0;
        boolean foundWildcard = false;
        //Lucene tokenizer are really low level ...
        try {
          tokenizer.reset(); //starting with Solr4 reset MUST BE called before using
            while(tokenizer.incrementToken()){
                //only interested in the start/end indexes of tokens
                OffsetAttribute offset = tokenizer.addAttribute(OffsetAttribute.class);
                if(lastAdded < 0){ //rest with this token
                    lastAdded = offset.startOffset();
View Full Code Here

Examples of org.apache.lucene.analysis.payloads.DelimitedPayloadTokenFilter.reset()

41
42
43
44
45
46
47
48
49
50
51
    ResourceLoader loader = new StringMockResourceLoader("solr/collection1");
    factory.inform(loader);

    TokenStream input = new MockTokenizer(new StringReader("the|0.1 quick|0.1 red|0.1"), MockTokenizer.WHITESPACE, false);
    DelimitedPayloadTokenFilter tf = factory.create(input);
    tf.reset();
    while (tf.incrementToken()){
      PayloadAttribute payAttr = tf.getAttribute(PayloadAttribute.class);
      assertTrue("payAttr is null and it shouldn't be", payAttr != null);
      byte[] payData = payAttr.getPayload().bytes;
      assertTrue("payData is null and it shouldn't be", payData != null);
View Full Code Here

Examples of org.apache.lucene.analysis.shingle.ShingleFilter.reset()

69
70
71
72
73
74
75
76
77
78
79
    Vector vector = new RandomAccessSparseVector(dimension, value.length()); // guess at initial size

    if (maxNGramSize >= 2) {
      ShingleFilter sf = new ShingleFilter(new IteratorTokenStream(value.getEntries().iterator()), maxNGramSize);
      sf.reset();
      try {
        do {
          String term = sf.getAttribute(CharTermAttribute.class).toString();
          if (!term.isEmpty() && dictionary.containsKey(term)) { // ngram
            int termId = dictionary.get(term);
View Full Code Here

Examples of org.apache.lucene.analysis.standard.StandardTokenizer.reset()

119
120
121
122
123
124
125
126
127
128
129
    StandardTokenizer st = new StandardTokenizer(Version.LUCENE_30, new StringReader(ContentUtils.stripDiacritics(term)));
    CharTermAttribute termAtt = st.addAttribute(CharTermAttribute.class);
    StringBuffer sb = new StringBuffer();
    try {
      try {
        st.reset();
        while (st.incrementToken()) {
          if (sb.length() > 0) {
            sb.append(" +");
          }
          else {
View Full Code Here

Examples of org.apache.lucene.analysis.wikipedia.WikipediaTokenizer.reset()

163
164
165
166
167
168
169
170
171
172
173
    // now check the flags, TODO: add way to check flags from BaseTokenStreamTestCase?
    tf = new WikipediaTokenizer(new StringReader(test), WikipediaTokenizer.BOTH, untoks);
    int expectedFlags[] = new int[] { UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0,
        0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0 };
    FlagsAttribute flagsAtt = tf.addAttribute(FlagsAttribute.class);
    tf.reset();
    for (int i = 0; i < expectedFlags.length; i++) {
      assertTrue(tf.incrementToken());
      assertEquals("flags " + i, expectedFlags[i], flagsAtt.getFlags());
    }
    assertFalse(tf.incrementToken());
View Full Code Here

Examples of org.apache.lucene.benchmark.stats.TimeData.reset()

331
332
333
334
335
336
337
338
339
340
341
                cnt++;
                if (cnt % logStep == 0)
                {
                    System.err.println(" - processed " + cnt + ", run id=" + trd.getId());
                    trd.addData(td);
                    td.reset();
                }
            }
        }
        trd.addData(td);
    }
View Full Code Here

Examples of org.apache.lucene.facet.index.CategoryListPayloadStream.reset()

70
71
72
73
74
75
76
77
78
79
80
    assertEquals("Wrong value in byte stream", 100000000, decoder.decode());
    assertEquals("Wrong value in byte stream", 1000000000, decoder.decode());
    assertEquals("Wrong value in byte stream", Integer.MAX_VALUE, decoder.decode());
    assertEquals("End of stream not reached", IntDecoder.EOS, decoder.decode());

    clps.reset();
    decoder.reInit(bais);
    assertEquals("End of stream not reached", IntDecoder.EOS, decoder.decode());
  }

}
View Full Code Here

Examples of org.apache.lucene.facet.index.streaming.CategoryAttributesStream.reset()

53
54
55
56
57
58
59
60
61
62
63
      nTokens++;
    }
    assertEquals("Wrong number of tokens", 3, nTokens);

    // test reset
    stream.reset();
    nTokens = 0;
    while (stream.incrementToken()) {
      nTokens++;
    }
    assertEquals("Wrong number of tokens", 3, nTokens);
View Full Code Here

Examples of org.apache.lucene.store.ByteArrayDataInput.reset()

144
145
146
147
148
149
150
151
152
153
154
      AtomicReader reader = context.reader();
      BytesRef scratch = new BytesRef(bytes);
      BinaryDocValues dv = reader.getBinaryDocValues("dv");
      for (int i = 0; i < reader.maxDoc(); i++) {
        dv.get(i, scratch);
        input.reset(scratch.bytes, scratch.offset, scratch.length);
        assertEquals(expectedValue % 65535, input.readVInt());
        assertTrue(input.eof());
        expectedValue++;
      }
    }
View Full Code Here

Examples of org.apache.lucene.store.ByteArrayDataOutput.reset()

119
120
121
122
123
124
125
126
127
128
129
    BytesRef data = new BytesRef(bytes);
    BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data);
    doc.add(dvField);
   
    for (int i = 0; i < Integer.MAX_VALUE; i++) {
      encoder.reset(bytes);
      encoder.writeVInt(i % 65535); // 1, 2, or 3 bytes
      data.length = encoder.getPosition();
      w.addDocument(doc);
      if (i % 100000 == 0) {
        System.out.println("indexed: " + i);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.