Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Tokenizer.reset()

86
87
88
89
90
91
92
93
94
95
96
    assertEquals("now", term.term());
    assertTrue(ts.incrementToken());
    assertEquals("A", term.term());
    // reset back to input,
    // if reset() does not work correctly then previous buffered tokens will remain
    tokenizer.reset(new StringReader(input));
    ts.reset();
    assertTrue(ts.incrementToken());
    assertEquals("How", term.term());
  }
}
View Full Code Here
38
39
40
41
42
43
44
45
46
47
        Tokenizer tokenizer = (Tokenizer)getPreviousTokenStream();
        if (tokenizer == null) {
            tokenizer = new TagTokenizer(reader);
            setPreviousTokenStream(tokenizer);
        } else
            tokenizer.reset(reader);
        return tokenizer;
    }

}
View Full Code Here
224
225
226
227
228
229
230
231
232
233
        Tokenizer tokenizer = (Tokenizer)getPreviousTokenStream();
        if (tokenizer == null) {
            tokenizer = new DomainPathTokenizer(reader);
            setPreviousTokenStream(tokenizer);
        } else
            tokenizer.reset(reader);
        return tokenizer;
    }

}
View Full Code Here
39
40
41
42
43
44
45
46
47
48
        Tokenizer tokenizer = (Tokenizer)getPreviousTokenStream();
        if (tokenizer == null) {
            tokenizer = new NamingTokenizer(reader);
            setPreviousTokenStream(tokenizer);
        } else
            tokenizer.reset(reader);
        return tokenizer;
    }

}
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
    CharTermAttribute termAtt = tf.getAttribute(CharTermAttribute.class);
    assertTrue(tf.incrementToken());
    assertEquals("Rindfleischüberwachungsgesetz", termAtt.toString());
    assertTrue(tf.incrementToken());
    assertEquals("Rind", termAtt.toString());
    wsTokenizer.reset(new StringReader("Rindfleischüberwachungsgesetz"));
    tf.reset();
    assertTrue(tf.incrementToken());
    assertEquals("Rindfleischüberwachungsgesetz", termAtt.toString());
  }
View Full Code Here
210
211
212
213
214
215
216
217
218
219
220
        int lastAdded = -1;
        int lastOffset = 0;
        boolean foundWildcard = false;
        //Lucene tokenizer are really low level ...
        try {
          tokenizer.reset(); //starting with Solr4 reset MUST BE called before using
            while(tokenizer.incrementToken()){
                //only interested in the start/end indexes of tokens
                OffsetAttribute offset = tokenizer.addAttribute(OffsetAttribute.class);
                if(lastAdded < 0){ //rest with this token
                    lastAdded = offset.startOffset();
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
    assertEquals("now", term.term());
    assertTrue(ts.incrementToken());
    assertEquals("A", term.term());
    // reset back to input,
    // if reset() does not work correctly then previous buffered tokens will remain
    tokenizer.reset(new StringReader(input));
    ts.reset();
    assertTrue(ts.incrementToken());
    assertEquals("How", term.term());
  }
}
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
        "꽃:N:NNG:null:1:1:0:1,배달:N:NNG:null:1:1:1:3,"
        + "꽃:N:NNG:null:1:1:4:5,꽃망울:COMPOUND:Compound:null:0:2:4:7,"
        + "망울:N:NNG:null:1:1:5:7,오토바이:N:NNG:null:1:1:8:12,",
        tokenizerToString(tokenizer));
  
    tokenizer.reset();
    tokenizer.setReader(new StringReader("소설 무궁화꽃이 피었습니다."));
    assertEquals(
        "소설:N:NNG:null:1:1:0:2,무궁:N:NNG:null:1:1:3:5,"
        + "무궁화:COMPOUND:Compound:null:0:2:3:6,화:N:NNG:null:1:1:5:6,"
        + "꽃이:EOJEOL:NNG+JKS:null:1:1:6:8,꽃:N:NNG:null:0:1:6:7,"
View Full Code Here
39
40
41
42
43
44
45
46
47
48
        Tokenizer tokenizer = (Tokenizer)getPreviousTokenStream();
        if (tokenizer == null) {
            tokenizer = new NamingTokenizer(reader);
            setPreviousTokenStream(tokenizer);
        } else
            tokenizer.reset(reader);
        return tokenizer;
    }

}
View Full Code Here
224
225
226
227
228
229
230
231
232
233
        Tokenizer tokenizer = (Tokenizer)getPreviousTokenStream();
        if (tokenizer == null) {
            tokenizer = new DomainPathTokenizer(reader);
            setPreviousTokenStream(tokenizer);
        } else
            tokenizer.reset(reader);
        return tokenizer;
    }

}
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.