Examples of tokenStream()

Examples of org.apache.lucene.analysis.SimpleAnalyzer.tokenStream()

98
99
100
101
102
103
104
105
106
107
108
  }

  static void test(Reader reader, boolean verbose, long bytes)
       throws Exception {
    Analyzer analyzer = new SimpleAnalyzer();
    TokenStream stream = analyzer.tokenStream(null, reader);

    Date start = new Date();

    int count = 0;
    for (Token t = stream.next(); t!=null; t = stream.next()) {
View Full Code Here

Examples of org.apache.lucene.analysis.WhitespaceAnalyzer.tokenStream()

4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
  public void testEndOffsetPositionWithCachingTokenFilter() throws Exception {
    MockRAMDirectory dir = new MockRAMDirectory();
    Analyzer analyzer = new WhitespaceAnalyzer();
    IndexWriter w = new IndexWriter(dir, analyzer, IndexWriter.MaxFieldLength.LIMITED);
    Document doc = new Document();
    TokenStream stream = new CachingTokenFilter(analyzer.tokenStream("field", new StringReader("abcd   ")));
    Field f = new Field("field", stream, Field.TermVector.WITH_POSITIONS_OFFSETS);
    doc.add(f);
    doc.add(f);
    w.addDocument(doc);
    w.close();
View Full Code Here

Examples of org.apache.lucene.analysis.WhitespaceAnalyzer.tokenStream()

4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
  public void testEndOffsetPositionWithTeeSinkTokenFilter() throws Exception {
    MockRAMDirectory dir = new MockRAMDirectory();
    Analyzer analyzer = new WhitespaceAnalyzer();
    IndexWriter w = new IndexWriter(dir, analyzer, IndexWriter.MaxFieldLength.LIMITED);
    Document doc = new Document();
    TeeSinkTokenFilter tee = new TeeSinkTokenFilter(analyzer.tokenStream("field", new StringReader("abcd   ")));
    TokenStream sink = tee.newSinkTokenStream();
    Field f1 = new Field("field", tee, Field.TermVector.WITH_POSITIONS_OFFSETS);
    Field f2 = new Field("field", sink, Field.TermVector.WITH_POSITIONS_OFFSETS);
    doc.add(f1);
    doc.add(f2);
View Full Code Here

Examples of org.apache.lucene.analysis.WhitespaceAnalyzer.tokenStream()

4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
  public void testEndOffsetPositionWithTeeSinkTokenFilter() throws Exception {
    MockRAMDirectory dir = new MockRAMDirectory();
    Analyzer analyzer = new WhitespaceAnalyzer();
    IndexWriter w = new IndexWriter(dir, analyzer, IndexWriter.MaxFieldLength.LIMITED);
    Document doc = new Document();
    TeeSinkTokenFilter tee = new TeeSinkTokenFilter(analyzer.tokenStream("field", new StringReader("abcd   ")));
    TokenStream sink = tee.newSinkTokenStream();
    Field f1 = new Field("field", tee, Field.TermVector.WITH_POSITIONS_OFFSETS);
    Field f2 = new Field("field", sink, Field.TermVector.WITH_POSITIONS_OFFSETS);
    doc.add(f1);
    doc.add(f2);
View Full Code Here

Examples of org.apache.lucene.analysis.WhitespaceAnalyzer.tokenStream()

4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
  public void testEndOffsetPositionWithCachingTokenFilter() throws Exception {
    MockRAMDirectory dir = new MockRAMDirectory();
    Analyzer analyzer = new WhitespaceAnalyzer();
    IndexWriter w = new IndexWriter(dir, analyzer, IndexWriter.MaxFieldLength.LIMITED);
    Document doc = new Document();
    TokenStream stream = new CachingTokenFilter(analyzer.tokenStream("field", new StringReader("abcd   ")));
    Field f = new Field("field", stream, Field.TermVector.WITH_POSITIONS_OFFSETS);
    doc.add(f);
    doc.add(f);
    w.addDocument(doc);
    w.close();
View Full Code Here

Examples of org.apache.lucene.analysis.WhitespaceAnalyzer.tokenStream()

1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
    Directory dir = newDirectory();
    Analyzer analyzer = new Analyzer(){
      Analyzer a = new WhitespaceAnalyzer( TEST_VERSION_CURRENT );
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader){
        return a.tokenStream(fieldName, reader);
      }
      @Override
      public int getPositionIncrementGap(String fieldName) {
        return 100;
      }
View Full Code Here

Examples of org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer.tokenStream()

77
78
79
80
81
82
83
84
85
86
87
      e1.printStackTrace();
    }
   
    long begin = System.currentTimeMillis();  
   
    TokenStream tokenStream = ss.tokenStream("", reader);
    try {
      while(tokenStream.incrementToken()){
//        TermAttribute termAttribute = tokenStream.getAttribute(TermAttribute.class);
//        System.out.println(termAttribute.toString());
       
View Full Code Here

Examples of org.apache.lucene.analysis.core.KeywordAnalyzer.tokenStream()

371
372
373
374
375
376
377
378
379
380
381
    if (value != null) {
      Analyzer analyzer = new KeywordAnalyzer();

      try {
        TokenStream tokenStream =
          analyzer.tokenStream(fieldName, new StringReader(value));
        tokenStream.reset();
        CharTermAttribute attr =
          tokenStream.getAttribute(CharTermAttribute.class);

        while (tokenStream.incrementToken()) {
View Full Code Here

Examples of org.apache.lucene.analysis.core.SimpleAnalyzer.tokenStream()

243
244
245
246
247
248
249
250
251
252
253
        // "localPart":s* OR "label":s* OR "description":s*)
        BooleanQuery q3 = new BooleanQuery();
        q3.add(new WildcardQuery(new Term("prefix", s + "*")),
            Occur.SHOULD);

        TokenStream stream = analyzer.tokenStream("localPart",
            new StringReader(s));
        // get the TermAttribute from the TokenStream
        CharTermAttribute termAtt = (CharTermAttribute) stream
            .addAttribute(CharTermAttribute.class);

View Full Code Here

Examples of org.apache.lucene.analysis.core.StopAnalyzer.tokenStream()

27
28
29
30
31
32
33
34
35
36
37
        Analyzer analyzer = new StopAnalyzer(Version.LUCENE_4_9);

        List<String> result = new ArrayList<>();

        try {
            TokenStream stream = analyzer.tokenStream(null, new StringReader(linha));
            stream.reset();
            while (stream.incrementToken()) {

                result.add(stream.getAttribute(CharTermAttribute.class).toString());
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.