Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.CannedTokenStream

485
486
487
488
489
490
491
492
493
494
495
      ft.setStoreTermVectors(true);
      ft.setStoreTermVectorPositions(true);
      ft.setStoreTermVectorOffsets(true);
    
      Document doc = new Document();
      doc.add(new Field("body", new CannedTokenStream(tokens), ft));
      riw.addDocument(doc);
      success = true;
    } finally {
      if (success) {
        IOUtils.close(riw, dir);
View Full Code Here
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
    Document doc = new Document();
    Token t1 = new Token("foo", 0, 3);
    t1.setPositionIncrement(Integer.MAX_VALUE);
    Token t2 = new Token("bar", 4, 7);
    t2.setPositionIncrement(200);
    TokenStream overflowingTokenStream = new CannedTokenStream(
        new Token[] { t1, t2 }
    );
    Field field = new TextField("foo", overflowingTokenStream);
    doc.add(field);
    try {
View Full Code Here
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
    Token t1 = new Token("foo", 0, 3);
    t1.setPositionIncrement(Integer.MAX_VALUE-500);
    if (random().nextBoolean()) {
      t1.setPayload(new BytesRef(new byte[] { 0x1 } ));
    }
    TokenStream overflowingTokenStream = new CannedTokenStream(
        new Token[] { t1 }
    );
    Field field = new TextField("foo", overflowingTokenStream);
    doc.add(field);
    iw.addDocument(doc);
View Full Code Here
465
466
467
468
469
470
471
472
473
474
  }
 
  // LUCENE-4880
  public void testEmptyString() throws IOException {
    MemoryIndex memory = new MemoryIndex();
    memory.addField("foo", new CannedTokenStream(new Token("", 0, 5)));
    IndexSearcher searcher = memory.createSearcher();
    TopDocs docs = searcher.search(new TermQuery(new Term("foo", "")), 10);
    assertEquals(1, docs.totalHits);
  }
View Full Code Here
367
368
369
370
371
372
373
374
375
376
377
378
379
380
    tokens[2].append("c");
    tokens[2].setPositionIncrement(0);

    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
    Document doc = new Document();
    doc.add(new TextField("field", new CannedTokenStream(tokens)));
    writer.addDocument(doc);
    doc = new Document();
    doc.add(new TextField("field", new CannedTokenStream(tokens)));
    writer.addDocument(doc);
    IndexReader r = writer.getReader();
    writer.close();
    IndexSearcher s = newSearcher(r);
    MultiPhraseQuery mpq = new MultiPhraseQuery();
View Full Code Here
468
469
470
471
472
473
474
475
476
477
478
  private void doTestZeroPosIncrSloppy(Query q, int nExpected) throws IOException {
    Directory dir = newDirectory(); // random dir
    IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
    IndexWriter writer = new IndexWriter(dir, cfg);
    Document doc = new Document();
    doc.add(new TextField("field", new CannedTokenStream(INCR_0_DOC_TOKENS)));
    writer.addDocument(doc);
    IndexReader r = DirectoryReader.open(writer,false);
    writer.close();
    IndexSearcher s = newSearcher(r);
   
View Full Code Here
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
       
        return new TokenStreamComponents(tokenizer) {
          int tokenStreamCounter = 0;
          final TokenStream[] tokenStreams = new TokenStream[] {
            new CannedTokenStream(new Token[] {
                token("wifi",1,1),
                token("hotspot",0,2),
                token("network",1,1),
                token("is",1,1),
                token("slow",1,1)
              }),
            new CannedTokenStream(new Token[] {
                token("wi",1,1),
                token("hotspot",0,3),
                token("fi",1,1),
                token("network",1,1),
                token("is",1,1),
                token("fast",1,1)

              }),
            new CannedTokenStream(new Token[] {
                token("wifi",1,1),
                token("hotspot",0,2),
                token("network",1,1)
              }),
          };
View Full Code Here
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
       
        return new TokenStreamComponents(tokenizer) {
          int tokenStreamCounter = 0;
          final TokenStream[] tokenStreams = new TokenStream[] {
            new CannedTokenStream(new Token[] {
                token("ab",1,1),
                token("ba",0,1),
                token("xc",1,1)
              }),
            new CannedTokenStream(new Token[] {
                token("ba",1,1),         
                token("xd",1,1)
              }),
            new CannedTokenStream(new Token[] {
                token("ab",1,1),
                token("ba",0,1),
                token("x",1,1)
              })
          };
View Full Code Here
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
          @Override
          public TokenStream getTokenStream() {
            // 4th time we are called, return tokens a b,
            // else just a:
            if (count++ != 3) {
              return new CannedTokenStream(new Token[] {
                  token("a", 1, 1),
                });
            } else {
              // After that "a b":
              return new CannedTokenStream(new Token[] {
                  token("a", 1, 1),
                  token("b", 1, 1),
                });
            }
          }
View Full Code Here
955
956
957
958
959
960
961
962
963
964
965
       
        return new TokenStreamComponents(tokenizer) {

          @Override
          public TokenStream getTokenStream() {
            return new CannedTokenStream(new Token[] {
                token("hairy", 1, 1),
                token("smelly", 0, 1),
                token("dog", 1, 1),
              });
          }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.CannedTokenStream

  • org.apache.lucene.analysis.miscellaneous.TestRemoveDuplicatesTokenFilterFactory
  • org.apache.lucene.analysis.shingle.ShingleFilterTest
  • org.apache.lucene.document.TestField
  • org.apache.lucene.index.memory.MemoryIndexTest
  • org.apache.lucene.index.TestCheckIndex
  • org.apache.lucene.index.TestFieldReuse
  • org.apache.lucene.index.TestFieldReuse$MyField
  • org.apache.lucene.index.TestIndexWriterExceptions
  • org.apache.lucene.index.TestPayloadsOnVectors
  • org.apache.lucene.index.TestPostingsOffsets
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.