Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.CannedTokenStream

1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
          @Override
          public TokenStream getTokenStream() {
            if (count == 0) {
              count++;
              return new CannedTokenStream(new Token[] {
                  token("p", 1, 1),
                  token("q", 1, 1),
                  token("r", 0, 1),
                  token("s", 0, 1),
                });
            } else {
              return new CannedTokenStream(new Token[] {
                  token("p", 1, 1),
                });
            }
          }
        
View Full Code Here
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
            public TokenStream getTokenStream() {
              Token a = new Token("a", 0, 1);
              a.setPositionIncrement(1);
              Token b = new Token("b", 0, 1);
              b.setPositionIncrement(0);
              return new CannedTokenStream(new Token[] {a, b});
            }
        
            @Override
            protected void setReader(final Reader reader) throws IOException {
            }
View Full Code Here
459
460
461
462
463
464
465
466
467
468
  }
 
  // LUCENE-4880
  public void testEmptyString() throws IOException {
    MemoryIndex memory = new MemoryIndex();
    memory.addField("foo", new CannedTokenStream(new Token("", 0, 5)));
    IndexSearcher searcher = memory.createSearcher();
    TopDocs docs = searcher.search(new TermQuery(new Term("foo", "")), 10);
    assertEquals(1, docs.totalHits);
  }
View Full Code Here
328
329
330
331
332
333
334
335
336
337
338
      getToken("jump"),
      getToken("high")
    };

    Document doc = new Document();
    doc.add(new Field("field", new CannedTokenStream(tokens), myFieldType));
    writer.addDocument(doc);
 
    IndexReader reader = writer.getReader();
    writer.close();
    assertEquals(1, reader.numDocs());
View Full Code Here
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
    Document doc = new Document();
    Token t1 = new Token("foo", 0, 3);
    t1.setPositionIncrement(Integer.MAX_VALUE);
    Token t2 = new Token("bar", 4, 7);
    t2.setPositionIncrement(200);
    TokenStream overflowingTokenStream = new CannedTokenStream(
        new Token[] { t1, t2 }
    );
    Field field = new TextField("foo", overflowingTokenStream);
    doc.add(field);
    try {
View Full Code Here
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
    Token t1 = new Token("foo", 0, 3);
    t1.setPositionIncrement(Integer.MAX_VALUE-500);
    if (random().nextBoolean()) {
      t1.setPayload(new BytesRef(new byte[] { 0x1 } ));
    }
    TokenStream overflowingTokenStream = new CannedTokenStream(
        new Token[] { t1 }
    );
    Field field = new TextField("foo", overflowingTokenStream);
    doc.add(field);
    iw.addDocument(doc);
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
    assertSame(ts, ts2);
    assertNumericContents(20, ts);
   
    // pass a bogus stream and ensure its still ok
    intField = new IntField("foo", 2343, Field.Store.NO);
    TokenStream bogus = new CannedTokenStream(new Token("bogus", 0, 5));
    ts = intField.tokenStream(null, bogus);
    assertNotSame(bogus, ts);
    assertNumericContents(2343, ts);
   
    // pass another bogus stream (numeric, but different precision step!)
View Full Code Here
117
118
119
120
121
122
123
    }
   
    @Override
    public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) throws IOException {
      lastSeen = reuse;
      return lastReturned = new CannedTokenStream(new Token("unimportant", 0, 10));
    }
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
    field.setTokenStream(ts);
    doc.add(field);
    Field field2 = new Field("field", "", customType);
    Token withPayload = new Token("withPayload", 0, 11);
    withPayload.setPayload(new BytesRef("test"));
    ts = new CannedTokenStream(withPayload);
    assertTrue(ts.hasAttribute(PayloadAttribute.class));
    field2.setTokenStream(ts);
    doc.add(field2);
    Field field3 = new Field("field", "", customType);
    ts = new MockTokenizer(new StringReader("nopayload"), MockTokenizer.WHITESPACE, true);
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
      makeToken("a", 1, 0, 6),
      makeToken("b", 1, 8, 9),
      makeToken("a", 1, 9, 17),
      makeToken("c", 1, 19, 50),
    };
    doc.add(new Field("content", new CannedTokenStream(tokens), ft));

    w.addDocument(doc);
    IndexReader r = w.getReader();
    w.close();
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.CannedTokenStream

  • org.apache.lucene.analysis.miscellaneous.TestRemoveDuplicatesTokenFilterFactory
  • org.apache.lucene.analysis.shingle.ShingleFilterTest
  • org.apache.lucene.document.TestField
  • org.apache.lucene.index.memory.MemoryIndexTest
  • org.apache.lucene.index.TestCheckIndex
  • org.apache.lucene.index.TestFieldReuse
  • org.apache.lucene.index.TestFieldReuse$MyField
  • org.apache.lucene.index.TestIndexWriterExceptions
  • org.apache.lucene.index.TestPayloadsOnVectors
  • org.apache.lucene.index.TestPostingsOffsets
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.