Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.TokenStream$MethodSupport

174
175
176
177
178
179
180
181
182
183
184
        TopDocs hits = indexSearcher.search(query, 1);
        assertEquals(1, hits.totalHits);
        final Highlighter highlighter = new Highlighter(
            new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),
            new QueryScorer(query));
        final TokenStream tokenStream = TokenSources
            .getTokenStream(
                (TermPositionVector) indexReader.getTermFreqVector(0, FIELD),
                false);
        assertEquals("<B>the fox</B> did not jump",
            highlighter.getBestFragment(tokenStream, TEXT));
View Full Code Here
220
221
222
223
224
225
226
227
228
229
230
        TopDocs hits = indexSearcher.search(phraseQuery, 1);
        assertEquals(1, hits.totalHits);
        final Highlighter highlighter = new Highlighter(
            new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),
            new QueryScorer(phraseQuery));
        final TokenStream tokenStream = TokenSources
            .getTokenStream(
                (TermPositionVector) indexReader.getTermFreqVector(0, FIELD),
                false);
        assertEquals("<B>the fox</B> did not jump",
            highlighter.getBestFragment(tokenStream, TEXT));
View Full Code Here
267
268
269
270
271
272
273
274
275
276
277
        TopDocs hits = indexSearcher.search(phraseQuery, 1);
        assertEquals(1, hits.totalHits);
        final Highlighter highlighter = new Highlighter(
            new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),
            new QueryScorer(phraseQuery));
        final TokenStream tokenStream = TokenSources
            .getTokenStream(
                (TermPositionVector) indexReader.getTermFreqVector(0, FIELD),
                false);
        assertEquals("<B>the fox</B> did not jump",
            highlighter.getBestFragment(tokenStream, TEXT));
View Full Code Here
211
212
213
214
215
216
217
218
219
220
221
  public void testPreAnalyzedField() throws IOException {
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
        TEST_VERSION_CURRENT, new MockAnalyzer(random)));
    Document doc = new Document();
   
    doc.add(new Field("preanalyzed", new TokenStream() {
      private String[] tokens = new String[] {"term1", "term2", "term3", "term2"};
      private int index = 0;
     
      private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
     
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
    for (int i = 0; i < hits.scoreDocs.length; i++) {
      Document doc = searcher.doc(hits.scoreDocs[i].doc);
      String storedField = doc.get(FIELD_NAME);

      TokenStream stream = TokenSources.getAnyTokenStream(searcher
          .getIndexReader(), hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);

      Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);

      highlighter.setTextFragmenter(fragmenter);
View Full Code Here
173
174
175
176
177
178
179
180
181
182
183
   * This method intended for use with <tt>testHighlightingWithDefaultField()</tt>
* @throws InvalidTokenOffsetsException
   */
  private static String highlightField(Query query, String fieldName, String text)
      throws IOException, InvalidTokenOffsetsException {
    TokenStream tokenStream = new StandardAnalyzer(TEST_VERSION_CURRENT).tokenStream(fieldName, new StringReader(text));
    // Assuming "<B>", "</B>" used to highlight
    SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
    QueryScorer scorer = new QueryScorer(query, fieldName, FIELD_NAME);
    Highlighter highlighter = new Highlighter(formatter, scorer);
    highlighter.setTextFragmenter(new SimpleFragmenter(Integer.MAX_VALUE));
View Full Code Here
194
195
196
197
198
199
200
201
202
203
204
    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
    Highlighter highlighter = new Highlighter(scorer);
   
    for (int i = 0; i < hits.totalHits; i++) {
      String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
          new StringReader(text));
      highlighter.setTextFragmenter(new SimpleFragmenter(40));

      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
          "...");
View Full Code Here
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
    Highlighter highlighter = new Highlighter(this, scorer);
   
    for (int i = 0; i < hits.totalHits; i++) {
      String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));

      highlighter.setTextFragmenter(new SimpleFragmenter(40));

      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
          "...");
      if (VERBOSE) System.out.println("\t" + result);
    }

    assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
        numHighlights == 3);
   
    numHighlights = 0;
    doSearching("\"This piece of text refers to Kennedy\"");

    maxNumFragmentsRequired = 2;

    scorer = new QueryScorer(query, FIELD_NAME);
    highlighter = new Highlighter(this, scorer);
   
    for (int i = 0; i < hits.totalHits; i++) {
      String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));

      highlighter.setTextFragmenter(new SimpleFragmenter(40));

      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
          "...");
      if (VERBOSE) System.out.println("\t" + result);
    }

    assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
        numHighlights == 4);
   
    numHighlights = 0;
    doSearching("\"lets is a the lets is a the lets is a the lets\"");

    maxNumFragmentsRequired = 2;

    scorer = new QueryScorer(query, FIELD_NAME);
    highlighter = new Highlighter(this, scorer);
   
    for (int i = 0; i < hits.totalHits; i++) {
      String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));

      highlighter.setTextFragmenter(new SimpleFragmenter(40));

      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
          "...");
View Full Code Here
313
314
315
316
317
318
319
320
321
322
323
    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
    Highlighter highlighter = new Highlighter(this, scorer);
   
    for (int i = 0; i < hits.totalHits; i++) {
      String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));

      highlighter.setTextFragmenter(new SimpleFragmenter(40));

      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
          "...");
View Full Code Here
337
338
339
340
341
342
343
344
345
346
347
    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
    Highlighter highlighter = new Highlighter(this, scorer);
   
    for (int i = 0; i < hits.totalHits; i++) {
      String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));

      highlighter.setTextFragmenter(new SimpleFragmenter(40));

      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
          "...");
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.TokenStream$MethodSupport

  • com.code972.elasticsearch.rest.action.RestHebrewAnalyzerCheckWordAction
  • com.gentics.cr.lucene.analysis.CustomPatternAnalyzerTest
  • com.github.pmerienne.trident.ml.preprocessing.EnglishTokenizer
  • com.o19s.RegexPathHierarchyTokenizerTest
  • com.stimulus.archiva.search.FileNameAnalyzer
  • com.stimulus.archiva.search.FileNameAnalyzer$LowercaseDelimiterAnalyzer
  • ivory.core.tokenize.LuceneAnalyzer
  • ivory.core.tokenize.LuceneArabicAnalyzer
  • ivory.core.tokenize.LuceneTokenizer
  • org.apache.jackrabbit.core.query.lucene.SearchIndex
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.