Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.TokenStream$MethodSupport

1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
  public void assertExpectedHighlightCount(final int maxNumFragmentsRequired,
      final int expectedHighlights) throws Exception {
    for (int i = 0; i < hits.totalHits; i++) {
      String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
      QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
      Highlighter highlighter = new Highlighter(this, scorer);

      highlighter.setTextFragmenter(new SimpleFragmenter(40));
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
     *   and {@link RussianStemFilter}
     */
    @Override
    public TokenStream tokenStream(String fieldName, Reader reader)
    {
        TokenStream result = new RussianLetterTokenizer(reader);
        result = new LowerCaseFilter(result);
        result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                result, stopSet);
        result = new RussianStemFilter(result);
        return result;
View Full Code Here
207
208
209
210
211
212
213
214
215
216
217
   *         filtered with {@link StandardFilter}, {@link StopFilter},
   *         {@link FrenchStemFilter} and {@link LowerCaseFilter}
   */
  @Override
  public final TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new StandardTokenizer(matchVersion, reader);
    result = new StandardFilter(result);
    result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                            result, stoptable);
    result = new FrenchStemFilter(result, excltable);
    // Convert to lowercase after stemming!
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
    this.outputUnigrams = outputUnigrams;
  }

  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream wrapped;
    try {
      wrapped = defaultAnalyzer.reusableTokenStream(fieldName, reader);
    } catch (IOException e) {
      wrapped = defaultAnalyzer.tokenStream(fieldName, reader);
    }
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
      streams = new SavedStreams();
      streams.wrapped = defaultAnalyzer.reusableTokenStream(fieldName, reader);
      streams.shingle = new ShingleFilter(streams.wrapped);
      setPreviousTokenStream(streams);
    } else {
      TokenStream result = defaultAnalyzer.reusableTokenStream(fieldName, reader);
      if (result == streams.wrapped) {
        /* the wrapped analyzer reused the stream */
        streams.shingle.reset();
      } else {
        /* the wrapped analyzer did not, create a new shingle around the new one */
 
View Full Code Here
164
165
166
167
168
169
170
171
172
173
174
   *         {@link ArabicNormalizationFilter},
   *         {@link PersianNormalizationFilter} and Persian Stop words
   */
  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new ArabicLetterTokenizer(reader);
    result = new LowerCaseFilter(result);
    result = new ArabicNormalizationFilter(result);
    /* additional persian-specific normalization */
    result = new PersianNormalizationFilter(result);
    /*
 
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
   *   filtered with {@link StandardFilter}, {@link StopFilter},
   *   and {@link DutchStemFilter}
   */
  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new StandardTokenizer(matchVersion, reader);
    result = new StandardFilter(result);
    result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                            result, stoptable);
    result = new DutchStemFilter(result, excltable, stemdict);
    return result;
View Full Code Here
194
195
196
197
198
199
200
201
202
203
204
  }
 
  public void testTokenStream() throws Exception {
    QueryAutoStopWordAnalyzer a = new QueryAutoStopWordAnalyzer(Version.LUCENE_CURRENT, new WhitespaceAnalyzer());
    a.addStopWords(reader, 10);
    TokenStream ts = a.tokenStream("repetitiveField", new StringReader("this boring"));
    TermAttribute termAtt = ts.getAttribute(TermAttribute.class);
    assertTrue(ts.incrementToken());
    assertEquals("this", termAtt.term());
    assertFalse(ts.incrementToken());
  }
View Full Code Here
40
41
42
43
44
45
46
47
48
    * @return  A {@link TokenStream} built from a {@link ChineseTokenizer}
    *   filtered with {@link ChineseFilter}.
    */
    @Override
    public final TokenStream tokenStream(String fieldName, Reader reader) {
        TokenStream result = new ChineseTokenizer(reader);
        result = new ChineseFilter(result);
        return result;
    }
View Full Code Here
76
77
78
79
80
81
82
83
84
    this.collator = collator;
  }

  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new KeywordTokenizer(reader);
    result = new ICUCollationKeyFilter(result, collator);
    return result;
  }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.TokenStream$MethodSupport

  • com.code972.elasticsearch.rest.action.RestHebrewAnalyzerCheckWordAction
  • com.gentics.cr.lucene.analysis.CustomPatternAnalyzerTest
  • com.github.pmerienne.trident.ml.preprocessing.EnglishTokenizer
  • com.o19s.RegexPathHierarchyTokenizerTest
  • com.stimulus.archiva.search.FileNameAnalyzer
  • com.stimulus.archiva.search.FileNameAnalyzer$LowercaseDelimiterAnalyzer
  • ivory.core.tokenize.LuceneAnalyzer
  • ivory.core.tokenize.LuceneArabicAnalyzer
  • ivory.core.tokenize.LuceneTokenizer
  • org.apache.jackrabbit.core.query.lucene.SearchIndex
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.