Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.StopFilter

61
62
63
64
65
66
67
68
69
70
71
    if (streams == null) {
      streams = new SavedStreams();
      streams.source = new StandardTokenizer(reader);
      streams.result = new StandardFilter(streams.source);
      streams.result = new ThaiWordFilter(streams.result);
      streams.result = new StopFilter(streams.result, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
      setPreviousTokenStream(streams);
    } else {
      streams.source.reset(reader);
      streams.result.reset(); // reset the ThaiWordFilter's state
    }
View Full Code Here
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
      streams.wrapped = delegate.reusableTokenStream(fieldName, reader);

      /* if there are any stopwords for the field, save the stopfilter */
      HashSet stopWords = (HashSet) stopWordsPerField.get(fieldName);
      if (stopWords != null)
        streams.withStopFilter = new StopFilter(streams.wrapped, stopWords);
      else
        streams.withStopFilter = streams.wrapped;

    } else {
      /*
       * an entry for this field exists, verify the wrapped stream has not
       * changed. if it has not, reuse it, otherwise wrap the new stream.
       */
      TokenStream result = delegate.reusableTokenStream(fieldName, reader);
      if (result == streams.wrapped) {
        /* the wrapped analyzer reused the stream */
        streams.withStopFilter.reset();
      } else {
        /*
         * the wrapped analyzer did not. if there are any stopwords for the
         * field, create a new StopFilter around the new stream
         */
        streams.wrapped = result;
        HashSet stopWords = (HashSet) stopWordsPerField.get(fieldName);
        if (stopWords != null)
          streams.withStopFilter = new StopFilter(streams.wrapped, stopWords);
        else
          streams.withStopFilter = streams.wrapped;
      }
    }

View Full Code Here
283
284
285
286
287
288
289
290
291
     */
    public TokenStream tokenStream(String fieldName, Reader reader)
    {
        TokenStream result = new RussianLetterTokenizer(reader, charset);
        result = new RussianLowerCaseFilter(result, charset);
        result = new StopFilter(result, stopSet);
        result = new RussianStemFilter(result, charset);
        return result;
    }
View Full Code Here
309
310
311
312
313
314
315
316
317
318
319
    SavedStreams streams = (SavedStreams) getPreviousTokenStream();
    if (streams == null) {
      streams = new SavedStreams();
      streams.source = new RussianLetterTokenizer(reader, charset);
      streams.result = new RussianLowerCaseFilter(streams.source, charset);
      streams.result = new StopFilter(streams.result, stopSet);
      streams.result = new RussianStemFilter(streams.result, charset);
      setPreviousTokenStream(streams);
    } else {
      streams.source.reset(reader);
    }
View Full Code Here
137
138
139
140
141
142
143
144
145
   */
  public final TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new StandardTokenizer( reader );
    result = new LowerCaseFilter( result );
    result = new StandardFilter( result );
    result = new StopFilter( result, stoptable );
    result = new BrazilianStemFilter( result, excltable );
    return result;
  }
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
      if (streams == null) {
        streams = new SavedStreams();
        streams.source = new StandardTokenizer(reader);
        streams.result = new LowerCaseFilter(streams.source);
        streams.result = new StandardFilter(streams.result);
        streams.result = new StopFilter(streams.result, stoptable);
        streams.result = new BrazilianStemFilter(streams.result, excltable);
        setPreviousTokenStream(streams);
      } else {
        streams.source.reset(reader);
      }
View Full Code Here
151
152
153
154
155
156
157
158
159
160
    // result = new LowerCaseFilter(result);
    // LowerCaseFilter is not needed, as SegTokenFilter lowercases Basic Latin text.
    // The porter stemming is too strict, this is not a bug, this is a feature:)
    result = new PorterStemFilter(result);
    if (!stopWords.isEmpty()) {
      result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                              result, stopWords, false);
    }
    return result;
  }
View Full Code Here
173
174
175
176
177
178
179
180
181
182
183
      setPreviousTokenStream(streams);
      streams.tokenStream = new SentenceTokenizer(reader);
      streams.filteredTokenStream = new WordTokenFilter(streams.tokenStream);
      streams.filteredTokenStream = new PorterStemFilter(streams.filteredTokenStream);
      if (!stopWords.isEmpty()) {
        streams.filteredTokenStream = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                                     streams.filteredTokenStream, stopWords, false);
      }
    } else {
      streams.tokenStream.reset(reader);
      streams.filteredTokenStream.reset(); // reset WordTokenFilter's state
View Full Code Here
231
232
233
234
235
236
237
      this.enablePositionIncrements = enablePositionIncrements;
    }
    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      TokenStream ts = a.tokenStream(fieldName,reader);
      return new StopFilter(enablePositionIncrements, ts, new CharArraySet(Collections.singleton("stop"), true));
    }
View Full Code Here
484
485
486
487
488
489
490
491
492
493
494
495
496
        new int[] { 1, 11, 1 });

    Analyzer a3 = new Analyzer() {
      @Override
      public TokenStream tokenStream(String field, Reader reader) {
        StopFilter filter = new StopFilter(TEST_VERSION_CURRENT,
            new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), StandardAnalyzer.STOP_WORDS_SET);
        filter.setEnablePositionIncrements(true);
        return new WordDelimiterFilter(filter,
            1, 1, 0, 0, 1, 1, 0, 1, 1, protWords);
      }
    };
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.StopFilter

  • BugAnalyzer.PorterStemAnalyzer
  • com.gentics.cr.lucene.analysis.CustomPatternAnalyzer
  • com.gentics.cr.lucene.autocomplete.AutocompleteAnalyzer
  • com.github.pmerienne.trident.ml.preprocessing.EnglishTokenizer$EnglishSpecialAnalyzer
  • com.infoclinika.mssharing.search.NoStopWordsAnalyzer
  • com.mozilla.grouperfish.lucene.analysis.en.EnglishAnalyzer
  • com.mozilla.grouperfish.transforms.coclustering.lucene.analysis.en.EnglishAnalyzer
  • edu.wiki.index.WikipediaAnalyzer
  • ivory.core.tokenize.LuceneArabicAnalyzer
  • ivory.core.tokenize.LuceneSpanishAnalyzer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.