Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.StopFilter

79
80
81
82
83
84
85
86
87
  public Set<?> getStopWords() {
    return stopWords;
  }

  public StopFilter create(TokenStream input) {
    StopFilter stopFilter = new StopFilter(luceneMatchVersion,input,stopWords,ignoreCase);
    stopFilter.setEnablePositionIncrements(enablePositionIncrements);
    return stopFilter;
  }
View Full Code Here
195
196
197
198
199
200
201
202
203
204
    else if (pattern == WHITESPACE_PATTERN) { // fast path
      stream = new FastStringTokenizer(text, false, toLowerCase, stopWords);
    }
    else {
      stream = new PatternTokenizer(text, pattern, toLowerCase);
      if (stopWords != null) stream = new StopFilter(false, stream, stopWords);
    }
   
    return stream;
  }
View Full Code Here
179
180
181
182
183
184
185
186
187
   *   and {@link DutchStemFilter}
   */
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new StandardTokenizer(reader);
    result = new StandardFilter(result);
    result = new StopFilter(result, stoptable);
    result = new DutchStemFilter(result, excltable, stemdict);
    return result;
  }
View Full Code Here
211
212
213
214
215
216
217
218
219
220
221
    SavedStreams streams = (SavedStreams) getPreviousTokenStream();
    if (streams == null) {
      streams = new SavedStreams();
      streams.source = new StandardTokenizer(reader);
      streams.result = new StandardFilter(streams.source);
      streams.result = new StopFilter(streams.result, stoptable);
      streams.result = new DutchStemFilter(streams.result, excltable, stemdict);
      setPreviousTokenStream(streams);
    } else {
      streams.source.reset(reader);
    }
View Full Code Here
224
225
226
227
228
229
230
    public StopWhitespaceAnalyzer(boolean enablePositionIncrements) {
      this.enablePositionIncrements = enablePositionIncrements;
    }
    public TokenStream tokenStream(String fieldName, Reader reader) {
      TokenStream ts = a.tokenStream(fieldName,reader);
      return new StopFilter(enablePositionIncrements, ts, new String[]{"stop"});
    }
View Full Code Here
112
113
114
115
116
117
118
119
120
    // result = new LowerCaseFilter(result);
    // LowerCaseFilter is not needed, as SegTokenFilter lowercases Basic Latin text.
    // The porter stemming is too strict, this is not a bug, this is a feature:)
    result = new PorterStemFilter(result);
    if (stopWords != null) {
      result = new StopFilter(result, stopWords, false);
    }
    return result;
  }
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
      setPreviousTokenStream(streams);
      streams.tokenStream = new SentenceTokenizer(reader);
      streams.filteredTokenStream = new WordTokenFilter(streams.tokenStream);
      streams.filteredTokenStream = new PorterStemFilter(streams.filteredTokenStream);
      if (stopWords != null) {
        streams.filteredTokenStream = new StopFilter(streams.filteredTokenStream, stopWords, false);
      }
    } else {
      streams.tokenStream.reset(reader);
      streams.filteredTokenStream.reset(); // reset WordTokenFilter's state
    }
View Full Code Here
43
44
45
46
47
48
49
50
  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new WikipediaTokenizer(reader);
    result = new StandardFilter(result);
    result = new LowerCaseFilter(result);
    result = new StopFilter(true, result, stopSet);
    return result;
  }
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
      expect(plainFileReaderFactory.createPlainFileReader(TEST_FILE_1)).andReturn(plainFileReader);
      expect(plainFileReader.readLines()).andReturn(lines);
        replay(plainFileReaderFactory);
      replay(plainFileReader);
     
      StopFilter stopFilter = (StopFilter) stopwordFilterFactory.createTokenFilter(tokenStream, properties);
      assertNotNull(stopFilter); 
      verify(plainFileReaderFactory);
      verify(plainFileReader);
     
      reset(plainFileReaderFactory);
View Full Code Here
450
451
452
453
454
455
456
457
458
459
    } else if (pattern == WHITESPACE_PATTERN) { // fast path
      stream = new FastStringTokenizer(text, false, toLowerCase, stopWords);
    } else {
      stream = new PatternTokenizer(text, pattern, toLowerCase);
      if (stopWords != null)
        stream = new StopFilter(matchVersion, stream, stopWords);
    }

    return stream;
  }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.StopFilter

  • BugAnalyzer.PorterStemAnalyzer
  • com.gentics.cr.lucene.analysis.CustomPatternAnalyzer
  • com.gentics.cr.lucene.autocomplete.AutocompleteAnalyzer
  • com.github.pmerienne.trident.ml.preprocessing.EnglishTokenizer$EnglishSpecialAnalyzer
  • com.infoclinika.mssharing.search.NoStopWordsAnalyzer
  • com.mozilla.grouperfish.lucene.analysis.en.EnglishAnalyzer
  • com.mozilla.grouperfish.transforms.coclustering.lucene.analysis.en.EnglishAnalyzer
  • edu.wiki.index.WikipediaAnalyzer
  • ivory.core.tokenize.LuceneArabicAnalyzer
  • ivory.core.tokenize.LuceneSpanishAnalyzer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.