Package org.apache.lucene.analysis.standard

Examples of org.apache.lucene.analysis.standard.StandardFilter

185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
  @Override
  protected TokenStreamComponents createComponents(String fieldName,
      Reader aReader) {
    if (matchVersion.onOrAfter(Version.LUCENE_31)) {
      final Tokenizer source = new StandardTokenizer(matchVersion, aReader);
      TokenStream result = new StandardFilter(matchVersion, source);
      result = new LowerCaseFilter(matchVersion, result);
      result = new StopFilter(matchVersion, result, stoptable);
      if (!excltable.isEmpty())
        result = new SetKeywordMarkerFilter(result, excltable);
      if (stemdict != null)
        result = new StemmerOverrideFilter(result, stemdict);
      result = new SnowballFilter(result, new org.tartarus.snowball.ext.DutchStemmer());
      return new TokenStreamComponents(source, result);
    } else {
      final Tokenizer source = new StandardTokenizer(matchVersion, aReader);
      TokenStream result = new StandardFilter(matchVersion, source);
      result = new StopFilter(matchVersion, result, stoptable);
      if (!excltable.isEmpty())
        result = new SetKeywordMarkerFilter(result, excltable);
      result = new DutchStemFilter(result, origStemdict);
      return new TokenStreamComponents(source, result);
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
   */
  @Override
  protected TokenStreamComponents createComponents(String fieldName,
      Reader reader) {
    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    if (matchVersion.onOrAfter(Version.LUCENE_36)) {
      result = new ElisionFilter(result, DEFAULT_ARTICLES);
    }
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
View Full Code Here
166
167
168
169
170
171
172
173
174
175
176
     *    StandardFilter, StopFilter, GermanStemFilter
     */
    public TokenStream tokenStream( String fieldName, Reader reader )
    {
  TokenStream result = new StandardTokenizer( reader );
  result = new StandardFilter( result );
  // shouldn't there be a lowercaser before stop word filtering?
  result = new StopFilter( result, stoptable );
  result = new GermanStemFilter( result, excltable );
  return result;
    }
View Full Code Here
215
216
217
218
219
220
221
222
223
224
225
  @Override
  protected TokenStreamComponents createComponents(String fieldName,
      Reader reader) {
    Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new LowerCaseFilter(matchVersion, source);
    result = new StandardFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(excltable != null && !excltable.isEmpty())
      result = new KeywordMarkerFilter(result, excltable);
    return new TokenStreamComponents(source, new BrazilianStemFilter(result));
  }
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
   */
  @Override
  protected TokenStreamComponents createComponents(String fieldName,
      Reader reader) {
    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    if (matchVersion.onOrAfter(Version.LUCENE_32)) {
      result = new ElisionFilter(matchVersion, result, DEFAULT_ARTICLES);
    }
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
View Full Code Here
63
64
65
66
67
68
69
70
71
     * Constructs a {@link StandardTokenizer} filtered by a {@link
     * StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}.
     */
    public TokenStream tokenStream(String fieldName, Reader reader) {
        TokenStream result = new WhitespaceTokenizer(reader);
        result = new StandardFilter(result);
        result = new StopFilter(result, stopSet);
        return result;
    }
View Full Code Here
23
24
25
26
27
28
29
/**
* @version $Id: StandardFilterFactory.java 591158 2007-11-01 22:37:42Z hossman $
*/
public class StandardFilterFactory extends BaseTokenFilterFactory {
  public StandardFilter create(TokenStream input) {
    return new StandardFilter(input);
  }
View Full Code Here
368
369
370
371
372
373
374
375
376
377
378
            @Override public String name() {
                return "standard";
            }

            @Override public TokenStream create(TokenStream tokenStream) {
                return new StandardFilter(Lucene.ANALYZER_VERSION, tokenStream);
            }
        }));

        tokenFilterFactories.put("nGram", new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() {
            @Override public String name() {
View Full Code Here
36
37
38
39
40
41
42
    @Inject public StandardTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) {
        super(index, indexSettings, name, settings);
    }

    @Override public TokenStream create(TokenStream tokenStream) {
        return new StandardFilter(version, tokenStream);
    }
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
    }

    @Override protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
        final StandardTokenizer src = new StandardTokenizer(matchVersion, reader);
        src.setMaxTokenLength(StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
        TokenStream tok = new StandardFilter(matchVersion, src);
        tok = new LowerCaseFilter(matchVersion, tok);
        tok = new StopFilter(matchVersion, tok, stopwords);
        return new TokenStreamComponents(src, tok) {
            @Override
            protected boolean reset(final Reader reader) throws IOException {
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.standard.StandardFilter

  • com.foundationdb.server.service.text.SelectiveCaseAnalyzer
  • com.gentics.cr.lucene.autocomplete.AutocompleteAnalyzer
  • com.github.pmerienne.trident.ml.preprocessing.EnglishTokenizer$EnglishSpecialAnalyzer
  • com.googlecode.lucene.PorterAnalyzer
  • com.livingsocial.hive.udf.Tokenize$MyAnalyzer
  • com.mozilla.grouperfish.lucene.analysis.en.EnglishAnalyzer
  • com.mozilla.grouperfish.lucene.analysis.en.NGramEnglishAnalyzer
  • com.mozilla.grouperfish.transforms.coclustering.lucene.analysis.en.EnglishAnalyzer
  • com.mozilla.grouperfish.transforms.coclustering.lucene.analysis.en.NGramEnglishAnalyzer
  • edu.wiki.index.WikipediaAnalyzer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.