Package org.apache.lucene.analysis.ngram

Examples of org.apache.lucene.analysis.ngram.NGramTokenFilter

49
50
51
52
53
54
55
    minGramSize = (minArg != null ? Integer.parseInt(minArg)
        : NGramTokenFilter.DEFAULT_MIN_NGRAM_SIZE);
  }

  public NGramTokenFilter create(TokenStream input) {
    return new NGramTokenFilter(input, minGramSize, maxGramSize);
  }
View Full Code Here
245
246
247
248
249
250
251
252
253
254
255
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, 2, 94);
        //TokenStream stream = new SopTokenFilter(tokenizer);
        TokenStream stream = new ShingleFilter(tokenizer, 5);
        //stream = new SopTokenFilter(stream);
        stream = new NGramTokenFilter(TEST_VERSION_CURRENT, stream, 55, 83);
        //stream = new SopTokenFilter(stream);
        return new TokenStreamComponents(tokenizer, stream);
      } 
    };
    checkRandomData(random(), analyzer, 2000);
View Full Code Here
247
248
249
250
251
252
253
254
255
256
257
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, 2, 94);
        //TokenStream stream = new SopTokenFilter(tokenizer);
        TokenStream stream = new ShingleFilter(tokenizer, 5);
        //stream = new SopTokenFilter(stream);
        stream = new NGramTokenFilter(TEST_VERSION_CURRENT, stream, 55, 83);
        //stream = new SopTokenFilter(stream);
        return new TokenStreamComponents(tokenizer, stream);
      } 
    };
    checkRandomData(random(), analyzer, 2000);
View Full Code Here
41
42
43
44
45
46
47
    minGramSize = (minArg != null ? Integer.parseInt(minArg)
        : NGramTokenFilter.DEFAULT_MIN_NGRAM_SIZE);
  }

  public NGramTokenFilter create(TokenStream input) {
    return new NGramTokenFilter(input, minGramSize, maxGramSize);
  }
View Full Code Here
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
            @Override public String name() {
                return "nGram";
            }

            @Override public TokenStream create(TokenStream tokenStream) {
                return new NGramTokenFilter(tokenStream);
            }
        }));

        tokenFilterFactories.put("ngram", new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() {
            @Override public String name() {
                return "ngram";
            }

            @Override public TokenStream create(TokenStream tokenStream) {
                return new NGramTokenFilter(tokenStream);
            }
        }));

        tokenFilterFactories.put("edgeNGram", new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() {
            @Override public String name() {
View Full Code Here
43
44
45
46
47
48
49
        this.minGram = settings.getAsInt("min_gram", NGramTokenFilter.DEFAULT_MIN_NGRAM_SIZE);
        this.maxGram = settings.getAsInt("max_gram", NGramTokenFilter.DEFAULT_MAX_NGRAM_SIZE);
    }

    @Override public TokenStream create(TokenStream tokenStream) {
        return new NGramTokenFilter(tokenStream, minGram, maxGram);
    }
View Full Code Here
51
52
53
54
55
56
57
   /*
    * (non-Javadoc)
    * @see org.apache.lucene.analysis.Analyzer#tokenStream(java.lang.String, java.io.Reader)
    */
   public TokenStream tokenStream(String fieldName, Reader reader) {
       return new NGramTokenFilter(new UpperCaseFilter(new SentenceTokenizer(reader)), minTokenLength, maxTokenLength);
   }
View Full Code Here
51
52
53
54
55
56
57
        : NGramTokenFilter.DEFAULT_MIN_NGRAM_SIZE);
  }

  @Override
  public NGramTokenFilter create(TokenStream input) {
    return new NGramTokenFilter(input, minGramSize, maxGramSize);
  }
View Full Code Here
251
252
253
254
255
256
257
258
259
260
261
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, 2, 94);
        //TokenStream stream = new SopTokenFilter(tokenizer);
        TokenStream stream = new ShingleFilter(tokenizer, 5);
        //stream = new SopTokenFilter(stream);
        stream = new NGramTokenFilter(TEST_VERSION_CURRENT, stream, 55, 83);
        //stream = new SopTokenFilter(stream);
        return new TokenStreamComponents(tokenizer, stream);
      } 
    };
    checkRandomData(random(), analyzer, 2000);
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
    loader.assertAwareCompatibility( clazz, new NGramFilterFactory() );
    loader.assertAwareCompatibility( clazz, new KeywordTokenizerFactory() );
   
    // Make sure it throws an error for invalid objects
    Object[] invalid = new Object[] {
        new NGramTokenFilter( null ),
        "hello",  new Float( 12.3f ),
        new LukeRequestHandler(),
        new JSONResponseWriter()
    };
    for( Object obj : invalid ) {
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.ngram.NGramTokenFilter

  • org.apache.james.mailbox.lucene.search.StrictImapSearchAnalyzer
  • org.apache.james.mailbox.store.lucene.ImapSearchAnalyzer
  • org.apache.lucene.analysis.core.TestBugInSomething
  • org.apache.lucene.analysis.miscellaneous.CodepointCountFilter
  • org.apache.lucene.analysis.miscellaneous.LengthFilter
  • org.apache.lucene.analysis.ngram.NGramFilterFactory
  • org.apache.lucene.analysis.Token
  • org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute
  • org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute
  • org.apache.solr.analysis.NGramFilterFactory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.