Package org.apache.lucene.analysis.ngram

Examples of org.apache.lucene.analysis.ngram.EdgeNGramTokenizer

43
44
45
46
47
48
49
50
  }

  private class NGramAnalyzer extends Analyzer {
    @Override
    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
      return new TokenStreamComponents(new EdgeNGramTokenizer(reader, EdgeNGramTokenizer.Side.BACK,
          10, 20));
    }
View Full Code Here
48
49
50
51
52
53
54
            side = EdgeNGramTokenizer.Side.FRONT.getLabel();
        }
    }

    public TokenStream create(Reader input) {
        return new EdgeNGramTokenizer(input, side, minGramSize, maxGramSize);
    }
View Full Code Here
247
248
249
250
251
252
253
254
255
256
257
  // LUCENE-5269
  public void testUnicodeShinglesAndNgrams() throws Exception {
    Analyzer analyzer = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, 2, 94);
        //TokenStream stream = new SopTokenFilter(tokenizer);
        TokenStream stream = new ShingleFilter(tokenizer, 5);
        //stream = new SopTokenFilter(stream);
        stream = new NGramTokenFilter(TEST_VERSION_CURRENT, stream, 55, 83);
        //stream = new SopTokenFilter(stream);
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
                        + " To obtain the same behavior as the previous version please use \"edgeNGram\" filter which still supports side=back"
                        + " in combination with a \"keyword\" tokenizer");
            }
            final Version version = this.version == Version.LUCENE_4_3 ? Version.LUCENE_4_4 : this.version; // always use 4.4 or higher
            if (matcher == null) {
                return new EdgeNGramTokenizer(minGram, maxGram);
            } else {
                return new EdgeNGramTokenizer(minGram, maxGram) {
                    @Override
                    protected boolean isTokenChar(int chr) {
                        return matcher.isTokenChar(chr);
                    }
                };
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.ngram.EdgeNGramTokenizer

  • org.apache.lucene.analysis.core.TestBugInSomething
  • org.apache.lucene.analysis.ngram.EdgeNGramTokenizerFactory
  • org.apache.lucene.analysis.Token
  • org.apache.lucene.classification.SimpleNaiveBayesClassifierTest$NGramAnalyzer
  • org.apache.solr.analysis.EdgeNGramTokenizerFactory
  • org.elasticsearch.index.analysis.EdgeNGramTokenizerFactory
  • org.elasticsearch.indices.analysis.IndicesAnalysisService
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.