Package org.apache.lucene.analysis.ngram

Examples of org.apache.lucene.analysis.ngram.EdgeNGramTokenizer

53
54
55
56
57
58
59
            side = EdgeNGramTokenizer.Side.FRONT.getLabel();
        }
    }

    public EdgeNGramTokenizer create(Reader input) {
        return new EdgeNGramTokenizer(input, side, minGramSize, maxGramSize);
    }
View Full Code Here
241
242
243
244
245
246
247
248
249
250
251
  // LUCENE-5269
  public void testUnicodeShinglesAndNgrams() throws Exception {
    Analyzer analyzer = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, 2, 94);
        //TokenStream stream = new SopTokenFilter(tokenizer);
        TokenStream stream = new ShingleFilter(tokenizer, 5);
        //stream = new SopTokenFilter(stream);
        stream = new NGramTokenFilter(TEST_VERSION_CURRENT, stream, 55, 83);
        //stream = new SopTokenFilter(stream);
View Full Code Here
243
244
245
246
247
248
249
250
251
252
253
  // LUCENE-5269
  public void testUnicodeShinglesAndNgrams() throws Exception {
    Analyzer analyzer = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, 2, 94);
        //TokenStream stream = new SopTokenFilter(tokenizer);
        TokenStream stream = new ShingleFilter(tokenizer, 5);
        //stream = new SopTokenFilter(stream);
        stream = new NGramTokenFilter(TEST_VERSION_CURRENT, stream, 55, 83);
        //stream = new SopTokenFilter(stream);
View Full Code Here
46
47
48
49
50
51
52
            side = EdgeNGramTokenizer.Side.FRONT.getLabel();
        }
    }

    public EdgeNGramTokenizer create(Reader input) {
        return new EdgeNGramTokenizer(input, side, minGramSize, maxGramSize);
    }
View Full Code Here
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
            @Override public String name() {
                return "edgeNGram";
            }

            @Override public Tokenizer create(Reader reader) {
                return new EdgeNGramTokenizer(reader, EdgeNGramTokenizer.DEFAULT_SIDE, EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE);
            }
        }));

        tokenizerFactories.put("edge_ngram", new PreBuiltTokenizerFactoryFactory(new TokenizerFactory() {
            @Override public String name() {
                return "edge_ngram";
            }

            @Override public Tokenizer create(Reader reader) {
                return new EdgeNGramTokenizer(reader, EdgeNGramTokenizer.DEFAULT_SIDE, EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE);
            }
        }));

        tokenizerFactories.put("pattern", new PreBuiltTokenizerFactoryFactory(new TokenizerFactory() {
            @Override public String name() {
View Full Code Here
47
48
49
50
51
52
53
        this.maxGram = settings.getAsInt("max_gram", NGramTokenizer.DEFAULT_MAX_NGRAM_SIZE);
        this.side = EdgeNGramTokenizer.Side.getSide(settings.get("side", EdgeNGramTokenizer.DEFAULT_SIDE.getLabel()));
    }

    @Override public Tokenizer create(Reader reader) {
        return new EdgeNGramTokenizer(reader, side, minGram, maxGram);
    }
View Full Code Here
55
56
57
58
59
60
61
    }
  }
 
  @Override
  public EdgeNGramTokenizer create(Reader input) {
    return new EdgeNGramTokenizer(input, side, minGramSize, maxGramSize);
  }
View Full Code Here
42
43
44
45
46
47
48
49
  }

  private class NGramAnalyzer extends Analyzer {
    @Override
    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
      return new TokenStreamComponents(new EdgeNGramTokenizer(reader, EdgeNGramTokenizer.Side.BACK,
          10, 20));
    }
View Full Code Here
247
248
249
250
251
252
253
254
255
256
257
  // LUCENE-5269
  public void testUnicodeShinglesAndNgrams() throws Exception {
    Analyzer analyzer = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, 2, 94);
        //TokenStream stream = new SopTokenFilter(tokenizer);
        TokenStream stream = new ShingleFilter(tokenizer, 5);
        //stream = new SopTokenFilter(stream);
        stream = new NGramTokenFilter(TEST_VERSION_CURRENT, stream, 55, 83);
        //stream = new SopTokenFilter(stream);
View Full Code Here
47
48
49
50
51
52
53
      side = EdgeNGramTokenizer.Side.FRONT.getLabel();
    }
  }

  public EdgeNGramTokenizer create(Reader input) {
    return new EdgeNGramTokenizer( input, side, minGramSize, maxGramSize );
  }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.ngram.EdgeNGramTokenizer

  • org.apache.lucene.analysis.core.TestBugInSomething
  • org.apache.lucene.analysis.ngram.EdgeNGramTokenizerFactory
  • org.apache.lucene.analysis.Token
  • org.apache.lucene.classification.SimpleNaiveBayesClassifierTest$NGramAnalyzer
  • org.apache.solr.analysis.EdgeNGramTokenizerFactory
  • org.elasticsearch.index.analysis.EdgeNGramTokenizerFactory
  • org.elasticsearch.indices.analysis.IndicesAnalysisService
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.