Package org.apache.lucene.analysis.ngram

Examples of org.apache.lucene.analysis.ngram.NGramTokenizer

33
34
35
36
37
38
39
40
41
42
43
44
45
46
    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer source;
        TokenStream stream;

        if("_name_prefix".equals(fieldName)) {
            source = new NGramTokenizer(reader,mingram,maxgram);
            stream = new WordDelimiterFilter(new LowerCaseFilter(Version.LUCENE_43,source), WordDelimiterFilter.ALPHANUM, CharArraySet.EMPTY_SET);
        } else if(fieldName.startsWith("_ngram_")) {
            source = new NGramTokenizer(reader,3,4);
            stream = null;
        } else {
            source = new WhitespaceTokenizer(Version.LUCENE_43,reader);
            stream = new WordDelimiterFilter(new LowerCaseFilter(Version.LUCENE_43,source), WordDelimiterFilter.ALPHANUM, CharArraySet.EMPTY_SET);
        }
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
118
119
             * We added this in 0.90.2 but 0.90.1 used LUCENE_43 already so we can not rely on the lucene version.
             * Yet if somebody uses 0.90.2 or higher with a prev. lucene version we should also use the deprecated version.
             */
            final Version version = this.version == Version.LUCENE_4_3 ? Version.LUCENE_4_4 : this.version; // always use 4.4 or higher
            if (matcher == null) {
                return new NGramTokenizer(minGram, maxGram);
            } else {
                return new NGramTokenizer(minGram, maxGram) {
                    @Override
                    protected boolean isTokenChar(int chr) {
                        return matcher.isTokenChar(chr);
                    }
                };
View Full Code Here
42
43
44
45
46
47
48
        minGramSize = (minArg != null ? Integer.parseInt(minArg) : NGramTokenizer.DEFAULT_MIN_NGRAM_SIZE);
    }

    /** Creates the {@link TokenStream} of n-grams from the given {@link Reader}. */
    public TokenStream create(Reader input) {
        return new NGramTokenizer(input, minGramSize, maxGramSize);
    }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.ngram.NGramTokenizer

  • de.arago.lucene.util.LowCaseAnalyzer
  • org.apache.lucene.analysis.icu.TestICUNormalizer2CharFilter
  • org.apache.lucene.analysis.ngram.NGramTokenizerFactory
  • org.apache.lucene.analysis.Token
  • org.apache.solr.analysis.NGramTokenizerFactory
  • org.dbpedia.spotlight.lucene.analysis.NGramAnalyzer
  • org.elasticsearch.index.analysis.NGramTokenizerFactory
  • org.elasticsearch.indices.analysis.IndicesAnalysisService
  • org.zanata.hibernate.search.ConfigurableNgramAnalyzer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.