Package org.apache.lucene.analysis.core

Examples of org.apache.lucene.analysis.core.KeywordTokenizer

86
87
88
89
90
91
92
93
94
95
96
 
  public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new KeywordTokenizer(reader);
        return new TokenStreamComponents(tokenizer, new PortugueseStemFilter(tokenizer));
      }
    };
    checkOneTerm(a, "", "");
  }
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
 
  public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new KeywordTokenizer(reader);
        final boolean updateOffsets = random().nextBoolean();
        final Version version = updateOffsets ? Version.LUCENE_4_3 : TEST_VERSION_CURRENT;
        return new TokenStreamComponents(tokenizer, new TrimFilter(version, tokenizer, updateOffsets));
      }
    };
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
 
  public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new KeywordTokenizer(reader);
        return new TokenStreamComponents(tokenizer, new PortugueseLightStemFilter(tokenizer));
      }
    };
    checkOneTerm(a, "", "");
  }
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
 
  public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new KeywordTokenizer(reader);
        return new TokenStreamComponents(tokenizer, new PortugueseMinimalStemFilter(tokenizer));
      }
    };
    checkOneTerm(a, "", "");
  }
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
 
  public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new KeywordTokenizer(reader);
        return new TokenStreamComponents(tokenizer, new ReverseStringFilter(TEST_VERSION_CURRENT, tokenizer));
      }
    };
    checkOneTerm(a, "", "");
  }
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
 
  public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new KeywordTokenizer(reader);
        return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer));
      }
    };
    checkOneTerm(a, "", "");
  }
View Full Code Here
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
    final CharArraySet dict = makeDictionary("a", "e", "i", "o", "u", "y", "bc", "def");
    Analyzer a = new Analyzer() {

      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new KeywordTokenizer(reader);
        return new TokenStreamComponents(tokenizer, new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT, tokenizer, dict));
      }
    };
    checkOneTerm(a, "", "");
   
    InputSource is = new InputSource(getClass().getResource("da_UTF8.xml").toExternalForm());
    final HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter.getHyphenationTree(is);
    Analyzer b = new Analyzer() {

      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new KeywordTokenizer(reader);
        TokenFilter filter = new HyphenationCompoundWordTokenFilter(TEST_VERSION_CURRENT, tokenizer, hyphenator);
        return new TokenStreamComponents(tokenizer, filter);
      }
    };
    checkOneTerm(b, "", "");
View Full Code Here
402
403
404
405
406
407
408
409
410
411
412
      }
   
      Analyzer a = new Analyzer() {
        @Override
        protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
          Tokenizer tokenizer = new KeywordTokenizer(reader);
          return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, tokenizer, flags, protectedWords));
        }
      };
      // depending upon options, this thing may or may not preserve the empty term
      checkAnalysisConsistency(random, a, random.nextBoolean(), "");
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
 
  public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new KeywordTokenizer(reader);
        return new TokenStreamComponents(tokenizer, new CapitalizationFilter(tokenizer));
      }
    };
    checkOneTerm(a, "", "");
  }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
 
  public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new KeywordTokenizer(reader);
        return new TokenStreamComponents(tokenizer, new GalicianStemFilter(tokenizer));
      }
    };
    checkOneTerm(a, "", "");
  }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.core.KeywordTokenizer

  • at.molindo.esi4j.util.NullAnalyzer
  • org.apache.jackrabbit.oak.plugins.index.solr.configuration.DefaultAnalyzersConfigurationTest
  • org.apache.lucene.analysis.ar.TestArabicNormalizationFilter
  • org.apache.lucene.analysis.ar.TestArabicStemFilter
  • org.apache.lucene.analysis.bg.TestBulgarianStemmer
  • org.apache.lucene.analysis.br.TestBrazilianStemmer
  • org.apache.lucene.analysis.cjk.TestCJKAnalyzer
  • org.apache.lucene.analysis.cjk.TestCJKWidthFilter
  • org.apache.lucene.analysis.ckb.TestSoraniNormalizationFilter
  • org.apache.lucene.analysis.ckb.TestSoraniStemFilter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.