Package org.apache.lucene.analysis.Analyzer

Examples of org.apache.lucene.analysis.Analyzer.TokenStreamComponents

403
404
405
406
407
408
409
410
411
412
413
   
      Analyzer a = new Analyzer() {
        @Override
        protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
          Tokenizer tokenizer = new KeywordTokenizer(reader);
          return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, tokenizer, flags, protectedWords));
        }
      };
      // depending upon options, this thing may or may not preserve the empty term
      checkAnalysisConsistency(random, a, random.nextBoolean(), "");
    }
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
    for (final String lang : SNOWBALL_LANGS) {
      Analyzer a = new Analyzer() {
        @Override
        protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
          Tokenizer tokenizer = new KeywordTokenizer(reader);
          return new TokenStreamComponents(tokenizer, new SnowballFilter(tokenizer, lang));
        }
      };
      checkOneTerm(a, "", "");
    }
  }
View Full Code Here
174
175
176
177
178
179
180
181
182
183
  public void checkRandomStrings(final String snowballLanguage) throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer t = new MockTokenizer(reader);
        return new TokenStreamComponents(t, new SnowballFilter(t, snowballLanguage));
      } 
    };
    checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
  }
View Full Code Here
459
460
461
462
463
464
465
466
467
468
469
        @Override
        public TokenStreamComponents createComponents(String fieldName, Reader reader) {
          MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true, 100);
          tokenizer.setEnableChecks(true);
          MockTokenFilter filt = new MockTokenFilter(tokenizer, MockTokenFilter.EMPTY_STOPSET);
          return new TokenStreamComponents(tokenizer, filt);
        }
      };
    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, a);
    iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
View Full Code Here
102
103
104
105
106
107
108
109
110
111
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
        return new TokenStreamComponents(source, new PortugueseLightStemFilter(sink));
      }
    };
    checkOneTerm(a, "quilométricas", "quilométricas");
  }
View Full Code Here
118
119
120
121
122
123
124
125
126
127
  public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new KeywordTokenizer(reader);
        return new TokenStreamComponents(tokenizer, new PortugueseLightStemFilter(tokenizer));
      }
    };
    checkOneTermReuse(a, "", "");
  }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
        return new TokenStreamComponents(source, new PortugueseMinimalStemFilter(sink));
      }
    };
    checkOneTerm(a, "quilométricas", "quilométricas");
  }
View Full Code Here
92
93
94
95
96
97
98
99
100
101
  public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new KeywordTokenizer(reader);
        return new TokenStreamComponents(tokenizer, new PortugueseMinimalStemFilter(tokenizer));
      }
    };
    checkOneTermReuse(a, "", "");
  }
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
    Analyzer analyzer = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer(Version.LUCENE_43, reader) : factory.create(reader);
        TokenStream stream = ignoreCase ? new LowerCaseFilter(Version.LUCENE_43, tokenizer) : tokenizer;
        return new TokenStreamComponents(tokenizer, stream);
      }
    };

    try {
      if (format == null || format.equals("solr")) {
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
    for (final String lang : SNOWBALL_LANGS) {
      Analyzer a = new Analyzer() {
        @Override
        protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
          Tokenizer tokenizer = new KeywordTokenizer(reader);
          return new TokenStreamComponents(tokenizer, new SnowballFilter(tokenizer, lang));
        }
      };
      checkOneTermReuse(a, "", "");
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.Analyzer.TokenStreamComponents

  • org.apache.lucene.analysis.miscellaneous.TestWordDelimiterFilter
  • org.apache.lucene.analysis.ngram.EdgeNGramTokenizerTest
  • org.apache.lucene.analysis.path.TestReversePathHierarchyTokenizer
  • org.apache.lucene.analysis.pattern.TestPatternTokenizer
  • org.apache.lucene.analysis.pt.TestPortugueseLightStemFilter
  • org.apache.lucene.analysis.pt.TestPortugueseMinimalStemFilter
  • org.apache.lucene.analysis.snowball.TestSnowball
  • org.apache.lucene.analysis.synonym.FSTSynonymFilterFactory
  • org.apache.lucene.codecs.idversion.TestIDVersionPostingsFormat
  • org.apache.lucene.index.TestPostingsOffsets
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.