Package org.apache.lucene.analysis.Analyzer

Examples of org.apache.lucene.analysis.Analyzer.TokenStreamComponents

327
328
329
330
331
332
333
334
335
336
337
    /* analyzer that uses whitespace + wdf */
    Analyzer a = new Analyzer() {
      @Override
      public TokenStreamComponents createComponents(String field, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, tokenizer, flags, null));
      }
    };
   
    assertAnalyzesTo(a, "abc-def-123-456",
        new String[] { "abc-def-123-456", "abc", "abcdef", "abcdef123456", "def", "123", "123456", "456" },
View Full Code Here
355
356
357
358
359
360
361
362
363
364
365
      Analyzer a = new Analyzer() {
       
        @Override
        protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
          Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
          return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, tokenizer, flags, protectedWords));
        }
      };
      checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
    }
  }
View Full Code Here
379
380
381
382
383
384
385
386
387
388
389
      Analyzer a = new Analyzer() {
       
        @Override
        protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
          Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
          return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, tokenizer, flags, protectedWords));
        }
      };
      checkRandomData(random(), a, 100*RANDOM_MULTIPLIER, 8192);
    }
  }
View Full Code Here
401
402
403
404
405
406
407
408
409
410
411
   
      Analyzer a = new Analyzer() {
        @Override
        protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
          Tokenizer tokenizer = new KeywordTokenizer(reader);
          return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, tokenizer, flags, protectedWords));
        }
      };
      // depending upon options, this thing may or may not preserve the empty term
      checkAnalysisConsistency(random, a, random.nextBoolean(), "");
    }
View Full Code Here
43
44
45
46
47
48
49
50
51
        analyzer = new Analyzer() {
            @Override
            protected TokenStreamComponents createComponents(String fieldName,
                    Reader reader) {
                Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
                return new TokenStreamComponents(source, new UrlTokenizingFilter(source));
            }
        };
    }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
    public void testEmptyTerm() throws IOException {
        Analyzer a = new Analyzer() {
            @Override
            protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
                Tokenizer tokenizer = new KeywordTokenizer(reader);
                return new TokenStreamComponents(tokenizer, new UrlTokenizingFilter(tokenizer));
            }
        };
        checkOneTermReuse(a, "", "");
    }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.Analyzer.TokenStreamComponents

  • org.apache.lucene.analysis.miscellaneous.TestWordDelimiterFilter
  • org.apache.lucene.analysis.ngram.EdgeNGramTokenizerTest
  • org.apache.lucene.analysis.path.TestReversePathHierarchyTokenizer
  • org.apache.lucene.analysis.pattern.TestPatternTokenizer
  • org.apache.lucene.analysis.pt.TestPortugueseLightStemFilter
  • org.apache.lucene.analysis.pt.TestPortugueseMinimalStemFilter
  • org.apache.lucene.analysis.snowball.TestSnowball
  • org.apache.lucene.analysis.synonym.FSTSynonymFilterFactory
  • org.apache.lucene.codecs.idversion.TestIDVersionPostingsFormat
  • org.apache.lucene.index.TestPostingsOffsets
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.