Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.WhitespaceTokenizer

29
30
31
32
33
34
35
public class EdgeNGramTokenFilterTest extends BaseTokenStreamTestCase {
  private TokenStream input;

  public void setUp() throws Exception {
    super.setUp();
    input = new WhitespaceTokenizer(new StringReader("abcde"));
  }
View Full Code Here
88
89
90
91
92
93
94
95
96
    EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.BACK, 1, 3);
    assertTokenStreamContents(tokenizer, new String[]{"e","de","cde"}, new int[]{4,3,2}, new int[]{5,5,5});
  }
 
  public void testSmallTokenInStream() throws Exception {
    input = new WhitespaceTokenizer(new StringReader("abc de fgh"));
    EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.FRONT, 3, 3);
    assertTokenStreamContents(tokenizer, new String[]{"abc","fgh"}, new int[]{0,0}, new int[]{3,3});
  }
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
    EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.FRONT, 3, 3);
    assertTokenStreamContents(tokenizer, new String[]{"abc","fgh"}, new int[]{0,0}, new int[]{3,3});
  }
 
  public void testReset() throws Exception {
    WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(new StringReader("abcde"));
    EdgeNGramTokenFilter filter = new EdgeNGramTokenFilter(tokenizer, EdgeNGramTokenFilter.Side.FRONT, 1, 3);
    assertTokenStreamContents(filter, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3});
    tokenizer.reset(new StringReader("abcde"));
    assertTokenStreamContents(filter, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3});
  }
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
  public void test() throws IOException {

    PrefixAndSuffixAwareTokenFilter ts = new PrefixAndSuffixAwareTokenFilter(
        new SingleTokenTokenStream(createToken("^", 0, 0)),
        new WhitespaceTokenizer(new StringReader("hello world")),
        new SingleTokenTokenStream(createToken("$", 0, 0)));

    assertNext(ts, "^", 0, 0);
    assertNext(ts, "hello", 0, 5);
    assertNext(ts, "world", 6, 11);
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
    assertNext(ts, "b", 1, 2);
    assertFalse(ts.incrementToken());

    // prefix and suffix using 2x prefix

    ts = new PrefixAwareTokenFilter(new SingleTokenTokenStream(createToken("^", 0, 0)), new WhitespaceTokenizer(new StringReader("hello world")));
    ts = new PrefixAwareTokenFilter(ts, new SingleTokenTokenStream(createToken("$", 0, 0)));

    assertNext(ts, "^", 0, 0);
    assertNext(ts, "hello", 0, 5);
    assertNext(ts, "world", 6, 11);
View Full Code Here
93
94
95
96
97
98
99
100
101
102
      this.maxSynonyms = maxSynonyms;
    }
   
    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      TokenStream ts = new WhitespaceTokenizer(reader);
      ts = new LowerCaseFilter(ts);
      ts = new SynonymTokenFilter(ts, synonyms, maxSynonyms);
      return ts;
    }
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
    public TokenStream reusableTokenStream(String fieldName, Reader reader)
        throws IOException {
      SavedStreams streams = (SavedStreams) getPreviousTokenStream();
      if (streams == null) {
        streams = new SavedStreams();
        streams.source = new WhitespaceTokenizer(reader);
        streams.result = new LowerCaseFilter(streams.source);
        streams.result = new SynonymTokenFilter(streams.result, synonyms, maxSynonyms);
        setPreviousTokenStream(streams);
      } else {
        streams.source.reset(reader);
View Full Code Here
65
66
67
68
69
70
71
      super(Version.LUCENE_CURRENT, name);
    }
   
    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      return new WhitespaceTokenizer(reader);
    }
View Full Code Here
31
32
33
34
35
36
37
    private TokenStream input;
   
    @Override
    public void setUp() throws Exception {
        super.setUp();
        input = new WhitespaceTokenizer(new StringReader("abcde"));
    }
View Full Code Here
79
80
81
82
83
84
85
86
87
      NGramTokenFilter filter = new NGramTokenFilter(input, 6, 7);
      assertTokenStreamContents(filter, new String[0], new int[0], new int[0]);
    }
   
    public void testSmallTokenInStream() throws Exception {
      input = new WhitespaceTokenizer(new StringReader("abc de fgh"));
      NGramTokenFilter filter = new NGramTokenFilter(input, 3, 3);
      assertTokenStreamContents(filter, new String[]{"abc","fgh"}, new int[]{0,7}, new int[]{3,10});
    }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.WhitespaceTokenizer

  • org.apache.james.mailbox.store.lucene.ImapSearchAnalyzer
  • org.apache.lucene.analysis.cn.TestChineseTokenizer$JustChineseFilterAnalyzer
  • org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter
  • org.apache.lucene.analysis.de.TestGermanStemFilter$GermanSubclassAnalyzer
  • org.apache.lucene.analysis.icu.TestICUNormalizer2Filter
  • org.apache.lucene.analysis.icu.TestICUTransformFilter
  • org.apache.lucene.analysis.miscellaneous.TestPrefixAndSuffixAwareTokenFilter
  • org.apache.lucene.analysis.miscellaneous.TestPrefixAwareTokenFilter
  • org.apache.lucene.analysis.ngram.EdgeNGramTokenFilterTest
  • org.apache.lucene.analysis.ngram.NGramTokenFilterTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.