Package org.apache.lucene.analysis.core

Examples of org.apache.lucene.analysis.core.WhitespaceTokenizer
30
31
32
33
34
35
36
37
38
39
40
*/
public class TestSmartChineseFactories extends BaseTokenStreamTestCase {
  /** Test showing the behavior with whitespace */
  public void testSimple() throws Exception {
    String sentence = "我购买了道具和服装。";
    WhitespaceTokenizer ws = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(sentence));
    SmartChineseWordTokenFilterFactory factory = new SmartChineseWordTokenFilterFactory();
    TokenStream ts = factory.create(ws);
    // TODO: fix smart chinese to not emit punctuation tokens
    // at the moment: you have to clean up with WDF, or use the stoplist, etc
    assertTokenStreamContents(ts,
View Full Code Here
35
36
37
38
39
40
41
42
43
    initParams.put(MorfologikFilterFactory.DICTIONARY_SCHEMA_ATTRIBUTE,
        "morfologik");
    MorfologikFilterFactory factory = new MorfologikFilterFactory();
    factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
    factory.init(initParams);
    TokenStream ts = factory.create(new WhitespaceTokenizer(TEST_VERSION_CURRENT,
        reader));
    assertTokenStreamContents(ts, new String[] {"rower", "bilet"});
  }
View Full Code Here
28
29
30
31
32
33
34
35
36
*/
public class TestStempelPolishStemFilterFactory extends BaseTokenStreamTestCase {
  public void testBasics() throws Exception {
    StringReader document = new StringReader("studenta studenci");
    StempelPolishStemFilterFactory factory = new StempelPolishStemFilterFactory();
    TokenStream ts = factory.create(new WhitespaceTokenizer(TEST_VERSION_CURRENT, document));
    assertTokenStreamContents(ts,
        new String[] { "student", "student" });
  }
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
    final TokenizerFactory factory = tf == null ? null : loadTokenizerFactory(loader, tf);
   
    Analyzer analyzer = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer(Version.LUCENE_31, reader) : factory.create(reader);
        TokenStream stream = ignoreCase ? new LowerCaseFilter(Version.LUCENE_31, tokenizer) : tokenizer;
        return new TokenStreamComponents(tokenizer, stream);
      }
    };
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
    EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.FRONT, 3, 3);
    assertTokenStreamContents(tokenizer, new String[]{"abc","fgh"}, new int[]{0,7}, new int[]{3,10});
  }
 
  public void testReset() throws Exception {
    WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde"));
    EdgeNGramTokenFilter filter = new EdgeNGramTokenFilter(tokenizer, EdgeNGramTokenFilter.Side.FRONT, 1, 3);
    assertTokenStreamContents(filter, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3});
    tokenizer.setReader(new StringReader("abcde"));
    assertTokenStreamContents(filter, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3});
  }
View Full Code Here
35
36
37
38
39
40
41
42
43
    Reader reader = new StringReader("This is a Ｔｅｓｔ");
    ICUNormalizer2FilterFactory factory = new ICUNormalizer2FilterFactory();
    factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
    Map<String, String> args = Collections.emptyMap();
    factory.init(args);
    Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader);
    TokenStream stream = factory.create(tokenizer);
    assertTokenStreamContents(stream, new String[] { "this", "is", "a", "test" });
  }
View Full Code Here
35
36
37
38
39
40
41
42
43
    Reader reader = new StringReader("簡化字");
    ICUTransformFilterFactory factory = new ICUTransformFilterFactory();
    Map<String,String> args = new HashMap<String,String>();
    args.put("id", "Traditional-Simplified");
    factory.init(args);
    Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader);
    TokenStream stream = factory.create(tokenizer);
    assertTokenStreamContents(stream, new String[] { "简化字" });
  }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
    Reader reader = new StringReader("Российская Федерация");
    ICUTransformFilterFactory factory = new ICUTransformFilterFactory();
    Map<String,String> args = new HashMap<String,String>();
    args.put("id", "Cyrillic-Latin");
    factory.init(args);
    Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader);
    TokenStream stream = factory.create(tokenizer);
    assertTokenStreamContents(stream, new String[] { "Rossijskaâ",  "Federaciâ" });
   
    // backward (invokes Latin-Cyrillic)
    reader = new StringReader("Rossijskaâ Federaciâ");
    args.put("direction", "reverse");
    factory.init(args);
    tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader);
    stream = factory.create(tokenizer);
    assertTokenStreamContents(stream, new String[] { "Российская", "Федерация" });
  }
View Full Code Here
31
32
33
34
35
36
37
38
39
  /** basic tests to ensure the folding is working */
  public void test() throws Exception {
    Reader reader = new StringReader("Résumé");
    ICUFoldingFilterFactory factory = new ICUFoldingFilterFactory();
    factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
    Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader);
    TokenStream stream = factory.create(tokenizer);
    assertTokenStreamContents(stream, new String[] { "resume" });
  }
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
  public void testTokenEndingWithWordComponentOfMinimumLength() throws Exception {
    CharArraySet dict = makeDictionary("ab", "cd", "ef");

    Tokenizer tokenizer = new MockTokenizer(new StringReader("abcdef"), MockTokenizer.WHITESPACE, false);
    DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT,
      new WhitespaceTokenizer(TEST_VERSION_CURRENT,
        new StringReader(
          "abcdef")
        ),
      dict,
      CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.core.WhitespaceTokenizer

  • cc.twittertools.index.TweetAnalyzer
  • cc.twittertools.search.indexing.TweetAnalyzer
  • cc.twittertools.udf.LuceneTokenizer
  • de.arago.lucene.util.LowCaseAnalyzer
  • org.apache.jackrabbit.oak.plugins.index.lucene.OakAnalyzer
  • org.apache.lucene.analysis.cn.smart.TestSmartChineseFactories
  • org.apache.lucene.analysis.cn.TestChineseTokenizer$JustChineseFilterAnalyzer
  • org.apache.lucene.analysis.commongrams.CommonGramsFilterTest
  • org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter
  • org.apache.lucene.analysis.core.WhitespaceTokenizerFactory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.