Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.MockTokenizer

910
911
912
913
914
915
916
917
918
919
920
    Directory dir = newDirectory();
    // note this test explicitly disables payloads
    final Analyzer analyzer = new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
      }
    };
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setRAMBufferSizeMB(1.0).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH));
    w.setInfoStream(VERBOSE ? System.out : null);
    Document doc = new Document();
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
    final int codeLen = _TestUtil.nextInt(random(), 1, 8);
    Analyzer a = new Analyzer() {

      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        return new TokenStreamComponents(tokenizer, new DoubleMetaphoneFilter(tokenizer, codeLen, false));
      }
     
    };
    checkRandomData(random(), a, 1000 * RANDOM_MULTIPLIER);
   
    Analyzer b = new Analyzer() {

      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        return new TokenStreamComponents(tokenizer, new DoubleMetaphoneFilter(tokenizer, codeLen, true));
      }
     
    };
    checkRandomData(random(), b, 1000 * RANDOM_MULTIPLIER);
View Full Code Here
179
180
181
182
183
184
185
186
187
188
189
    }
  }
 
  static void assertAlgorithm(String algName, String inject, String input,
      String[] expected) throws Exception {
    Tokenizer tokenizer = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
    Map<String,String> args = new HashMap<String,String>();
    args.put("encoder", algName);
    args.put("inject", inject);
    PhoneticFilterFactory factory = new PhoneticFilterFactory(args);
    factory.inform(new ClasspathResourceLoader(factory.getClass()));
View Full Code Here
29
30
31
32
33
34
35
36
37
38
39
public class TestDoubleMetaphoneFilterFactory extends BaseTokenStreamTestCase {

  public void testDefaults() throws Exception {
    DoubleMetaphoneFilterFactory factory = new DoubleMetaphoneFilterFactory(new HashMap<String, String>());
    TokenStream inputStream = new MockTokenizer(new StringReader("international"), MockTokenizer.WHITESPACE, false);

    TokenStream filteredStream = factory.create(inputStream);
    assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());
    assertTokenStreamContents(filteredStream, new String[] { "international", "ANTR" });
  }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
    Map<String,String> parameters = new HashMap<String,String>();
    parameters.put("inject", "false");
    parameters.put("maxCodeLength", "8");
    DoubleMetaphoneFilterFactory factory = new DoubleMetaphoneFilterFactory(parameters);

    TokenStream inputStream = new MockTokenizer(new StringReader("international"), MockTokenizer.WHITESPACE, false);

    TokenStream filteredStream = factory.create(inputStream);
    assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());
    assertTokenStreamContents(filteredStream, new String[] { "ANTRNXNL" });
  }
View Full Code Here
27
28
29
30
31
32
33
34
35
36
37
/** Simple tests for {@link BeiderMorseFilterFactory} */
public class TestBeiderMorseFilterFactory extends BaseTokenStreamTestCase {
  public void testBasics() throws Exception {
    BeiderMorseFilterFactory factory = new BeiderMorseFilterFactory(new HashMap<String,String>());
    TokenStream ts = factory.create(new MockTokenizer(new StringReader("Weinberg"), MockTokenizer.WHITESPACE, false));
    assertTokenStreamContents(ts,
        new String[] { "vDnbirk", "vanbirk", "vinbirk", "wDnbirk", "wanbirk", "winbirk" },
        new int[] { 0, 0, 0, 0, 0, 0 },
        new int[] { 8, 8, 8, 8, 8, 8 },
        new int[] { 1, 0, 0, 0, 0, 0 });
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
 
  public void testLanguageSet() throws Exception {
    Map<String,String> args = new HashMap<String,String>();
    args.put("languageSet", "polish");
    BeiderMorseFilterFactory factory = new BeiderMorseFilterFactory(args);
    TokenStream ts = factory.create(new MockTokenizer(new StringReader("Weinberg"), MockTokenizer.WHITESPACE, false));
    assertTokenStreamContents(ts,
        new String[] { "vDmbYrk", "vDmbirk", "vambYrk", "vambirk", "vimbYrk", "vimbirk" },
        new int[] { 0, 0, 0, 0, 0, 0 },
        new int[] { 8, 8, 8, 8, 8, 8 },
        new int[] { 1, 0, 0, 0, 0, 0 });
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
  public void testOptions() throws Exception {
    Map<String,String> args = new HashMap<String,String>();
    args.put("nameType", "ASHKENAZI");
    args.put("ruleType", "EXACT");
    BeiderMorseFilterFactory factory = new BeiderMorseFilterFactory(args);
    TokenStream ts = factory.create(new MockTokenizer(new StringReader("Weinberg"), MockTokenizer.WHITESPACE, false));
    assertTokenStreamContents(ts,
        new String[] { "vajnberk" },
        new int[] { 0 },
        new int[] { 8 },
        new int[] { 1 });
View Full Code Here
37
38
39
40
41
42
43
44
45
46
47
  static private Analyzer newTestAnalyzer() {
    return new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        return new TokenStreamComponents(tokenizer, tokenizer);
      }

      @Override
      protected Reader initReader(String fieldName, Reader reader) {
View Full Code Here
41
42
43
44
45
46
47
48
49
    List<String> rules = new ArrayList<String>();
    rules.add("a b c,d");
    SlowSynonymMap synMap = new SlowSynonymMap(true);
    SlowSynonymFilterFactory.parseRules(rules, synMap, "=>", ",", true, null);

    SlowSynonymFilter ts = new SlowSynonymFilter(new MockTokenizer(new StringReader("a e"), MockTokenizer.WHITESPACE, false), synMap);
    // This fails because ["e","e"] is the value of the token stream
    assertTokenStreamContents(ts, new String[] { "a", "e" });
  }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.MockTokenizer

  • org.apache.lucene.analysis.bg.TestBulgarianStemmer
  • org.apache.lucene.analysis.charfilter.TestHTMLStripCharFilterFactory
  • org.apache.lucene.analysis.charfilter.TestMappingCharFilter
  • org.apache.lucene.analysis.cjk.TestCJKAnalyzer
  • org.apache.lucene.analysis.cn.smart.TestSmartChineseAnalyzer
  • org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter
  • org.apache.lucene.analysis.compound.TestHyphenationCompoundWordTokenFilterFactory
  • org.apache.lucene.analysis.core.TestStopFilter
  • org.apache.lucene.analysis.de.TestGermanStemFilter
  • org.apache.lucene.analysis.en.TestPorterStemFilter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.