Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.CharFilter

55
56
57
58
59
60
61
62
63
64
65
  public void testTokenStream() throws IOException {
    // '℃', '№', '㈱', '㌘', 'ｻ'+'<<', 'ｿ'+'<<', '㌰'+'<<'
    String input = "℃ № ㈱ ㌘ ｻﾞ ｿﾞ ㌰ﾞ";

    CharFilter reader = new ICUNormalizer2CharFilter(new StringReader(input),
      Normalizer2.getInstance(null, "nfkc", Normalizer2.Mode.COMPOSE));

    Tokenizer tokenStream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);

    assertTokenStreamContents(tokenStream,
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
  public void testTokenStream2() throws IOException {
    // '㌰', '<<'゙, '5', '℃', '№', '㈱', '㌘', 'ｻ', '<<', 'ｿ', '<<'
    String input = "㌰゙5℃№㈱㌘ｻﾞｿﾞ";

    CharFilter reader = new ICUNormalizer2CharFilter(new StringReader(input),
      Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));

    Tokenizer tokenStream = new NGramTokenizer(TEST_VERSION_CURRENT, newAttributeFactory(), reader, 1, 1);

    assertTokenStreamContents(tokenStream,
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
  }
 
  public void testMassiveLigature() throws IOException {
    String input = "\uFDFA";

    CharFilter reader = new ICUNormalizer2CharFilter(new StringReader(input),
      Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));

    Tokenizer tokenStream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);

    assertTokenStreamContents(tokenStream,
View Full Code Here
33
34
35
36
37
38
39
40
41
public class TestJapaneseIterationMarkCharFilterFactory extends BaseTokenStreamTestCase {

  public void testIterationMarksWithKeywordTokenizer() throws IOException {
    final String text = "時々馬鹿々々しいところゞゝゝミスヾ";
    JapaneseIterationMarkCharFilterFactory filterFactory = new JapaneseIterationMarkCharFilterFactory(new HashMap<String,String>());
    CharFilter filter = filterFactory.create(new StringReader(text));
    TokenStream tokenStream = new MockTokenizer(filter, MockTokenizer.KEYWORD, false);
    assertTokenStreamContents(tokenStream, new String[]{"時時馬鹿馬鹿しいところどころミスズ"});
  }
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
  public void testIterationMarksWithJapaneseTokenizer() throws IOException {
    JapaneseTokenizerFactory tokenizerFactory = new JapaneseTokenizerFactory(new HashMap<String,String>());
    tokenizerFactory.inform(new StringMockResourceLoader(""));

    JapaneseIterationMarkCharFilterFactory filterFactory = new JapaneseIterationMarkCharFilterFactory(new HashMap<String,String>());
    CharFilter filter = filterFactory.create(
        new StringReader("時々馬鹿々々しいところゞゝゝミスヾ")
    );
    TokenStream tokenStream = tokenizerFactory.create(newAttributeFactory(), filter);
    assertTokenStreamContents(tokenStream, new String[]{"時時", "馬鹿馬鹿しい", "ところどころ", "ミ", "スズ"});
  }
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
    Map<String, String> filterArgs = new HashMap<>();
    filterArgs.put("normalizeKanji", "true");
    filterArgs.put("normalizeKana", "false");
    JapaneseIterationMarkCharFilterFactory filterFactory = new JapaneseIterationMarkCharFilterFactory(filterArgs);
   
    CharFilter filter = filterFactory.create(
        new StringReader("時々馬鹿々々しいところゞゝゝミスヾ")
    );
    TokenStream tokenStream = tokenizerFactory.create(newAttributeFactory(), filter);
    assertTokenStreamContents(tokenStream, new String[]{"時時", "馬鹿馬鹿しい", "ところ", "ゞ", "ゝ", "ゝ", "ミス", "ヾ"});
  }
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
    Map<String, String> filterArgs = new HashMap<>();
    filterArgs.put("normalizeKanji", "false");
    filterArgs.put("normalizeKana", "true");
    JapaneseIterationMarkCharFilterFactory filterFactory = new JapaneseIterationMarkCharFilterFactory(filterArgs);

    CharFilter filter = filterFactory.create(
        new StringReader("時々馬鹿々々しいところゞゝゝミスヾ")
    );
    TokenStream tokenStream = tokenizerFactory.create(newAttributeFactory(), filter);
    assertTokenStreamContents(tokenStream, new String[]{"時々", "馬鹿", "々", "々", "しい", "ところどころ", "ミ", "スズ"});
  }
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
    assertAnalyzesTo(keywordAnalyzer, "。。。", new String[]{"。。。"});
  }

  public void testKanjiOnly() throws IOException {
    // Test kanji only repetition marks
    CharFilter filter = new JapaneseIterationMarkCharFilter(
        new StringReader("時々、おゝのさんと一緒にお寿司が食べたいです。abcところゞゝゝ。"),
        true, // kanji
        false // no kana
    );
    assertCharFilterEquals(filter, "時時、おゝのさんと一緒にお寿司が食べたいです。abcところゞゝゝ。");
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
    assertCharFilterEquals(filter, "時時、おゝのさんと一緒にお寿司が食べたいです。abcところゞゝゝ。");
  }

  public void testKanaOnly() throws IOException {
    // Test kana only repetition marks
    CharFilter filter = new JapaneseIterationMarkCharFilter(
        new StringReader("時々、おゝのさんと一緒にお寿司が食べたいです。abcところゞゝゝ。"),
        false, // no kanji
        true   // kana
    );
    assertCharFilterEquals(filter, "時々、おおのさんと一緒にお寿司が食べたいです。abcところどころ。");
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
    assertCharFilterEquals(filter, "時々、おおのさんと一緒にお寿司が食べたいです。abcところどころ。");
  }

  public void testNone() throws IOException {
    // Test no repetition marks
    CharFilter filter = new JapaneseIterationMarkCharFilter(
        new StringReader("時々、おゝのさんと一緒にお寿司が食べたいです。abcところゞゝゝ。"),
        false, // no kanji
        false  // no kana
    );
    assertCharFilterEquals(filter, "時々、おゝのさんと一緒にお寿司が食べたいです。abcところゞゝゝ。");
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.CharFilter

  • org.apache.lucene.analysis.charfilter.TestMappingCharFilter
  • org.apache.lucene.analysis.core.TestBugInSomething
  • org.apache.lucene.analysis.icu.TestICUNormalizer2CharFilter
  • org.apache.lucene.analysis.ja.TestJapaneseIterationMarkCharFilter
  • org.apache.lucene.analysis.ja.TestJapaneseIterationMarkCharFilterFactory
  • org.apache.lucene.analysis.pattern.TestPatternReplaceCharFilter
  • org.apache.lucene.analysis.pattern.TestPatternTokenizer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.