Package org.apache.lucene.analysis.cjk

Examples of org.apache.lucene.analysis.cjk.CJKWidthFilter

88
89
90
91
92
93
94
95
96
97
98
  @Override
  protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
    Tokenizer tokenizer = new JapaneseTokenizer(reader, userDict, true, mode);
    TokenStream stream = new JapaneseBaseFormFilter(tokenizer);
    stream = new JapanesePartOfSpeechStopFilter(matchVersion, stream, stoptags);
    stream = new CJKWidthFilter(stream);
    stream = new StopFilter(matchVersion, stream, stopwords);
    stream = new JapaneseKatakanaStemFilter(stream);
    stream = new LowerCaseFilter(matchVersion, stream);
    return new TokenStreamComponents(tokenizer, stream);
  }
View Full Code Here
38
39
40
41
42
43
44
public class CJKWidthFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
 
  @Override
  public TokenStream create(TokenStream input) {
    return new CJKWidthFilter(input);
  }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
  public void testKatakanaReadingsHalfWidth() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), reader, null, true, JapaneseTokenizer.Mode.SEARCH);
        TokenStream stream = new CJKWidthFilter(tokenizer);
        return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, false));
      }
    };
    assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
        new String[] { "コンヤ", "ハ", "ロバート", "センセイ", "ト", "ハナシ", "タ" }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
  public void testRomajiReadingsHalfWidth() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), reader, null, true, JapaneseTokenizer.Mode.SEARCH);
        TokenStream stream = new CJKWidthFilter(tokenizer);
        return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, true));
      }
    };
    assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
        new String[] { "kon'ya", "ha", "robato", "sensei", "to", "hanashi", "ta" }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
  public void testKatakanaReadingsHalfWidth() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.Mode.SEARCH);
        TokenStream stream = new CJKWidthFilter(tokenizer);
        return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, false));
      }
    };
    assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
        new String[] { "コンヤ", "ハ", "ロバート", "センセイ", "ト", "ハナシ", "タ" }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
  public void testRomajiReadingsHalfWidth() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.Mode.SEARCH);
        TokenStream stream = new CJKWidthFilter(tokenizer);
        return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, true));
      }
    };
    assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
        new String[] { "kon'ya", "ha", "robato", "sensei", "to", "hanashi", "ta" }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
  @Override
  protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
    Tokenizer tokenizer = new JapaneseTokenizer(reader, userDict, true, mode);
    TokenStream stream = new JapaneseBaseFormFilter(tokenizer);
    stream = new JapanesePartOfSpeechStopFilter(true, stream, stoptags);
    stream = new CJKWidthFilter(stream);
    stream = new StopFilter(matchVersion, stream, stopwords);
    stream = new JapaneseKatakanaStemFilter(stream);
    stream = new LowerCaseFilter(matchVersion, stream);
    return new TokenStreamComponents(tokenizer, stream);
  }
View Full Code Here
47
48
49
50
51
52
53
    }
  }
 
  @Override
  public TokenStream create(TokenStream input) {
    return new CJKWidthFilter(input);
  }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
  public void testKatakanaReadingsHalfWidth() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.Mode.SEARCH);
        TokenStream stream = new CJKWidthFilter(tokenizer);
        return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, false));
      }
    };
    assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
        new String[] { "コンヤ", "ハ", "ロバート", "センセイ", "ト", "ハナシ", "タ" }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
  @Override
  protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
    Tokenizer tokenizer = new JapaneseTokenizer(reader, userDict, true, mode);
    TokenStream stream = new JapaneseBaseFormFilter(tokenizer);
    stream = new JapanesePartOfSpeechStopFilter(matchVersion, stream, stoptags);
    stream = new CJKWidthFilter(stream);
    stream = new StopFilter(matchVersion, stream, stopwords);
    stream = new JapaneseKatakanaStemFilter(stream);
    stream = new LowerCaseFilter(matchVersion, stream);
    return new TokenStreamComponents(tokenizer, stream);
  }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.cjk.CJKWidthFilter

  • org.apache.lucene.analysis.cjk.CJKWidthFilterFactory
  • org.apache.lucene.analysis.ja.JapaneseAnalyzer
  • org.apache.lucene.analysis.ja.TestJapaneseReadingFormFilter
  • org.elasticsearch.index.analysis.CJKWidthFilterFactory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.