Examples of CJKWidthFilter

Examples of org.apache.lucene.analysis.cjk.CJKWidthFilter

88
89
90
91
92
93
94
95
96
97
98
  @Override
  protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
    Tokenizer tokenizer = new JapaneseTokenizer(reader, userDict, true, mode);
    TokenStream stream = new JapaneseBaseFormFilter(tokenizer);
    stream = new JapanesePartOfSpeechStopFilter(matchVersion, stream, stoptags);
    stream = new CJKWidthFilter(stream);
    stream = new StopFilter(matchVersion, stream, stopwords);
    stream = new JapaneseKatakanaStemFilter(stream);
    stream = new LowerCaseFilter(matchVersion, stream);
    return new TokenStreamComponents(tokenizer, stream);
  }
View Full Code Here

Examples of org.apache.lucene.analysis.cjk.CJKWidthFilter

38
39
40
41
42
43
44
public class CJKWidthFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
 
  @Override
  public TokenStream create(TokenStream input) {
    return new CJKWidthFilter(input);
  }
View Full Code Here

Examples of org.apache.lucene.analysis.cjk.CJKWidthFilter

58
59
60
61
62
63
64
65
66
67
68
  public void testKatakanaReadingsHalfWidth() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), reader, null, true, JapaneseTokenizer.Mode.SEARCH);
        TokenStream stream = new CJKWidthFilter(tokenizer);
        return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, false));
      }
    };
    assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
        new String[] { "コンヤ", "ハ", "ロバート", "センセイ", "ト", "ハナシ", "タ" }
View Full Code Here

Examples of org.apache.lucene.analysis.cjk.CJKWidthFilter

78
79
80
81
82
83
84
85
86
87
88
  public void testRomajiReadingsHalfWidth() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), reader, null, true, JapaneseTokenizer.Mode.SEARCH);
        TokenStream stream = new CJKWidthFilter(tokenizer);
        return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, true));
      }
    };
    assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
        new String[] { "kon'ya", "ha", "robato", "sensei", "to", "hanashi", "ta" }
View Full Code Here

Examples of org.apache.lucene.analysis.cjk.CJKWidthFilter

58
59
60
61
62
63
64
65
66
67
68
  public void testKatakanaReadingsHalfWidth() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.Mode.SEARCH);
        TokenStream stream = new CJKWidthFilter(tokenizer);
        return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, false));
      }
    };
    assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
        new String[] { "コンヤ", "ハ", "ロバート", "センセイ", "ト", "ハナシ", "タ" }
View Full Code Here

Examples of org.apache.lucene.analysis.cjk.CJKWidthFilter

78
79
80
81
82
83
84
85
86
87
88
  public void testRomajiReadingsHalfWidth() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.Mode.SEARCH);
        TokenStream stream = new CJKWidthFilter(tokenizer);
        return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, true));
      }
    };
    assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
        new String[] { "kon'ya", "ha", "robato", "sensei", "to", "hanashi", "ta" }
View Full Code Here

Examples of org.apache.lucene.analysis.cjk.CJKWidthFilter

88
89
90
91
92
93
94
95
96
97
98
  @Override
  protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
    Tokenizer tokenizer = new JapaneseTokenizer(reader, userDict, true, mode);
    TokenStream stream = new JapaneseBaseFormFilter(tokenizer);
    stream = new JapanesePartOfSpeechStopFilter(true, stream, stoptags);
    stream = new CJKWidthFilter(stream);
    stream = new StopFilter(matchVersion, stream, stopwords);
    stream = new JapaneseKatakanaStemFilter(stream);
    stream = new LowerCaseFilter(matchVersion, stream);
    return new TokenStreamComponents(tokenizer, stream);
  }
View Full Code Here

Examples of org.apache.lucene.analysis.cjk.CJKWidthFilter

47
48
49
50
51
52
53
    }
  }
 
  @Override
  public TokenStream create(TokenStream input) {
    return new CJKWidthFilter(input);
  }
View Full Code Here

Examples of org.apache.lucene.analysis.cjk.CJKWidthFilter

58
59
60
61
62
63
64
65
66
67
68
  public void testKatakanaReadingsHalfWidth() throws IOException {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.Mode.SEARCH);
        TokenStream stream = new CJKWidthFilter(tokenizer);
        return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, false));
      }
    };
    assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
        new String[] { "コンヤ", "ハ", "ロバート", "センセイ", "ト", "ハナシ", "タ" }
View Full Code Here

Examples of org.apache.lucene.analysis.cjk.CJKWidthFilter

88
89
90
91
92
93
94
95
96
97
98
  @Override
  protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
    Tokenizer tokenizer = new JapaneseTokenizer(reader, userDict, true, mode);
    TokenStream stream = new JapaneseBaseFormFilter(tokenizer);
    stream = new JapanesePartOfSpeechStopFilter(matchVersion, stream, stoptags);
    stream = new CJKWidthFilter(stream);
    stream = new StopFilter(matchVersion, stream, stopwords);
    stream = new JapaneseKatakanaStemFilter(stream);
    stream = new LowerCaseFilter(matchVersion, stream);
    return new TokenStreamComponents(tokenizer, stream);
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.