Package org.apache.lucene.analysis.ar

Examples of org.apache.lucene.analysis.ar.ArabicLetterTokenizer

119
120
121
122
123
124
125
126
127
128
129
      Reader reader) {
    final Tokenizer source;
    if (matchVersion.onOrAfter(Version.LUCENE_31)) {
      source = new StandardTokenizer(matchVersion, reader);
    } else {
      source = new ArabicLetterTokenizer(matchVersion, reader);
    }
    TokenStream result = new LowerCaseFilter(matchVersion, source);
    result = new ArabicNormalizationFilter(result);
    /* additional persian-specific normalization */
    result = new PersianNormalizationFilter(result);
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
  public void testHehGoal() throws IOException {
    check("زادہ", "زاده");
  }

  private void check(final String input, final String expected) throws IOException {
    ArabicLetterTokenizer tokenStream = new ArabicLetterTokenizer(TEST_VERSION_CURRENT,
        new StringReader(input));
    PersianNormalizationFilter filter = new PersianNormalizationFilter(
        tokenStream);
    assertTokenStreamContents(filter, new String[]{expected});
  }
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
   *         filtered with {@link LowerCaseFilter},
   *         {@link ArabicNormalizationFilter},
   *         {@link PersianNormalizationFilter} and Persian Stop words
   */
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new ArabicLetterTokenizer(reader);
    result = new LowerCaseFilter(result);
    result = new ArabicNormalizationFilter(result);
    /* additional persian-specific normalization */
    result = new PersianNormalizationFilter(result);
    /*
 
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
  public TokenStream reusableTokenStream(String fieldName, Reader reader)
      throws IOException {
    SavedStreams streams = (SavedStreams) getPreviousTokenStream();
    if (streams == null) {
      streams = new SavedStreams();
      streams.source = new ArabicLetterTokenizer(reader);
      streams.result = new LowerCaseFilter(streams.source);
      streams.result = new ArabicNormalizationFilter(streams.result);
      /* additional persian-specific normalization */
      streams.result = new PersianNormalizationFilter(streams.result);
      /*
 
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
  public void testHehGoal() throws IOException {
    check("زادہ", "زاده");
  }

  private void check(final String input, final String expected) throws IOException {
    ArabicLetterTokenizer tokenStream = new ArabicLetterTokenizer(
        new StringReader(input));
    PersianNormalizationFilter filter = new PersianNormalizationFilter(
        tokenStream);
    assertTokenStreamContents(filter, new String[]{expected});
  }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
  public void testHehGoal() throws IOException {
    check("زادہ", "زاده");
  }

  private void check(final String input, final String expected) throws IOException {
    ArabicLetterTokenizer tokenStream = new ArabicLetterTokenizer(TEST_VERSION_CURRENT,
        new StringReader(input));
    PersianNormalizationFilter filter = new PersianNormalizationFilter(
        tokenStream);
    assertTokenStreamContents(filter, new String[]{expected});
  }
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
      Reader reader) {
    final Tokenizer source;
    if (matchVersion.onOrAfter(Version.LUCENE_31)) {
      source = new StandardTokenizer(matchVersion, reader);
    } else {
      source = new ArabicLetterTokenizer(matchVersion, reader);
    }
    TokenStream result = new LowerCaseFilter(matchVersion, source);
    result = new ArabicNormalizationFilter(result);
    /* additional persian-specific normalization */
    result = new PersianNormalizationFilter(result);
View Full Code Here
152
153
154
155
156
157
158
159
160
161
162
      Reader reader) {
    final Tokenizer source;
    if (matchVersion.onOrAfter(Version.LUCENE_31)) {
      source = new StandardTokenizer(matchVersion, reader);
    } else {
      source = new ArabicLetterTokenizer(matchVersion, reader);
    }
    TokenStream result = new LowerCaseFilter(matchVersion, source);
    result = new ArabicNormalizationFilter(result);
    /* additional persian-specific normalization */
    result = new PersianNormalizationFilter(result);
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
  public void testHehGoal() throws IOException {
    check("زادہ", "زاده");
  }

  private void check(final String input, final String expected) throws IOException {
    ArabicLetterTokenizer tokenStream = new ArabicLetterTokenizer(TEST_VERSION_CURRENT,
        new StringReader(input));
    PersianNormalizationFilter filter = new PersianNormalizationFilter(
        tokenStream);
    assertTokenStreamContents(filter, new String[]{expected});
  }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.ar.ArabicLetterTokenizer

  • org.apache.lucene.analysis.ar.ArabicLetterTokenizerFactory
  • org.apache.lucene.analysis.fa.PersianAnalyzer
  • org.apache.lucene.analysis.fa.TestPersianNormalizationFilter
  • org.apache.solr.analysis.ArabicLetterTokenizerFactory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.