Examples of KeywordTokenizer


Examples of org.apache.lucene.analysis.KeywordTokenizer

    this.collator = collator;
  }

  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new KeywordTokenizer(reader);
    result = new ICUCollationKeyFilter(result, collator);
    return result;
  }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordTokenizer

    throws IOException {
   
    SavedStreams streams = (SavedStreams)getPreviousTokenStream();
    if (streams == null) {
      streams = new SavedStreams();
      streams.source = new KeywordTokenizer(reader);
      streams.result = new ICUCollationKeyFilter(streams.source, collator);
      setPreviousTokenStream(streams);
    } else {
      streams.source.reset(reader);
    }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordTokenizer

    this.collator = collator;
  }

  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new KeywordTokenizer(reader);
    result = new CollationKeyFilter(result, collator);
    return result;
  }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordTokenizer

    throws IOException {
   
    SavedStreams streams = (SavedStreams)getPreviousTokenStream();
    if (streams == null) {
      streams = new SavedStreams();
      streams.source = new KeywordTokenizer(reader);
      streams.result = new CollationKeyFilter(streams.source, collator);
      setPreviousTokenStream(streams);
    } else {
      streams.source.reset(reader);
    }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordTokenizer

      this.collator = collator;
    }

    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      TokenStream result = new KeywordTokenizer(reader);
      result = new ICUCollationKeyFilter(result, collator);
      return result;
    }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordTokenizer

      this.collator = collator;
    }

    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      TokenStream result = new KeywordTokenizer(reader);
      result = new CollationKeyFilter(result, collator);
      return result;
    }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordTokenizer

* &lt;/fieldType&gt;</pre>
* @version $Id: LowerCaseTokenizerFactory.java 382610 2006-03-03 01:43:03Z yonik $
*/
public class KeywordTokenizerFactory extends BaseTokenizerFactory {
  public KeywordTokenizer create(Reader input) {
    return new KeywordTokenizer(input);
  }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordTokenizer

 
  public void testOptimizer() throws Exception {
    String rules = "a > b; b > c;"; // convert a's to b's and b's to c's
    Transliterator custom = Transliterator.createFromRules("test", rules, Transliterator.FORWARD);
    assertTrue(custom.getFilter() == null);
    new ICUTransformFilter(new KeywordTokenizer(new StringReader("")), custom);
    assertTrue(custom.getFilter().equals(new UnicodeSet("[ab]")));
  }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordTokenizer

 
  public void testOptimizerSurrogate() throws Exception {
    String rules = "\\U00020087 > x;"; // convert CJK UNIFIED IDEOGRAPH-20087 to an x
    Transliterator custom = Transliterator.createFromRules("test", rules, Transliterator.FORWARD);
    assertTrue(custom.getFilter() == null);
    new ICUTransformFilter(new KeywordTokenizer(new StringReader("")), custom);
    assertTrue(custom.getFilter().equals(new UnicodeSet("[\\U00020087]")));
  }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordTokenizer

    new ICUTransformFilter(new KeywordTokenizer(new StringReader("")), custom);
    assertTrue(custom.getFilter().equals(new UnicodeSet("[\\U00020087]")));
  }

  private void checkToken(Transliterator transform, String input, String expected) throws IOException {
    TokenStream ts = new ICUTransformFilter(new KeywordTokenizer((new StringReader(input))), transform);
    assertTokenStreamContents(ts, new String[] { expected });
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.