Examples of org.apache.lucene.analysis.LetterTokenizer

org.apache.lucene.analysis.LetterTokenizer
A LetterTokenizer is a tokenizer that divides text at non-letters. That's to say, it defines tokens as maximal strings of adjacent letters, as defined by java.lang.Character.isLetter() predicate. Note: this does a decent job for most European languages, but does a terrible job for some Asian languages, where words are not separated by spaces.

  }
  
  private TokenStream luceneTokenStream(String text, boolean letters, boolean toLowerCase, Set stopWords) {
    TokenStream stream;
    if (letters) 
      stream = new LetterTokenizer(new StringReader(text));
    else
      stream = new WhitespaceTokenizer(new StringReader(text));
    if (toLowerCase)  stream = new LowerCaseFilter(stream);
    if (stopWords != null) stream = new StopFilter(stream, stopWords);
    return stream;

View Full Code Here

    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      if (++invocationCount % 2 == 0)
        return new WhitespaceTokenizer(reader);
      else
        return new LetterTokenizer(reader);
    }

View Full Code Here

    super.init(args);
    assureMatchVersion();
  }


  public LetterTokenizer create(Reader input) {
    return new LetterTokenizer(luceneMatchVersion, input);
  }

View Full Code Here

    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      if (++invocationCount % 2 == 0)
        return new WhitespaceTokenizer(reader);
      else
        return new LetterTokenizer(reader);
    }

View Full Code Here

/**
 * @version $Id: LetterTokenizerFactory.java 591158 2007-11-01 22:37:42Z hossman $
 */
public class LetterTokenizerFactory extends BaseTokenizerFactory {
  public LetterTokenizer create(Reader input) {
    return new LetterTokenizer(input);
  }

View Full Code Here

    @Inject public LetterTokenizerFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) {
        super(index, indexSettings, name, settings);
    }


    @Override public Tokenizer create(Reader reader) {
        return new LetterTokenizer(version, reader);
    }

View Full Code Here

  }
  
  private TokenStream luceneTokenStream(String text, boolean letters, boolean toLowerCase, Set stopWords) {
    TokenStream stream;
    if (letters) 
      stream = new LetterTokenizer(new StringReader(text));
    else
      stream = new WhitespaceTokenizer(new StringReader(text));
    if (toLowerCase)  stream = new LowerCaseFilter(stream);
    if (stopWords != null) stream = new StopFilter(stream, stopWords);
    return stream;

View Full Code Here

  }
  
  private TokenStream luceneTokenStream(String text, boolean letters, boolean toLowerCase, Set stopWords) {
    TokenStream stream;
    if (letters) 
      stream = new LetterTokenizer(new StringReader(text));
    else
      stream = new WhitespaceTokenizer(new StringReader(text));
    if (toLowerCase)  stream = new LowerCaseFilter(stream);
    if (stopWords != null) stream = new StopFilter(stream, stopWords);
    return stream;

View Full Code Here

/**
 * @version $Id: LetterTokenizerFactory.java 591158 2007-11-01 22:37:42Z hossman $
 */
public class LetterTokenizerFactory extends BaseTokenizerFactory {
  public LetterTokenizer create(Reader input) {
    return new LetterTokenizer( input );
  }

View Full Code Here

  }
  
  private TokenStream luceneTokenStream(String text, boolean letters, boolean toLowerCase, Set stopWords) {
    TokenStream stream;
    if (letters) 
      stream = new LetterTokenizer(new StringReader(text));
    else
      stream = new WhitespaceTokenizer(new StringReader(text));
    if (toLowerCase)  stream = new LowerCaseFilter(stream);
    if (stopWords != null) stream = new StopFilter(stream, stopWords);
    return stream;

View Full Code Here

0 1

TOP

Related Classes of org.apache.lucene.analysis.LetterTokenizer

org.apache.lucene.analysis.query.QueryAutoStopWordAnalyzerTest$NonreusableAnalyzer

org.apache.lucene.analysis.shingle.ShingleAnalyzerWrapperTest$NonreusableAnalyzer

org.apache.lucene.index.memory.PatternAnalyzerTest

org.apache.solr.analysis.LetterTokenizerFactory

org.elasticsearch.index.analysis.LetterTokenizerFactory

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.