Examples of org.apache.lucene.analysis.WhitespaceTokenizer

org.apache.lucene.analysis.WhitespaceTokenizer
A WhitespaceTokenizer is a tokenizer that divides text at whitespace. Adjacent sequences of non-Whitespace characters form tokens.


  public void testPositionIncrementGap() throws IOException {
    Analyzer analyzer = new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new WhitespaceTokenizer(reader);
      }


      @Override
      public int getPositionIncrementGap(String fieldName) {
        return 500;

View Full Code Here


  public void testTokenReuse() throws IOException {
    Analyzer analyzer = new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new TokenFilter(new WhitespaceTokenizer(reader)) {
          boolean first=true;
          AttributeSource.State state;


          @Override
          public boolean incrementToken() throws IOException {

View Full Code Here

  }


  public void testDocumentsWriterExceptions() throws IOException {
    Analyzer analyzer = new Analyzer() {
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new CrashingFilter(fieldName, new WhitespaceTokenizer(reader));
      }
    };


    for(int i=0;i<2;i++) {
      MockRAMDirectory dir = new MockRAMDirectory();

View Full Code Here

  }


  public void testDocumentsWriterExceptionThreads() throws Exception {
    Analyzer analyzer = new Analyzer() {
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new CrashingFilter(fieldName, new WhitespaceTokenizer(reader));
      }
    };


    final int NUM_THREAD = 3;
    final int NUM_ITER = 100;

View Full Code Here

                      Field.Index.ANALYZED));
    w.addDocument(doc);


    Analyzer analyzer = new Analyzer() {
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new CrashingFilter(fieldName, new WhitespaceTokenizer(reader));
      }
    };


    Document crashDoc = new Document();
    crashDoc.add(new Field("crash", "do it on token 4", Field.Store.YES,

View Full Code Here

    for(int i=0;i<50;i++)
      b[i] = (byte) (i+77);


    Document doc = new Document();
    Field f = new Field("binary", b, 10, 17, Field.Store.YES);
    f.setTokenStream(new WhitespaceTokenizer(new StringReader("doc1field1")));
    Field f2 = new Field("string", "value", Field.Store.YES,Field.Index.ANALYZED);
    f2.setTokenStream(new WhitespaceTokenizer(new StringReader("doc1field2")));
    doc.add(f);
    doc.add(f2);
    w.addDocument(doc);
    
    // add 2 docs to test in-memory merging
    f.setTokenStream(new WhitespaceTokenizer(new StringReader("doc2field1")));
    f2.setTokenStream(new WhitespaceTokenizer(new StringReader("doc2field2")));
    w.addDocument(doc);
  
    // force segment flush so we can force a segment merge with doc3 later.
    w.commit();


    f.setTokenStream(new WhitespaceTokenizer(new StringReader("doc3field1")));
    f2.setTokenStream(new WhitespaceTokenizer(new StringReader("doc3field2")));


    w.addDocument(doc);
    w.commit();
    w.optimize();   // force segment merge.

View Full Code Here

        
      }
    } catch (IOException e) {
      e.printStackTrace();
    }
    return new WhitespaceTokenizer(new StringReader(buf.toString()));
  }

View Full Code Here

      }
    } catch (IOException e) {
      e.printStackTrace();
    }
    
    return new WhitespaceTokenizer(new StringReader(buf.toString()));
  }

View Full Code Here

    super.init(args);
    assureMatchVersion();
  }


  public WhitespaceTokenizer create(Reader input) {
    return new WhitespaceTokenizer(luceneMatchVersion,input);
  }

View Full Code Here

    final TokenizerFactory factory = tf == null ? null : loadTokenizerFactory(loader, tf, args);
    
    Analyzer analyzer = new ReusableAnalyzerBase() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer(Version.LUCENE_31, reader) : factory.create(reader);
        TokenStream stream = ignoreCase ? new LowerCaseFilter(Version.LUCENE_31, tokenizer) : tokenizer;
        return new TokenStreamComponents(tokenizer, stream);
      }
    };

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.analysis.WhitespaceTokenizer

org.apache.james.mailbox.store.lucene.ImapSearchAnalyzer

org.apache.lucene.analysis.cn.TestChineseTokenizer$JustChineseFilterAnalyzer

org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter

org.apache.lucene.analysis.de.TestGermanStemFilter$GermanSubclassAnalyzer

org.apache.lucene.analysis.icu.TestICUNormalizer2Filter

org.apache.lucene.analysis.icu.TestICUTransformFilter

org.apache.lucene.analysis.miscellaneous.TestPrefixAndSuffixAwareTokenFilter

org.apache.lucene.analysis.miscellaneous.TestPrefixAwareTokenFilter

org.apache.lucene.analysis.ngram.EdgeNGramTokenFilterTest

org.apache.lucene.analysis.ngram.NGramTokenFilterTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.