Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.WhitespaceTokenizer

468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
        tokenBuilder.append((String) groupIter.next());
        tokenBuilder.append(" ");
      }
   
      //doc.add(new Field("groups", new IteratorTokenStream(groupIter)));
      doc.add(new Field("groups", new WhitespaceTokenizer(new StringReader(tokenBuilder.toString()))));
    }

    // Add the URL of the document
    doc.add(new Field("url", url, Field.Store.YES, Field.Index.NOT_ANALYZED));
   
    // Add the file name (without protocol, drive-letter and path)
    String filenameWithVariants = RegainToolkit.urlToWhitespacedFileName(url);
    doc.add(new Field("filename", new WhitespaceTokenizer(new StringReader(filenameWithVariants))));
    PathFilenamePair pfPair = RegainToolkit.fragmentUrl(url);

    // Add the filename field for sorting
    doc.add(new Field("filename_sort", pfPair.getFilename(), Field.Store.YES, Field.Index.NOT_ANALYZED));
View Full Code Here
43
44
45
46
47
48
49
50
51
52
   * @see org.apache.lucene.analysis.KeywordAnalyzer#tokenStream(java.lang.String,
   * java.io.Reader)
   */
  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new WhitespaceTokenizer(reader);
    result = new LowerCaseFilter(result);

    return result;
  }
View Full Code Here
181
182
183
184
185
186
187
188
189
190
  private TokenStream luceneTokenStream(String text, boolean letters, boolean toLowerCase, Set stopWords) {
    TokenStream stream;
    if (letters)
      stream = new LetterTokenizer(new StringReader(text));
    else
      stream = new WhitespaceTokenizer(new StringReader(text));
    if (toLowerCase)  stream = new LowerCaseFilter(stream);
    if (stopWords != null) stream = new StopFilter(stream, stopWords);
    return stream;           
  }
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
  public void testPositionIncrementGap() throws IOException {
    Analyzer analyzer = new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader);
      }

      @Override
      public int getPositionIncrementGap(String fieldName) {
        return 500;
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
  public void testTokenReuse() throws IOException {
    Analyzer analyzer = new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new TokenFilter(new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader)) {
          boolean first=true;
          AttributeSource.State state;

          @Override
          public boolean incrementToken() throws IOException {
View Full Code Here
156
157
158
159
160
161
162
      super(matchVersion, new WhitespaceAnalyzer());
    }
   
    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      return new WhitespaceTokenizer(reader);
    }   
View Full Code Here
176
177
178
179
180
181
182
183
184
  private class NonreusableAnalyzer extends Analyzer {
    int invocationCount = 0;
    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      if (++invocationCount % 2 == 0)
        return new WhitespaceTokenizer(reader);
      else
        return new LetterTokenizer(reader);
    }
View Full Code Here
134
135
136
137
138
139
140
    public DutchSubclassAnalyzer(Version matchVersion) {
      super(matchVersion);
    }
    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      return new WhitespaceTokenizer(reader);
    }
View Full Code Here
132
133
134
135
136
137
138
          public ThaiSubclassAnalyzer(Version matchVersion) {
            super(matchVersion);
          }
    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      return new WhitespaceTokenizer(reader);
    }
View Full Code Here
393
394
395
396
397
398
399
400
401
402
403
        }
       
        @Override
        public TokenStream tokenStream(String fieldName, Reader reader) {
            PayloadData payload = (PayloadData) fieldToData.get(fieldName);
            TokenStream ts = new WhitespaceTokenizer(reader);
            if (payload != null) {
                if (payload.numFieldInstancesToSkip == 0) {
                    ts = new PayloadFilter(ts, payload.data, payload.offset, payload.length);
                } else {
                    payload.numFieldInstancesToSkip--;
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.WhitespaceTokenizer

  • org.apache.james.mailbox.store.lucene.ImapSearchAnalyzer
  • org.apache.lucene.analysis.cn.TestChineseTokenizer$JustChineseFilterAnalyzer
  • org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter
  • org.apache.lucene.analysis.de.TestGermanStemFilter$GermanSubclassAnalyzer
  • org.apache.lucene.analysis.icu.TestICUNormalizer2Filter
  • org.apache.lucene.analysis.icu.TestICUTransformFilter
  • org.apache.lucene.analysis.miscellaneous.TestPrefixAndSuffixAwareTokenFilter
  • org.apache.lucene.analysis.miscellaneous.TestPrefixAwareTokenFilter
  • org.apache.lucene.analysis.ngram.EdgeNGramTokenFilterTest
  • org.apache.lucene.analysis.ngram.NGramTokenFilterTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.