Examples of org.apache.lucene.analysis.TokenStream.end()

Class org.apache.lucene.analysis.TokenStream

Examples of org.apache.lucene.analysis.TokenStream.end()

org.apache.lucene.analysis.TokenStream.end()
This method is called by the consumer after the last token has been consumed, after {@link #incrementToken()} returned false(using the new TokenStream API). Streams implementing the old API should upgrade to use this feature.
This method can be used to perform any end-of-stream operations, such as setting the final offset of a stream. The final offset of a stream might differ from the offset of the last token eg in case one or more whitespaces followed after the last token, but a {@link WhitespaceTokenizer} was used. @throws IOException

      while (ts.incrementToken()) {
        j += posIncrAtt.getPositionIncrement();
        String termText = termAtt.toString();
        q.add(new Term("content", termText), j);
      }
      ts.end();
    } finally {
      IOUtils.closeWhileHandlingException(ts);
    }


    ScoreDoc[] hits = searcher.search(q, null, 1000).scoreDocs;

View Full Code Here

      while (ts.incrementToken()) {
        String termText =  termAtt.toString();
        q.add(new TermQuery(new Term("content", termText)),
            BooleanClause.Occur.SHOULD);
      }
      ts.end();
    } finally {
      IOUtils.closeWhileHandlingException(ts);
    }


    ScoreDoc[] hits = searcher.search(q, null, 1000).scoreDocs;

View Full Code Here

          throw new IllegalArgumentException("tokens must not contain separator byte; got token=" + tokenBytes + " but gramCount=" + gramCount + " does not match recalculated count=" + countGrams(tokenBytes));
        }
        maxEndOffset = Math.max(maxEndOffset, offsetAtt.endOffset());
        lastTokens[gramCount-1] = BytesRef.deepCopyOf(tokenBytes);
      }
      ts.end();
      
      if (!sawRealToken) {
        throw new IllegalArgumentException("no tokens produced by analyzer, or the only tokens were empty strings");
      }

View Full Code Here

              fieldState.length++;
              fieldState.position++;
            } while (stream.incrementToken());
          }
          // trigger streams to perform end-of-stream operations
          stream.end();
          // TODO: maybe add some safety? then again, its already checked 
          // when we come back around to the field...
          fieldState.position += posIncrAttribute.getPositionIncrement();
          fieldState.offset += offsetAttribute.endOffset();
          success2 = true;

View Full Code Here

      boolean expected = count >= min && count <= max;
      TokenStream stream = new KeywordTokenizer(new StringReader(text));
      stream = new CodepointCountFilter(TEST_VERSION_CURRENT, stream, min, max);
      stream.reset();
      assertEquals(expected, stream.incrementToken());
      stream.end();
      stream.close();
    }
  }
}

View Full Code Here

        lastToken = termAtt.toString();
        if (lastToken != null) {
          maxEndOffset = Math.max(maxEndOffset, offsetAtt.endOffset());
        }
      }
      ts.end();


      String prefixToken = null;
      if (lastToken != null) {
        Query lastQuery;
        if (maxEndOffset == offsetAtt.endOffset()) {

View Full Code Here

        } else if (prefixToken != null && token.startsWith(prefixToken)) {
          addPrefixMatch(sb, text.substring(startOffset, endOffset), token, prefixToken);
          upto = endOffset;
        }
      }
      ts.end();
      int endOffset = offsetAtt.endOffset();
      if (upto < endOffset) {
        addNonMatch(sb, text.substring(upto));
      }
      return sb.toString();

View Full Code Here

      for(String inflectionType : inflectionTypes) {
        assertTrue(ts.incrementToken());
        assertEquals(inflectionType, inflectionAtt.getInflectionType());
      }
      assertFalse(ts.incrementToken());
      ts.end();
    } finally {
      IOUtils.closeWhileHandlingException(ts);
    }
  }

View Full Code Here

      for(String inflectionForm : inflectionForms) {
        assertTrue(ts.incrementToken());
        assertEquals(inflectionForm, inflectionAtt.getInflectionForm());
      }
      assertFalse(ts.incrementToken());
      ts.end();
    } finally {
      IOUtils.closeWhileHandlingException(ts);
    }
  }

View Full Code Here

      for(String partOfSpeech : partsOfSpeech) {
        assertTrue(ts.incrementToken());
        assertEquals(partOfSpeech, partOfSpeechAtt.getPartOfSpeech());
      }
      assertFalse(ts.incrementToken());
      ts.end();
    } finally {
      IOUtils.closeWhileHandlingException(ts);
    }
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.