Examples of org.apache.lucene.analysis.TokenStream.end()

Class org.apache.lucene.analysis.TokenStream

Examples of org.apache.lucene.analysis.TokenStream.end()

org.apache.lucene.analysis.TokenStream.end()
This method is called by the consumer after the last token has been consumed, after {@link #incrementToken()} returned false(using the new TokenStream API). Streams implementing the old API should upgrade to use this feature.
This method can be used to perform any end-of-stream operations, such as setting the final offset of a stream. The final offset of a stream might differ from the offset of the last token eg in case one or more whitespaces followed after the last token, but a {@link WhitespaceTokenizer} was used. @throws IOException

    for (int i = 0; i < numIterations; i++) {
      TokenStream ts = analyzer.tokenStream("ignored", line);
      try {
        ts.reset();
        while(ts.incrementToken());
        ts.end();
      } finally {
        IOUtils.closeWhileHandlingException(ts);
      }
    }
    String[] sentences = line.split("、|。");

View Full Code Here

      for (String sentence: sentences) {
        TokenStream ts = analyzer.tokenStream("ignored", sentence);
        try {
          ts.reset();
          while(ts.incrementToken());
          ts.end();
        } finally {
          IOUtils.closeWhileHandlingException(ts);
        }
      }
    }

View Full Code Here

        CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
        ts.reset();
        while (ts.incrementToken()) {
          assertTrue(UnicodeUtil.validUTF16String(termAtt));
        }
        ts.end();
      } finally {
        IOUtils.closeWhileHandlingException(ts);
      }
    }
  }

View Full Code Here

        assertEquals(UScript.LATIN, scriptAtt.getCode());
        assertEquals(UScript.getName(UScript.LATIN), scriptAtt.getName());
        assertEquals(UScript.getShortName(UScript.LATIN), scriptAtt.getShortName());
        assertTrue(ts.reflectAsString(false).contains("script=Latin"));
      }
      ts.end();
    } finally {
      IOUtils.closeWhileHandlingException(ts);
    }
  }
}

View Full Code Here

      stream.reset();
      assertTrue(stream.incrementToken());
      assertEquals(0, offsetAtt.startOffset());
      assertEquals(4, offsetAtt.endOffset());
      assertFalse(stream.incrementToken());
      stream.end();
    } finally {
      IOUtils.closeWhileHandlingException(stream);
    }
  }

View Full Code Here

          termFreqMap.put(word, new Int());
        } else {
          cnt.x++;
        }
      }
      ts.end();
    } finally {
      IOUtils.closeWhileHandlingException(ts);
    }
  }

View Full Code Here

      if (!source.incrementToken())
        throw new IllegalArgumentException("analyzer returned no terms for multiTerm term: " + part);
      termAtt.fillBytesRef();
      if (source.incrementToken())
        throw new IllegalArgumentException("analyzer returned too many terms for multiTerm term: " + part);
      source.end();
      return BytesRef.deepCopyOf(bytes);
    } catch (IOException e) {
      throw new RuntimeException("Error analyzing multiTerm term: " + part, e);
    } finally {
      IOUtils.closeWhileHandlingException(source);

View Full Code Here

            reuse.length++;
          }
          System.arraycopy(termAtt.buffer(), 0, reuse.chars, end, length);
          reuse.length += length;
        }
        ts.end();
      } catch (IOException e) {
        priorException = e;
      } finally {
        IOUtils.closeWhileHandlingException(priorException, ts);
      }

View Full Code Here

        CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class);
        tokenStream.reset();
        while (tokenStream.incrementToken()) {
          result.add(charTermAttribute.toString());
        }
        tokenStream.end();
      } finally {
        IOUtils.closeWhileHandlingException(tokenStream);
      }
    }
    return result.toArray(new String[result.size()]);

View Full Code Here

      stream.reset();
    
      while (stream.incrementToken()) {
        assertFalse(inValidTokens.contains(termAtt.toString()));
      }
      stream.end();
    } finally {
      IOUtils.closeWhileHandlingException(stream);
    }
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.