Examples of org.apache.lucene.analysis.Token.startOffset()

org.apache.lucene.analysis.Token.startOffset()
Returns this Token's starting offset, the position of the first character corresponding to this token in the source text. Note that the difference between endOffset() and startOffset() may not be equal to {@link #termLength}, as the term text may have been altered by a stemmer or some other filter.

  @Override
  public Token next(Token token) throws IOException {
    if(tokenQueue.size() == 0) return null;
    if(token == null) token = new Token();
    Token fromQueue = tokenQueue.poll();
    token.reinit(fromQueue.term(), fromQueue.startOffset(), fromQueue.endOffset());
    return token;
  }
  
  @Override
  public Token next() throws IOException {

View Full Code Here

    int pos=0;
    for (Iterator iter = a.iterator(); iter.hasNext();) {
      Token tok = (Token)iter.next();
      pos += tok.getPositionIncrement();
      if (!tokAt(b, new String(tok.termBuffer(), 0, tok.termLength()), pos
              , checkOff ? tok.startOffset() : -1
              , checkOff ? tok.endOffset() : -1
              )) 
      {
        fail(a + "!=" + b);
      }

View Full Code Here

    int pos=0;
    for (Iterator iter = lst.iterator(); iter.hasNext();) {
      Token tok = (Token)iter.next();
      pos += tok.getPositionIncrement();
      if (pos==tokPos && new String(tok.termBuffer(), 0, tok.termLength()).equals(val)
          && (startOff==-1 || tok.startOffset()==startOff)
          && (endOff  ==-1 || tok.endOffset()  ==endOff  )
           )
      {
        return true;
      }

View Full Code Here

    
    Token t = wdf.next();
    
    assertNotNull(t);
    assertEquals("übelkeit", t.term());
    assertEquals(7, t.startOffset());
    assertEquals(15, t.endOffset());
  }
  
  public void testOffsetChange2() throws Exception
  {

View Full Code Here

    
    Token t = wdf.next();
    
    assertNotNull(t);
    assertEquals("übelkeit", t.term());
    assertEquals(8, t.startOffset());
    assertEquals(17, t.endOffset());
  }
  
  public void testOffsetChange3() throws Exception
  {

View Full Code Here

    
    Token t = wdf.next();
    
    assertNotNull(t);
    assertEquals("übelkeit", t.term());
    assertEquals(8, t.startOffset());
    assertEquals(16, t.endOffset());
  }
  
  public void testOffsetChange4() throws Exception
  {

View Full Code Here

    
    Token t = wdf.next();
    
    assertNotNull(t);
    assertEquals("foo", t.term());
    assertEquals(8, t.startOffset());
    assertEquals(11, t.endOffset());
    
    t = wdf.next();
    
    assertNotNull(t);

View Full Code Here

    
    t = wdf.next();
    
    assertNotNull(t);
    assertEquals("bar", t.term());
    assertEquals(12, t.startOffset());
    assertEquals(15, t.endOffset());
  }


  public void testAlphaNumericWords(){
     assertU(adoc("id",  "68","numericsubword","Java/J2SE"));

View Full Code Here

        t.setTermBuffer(t.termBuffer(), start, (end - start));
      } else {
        t.setTermLength(0);
      }
      if (updateOffsets) {
        t.setStartOffset(t.startOffset() + start);
        if (start < end) {
          t.setEndOffset(t.endOffset() - endOff);
        } //else if end is less than, start, then the term length is 0, so, no need to bother w/ the end offset
      }
      /*t = new Token( t.termText().substring( start, end ),

View Full Code Here

      Token inputToken = entry.getKey();
      Map<String, Integer> theSuggestions = entry.getValue();
      if (theSuggestions != null && theSuggestions.size() > 0) {
        SimpleOrderedMap suggestionList = new SimpleOrderedMap();
        suggestionList.add("numFound", theSuggestions.size());
        suggestionList.add("startOffset", inputToken.startOffset());
        suggestionList.add("endOffset", inputToken.endOffset());
        if (extendedResults && hasFreqInfo) {
          suggestionList.add("origFreq", spellingResult.getTokenFrequency(inputToken));
          for (Map.Entry<String, Integer> suggEntry : theSuggestions.entrySet()) {
            SimpleOrderedMap<Object> suggestionItem = new SimpleOrderedMap<Object>();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.