Examples of org.apache.lucene.analysis.Token.startOffset()

org.apache.lucene.analysis.Token.startOffset()
Returns this Token's starting offset, the position of the first character corresponding to this token in the source text. Note that the difference between endOffset() and startOffset() may not be equal to {@link #termLength}, as the term text may have been altered by a stemmer or some other filter.

      if (increment > 0) 
      { position = position + increment;
        System.out.println();
        System.out.print(position + ": ");
      }
      System.out.print("\t [" + token.termText() + ": " + token.type() + "] " + token.startOffset() + ":" + token.endOffset());
      
     } //*-- end of for
    System.out.println("");
  }

View Full Code Here

   Token ctoken = input.next();  if (ctoken == null) return (null);
   String ctext = ctoken.termText();
   
   //*-- check for an unigram entity
   String etype = ehash.get(ctext);
   if (etype != null) stoken = new Token(etype, ctoken.startOffset(), ctoken.endOffset(), "<ENTITY>"); 
   
   //*-- check for a bigram entity
   if (ptoken != null)    
   { String bigram = ptoken.termText() + " " + ctext;
     etype = ehash.get(bigram);

View Full Code Here

     if (etype != null)
      stoken = new Token(etype, ptoken.startOffset(), ctoken.endOffset(), "<ENTITY>");
   }
   
   if (number.matcher(ctext).matches())
     stoken = new Token("enumber", ctoken.startOffset(), ctoken.endOffset(), "<ENTITY>");
   
   ptoken = ctoken;
   return (ctoken);
  }
 }

View Full Code Here

    try {
      for (; i < size; i++) {
        Token t1 = (Token) tokens1.get(i);
        Token t2 = (Token) tokens2.get(i);
        if (!(t1.termText().equals(t2.termText()))) throw new IllegalStateException("termText");
        if (t1.startOffset() != t2.startOffset()) throw new IllegalStateException("startOffset");
        if (t1.endOffset() != t2.endOffset()) throw new IllegalStateException("endOffset");
        if (!(t1.type().equals(t2.type()))) throw new IllegalStateException("type");
      }
      if (tokens1.size() != tokens2.size())   throw new IllegalStateException("size1=" + tokens1.size() + ", size2=" + tokens2.size());
    }

View Full Code Here


  public void collect(String word, int offset, int end) {
    Token c = candidate != null ? candidate : last;
    if (c == null) {
      candidate = new Token(word, offset, end);
    } else if (offset == c.startOffset()) {
      if (end > c.endOffset()) {
        candidate = new Token(word, offset, end);
      }
    } else if (offset > c.startOffset()) {
      if (candidate != null) {

View Full Code Here

      candidate = new Token(word, offset, end);
    } else if (offset == c.startOffset()) {
      if (end > c.endOffset()) {
        candidate = new Token(word, offset, end);
      }
    } else if (offset > c.startOffset()) {
      if (candidate != null) {
        select(candidate);
      }
      if (end > c.endOffset()) {
        candidate = new Token(word, offset, end);

View Full Code Here

  public boolean incrementToken() throws IOException {
    clearAttributes();
    Token token = nextToken(reusableToken);
    if(token != null) {
      termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
      offsetAtt.setOffset(token.startOffset(), token.endOffset());
      typeAtt.setType(token.type());
      return true;
    } else {
      end();
      return false;

View Full Code Here

          return false;
        }
        Token token = tokens[currentToken++];
        clearAttributes();
        termAtt.setEmpty().append(token);
        offsetAtt.setOffset(token.startOffset(), token.endOffset());
        posincAtt
            .setPositionIncrement(currentToken <= 1
                || tokens[currentToken - 1].startOffset() > tokens[currentToken - 2]
                    .startOffset() ? 1 : 0);
        return true;

View Full Code Here

          return false;
        }
        Token token = tokens[currentToken++];
        clearAttributes();
        termAtt.setEmpty().append(token);
        offsetAtt.setOffset(token.startOffset(), token.endOffset());
        posincAtt
            .setPositionIncrement(currentToken <= 1
                || tokens[currentToken - 1].startOffset() > tokens[currentToken - 2]
                    .startOffset() ? 1 : 0);
        return true;

View Full Code Here

        if(iter.hasNext()) {
          Token token =  iter.next();
          clearAttributes();
          termAtt.setEmpty().append(token);
          posIncrAtt.setPositionIncrement(token.getPositionIncrement());
          offsetAtt.setOffset(token.startOffset(), token.endOffset());
          return true;
        }
        return false;
      }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.