Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.endOffset()


      if (increment > 0)
      { position = position + increment;
        System.out.println();
        System.out.print(position + ": ");
      }
      System.out.print("\t [" + token.termText() + ": " + token.type() + "] " + token.startOffset() + ":" + token.endOffset());
     
     } //*-- end of for
    System.out.println("");
  }

View Full Code Here


   Token ctoken = input.next()if (ctoken == null) return (null);
   String ctext = ctoken.termText();
  
   //*-- check for an unigram entity
   String etype = ehash.get(ctext);
   if (etype != null) stoken = new Token(etype, ctoken.startOffset(), ctoken.endOffset(), "<ENTITY>");
  
   //*-- check for a bigram entity
   if (ptoken != null)   
   { String bigram = ptoken.termText() + " " + ctext;
     etype = ehash.get(bigram);
View Full Code Here

   //*-- check for a bigram entity
   if (ptoken != null)   
   { String bigram = ptoken.termText() + " " + ctext;
     etype = ehash.get(bigram);
     if (etype != null)
      stoken = new Token(etype, ptoken.startOffset(), ctoken.endOffset(), "<ENTITY>");
   }
  
   if (number.matcher(ctext).matches())
     stoken = new Token("enumber", ctoken.startOffset(), ctoken.endOffset(), "<ENTITY>");
  
View Full Code Here

     if (etype != null)
      stoken = new Token(etype, ptoken.startOffset(), ctoken.endOffset(), "<ENTITY>");
   }
  
   if (number.matcher(ctext).matches())
     stoken = new Token("enumber", ctoken.startOffset(), ctoken.endOffset(), "<ENTITY>");
  
   ptoken = ctoken;
   return (ctoken);
  }
}
View Full Code Here

      for (; i < size; i++) {
        Token t1 = (Token) tokens1.get(i);
        Token t2 = (Token) tokens2.get(i);
        if (!(t1.termText().equals(t2.termText()))) throw new IllegalStateException("termText");
        if (t1.startOffset() != t2.startOffset()) throw new IllegalStateException("startOffset");
        if (t1.endOffset() != t2.endOffset()) throw new IllegalStateException("endOffset");
        if (!(t1.type().equals(t2.type()))) throw new IllegalStateException("type");
      }
      if (tokens1.size() != tokens2.size())   throw new IllegalStateException("size1=" + tokens1.size() + ", size2=" + tokens2.size());
    }
View Full Code Here

  public void collect(String word, int offset, int end) {
    Token c = candidate != null ? candidate : last;
    if (c == null) {
      candidate = new Token(word, offset, end);
    } else if (offset == c.startOffset()) {
      if (end > c.endOffset()) {
        candidate = new Token(word, offset, end);
      }
    } else if (offset > c.startOffset()) {
      if (candidate != null) {
        select(candidate);
View Full Code Here

      }
    } else if (offset > c.startOffset()) {
      if (candidate != null) {
        select(candidate);
      }
      if (end > c.endOffset()) {
        candidate = new Token(word, offset, end);
      } else {
        candidate = null;
      }
    } else if (end >= c.endOffset()) {
View Full Code Here

      if (end > c.endOffset()) {
        candidate = new Token(word, offset, end);
      } else {
        candidate = null;
      }
    } else if (end >= c.endOffset()) {
      if (last != null && last.startOffset() >= offset
          && last.endOffset() <= end) {
        for (Iterator iter = tokens.iterator(); iter.hasNext();) {
          last = (Token) iter.next();
          if (last.startOffset() >= offset && last.endOffset() <= end) {
View Full Code Here

  public boolean incrementToken() throws IOException {
    clearAttributes();
    Token token = nextToken(reusableToken);
    if(token != null) {
      termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
      offsetAtt.setOffset(token.startOffset(), token.endOffset());
      typeAtt.setType(token.type());
      return true;
    } else {
      end();
      return false;
View Full Code Here

          return false;
        }
        Token token = tokens[currentToken++];
        clearAttributes();
        termAtt.setEmpty().append(token);
        offsetAtt.setOffset(token.startOffset(), token.endOffset());
        posincAtt
            .setPositionIncrement(currentToken <= 1
                || tokens[currentToken - 1].startOffset() > tokens[currentToken - 2]
                    .startOffset() ? 1 : 0);
        return true;
View Full Code Here

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.