Examples of org.apache.lucene.analysis.Token.endOffset()

org.apache.lucene.analysis.Token.endOffset()
Returns this Token's ending offset, one greater than the position of the last character corresponding to this token in the source text. The length of the token in the source text is (endOffset - startOffset).

      if (tok.getPositionIncrement() == 0)
        continue;
      collation.replace(tok.startOffset() + offset, tok.endOffset() + offset,
          correction.getCorrection());
      offset += correction.getCorrection().length()
          - (tok.endOffset() - tok.startOffset());
    }
    return collation.toString();
  }


}

View Full Code Here

      Map<String, Integer> theSuggestions = entry.getValue();
      if (theSuggestions != null && (theSuggestions.size()>0 || shardRequest)) {
        SimpleOrderedMap suggestionList = new SimpleOrderedMap();
        suggestionList.add("numFound", theSuggestions.size());
        suggestionList.add("startOffset", inputToken.startOffset());
        suggestionList.add("endOffset", inputToken.endOffset());


        // Logical structure of normal (non-extended) results:
        // "suggestion":["alt1","alt2"]
        //
        // Logical structure of the extended results:

View Full Code Here

      public boolean incrementToken() throws IOException {
        if(iter.hasNext()) {
          Token token = (Token) iter.next();
          termAtt.setTermBuffer(token.term());
          posIncrAtt.setPositionIncrement(token.getPositionIncrement());
          offsetAtt.setOffset(token.startOffset(), token.endOffset());
          return true;
        }
        return false;
      }

View Full Code Here

      public boolean incrementToken() throws IOException {
        if(iter.hasNext()) {
          Token token = (Token) iter.next();
          termAtt.setTermBuffer(token.term());
          posIncrAtt.setPositionIncrement(token.getPositionIncrement());
          offsetAtt.setOffset(token.startOffset(), token.endOffset());
          return true;
        }
        return false;
      }
    };

View Full Code Here

      }
      Token prototype = (Token) iterator.next();
      termAtt.setTermBuffer(prototype.termBuffer(), 0, prototype.termLength());
      posIncrAtt.setPositionIncrement(prototype.getPositionIncrement());
      flagsAtt.setFlags(prototype.getFlags());
      offsetAtt.setOffset(prototype.startOffset(), prototype.endOffset());
      typeAtt.setType(prototype.type());
      payloadAtt.setPayload(prototype.getPayload());


      return true;
    }

View Full Code Here

    public final boolean incrementToken() throws IOException {
      clearAttributes();
      if (index < testToken.length) {
        Token t = testToken[index++];
        termAtt.setTermBuffer(t.termBuffer(), 0, t.termLength());
        offsetAtt.setOffset(t.startOffset(), t.endOffset());
        posIncrAtt.setPositionIncrement(t.getPositionIncrement());
        typeAtt.setType(TypeAttributeImpl.DEFAULT_TYPE);
        return true;
      } else {
        return false;

View Full Code Here

            if (currentToken >= tokens.length) {
              return false;
            }
            Token token = tokens[currentToken++];
            termAtt.setTermBuffer(token.term());
            offsetAtt.setOffset(token.startOffset(), token.endOffset());
            return true;
          }
        }      
        //code to reconstruct the original sequence of Tokens
        String[] terms=tpv.getTerms();

View Full Code Here

            Arrays.sort(tokensInOriginalOrder, new Comparator(){
                public int compare(Object o1, Object o2)
                {
                    Token t1=(Token) o1;
                    Token t2=(Token) o2;
                    if(t1.startOffset()>t2.endOffset())
                        return 1;
                    if(t1.startOffset()<t2.startOffset())
                        return -1;
                    return 0;
                }});

View Full Code Here

          termDocumentInformationFactory.payloads.add(null);
        }


        if (eField_Tokens.getKey().isStoreOffsetWithTermVector()) {


          termDocumentInformationFactory.termOffsets.add(new TermVectorOffsetInfo(fieldSetting.offset + token.startOffset(), fieldSetting.offset + token.endOffset()));
          lastOffset = fieldSetting.offset + token.endOffset();
        }




      }

View Full Code Here

        }


        if (eField_Tokens.getKey().isStoreOffsetWithTermVector()) {


          termDocumentInformationFactory.termOffsets.add(new TermVectorOffsetInfo(fieldSetting.offset + token.startOffset(), fieldSetting.offset + token.endOffset()));
          lastOffset = fieldSetting.offset + token.endOffset();
        }




      }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.