Examples of org.apache.lucene.analysis.Token.startOffset()

org.apache.lucene.analysis.Token.startOffset()
Returns this Token's starting offset, the position of the first character corresponding to this token in the source text. Note that the difference between endOffset() and startOffset() may not be equal to {@link #termLength}, as the term text may have been altered by a stemmer or some other filter.

        if(iter.hasNext()) {
          Token token = iter.next();
          clearAttributes();
          termAtt.setEmpty().append(token);
          posIncrAtt.setPositionIncrement(token.getPositionIncrement());
          offsetAtt.setOffset(token.startOffset(), token.endOffset());
          return true;
        }
        return false;
      }
    };

View Full Code Here


    clearAttributes();
    termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
    posIncrAtt.setPositionIncrement(token.getPositionIncrement());
    flagsAtt.setFlags(token.getFlags());
    offsetAtt.setOffset(token.startOffset(), token.endOffset());
    typeAtt.setType(token.type());
    payloadAtt.setPayload(token.getPayload());
    return true;
  }

View Full Code Here

          termDocumentInformationFactory.payloads.add(null);
        }


        if (eField_Tokens.getKey().isStoreOffsetWithTermVector()) {


          termDocumentInformationFactory.termOffsets.add(new TermVectorOffsetInfo(fieldSetting.offset + token.startOffset(), fieldSetting.offset + token.endOffset()));
          lastOffset = fieldSetting.offset + token.endOffset();
        }




      }

View Full Code Here

              return false;
            }
            clearAttributes();
            Token token = tokens[currentToken++];
            termAtt.setTermBuffer(token.term());
            offsetAtt.setOffset(token.startOffset(), token.endOffset());
            return true;
          }
        }      
        //code to reconstruct the original sequence of Tokens
        String[] terms=tpv.getTerms();

View Full Code Here

              return false;
            }
            clearAttributes();
            Token token = tokens[currentToken++];
            termAtt.setTermBuffer(token.term());
            offsetAtt.setOffset(token.startOffset(), token.endOffset());
            return true;
          }
        }      
        //code to reconstruct the original sequence of Tokens
        String[] terms=tpv.getTerms();

View Full Code Here

        return false;
      else {
        clearAttributes();
        Token token = tokens[index++];
        termAtt.setEmpty().append(token);
        offsetAtt.setOffset(token.startOffset(), token.endOffset());
        posIncAtt.setPositionIncrement(token.getPositionIncrement());
        flagsAtt.setFlags(token.getFlags());
        typeAtt.setType(token.type());
        payloadAtt.setPayload(token.getPayload());
        return true;

View Full Code Here

      Token tok = correction.getOriginal();
      // we are replacing the query in order, but injected terms might cause
      // illegal offsets due to previous replacements.
      if (tok.getPositionIncrement() == 0)
        continue;
      collation.replace(tok.startOffset() + offset, tok.endOffset() + offset,
          correction.getCorrection());
      offset += correction.getCorrection().length()
          - (tok.endOffset() - tok.startOffset());
    }
    return collation.toString();

View Full Code Here

      if (tok.getPositionIncrement() == 0)
        continue;
      collation.replace(tok.startOffset() + offset, tok.endOffset() + offset,
          correction.getCorrection());
      offset += correction.getCorrection().length()
          - (tok.endOffset() - tok.startOffset());
    }
    return collation.toString();
  }


}

View Full Code Here

      Token inputToken = entry.getKey();
      Map<String, Integer> theSuggestions = entry.getValue();
      if (theSuggestions != null && (theSuggestions.size()>0 || shardRequest)) {
        SimpleOrderedMap suggestionList = new SimpleOrderedMap();
        suggestionList.add("numFound", theSuggestions.size());
        suggestionList.add("startOffset", inputToken.startOffset());
        suggestionList.add("endOffset", inputToken.endOffset());


        // Logical structure of normal (non-extended) results:
        // "suggestion":["alt1","alt2"]
        //

View Full Code Here

      public boolean incrementToken() throws IOException {
        if(iter.hasNext()) {
          Token token = (Token) iter.next();
          termAtt.setTermBuffer(token.term());
          posIncrAtt.setPositionIncrement(token.getPositionIncrement());
          offsetAtt.setOffset(token.startOffset(), token.endOffset());
          return true;
        }
        return false;
      }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.