Examples of org.apache.lucene.analysis.Token.term()

org.apache.lucene.analysis.Token.term()
Returns the Token's term text. This method has a performance penalty because the text is stored internally in a char[]. If possible, use {@link #termBuffer()} and {@link #termLength()} directly instead. If you really need aString, use this method, which is nothing more than a convenience call to new String(token.termBuffer(), 0, token.termLength())

        fieldSetting.position += analyzer.getPositionIncrementGap(fieldSetting.fieldName);
      }


      for (Token token : eField_Tokens.getValue()) {


        TermDocumentInformationFactory termDocumentInformationFactory = termDocumentInformationFactoryByTermText.get(token.term());
        if (termDocumentInformationFactory == null) {
          termDocumentInformationFactory = new TermDocumentInformationFactory();
          termDocumentInformationFactoryByTermText.put(token.term(), termDocumentInformationFactory);
        }
        //termDocumentInformationFactory.termFrequency++;

View Full Code Here

      for (Token token : eField_Tokens.getValue()) {


        TermDocumentInformationFactory termDocumentInformationFactory = termDocumentInformationFactoryByTermText.get(token.term());
        if (termDocumentInformationFactory == null) {
          termDocumentInformationFactory = new TermDocumentInformationFactory();
          termDocumentInformationFactoryByTermText.put(token.term(), termDocumentInformationFactory);
        }
        //termDocumentInformationFactory.termFrequency++;


        fieldSetting.position += (token.getPositionIncrement() - 1);
        termDocumentInformationFactory.termPositions.add(fieldSetting.position++);

View Full Code Here

          public boolean incrementToken() throws IOException {
            if (currentToken >= tokens.length) {
              return false;
            }
            Token token = tokens[currentToken++];
            termAtt.setTermBuffer(token.term());
            offsetAtt.setOffset(token.startOffset(), token.endOffset());
            return true;
          }
        }      
        //code to reconstruct the original sequence of Tokens

View Full Code Here

        token.setStartOffset(savedStart);
        token.setEndOffset(savedEnd);
        return reusableToken;
      } else
        while ((token = this.input.next(reusableToken)) != null) {
          if (token.term().equals("phrase")) {
            inPhrase = true;
            savedStart = token.startOffset();
            savedEnd = token.endOffset();
            token.setTermBuffer("phrase1");
            token.setStartOffset(savedStart);

View Full Code Here

            savedEnd = token.endOffset();
            token.setTermBuffer("phrase1");
            token.setStartOffset(savedStart);
            token.setEndOffset(savedEnd);
            return token;
          } else if (!token.term().equals("stop"))
            return token;
        }


      return null;

View Full Code Here

      Token t = new Token();
      try
      {
         while ((t = ts.next(t)) != null)
         {
            String termText = t.term();
            TermVectorOffsetInfo[] info = (TermVectorOffsetInfo[])termMap.get(termText);
            if (info == null)
            {
               info = new TermVectorOffsetInfo[1];
            }

View Full Code Here

        Reader r = new StringReader(text);
        TokenStream ts = index.getTextAnalyzer().tokenStream("", r);
        Token t = new Token();
        try {
            while ((t = ts.next(t)) != null) {
                String termText = t.term();
                TermVectorOffsetInfo[] info = termMap.get(termText);
                if (info == null) {
                    info = new TermVectorOffsetInfo[1];
                } else {
                    TermVectorOffsetInfo[] tmp = info;

View Full Code Here

        while ((j < endToken) && (j - startToken < sumLength)) {
          //
          // Now grab the hit-element, if present
          //
          Token t = tokens[j];
          if (highlight.contains(t.term())) {
            excerpt.addToken(t.term());
            excerpt.add(new Fragment(text.substring(offset, t.startOffset())));
            excerpt.add(new Highlight(text.substring(t.startOffset(),t.endOffset())));
            offset = t.endOffset();
            endToken = Math.min(j + sumContext, tokens.length);

View Full Code Here

          //
          // Now grab the hit-element, if present
          //
          Token t = tokens[j];
          if (highlight.contains(t.term())) {
            excerpt.addToken(t.term());
            excerpt.add(new Fragment(text.substring(offset, t.startOffset())));
            excerpt.add(new Highlight(text.substring(t.startOffset(),t.endOffset())));
            offset = t.endOffset();
            endToken = Math.min(j + sumContext, tokens.length);
          }

View Full Code Here

        Reader r = new StringReader(text);
        TokenStream ts = index.getTextAnalyzer().tokenStream("", r);
        Token t = new Token();
        try {
            while ((t = ts.next(t)) != null) {
                String termText = t.term();
                TermVectorOffsetInfo[] info =
                        (TermVectorOffsetInfo[]) termMap.get(termText);
                if (info == null) {
                    info = new TermVectorOffsetInfo[1];
                } else {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.