Examples of org.apache.lucene.analysis.Token.term()

org.apache.lucene.analysis.Token.term()
Returns the Token's term text. This method has a performance penalty because the text is stored internally in a char[]. If possible, use {@link #termBuffer()} and {@link #termLength()} directly instead. If you really need aString, use this method, which is nothing more than a convenience call to new String(token.termBuffer(), 0, token.termLength())

    assert reusableToken != null;
    Token nextToken = input.next(reusableToken);
    if (nextToken == null)
      return null;


    String term = nextToken.term();


    // Check the exclusion table.
    if (exclusions == null || !exclusions.contains(term)) {
      String s = stemmer.stem(term);
      // If not stemmed, don't waste the time adjusting the token.

View Full Code Here

                assert reusableToken != null;
                Token nextToken = input.next(reusableToken);
    if (nextToken == null)
      return null;


    String term = nextToken.term();


    // Check the exclusion table
    if ( exclusions == null || !exclusions.contains( term ) ) {
      String s = stemmer.stem( term );
      // If not stemmed, don't waste the time  adjusting the token.

View Full Code Here

        assert reusableToken != null;
        Token nextToken = input.next(reusableToken);
        if (nextToken == null)
            return null;


        String term = nextToken.term();
        String s = stemmer.stem(term);
        if (s != null && !s.equals(term))
          nextToken.setTermBuffer(s);
        return nextToken;
    }

View Full Code Here

    Token nextToken = input.next(reusableToken);
    if (nextToken == null) {
      return null;
    }


    String type = this.tree.getOEMap().get(nextToken.term());
    if (type != null) {
      for (int i = 0; i < tree.getWeightBase() - 1; i++) {
        Token t = new Token();
        t.reinit(nextToken);
        queue.offer(t);

View Full Code Here

      Token nextToken = input.next(reusableToken);


      if (nextToken == null)
        return null;


      String term = nextToken.term();
      // Check the exclusion table.
      if (exclusionSet == null || !exclusionSet.contains(term)) {
        String s = stemmer.stem(term);
        // If not stemmed, don't waste the time adjusting the token.
        if ((s != null) && !s.equals(term))

View Full Code Here

    Token nextToken = input.next(reusableToken);
    if (nextToken == null || nextToken.termLength() == 0) {
      return null;
    }


    String text = nextToken.term();
    if (UnicodeBlock.of(text.charAt(0)) != UnicodeBlock.THAI) {
      nextToken.setTermBuffer(text.toLowerCase());
      return nextToken;
    }

View Full Code Here

    assert reusableToken != null;
    Token nextToken = input.next(reusableToken);
    if (nextToken == null)
      return null;


    String term = nextToken.term();


    // Check the exclusion table.
    if (exclusions == null || !exclusions.contains(term)) {
      String s = stemmer.stem(term);
      // If not stemmed, don't waste the time adjusting the token.

View Full Code Here

        Reader r = new StringReader(text);
        TokenStream ts = index.getTextAnalyzer().tokenStream("", r);
        Token t = new Token();
        try {
            while ((t = ts.next(t)) != null) {
                String termText = t.term();
                TermVectorOffsetInfo[] info =
                        (TermVectorOffsetInfo[]) termMap.get(termText);
                if (info == null) {
                    info = new TermVectorOffsetInfo[1];
                } else {

View Full Code Here

        Reader r = new StringReader(text);
        TokenStream ts = index.getTextAnalyzer().tokenStream("", r);
        Token t = new Token();
        try {
            while ((t = ts.next(t)) != null) {
                String termText = t.term();
                TermVectorOffsetInfo[] info =
                        (TermVectorOffsetInfo[]) termMap.get(termText);
                if (info == null) {
                    info = new TermVectorOffsetInfo[1];
                } else {

View Full Code Here

        Reader r = new StringReader(text);
        TokenStream ts = index.getTextAnalyzer().tokenStream("", r);
        Token t = new Token();
        try {
            while ((t = ts.next(t)) != null) {
                String termText = t.term();
                TermVectorOffsetInfo[] info = termMap.get(termText);
                if (info == null) {
                    info = new TermVectorOffsetInfo[1];
                } else {
                    TermVectorOffsetInfo[] tmp = info;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.