Examples of org.apache.lucene.analysis.Token.termText()

org.apache.lucene.analysis.Token.termText()
Returns the Token's term text. @deprecated This method now has a performance penaltybecause the text is stored internally in a char[]. If possible, use {@link #termBuffer()} and {@link #termLength()} directly instead. If you really need aString, use {@link #term()}

        Token t = input.next();


        if (t == null)
            return null;


        String txt = t.termText();


        char[] chArray = txt.toCharArray();
        for (int i = 0; i < chArray.length; i++)
        {
            chArray[i] = GreekCharsets.toLowerCase(chArray[i], charset);

View Full Code Here

            assertEquals(
                "Unicode",
                token.termText(),
                sampleToken == null
                ? null
                : sampleToken.termText());
        }


        inWords.close();
        sampleUnicode.close();
    }

View Full Code Here

            assertEquals(
                "KOI8",
                token.termText(),
                sampleToken == null
                ? null
                : sampleToken.termText());


        }


        inWordsKOI8.close();
        sampleKOI8.close();

View Full Code Here

            assertEquals(
                "1251",
                token.termText(),
                sampleToken == null
                ? null
                : sampleToken.termText());


        }


        inWords1251.close();
        sample1251.close();

View Full Code Here

                try
                {
                  Token stopToken=ts.next();
                  while(stopToken!=null)
                  {
                      stopWordsSet.add(stopToken.termText());
                      stopToken=ts.next();
                  }
                }
                catch(IOException ioe)
                {

View Full Code Here

    }
    Token tk = input.next();
    if (tk == null) {
      return null;
    }
    String text = tk.termText();
    if (UnicodeBlock.of(text.charAt(0)) != UnicodeBlock.THAI) {
      return new Token(text.toLowerCase(), tk.startOffset(), tk.endOffset(), tk.type());
    }
    thaiToken = tk;
    breaker.setText(text);

View Full Code Here

      int numTokens = 0;
      int pos = -1;
      Token token;
      
      while ((token = stream.next()) != null) {
        String term = token.termText();
        if (term.length() == 0) continue; // nothing to do
//        if (DEBUG) System.err.println("token='" + term + "'");
        numTokens++;
        pos += token.getPositionIncrement();

View Full Code Here

                                new StringReader("one two three four five"));


    while (true) {
      Token token = ts.next();
      if (token == null) break;
      assertEquals(token.termText(), 1, token.getPositionIncrement());
    }
  }
}

View Full Code Here

    int i=0;
    try {
      for (; i < size; i++) {
        Token t1 = (Token) tokens1.get(i);
        Token t2 = (Token) tokens2.get(i);
        if (!(t1.termText().equals(t2.termText()))) throw new IllegalStateException("termText");
        if (t1.startOffset() != t2.startOffset()) throw new IllegalStateException("startOffset");
        if (t1.endOffset() != t2.endOffset()) throw new IllegalStateException("endOffset");
        if (!(t1.type().equals(t2.type()))) throw new IllegalStateException("type");
      }
      if (tokens1.size() != tokens2.size())   throw new IllegalStateException("size1=" + tokens1.size() + ", size2=" + tokens2.size());

View Full Code Here

  private String toString(List tokens) {
    if (tokens == null) return "null";
    String str = "[";
    for (int i=0; i < tokens.size(); i++) {
      Token t1 = (Token) tokens.get(i);
      str = str + "'" + t1.termText() + "', ";
    }
    return str + "]";
  }
  
  // trick to detect default platform charset

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.