Examples of org.apache.lucene.analysis.Token.term()

org.apache.lucene.analysis.Token.term()
Returns the Token's term text. This method has a performance penalty because the text is stored internally in a char[]. If possible, use {@link #termBuffer()} and {@link #termLength()} directly instead. If you really need aString, use this method, which is nothing more than a convenience call to new String(token.termBuffer(), 0, token.termLength())

        Reader r = new StringReader(text);
        TokenStream ts = index.getTextAnalyzer().tokenStream("", r);
        Token t = new Token();
        try {
            while ((t = ts.next(t)) != null) {
                String termText = t.term();
                TermVectorOffsetInfo[] info = termMap.get(termText);
                if (info == null) {
                    info = new TermVectorOffsetInfo[1];
                } else {
                    TermVectorOffsetInfo[] tmp = info;

View Full Code Here

  }


  public Token next(final Token reusableToken) throws IOException {
    Token nextToken = input.next( reusableToken );
    if ( nextToken != null ) {
      nextToken.setTermBuffer( " " + nextToken.term() + " " );
      return nextToken;
    }
    else {
      return null;
    }

View Full Code Here

    if( startOffset == -1 )
      startOffset = nextToken.startOffset();
    
    while( nextToken != null ){
      lastToken = nextToken;
      stringBuffer.append(nextToken.term());
      nextToken = input.next(reusableToken);
      if( nextToken != null )
        stringBuffer.append(concatString);
    }

View Full Code Here

    );
    
    Token t = wdf.next();
    
    assertNotNull(t);
    assertEquals("übelkeit", t.term());
    assertEquals(7, t.startOffset());
    assertEquals(15, t.endOffset());
  }
  
  public void testOffsetChange2() throws Exception

View Full Code Here

    );
    
    Token t = wdf.next();
    
    assertNotNull(t);
    assertEquals("übelkeit", t.term());
    assertEquals(8, t.startOffset());
    assertEquals(17, t.endOffset());
  }
  
  public void testOffsetChange3() throws Exception

View Full Code Here

    );
    
    Token t = wdf.next();
    
    assertNotNull(t);
    assertEquals("übelkeit", t.term());
    assertEquals(8, t.startOffset());
    assertEquals(16, t.endOffset());
  }
  
  public void testOffsetChange4() throws Exception

View Full Code Here

    );
    
    Token t = wdf.next();
    
    assertNotNull(t);
    assertEquals("foo", t.term());
    assertEquals(8, t.startOffset());
    assertEquals(11, t.endOffset());
    
    t = wdf.next();

View Full Code Here

    assertEquals(11, t.endOffset());
    
    t = wdf.next();
    
    assertNotNull(t);
    assertEquals("bar", t.term());
    assertEquals(12, t.startOffset());
    assertEquals(15, t.endOffset());
  }


  public void testAlphaNumericWords(){

View Full Code Here

            ,1,1,0,0,0
    );


    for(String expected : output) {
      Token t = wdf.next();
      assertEquals(expected, t.term());
    }


    assertEquals(null, wdf.next());
  }

View Full Code Here

            ,1,1,0,0,0,1,0,1,stemPossessive,null
    );


    for(String expected : output) {
      Token t = wdf.next();
      assertEquals(expected, t.term());
    }


    assertEquals(null, wdf.next());
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.