Examples of org.apache.lucene.analysis.Token.term()

org.apache.lucene.analysis.Token.term()
Returns the Token's term text. This method has a performance penalty because the text is stored internally in a char[]. If possible, use {@link #termBuffer()} and {@link #termLength()} directly instead. If you really need aString, use this method, which is nothing more than a convenience call to new String(token.termBuffer(), 0, token.termLength())

    );
    
    Token t = wdf.next();
    
    assertNotNull(t);
    assertEquals("übelkeit", t.term());
    assertEquals(8, t.startOffset());
    assertEquals(16, t.endOffset());
  }
  
  public void testOffsetChange4() throws Exception

View Full Code Here

    );
    
    Token t = wdf.next();
    
    assertNotNull(t);
    assertEquals("foo", t.term());
    assertEquals(8, t.startOffset());
    assertEquals(11, t.endOffset());
    
    t = wdf.next();

View Full Code Here

    assertEquals(11, t.endOffset());
    
    t = wdf.next();
    
    assertNotNull(t);
    assertEquals("bar", t.term());
    assertEquals(12, t.startOffset());
    assertEquals(15, t.endOffset());
  }


  public void testAlphaNumericWords(){

View Full Code Here

            ,1,1,0,0,0
    );


    for(String expected : output) {
      Token t = wdf.next();
      assertEquals(expected, t.term());
    }


    assertEquals(null, wdf.next());
  }

View Full Code Here

            ,1,1,0,0,0,1,0,1,stemPossessive,null
    );


    for(String expected : output) {
      Token t = wdf.next();
      assertEquals(expected, t.term());
    }


    assertEquals(null, wdf.next());
  }

View Full Code Here

        break;
      for (int i = 0; i < t.getPositionIncrement(); i++)
        outBuf.append('/');
      if (t.getPositionIncrement() == 0)
        outBuf.append(',');
      outBuf.append(t.term());
    }


    String out = outBuf.toString();
    out = out.replaceAll(" ", "");
    return out;

View Full Code Here


    Token got = new Token();
    for( Token t : output ) {
      got = filter.next(got);
      // System.out.println("##### expect=" + t + " got="+got);
      assertEquals( t.term(), got.term());
    }
    assertNull( filter.next() );  // no more tokens
  }
  
  public void testEncodes() throws Exception {

View Full Code Here

    boolean readNext = true;
    Token t = null;
    while(readNext) {
      t = input.next(token);
      if (t == null) return null;
      readNext = !t.term().matches(".*[a-zA-Z0-9].*");
    }
    return t;
  }
  
}

View Full Code Here

  
  @Override
  public Token next(Token token) throws IOException {
    Token t = input.next(token);
    if (t == null) return null;
    t.setTermBuffer(StringTools.normaliseName2(t.term()));
    return t;
  }


  @Override
  public Token next() throws IOException {

View Full Code Here

        Reader r = new StringReader(text);
        TokenStream ts = index.getTextAnalyzer().tokenStream("", r);
        Token t = new Token();
        try {
            while ((t = ts.next(t)) != null) {
                String termText = t.term();
                TermVectorOffsetInfo[] info = termMap.get(termText);
                if (info == null) {
                    info = new TermVectorOffsetInfo[1];
                } else {
                    TermVectorOffsetInfo[] tmp = info;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.