Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.termBuffer()

455
456
457
458
459
460
461
462
463
464
465
        for (Iterator iterator = shingle.iterator(); iterator.hasNext();) {
          Token shingleToken = (Token) iterator.next();
          if (spacerCharacter != null && sb.length() > 0) {
            sb.append(spacerCharacter);
          }
          sb.append(shingleToken.termBuffer(), 0, shingleToken.termLength());
        }
        reusableToken.setTermBuffer(sb.toString());
        updateToken(reusableToken, shingle, currentPermutationTokensStartOffset, currentPermutationRows, currentPermuationTokens);

        return reusableToken;
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
    @Override
    public final boolean incrementToken() throws IOException {
      clearAttributes();
      if (index < testToken.length) {
        Token t = testToken[index++];
        termAtt.setTermBuffer(t.termBuffer(), 0, t.termLength());
        offsetAtt.setOffset(t.startOffset(), t.endOffset());
        posIncrAtt.setPositionIncrement(t.getPositionIncrement());
        typeAtt.setType(TypeAttributeImpl.DEFAULT_TYPE);
        return true;
      } else {
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
{
  public static String tsToString(TokenStream in) throws IOException {
    StringBuilder out = new StringBuilder();
    Token t = in.next();
    if (null != t)
      out.append(new String(t.termBuffer(), 0, t.termLength()));
   
    for (t = in.next(); null != t; t = in.next()) {
      out.append(" ").append(new String(t.termBuffer(), 0, t.termLength()));
    }
    in.close();
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
    Token t = in.next();
    if (null != t)
      out.append(new String(t.termBuffer(), 0, t.termLength()));
   
    for (t = in.next(); null != t; t = in.next()) {
      out.append(" ").append(new String(t.termBuffer(), 0, t.termLength()));
    }
    in.close();
    return out.toString();
  }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
  private void assertTokEq(List<Token> a, List<Token> b, boolean checkOff) {
    int pos=0;
    for (Iterator iter = a.iterator(); iter.hasNext();) {
      Token tok = (Token)iter.next();
      pos += tok.getPositionIncrement();
      if (!tokAt(b, new String(tok.termBuffer(), 0, tok.termLength()), pos
              , checkOff ? tok.startOffset() : -1
              , checkOff ? tok.endOffset() : -1
              ))
      {
        fail(a + "!=" + b);
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
  public boolean tokAt(List<Token> lst, String val, int tokPos, int startOff, int endOff) {
    int pos=0;
    for (Iterator iter = lst.iterator(); iter.hasNext();) {
      Token tok = (Token)iter.next();
      pos += tok.getPositionIncrement();
      if (pos==tokPos && new String(tok.termBuffer(), 0, tok.termLength()).equals(val)
          && (startOff==-1 || tok.startOffset()==startOff)
          && (endOff  ==-1 || tok.endOffset()  ==endOff  )
           )
      {
        return true;
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
  public static class AB_Q_Stream extends BufferedTokenStream {
    public AB_Q_Stream(TokenStream input) {super(input);}
    protected Token process(Token t) throws IOException {
      if ("A".equals(new String(t.termBuffer(), 0, t.termLength()))) {
        Token t2 = read();
        if (t2!=null && "B".equals(new String(t2.termBuffer(), 0, t2.termLength()))) t.setTermText("Q");
        if (t2!=null) pushBack(t2);
      }
      return t;
    }
  }
View Full Code Here
28
29
30
31
32
33
34
35
36
37
38
    TokenStream stream = new IterTokenStream("international");
    TokenStream filter = new DoubleMetaphoneFilter(stream, 4, false);

    Token token = filter.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("ANTR", new String(token.termBuffer(), 0, token.termLength()));

    assertNull(filter.next(new Token()));
  }

  public void testSize4TrueInject() throws Exception {
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
    TokenStream stream = new IterTokenStream("international");
    TokenStream filter = new DoubleMetaphoneFilter(stream, 4, true);

    Token token = filter.next(new Token());
    assertEquals(13, token.termLength());
    assertEquals("international", new String(token.termBuffer(), 0, token
        .termLength()));

    token = filter.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("ANTR", new String(token.termBuffer(), 0, token.termLength()));
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
    assertEquals("international", new String(token.termBuffer(), 0, token
        .termLength()));

    token = filter.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("ANTR", new String(token.termBuffer(), 0, token.termLength()));

    assertNull(filter.next(new Token()));
  }

  public void testAlternateInjectFalse() throws Exception {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.