Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.termLength()

36
37
38
39
40
41
42
43
44
45
46
    assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());

    Token token = filteredStream.next(new Token());
    assertEquals(13, token.termLength());
    assertEquals("international", new String(token.termBuffer(), 0, token
        .termLength()));

    token = filteredStream.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("ANTR", new String(token.termBuffer(), 0, token.termLength()));
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
    assertEquals(13, token.termLength());
    assertEquals("international", new String(token.termBuffer(), 0, token
        .termLength()));

    token = filteredStream.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("ANTR", new String(token.termBuffer(), 0, token.termLength()));

    assertNull(filteredStream.next(new Token()));
  }
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
    assertEquals("international", new String(token.termBuffer(), 0, token
        .termLength()));

    token = filteredStream.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("ANTR", new String(token.termBuffer(), 0, token.termLength()));

    assertNull(filteredStream.next(new Token()));
  }

  public void testSettingSizeAndInject() throws Exception {
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
    TokenStream filteredStream = factory.create(inputStream);

    assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());

    Token token = filteredStream.next(new Token());
    assertEquals(8, token.termLength());
    assertEquals("ANTRNXNL", new String(token.termBuffer(), 0, token
        .termLength()));

    assertNull(filteredStream.next(new Token()));
  }
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
    assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());

    Token token = filteredStream.next(new Token());
    assertEquals(8, token.termLength());
    assertEquals("ANTRNXNL", new String(token.termBuffer(), 0, token
        .termLength()));

    assertNull(filteredStream.next(new Token()));
  }
}
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
        // check the dictionary
        if (dictionary.contains(lowerCaseTermBuffer, start, partLength)) {
          if (this.onlyLongestMatch) {
            if (longestMatchToken != null) {
              if (longestMatchToken.termLength() < partLength) {
                longestMatchToken = createToken(start, partLength, token);
              }
            } else {
              longestMatchToken = createToken(start, partLength, token);
            }
View Full Code Here
212
213
214
215
216
217
218
219
220
221
222
          // shorter
          // to avoid problems with genitive 's characters and other binding
          // characters
          if (this.onlyLongestMatch) {
            if (longestMatchToken != null) {
              if (longestMatchToken.termLength() < partLength - 1) {
                longestMatchToken = createToken(start, partLength - 1, token);
              }
            } else {
              longestMatchToken = createToken(start, partLength - 1, token);
            }
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
      }

      // common case fast-path of first token not matching anything
      Token firstTok = nextTok(target);
      if (firstTok == null) return null;
      SynonymMap result = map.submap!=null ? map.submap.get(firstTok.termBuffer(), 0, firstTok.termLength()) : null;
      if (result == null) return firstTok;

      // OK, we matched a token, so find the longest match.

      matched = new LinkedList<Token>();
View Full Code Here
180
181
182
183
184
185
186
187
188
189
190
    if (map.submap != null) {
      Token tok = nextTok();
      if (tok != null) {
        // check for positionIncrement!=1?  if>1, should not match, if==0, check multiple at this level?
        SynonymMap subMap = map.submap.get(tok.termBuffer(), 0, tok.termLength());

        if (subMap != null) {
          // recurse
          result = match(subMap);
        }
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
  }

  @Override
  public final Token next(Token in) throws IOException {
    Token t = input.next(in);
    if (null == t || null == t.termBuffer() || t.termLength() == 0){
      return t;
    }
    char[] termBuffer = t.termBuffer();
    int len = t.termLength();
    int start = 0;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.