Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.type()

212
213
214
215
216
217
218
219
220
221
222
        Token t1 = (Token) tokens1.get(i);
        Token t2 = (Token) tokens2.get(i);
        if (!(t1.termText().equals(t2.termText()))) throw new IllegalStateException("termText");
        if (t1.startOffset() != t2.startOffset()) throw new IllegalStateException("startOffset");
        if (t1.endOffset() != t2.endOffset()) throw new IllegalStateException("endOffset");
        if (!(t1.type().equals(t2.type()))) throw new IllegalStateException("type");
      }
      if (tokens1.size() != tokens2.size())   throw new IllegalStateException("size1=" + tokens1.size() + ", size2=" + tokens2.size());
    }

    catch (IllegalStateException e) {
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
        Token token = null;

        while ((token = ts.next()) != null) {
            System.out.println("ReTokenizeFile.reTokenize(File): " + token.termText() + " " +
                token.startOffset() + " " + token.endOffset() + " " + token.type());
        }

        return file.getAbsolutePath();
    }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
    if (t == null)
      return null;

    char[] buffer = t.termBuffer();
    final int bufferLength = t.termLength();
    final String type = t.type();

    if (type == APOSTROPHE_TYPE &&      // remove 's
  bufferLength >= 2 &&
        buffer[bufferLength-2] == '\'' &&
        (buffer[bufferLength-1] == 's' || buffer[bufferLength-1] == 'S')) {
View Full Code Here
206
207
208
209
210
211
212
213
214
215
216
        Token t1 = (Token) tokens1.get(i);
        Token t2 = (Token) tokens2.get(i);
        if (!(t1.termText().equals(t2.termText()))) throw new IllegalStateException("termText");
        if (t1.startOffset() != t2.startOffset()) throw new IllegalStateException("startOffset");
        if (t1.endOffset() != t2.endOffset()) throw new IllegalStateException("endOffset");
        if (!(t1.type().equals(t2.type()))) throw new IllegalStateException("type");
      }
      if (tokens1.size() != tokens2.size())   throw new IllegalStateException("size1=" + tokens1.size() + ", size2=" + tokens2.size());
    }

    catch (IllegalStateException e) {
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
            if (token == null) return "[" + logName + ":EOS:" + fieldName + "]\n";
           
            position += token.getPositionIncrement();
            return "[" + logName + ":" + position + ":" + fieldName + ":"
                + token.termText() + ":" + token.startOffset()
                + "-" + token.endOffset() + ":" + token.type()
                + "]";
          }         
        };
      }
    };
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
    Token tok = new Token();
    boolean seenDogs = false;
    while ((tok = nptf.next(tok)) != null){
      if (tok.termText().equals("dogs")){
        seenDogs = true;
        assertTrue(tok.type() + " is not equal to " + "D", tok.type().equals("D") == true);
        assertTrue("tok.getPayload() is null and it shouldn't be", tok.getPayload() != null);
        byte [] bytes = tok.getPayload().getData();//safe here to just use the bytes, otherwise we should use offset, length
        assertTrue(bytes.length + " does not equal: " + tok.getPayload().length(), bytes.length == tok.getPayload().length());
        assertTrue(tok.getPayload().getOffset() + " does not equal: " + 0, tok.getPayload().getOffset() == 0);
        float pay = PayloadHelper.decodeFloat(bytes);
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
    Token tok = new Token();
    boolean seenDogs = false;
    while ((tok = nptf.next(tok)) != null){
      if (tok.termText().equals("dogs")){
        seenDogs = true;
        assertTrue(tok.type() + " is not equal to " + "D", tok.type().equals("D") == true);
        assertTrue("tok.getPayload() is null and it shouldn't be", tok.getPayload() != null);
        byte [] bytes = tok.getPayload().getData();//safe here to just use the bytes, otherwise we should use offset, length
        assertTrue(bytes.length + " does not equal: " + tok.getPayload().length(), bytes.length == tok.getPayload().length());
        assertTrue(tok.getPayload().getOffset() + " does not equal: " + 0, tok.getPayload().getOffset() == 0);
        float pay = PayloadHelper.decodeFloat(bytes);
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
    Token tok = null;
    int count = 0;
    while ((tok = tee.next()) != null){
      assertTrue("tok is null and it shouldn't be", tok != null);
      if (tok.termBuffer()[0] == '7'){
        assertTrue(tok.type() + " is not equal to " + DateRecognizerSinkTokenizer.DATE_TYPE,
                tok.type().equals(DateRecognizerSinkTokenizer.DATE_TYPE) == true);
      }
      count++;
    }
    assertTrue(count + " does not equal: " + 18, count == 18);
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
    int count = 0;
    while ((tok = tee.next()) != null){
      assertTrue("tok is null and it shouldn't be", tok != null);
      if (tok.termBuffer()[0] == '7'){
        assertTrue(tok.type() + " is not equal to " + DateRecognizerSinkTokenizer.DATE_TYPE,
                tok.type().equals(DateRecognizerSinkTokenizer.DATE_TYPE) == true);
      }
      count++;
    }
    assertTrue(count + " does not equal: " + 18, count == 18);
    assertTrue("sink Size: " + sink.getTokens().size() + " is not: " + 2, sink.getTokens().size() == 2);
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
        assertTrue(bytes.length + " does not equal: " + tok.getPayload().length(), bytes.length == tok.getPayload().length());
        assertTrue(tok.getPayload().getOffset() + " does not equal: " + 0, tok.getPayload().getOffset() == 0);
        float pay = PayloadHelper.decodeFloat(bytes);
        assertTrue(pay + " does not equal: " + 3, pay == 3);
      } else {
        assertTrue(tok.type() + " is not null and it should be", tok.type().equals("word"));
      }
    }
    assertTrue(seenDogs + " does not equal: " + true, seenDogs == true);
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.