Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.type()

54
55
56
57
58
59
60
61
62
63
64
        assertTrue(bytes.length + " does not equal: " + tok.getPayload().length(), bytes.length == tok.getPayload().length());
        assertTrue(tok.getPayload().getOffset() + " does not equal: " + 0, tok.getPayload().getOffset() == 0);
        float pay = PayloadHelper.decodeFloat(bytes);
        assertTrue(pay + " does not equal: " + 3, pay == 3);
      } else {
        assertTrue(tok.type() + " is not null and it should be", tok.type().equals("word"));
      }
    }
    assertTrue(seenDogs + " does not equal: " + true, seenDogs == true);
  }
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
    TypeAsPayloadTokenFilter nptf = new TypeAsPayloadTokenFilter(new WordTokenFilter(new WhitespaceTokenizer(new StringReader(test))));
    Token tok = new Token();
    int count = 0;
    while ((tok = nptf.next(tok)) != null){
      assertTrue(tok.type() + " is not null and it should be", tok.type().equals(String.valueOf(Character.toUpperCase(tok.termBuffer()[0]))));
      assertTrue("tok.getPayload() is null and it shouldn't be", tok.getPayload() != null);
      String type = new String(tok.getPayload().getData(), "UTF-8");
      assertTrue("type is null and it shouldn't be", type != null);
      assertTrue(type + " is not equal to " + tok.type(), type.equals(tok.type()) == true);
      count++;
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
    TypeAsPayloadTokenFilter nptf = new TypeAsPayloadTokenFilter(new WordTokenFilter(new WhitespaceTokenizer(new StringReader(test))));
    Token tok = new Token();
    int count = 0;
    while ((tok = nptf.next(tok)) != null){
      assertTrue(tok.type() + " is not null and it should be", tok.type().equals(String.valueOf(Character.toUpperCase(tok.termBuffer()[0]))));
      assertTrue("tok.getPayload() is null and it shouldn't be", tok.getPayload() != null);
      String type = new String(tok.getPayload().getData(), "UTF-8");
      assertTrue("type is null and it shouldn't be", type != null);
      assertTrue(type + " is not equal to " + tok.type(), type.equals(tok.type()) == true);
      count++;
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
    while ((tok = nptf.next(tok)) != null){
      assertTrue(tok.type() + " is not null and it should be", tok.type().equals(String.valueOf(Character.toUpperCase(tok.termBuffer()[0]))));
      assertTrue("tok.getPayload() is null and it shouldn't be", tok.getPayload() != null);
      String type = new String(tok.getPayload().getData(), "UTF-8");
      assertTrue("type is null and it shouldn't be", type != null);
      assertTrue(type + " is not equal to " + tok.type(), type.equals(tok.type()) == true);
      count++;
    }
    assertTrue(count + " does not equal: " + 10, count == 10);
  }
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
    while ((tok = nptf.next(tok)) != null){
      assertTrue(tok.type() + " is not null and it should be", tok.type().equals(String.valueOf(Character.toUpperCase(tok.termBuffer()[0]))));
      assertTrue("tok.getPayload() is null and it shouldn't be", tok.getPayload() != null);
      String type = new String(tok.getPayload().getData(), "UTF-8");
      assertTrue("type is null and it shouldn't be", type != null);
      assertTrue(type + " is not equal to " + tok.type(), type.equals(tok.type()) == true);
      count++;
    }
    assertTrue(count + " does not equal: " + 10, count == 10);
  }
View Full Code Here
90
91
92
93
94
95
96
97
98
        minPoz = (minPoz == -1) ? poz : Math.min(poz, minPoz);
    }
    if (minPoz != -1
        && articles.contains(text.substring(0, minPoz).toLowerCase()))
      text = text.substring(minPoz + 1);
    return new Token(text, t.startOffset(), t.endOffset(), t.type());
  }

}
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
      String tokText = token.termText();
      //System.out.println("Text: " + tokText + " Type: " + token.type());
      assertTrue("token is null and it shouldn't be", token != null);
      String expectedType = (String) tcm.get(tokText);
      assertTrue("expectedType is null and it shouldn't be for: " + token, expectedType != null);
      assertTrue(token.type() + " is not equal to " + expectedType + " for " + token, token.type().equals(expectedType) == true);
      count++;
      if (token.type().equals(WikipediaTokenizer.ITALICS)  == true){
        numItalics++;
      } else if (token.type().equals(WikipediaTokenizer.BOLD_ITALICS)  == true){
        numBoldItalics++;
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
      String tokText = token.termText();
      //System.out.println("Text: " + tokText + " Type: " + token.type());
      assertTrue("token is null and it shouldn't be", token != null);
      String expectedType = (String) tcm.get(tokText);
      assertTrue("expectedType is null and it shouldn't be for: " + token, expectedType != null);
      assertTrue(token.type() + " is not equal to " + expectedType + " for " + token, token.type().equals(expectedType) == true);
      count++;
      if (token.type().equals(WikipediaTokenizer.ITALICS)  == true){
        numItalics++;
      } else if (token.type().equals(WikipediaTokenizer.BOLD_ITALICS)  == true){
        numBoldItalics++;
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
      assertTrue("token is null and it shouldn't be", token != null);
      String expectedType = (String) tcm.get(tokText);
      assertTrue("expectedType is null and it shouldn't be for: " + token, expectedType != null);
      assertTrue(token.type() + " is not equal to " + expectedType + " for " + token, token.type().equals(expectedType) == true);
      count++;
      if (token.type().equals(WikipediaTokenizer.ITALICS)  == true){
        numItalics++;
      } else if (token.type().equals(WikipediaTokenizer.BOLD_ITALICS)  == true){
        numBoldItalics++;
      } else if (token.type().equals(WikipediaTokenizer.CATEGORY)  == true){
        numCategory++;
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
      assertTrue("expectedType is null and it shouldn't be for: " + token, expectedType != null);
      assertTrue(token.type() + " is not equal to " + expectedType + " for " + token, token.type().equals(expectedType) == true);
      count++;
      if (token.type().equals(WikipediaTokenizer.ITALICS)  == true){
        numItalics++;
      } else if (token.type().equals(WikipediaTokenizer.BOLD_ITALICS)  == true){
        numBoldItalics++;
      } else if (token.type().equals(WikipediaTokenizer.CATEGORY)  == true){
        numCategory++;
      }
      else if (token.type().equals(WikipediaTokenizer.CITATION)  == true){
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.