Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.type()

138
139
140
141
142
143
144
145
146
147
148
      count++;
      if (token.type().equals(WikipediaTokenizer.ITALICS)  == true){
        numItalics++;
      } else if (token.type().equals(WikipediaTokenizer.BOLD_ITALICS)  == true){
        numBoldItalics++;
      } else if (token.type().equals(WikipediaTokenizer.CATEGORY)  == true){
        numCategory++;
      }
      else if (token.type().equals(WikipediaTokenizer.CITATION)  == true){
        numCitation++;
      }
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
      } else if (token.type().equals(WikipediaTokenizer.BOLD_ITALICS)  == true){
        numBoldItalics++;
      } else if (token.type().equals(WikipediaTokenizer.CATEGORY)  == true){
        numCategory++;
      }
      else if (token.type().equals(WikipediaTokenizer.CITATION)  == true){
        numCitation++;
      }
    }
    assertTrue("We have not seen enough tokens: " + count + " is not >= " + tcm.size(), count >= tcm.size());
    assertTrue(numItalics + " does not equal: " + 4 + " for numItalics", numItalics == 4);
View Full Code Here
210
211
212
213
214
215
216
217
218
219
220
    Token token = new Token();
    token = tf.next(token);
    assertTrue("token is null and it shouldn't be", token != null);
    assertTrue(new String(token.termBuffer(), 0, token.termLength()) + " is not equal to " + "http://lucene.apache.org/java/docs/index.html#news",
            new String(token.termBuffer(), 0, token.termLength()).equals("http://lucene.apache.org/java/docs/index.html#news") == true);
    assertTrue(token.type() + " is not equal to " + WikipediaTokenizer.EXTERNAL_LINK_URL, token.type().equals(WikipediaTokenizer.EXTERNAL_LINK_URL) == true);
    tf.next(token);//skip here
    token = tf.next(token);
    assertTrue("token is null and it shouldn't be", token != null);
    assertTrue(new String(token.termBuffer(), 0, token.termLength()) + " is not equal to " + "http://lucene.apache.org/java/docs/index.html?b=c",
            new String(token.termBuffer(), 0, token.termLength()).equals("http://lucene.apache.org/java/docs/index.html?b=c") == true);
View Full Code Here
210
211
212
213
214
215
216
217
218
219
220
    Token token = new Token();
    token = tf.next(token);
    assertTrue("token is null and it shouldn't be", token != null);
    assertTrue(new String(token.termBuffer(), 0, token.termLength()) + " is not equal to " + "http://lucene.apache.org/java/docs/index.html#news",
            new String(token.termBuffer(), 0, token.termLength()).equals("http://lucene.apache.org/java/docs/index.html#news") == true);
    assertTrue(token.type() + " is not equal to " + WikipediaTokenizer.EXTERNAL_LINK_URL, token.type().equals(WikipediaTokenizer.EXTERNAL_LINK_URL) == true);
    tf.next(token);//skip here
    token = tf.next(token);
    assertTrue("token is null and it shouldn't be", token != null);
    assertTrue(new String(token.termBuffer(), 0, token.termLength()) + " is not equal to " + "http://lucene.apache.org/java/docs/index.html?b=c",
            new String(token.termBuffer(), 0, token.termLength()).equals("http://lucene.apache.org/java/docs/index.html?b=c") == true);
View Full Code Here
216
217
218
219
220
221
222
223
224
225
226
    tf.next(token);//skip here
    token = tf.next(token);
    assertTrue("token is null and it shouldn't be", token != null);
    assertTrue(new String(token.termBuffer(), 0, token.termLength()) + " is not equal to " + "http://lucene.apache.org/java/docs/index.html?b=c",
            new String(token.termBuffer(), 0, token.termLength()).equals("http://lucene.apache.org/java/docs/index.html?b=c") == true);
    assertTrue(token.type() + " is not equal to " + WikipediaTokenizer.EXTERNAL_LINK_URL, token.type().equals(WikipediaTokenizer.EXTERNAL_LINK_URL) == true);
    tf.next(token);//skip here
    token = tf.next(token);
    assertTrue("token is null and it shouldn't be", token != null);
    assertTrue(new String(token.termBuffer(), 0, token.termLength()) + " is not equal to " + "https://lucene.apache.org/java/docs/index.html?b=c",
            new String(token.termBuffer(), 0, token.termLength()).equals("https://lucene.apache.org/java/docs/index.html?b=c") == true);
View Full Code Here
216
217
218
219
220
221
222
223
224
225
226
    tf.next(token);//skip here
    token = tf.next(token);
    assertTrue("token is null and it shouldn't be", token != null);
    assertTrue(new String(token.termBuffer(), 0, token.termLength()) + " is not equal to " + "http://lucene.apache.org/java/docs/index.html?b=c",
            new String(token.termBuffer(), 0, token.termLength()).equals("http://lucene.apache.org/java/docs/index.html?b=c") == true);
    assertTrue(token.type() + " is not equal to " + WikipediaTokenizer.EXTERNAL_LINK_URL, token.type().equals(WikipediaTokenizer.EXTERNAL_LINK_URL) == true);
    tf.next(token);//skip here
    token = tf.next(token);
    assertTrue("token is null and it shouldn't be", token != null);
    assertTrue(new String(token.termBuffer(), 0, token.termLength()) + " is not equal to " + "https://lucene.apache.org/java/docs/index.html?b=c",
            new String(token.termBuffer(), 0, token.termLength()).equals("https://lucene.apache.org/java/docs/index.html?b=c") == true);
View Full Code Here
222
223
224
225
226
227
228
229
    tf.next(token);//skip here
    token = tf.next(token);
    assertTrue("token is null and it shouldn't be", token != null);
    assertTrue(new String(token.termBuffer(), 0, token.termLength()) + " is not equal to " + "https://lucene.apache.org/java/docs/index.html?b=c",
            new String(token.termBuffer(), 0, token.termLength()).equals("https://lucene.apache.org/java/docs/index.html?b=c") == true);
    assertTrue(token.type() + " is not equal to " + WikipediaTokenizer.EXTERNAL_LINK_URL, token.type().equals(WikipediaTokenizer.EXTERNAL_LINK_URL) == true);
  }
}
View Full Code Here
222
223
224
225
226
227
228
229
    tf.next(token);//skip here
    token = tf.next(token);
    assertTrue("token is null and it shouldn't be", token != null);
    assertTrue(new String(token.termBuffer(), 0, token.termLength()) + " is not equal to " + "https://lucene.apache.org/java/docs/index.html?b=c",
            new String(token.termBuffer(), 0, token.termLength()).equals("https://lucene.apache.org/java/docs/index.html?b=c") == true);
    assertTrue(token.type() + " is not equal to " + WikipediaTokenizer.EXTERNAL_LINK_URL, token.type().equals(WikipediaTokenizer.EXTERNAL_LINK_URL) == true);
  }
}
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
            if (token == null) return "[" + logName + ":EOS:" + fieldName + "]\n";
           
            position += token.getPositionIncrement();
            return "[" + logName + ":" + position + ":" + fieldName + ":"
                + token.termText() + ":" + token.startOffset()
                + "-" + token.endOffset() + ":" + token.type()
                + "]";
          }        
        };
      }
    };
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
        Token token = tokens[index++];
        termAtt.setEmpty().append(token);
        offsetAtt.setOffset(token.startOffset(), token.endOffset());
        posIncAtt.setPositionIncrement(token.getPositionIncrement());
        flagsAtt.setFlags(token.getFlags());
        typeAtt.setType(token.type());
        payloadAtt.setPayload(token.getPayload());
        return true;
      }
    }
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.