Examples of tokenStream()

Examples of org.apache.lucene.analysis.core.WhitespaceAnalyzer.tokenStream()

78
79
80
81
82
83
84
85
86
87
88
  /** normal case, unfiltered analyzer */
  @Test
  public void testAnalyzer() throws IOException {
    Reader reader = new StringReader(input);
    Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_43);
    TokenStream ts = analyzer.tokenStream(null, reader);
    ts.reset();
    validateTokens(allTokens, ts);
    ts.end();
    ts.close();
  }
View Full Code Here

Examples of org.apache.lucene.analysis.snowball.SnowballAnalyzer.tokenStream()

98
99
100
101
102
103
104
105
106
107
108
                                        org.apache.lucene.util.Version.LUCENE_CURRENT,
                                        "Spanish",
                                        SPANISH_STOP_WORDS);

                       
                        TokenStream tokenStream = analyzer.tokenStream(
                                        "content",
                                        new StringReader(indexCleanedOfHTMLTags));
                       
                        Token token = new Token();
View Full Code Here

Examples of org.apache.lucene.analysis.standard.StandardAnalyzer.tokenStream()

485
486
487
488
489
490
491
492
493
494
495
  }
 
  protected Set<String> getHighlightWords(String searchString) {
    try {
      Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
      TokenStream stream = analyzer.tokenStream("content", new StringReader(searchString));
      TermAttribute termAtt = (TermAttribute) stream.addAttribute(TermAttribute.class);
      for (boolean next = stream.incrementToken(); next; next = stream.incrementToken()) {
        String term = termAtt.term();
        if(log.isDebug()) log.debug(term);
      }
View Full Code Here

Examples of org.apache.lucene.analysis.standard.StandardAnalyzer.tokenStream()

1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
        int maxNumFragmentsRequired = 3;

        for (int i = 0; i < hits.totalHits; i++) {
          String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this, false);

          highlighter.setTextFragmenter(new SimpleFragmenter(40));

          String highlightedText = highlighter.getBestFragments(tokenStream, text,
View Full Code Here

Examples of org.apache.lucene.analysis.standard.StandardAnalyzer.tokenStream()

1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
        int maxNumFragmentsRequired = 3;

        for (int i = 0; i < hits.totalHits; i++) {
          String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this, false);

          highlighter.setTextFragmenter(new SimpleFragmenter(40));

          String highlightedText = highlighter.getBestFragments(tokenStream, text,
View Full Code Here

Examples of org.apache.lucene.analysis.standard.StandardAnalyzer.tokenStream()

23
24
25
26
27
28
29
30
31
32
33
  public static void main(String[] args) throws IOException {
    FeatureVectorEncoder encoder = new StaticWordValueEncoder("text");
    Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_31);    

    StringReader in = new StringReader("text to magically vectorize");
    TokenStream ts = analyzer.tokenStream("body", in);
    TermAttribute termAtt = ts.addAttribute(TermAttribute.class);

    Vector v1 = new RandomAccessSparseVector(100);                  
    while (ts.incrementToken()) {
      char[] termBuffer = termAtt.termBuffer();
View Full Code Here

Examples of org.apache.lucene.analysis.standard.StandardAnalyzer.tokenStream()

182
183
184
185
186
187
188
189
190
191
192
              String superColumnName = merge(superColumnList, " ");
              superColumnList.clear();
              if(importer_.columnFamily.superColumn.tokenize)
              {
                  Analyzer analyzer = new StandardAnalyzer();
                  TokenStream ts = analyzer.tokenStream("superColumn", new StringReader(superColumnName));
                  Token token = null;
                  token = ts.next();
                  while(token != null)
                  {
                    superColumnList.add(token.termText());
View Full Code Here

Examples of org.apache.lucene.analysis.standard.StandardAnalyzer.tokenStream()

123
124
125
126
127
128
129
130
131
132
133
      ClassifierContext ctx = new ClassifierContext(a,ds);
      ctx.initialize();
     
      //TODO: make the analyzer configurable
      StandardAnalyzer analyzer = new StandardAnalyzer(Version.LUCENE_36);
      TokenStream ts = analyzer.tokenStream(null, new InputStreamReader(new FileInputStream(inputFile), "UTF-8"));
    
      ArrayList<String> tokens = new ArrayList<String>(1000);
      while (ts.incrementToken()) {
        tokens.add(ts.getAttribute(CharTermAttribute.class).toString());
      }
View Full Code Here

Examples of org.apache.lucene.analysis.standard.StandardAnalyzer.tokenStream()

1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
        int maxNumFragmentsRequired = 3;

        for (int i = 0; i < hits.length(); i++) {
          String text = hits.doc(i).get(FIELD_NAME);
          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this, false);

          highlighter.setTextFragmenter(new SimpleFragmenter(40));

          String highlightedText = highlighter.getBestFragments(tokenStream, text,
View Full Code Here

Examples of org.apache.lucene.document.IntField.tokenStream()

73
74
75
76
77
78
79
80
81
82
83
 
  public void testNumericReuse() throws IOException {
    IntField intField = new IntField("foo", 5, Field.Store.NO);
   
    // passing null
    TokenStream ts = intField.tokenStream(null, null);
    assertTrue(ts instanceof NumericTokenStream);
    assertEquals(NumericUtils.PRECISION_STEP_DEFAULT_32, ((NumericTokenStream)ts).getPrecisionStep());
    assertNumericContents(5, ts);

    // now reuse previous stream
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.