Examples of tokenStream()

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
        int maxNumFragmentsRequired = 3;

        for (int i = 0; i < hits.length(); i++) {
          String text = hits.doc(i).get(FIELD_NAME);
          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this, false);

          highlighter.setTextFragmenter(new SimpleFragmenter(40));

          String highlightedText = highlighter.getBestFragments(tokenStream, text,
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
  public void testEndOffsetPositionWithCachingTokenFilter() throws Exception {
    MockRAMDirectory dir = new MockRAMDirectory();
    Analyzer analyzer = new WhitespaceAnalyzer();
    IndexWriter w = new IndexWriter(dir, analyzer, IndexWriter.MaxFieldLength.LIMITED);
    Document doc = new Document();
    TokenStream stream = new CachingTokenFilter(analyzer.tokenStream("field", new StringReader("abcd   ")));
    Field f = new Field("field", stream, Field.TermVector.WITH_POSITIONS_OFFSETS);
    doc.add(f);
    doc.add(f);
    w.addDocument(doc);
    w.close();
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
  public void testEndOffsetPositionWithTeeSinkTokenFilter() throws Exception {
    MockRAMDirectory dir = new MockRAMDirectory();
    Analyzer analyzer = new WhitespaceAnalyzer();
    IndexWriter w = new IndexWriter(dir, analyzer, IndexWriter.MaxFieldLength.LIMITED);
    Document doc = new Document();
    TeeSinkTokenFilter tee = new TeeSinkTokenFilter(analyzer.tokenStream("field", new StringReader("abcd   ")));
    TokenStream sink = tee.newSinkTokenStream();
    Field f1 = new Field("field", tee, Field.TermVector.WITH_POSITIONS_OFFSETS);
    Field f2 = new Field("field", sink, Field.TermVector.WITH_POSITIONS_OFFSETS);
    doc.add(f1);
    doc.add(f2);
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

67
68
69
70
71
72
73
74
75
76
77
     */
    public TokenStream tokenStream(String fieldName, Reader reader) {
        if (indexingConfig != null) {
            Analyzer propertyAnalyzer = indexingConfig.getPropertyAnalyzer(fieldName);
            if (propertyAnalyzer != null) {
                return propertyAnalyzer.tokenStream(fieldName, reader);
            }
        }
        return defaultAnalyzer.tokenStream(fieldName, reader);
    }

View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

104
105
106
107
108
109
110
111
    if ("anchor".equals(fieldName))
      analyzer = ANCHOR_ANALYZER;
    else
      analyzer = CONTENT_ANALYZER;

    return analyzer.tokenStream(fieldName, reader);
  }
}
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

69
70
71
72
73
74
75
76
77
78
79
     */
    public TokenStream tokenStream(String fieldName, Reader reader) {
        if (indexingConfig != null) {
            Analyzer propertyAnalyzer = indexingConfig.getPropertyAnalyzer(fieldName);
            if (propertyAnalyzer != null) {
                return propertyAnalyzer.tokenStream(fieldName, reader);
            }
        }
        return defaultAnalyzer.tokenStream(fieldName, reader);
    }

View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

76
77
78
79
80
81
82
83
84
85
86
      }
    };
    for (int i = 0; i < 1000; i++) {
      String s = _TestUtil.randomSimpleString(random);
      assertEquals(s, left.tokenStream("foo", newStringReader(s)),
                   right.tokenStream("foo", newStringReader(s)));
    }
  }
 
  // not so useful since its all one token?!
  public void testLetterAsciiHuge() throws Exception {
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

97
98
99
100
101
102
103
104
105
106
107
    };
    int numIterations = atLeast(50);
    for (int i = 0; i < numIterations; i++) {
      String s = _TestUtil.randomSimpleString(random, maxLength);
      assertEquals(s, left.tokenStream("foo", newStringReader(s)),
                   right.tokenStream("foo", newStringReader(s)));
    }
  }
 
  public void testLetterHtmlish() throws Exception {
    Random random = random();
View Full Code Here

Examples of org.apache.lucene.analysis.MockAnalyzer.tokenStream()

1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
      @Override
      public void run() throws Exception {
        numHighlights = 0;
        doSearching("meat");
        TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(texts[0]));
        Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream,
            HighlighterTest.this);// new Highlighter(this, new
        // QueryTermScorer(query));
        highlighter.setMaxDocCharsToAnalyze(30);
View Full Code Here

Examples of org.apache.lucene.analysis.MockAnalyzer.tokenStream()

41
42
43
44
45
46
47
48
49
50
51
    while(true) {
      String s = _TestUtil.randomRealisticUnicodeString(random);
      if (other != null && s.equals(other)) {
        continue;
      }
      final TokenStream ts = a.tokenStream("foo", new StringReader(s));
      final TermAttribute termAtt = ts.getAttribute(TermAttribute.class);
      int count = 0;
      ts.reset();
      while(ts.incrementToken()) {
        if (count == 0 && !termAtt.term().equals(s)) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.