Examples of tokenStream()

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

112
113
114
115
116
117
118
119
120
121
122
          app
                .showStatus("Couldn't instantiate analyzer - public 0-arg or 1-arg constructor(Version) required");
          return;
        }
      }
      TokenStream ts = analyzer.tokenStream("text", new StringReader(app
              .getString(inputText, "text")));
      app.removeAll(resultsList);
      ts.reset();

      while (ts.incrementToken()) {
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

132
133
134
135
136
137
138
139
140
141
142
              FieldType ft = schema.getFieldType(name);
              Analyzer analyzer = ft.getAnalyzer();
              Collection<Object> vals = doc.getFieldValues(name);
              for (Object val : vals) {
                Reader reader = new StringReader(val.toString());
                TokenStream tstream = analyzer.tokenStream(name, reader);
                NamedList<NamedList<Object>> tokens = getTokens(tstream);
                theTokens.add(name, tokens);
              }
            }
          }
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
      return queryAnalyzer.tokenStream(fieldName, reader);
    }
   
    Analyzer a = map.get(fieldName);
    if (a != null) {
      return a.tokenStream(fieldName, reader);
    }

    FieldType ft = parser.getReq().getSchema().getFieldType(fieldName);
    Analyzer qa = ft.getQueryAnalyzer();
    if (!(qa instanceof TokenizerChain)) {
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
    FieldType ft = parser.getReq().getSchema().getFieldType(fieldName);
    Analyzer qa = ft.getQueryAnalyzer();
    if (!(qa instanceof TokenizerChain)) {
      map.put(fieldName, qa);
      return qa.tokenStream(fieldName, reader);
    }
    TokenizerChain tcq = (TokenizerChain)qa;
    Analyzer ia = ft.getAnalyzer();
    if (ia == qa || !(ia instanceof TokenizerChain)) {
      map.put(fieldName, qa);
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

124
125
126
127
128
129
130
131
132
133
134
    String s = "Ab94aa4CdDbd34dfde082ed1b4c4d0c505b69";

    StringReader sr = new StringReader(s);
//    Analyzer analyzer =new StandardAnalyzer((Version) Enum.valueOf((Class) Class.forName("org.apache.lucene.util.Version"),  Version.LUCENE_35.name()));
    Analyzer analyzer = schema.getAnalyzer();//JobIndexPublic.setAnalyzer(conf);
    TokenStream tk=analyzer.tokenStream("rawquery", sr);

    boolean hasnext = tk.incrementToken();

    while(hasnext){
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

96
97
98
99
100
101
102
103
104
105
106
        }
      }
    }
    try {
      Analyzer analyzer = sp.getType().getQueryAnalyzer();
      TokenStream ts = analyzer.tokenStream(field,
              new StringReader(qstr));
      while (ts.incrementToken()) {//<co id="qqp.addTerms"/>
        String term = ((CharTermAttribute)
                ts.getAttribute(CharTermAttribute.class)).toString();
        sql.add(new SpanTermQuery(new Term(field, term)));
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

139
140
141
142
143
144
145
146
147
148
149
      int positionCount = 0;
      boolean severalTokensAtSamePosition = false;

      final TokenStream source;
      try {
        source = analyzer.tokenStream(field, new StringReader(text));
        source.reset();
      } catch (final IOException e1) {
        throw new RuntimeException(e1);
      }
      final CachingTokenFilter buffer = new CachingTokenFilter(source);
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

161
162
163
164
165
166
167
168
169
170
171
        if (reusableCharArray == null) {
          reusableCharArray = new ReusableCharArrayReader(termAtt.buffer(), 0, termAtt.length());
        } else {
          reusableCharArray.reset(termAtt.buffer(), 0, termAtt.length());
        }
        currentStream = analyzer.tokenStream("", reusableCharArray);
        currentStream.reset(); // reset to prepare the stream for consumption
        this.initTokenAttributes();
      }
      // Consume the token with the registered analyzer
      isConsumingToken = currentStream.incrementToken();
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

921
922
923
924
925
926
927
928
929
930
931
        String s = "football-soccer in the euro 2004 footie competition";
        QueryParser parser = new QueryParser("bookid", analyzer);
        Query query = parser.parse(srchkey);

        TokenStream tokenStream = analyzer.tokenStream(null, new StringReader(s));

        Highlighter highlighter = getHighlighter(query, null, tokenStream, HighlighterTest.this);

        // Get 3 best fragments and seperate with a "..."
        tokenStream = analyzer.tokenStream(null, new StringReader(s));
View Full Code Here

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

926
927
928
929
930
931
932
933
934
935
936
        TokenStream tokenStream = analyzer.tokenStream(null, new StringReader(s));

        Highlighter highlighter = getHighlighter(query, null, tokenStream, HighlighterTest.this);

        // Get 3 best fragments and seperate with a "..."
        tokenStream = analyzer.tokenStream(null, new StringReader(s));

        String result = highlighter.getBestFragments(tokenStream, s, 3, "...");
        String expectedResult = "<B>football</B>-<B>soccer</B> in the euro 2004 <B>footie</B> competition";
        assertTrue("overlapping analyzer should handle highlights OK, expected:" + expectedResult
            + " actual:" + result, expectedResult.equals(result));
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.