Package org.apache.lucene.analysis.miscellaneous

Examples of org.apache.lucene.analysis.miscellaneous.SingleTokenTokenStream

152
153
154
155
156
157
158
159
160
161
162
    tokens.add(tokenFactory("earth", 0, 1f, 5, 10, ShingleMatrixFilter.TokenPositioner.newRow));
    tokens.add(tokenFactory("tellus", 0, 1f, 5, 10, ShingleMatrixFilter.TokenPositioner.newRow));

    tls = new TokenListStream(tokens);

    ts = new PrefixAndSuffixAwareTokenFilter(new SingleTokenTokenStream(tokenFactory("^", 1, 100f, 0, 0)), tls, new SingleTokenTokenStream(tokenFactory("$", 1, 50f, 0, 0)));
    tls = new CachingTokenFilter(ts);

    // bi-grams, position incrememnt, weight, start offset, end offset

    ts = new ShingleMatrixFilter(tls, 2, 2, new Character('_'), false);
View Full Code Here
154
155
156
157
158
159
160
161
162
163
164
    tokens.add(tokenFactory("earth", 0, 1f, 5, 10, ShingleMatrixFilter.TokenPositioner.newRow));
    tokens.add(tokenFactory("tellus", 0, 1f, 5, 10, ShingleMatrixFilter.TokenPositioner.newRow));

    tls = new TokenListStream(tokens);

    ts = new PrefixAndSuffixAwareTokenFilter(new SingleTokenTokenStream(tokenFactory("^", 1, 100f, 0, 0)), tls, new SingleTokenTokenStream(tokenFactory("$", 1, 50f, 0, 0)));
    tls = new CachingTokenFilter(ts);

    // bi-grams, position incrememnt, weight, start offset, end offset

    ts = new ShingleMatrixFilter(tls, 2, 2, new Character('_'), false);
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
  public void testOffsets() throws IOException {

    // test that subwords and catenated subwords have
    // the correct offsets.
    WordDelimiterFilter wdf = new WordDelimiterFilter(
            new SingleTokenTokenStream(new Token("foo-bar", 5, 12)),
    1,1,0,0,1,1,0);

    assertTokenStreamContents(wdf,
        new String[] { "foo", "bar", "foobar" },
        new int[] { 5, 9, 5 },
        new int[] { 8, 12, 12 });

    wdf = new WordDelimiterFilter(
            new SingleTokenTokenStream(new Token("foo-bar", 5, 6)),
    1,1,0,0,1,1,0);
   
    assertTokenStreamContents(wdf,
        new String[] { "foo", "bar", "foobar" },
        new int[] { 5, 5, 5 },
View Full Code Here
184
185
186
187
188
189
190
191
192
193
194
 
  @Test
  public void testOffsetChange() throws Exception
  {
    WordDelimiterFilter wdf = new WordDelimiterFilter(
      new SingleTokenTokenStream(new Token("übelkeit)", 7, 16)),
      1,1,0,0,1,1,0
    );
   
    assertTokenStreamContents(wdf,
        new String[] { "übelkeit" },
View Full Code Here
198
199
200
201
202
203
204
205
206
207
208
 
  @Test
  public void testOffsetChange2() throws Exception
  {
    WordDelimiterFilter wdf = new WordDelimiterFilter(
      new SingleTokenTokenStream(new Token("(übelkeit", 7, 17)),
      1,1,0,0,1,1,0
    );
   
    assertTokenStreamContents(wdf,
        new String[] { "übelkeit" },
View Full Code Here
212
213
214
215
216
217
218
219
220
221
222
 
  @Test
  public void testOffsetChange3() throws Exception
  {
    WordDelimiterFilter wdf = new WordDelimiterFilter(
      new SingleTokenTokenStream(new Token("(übelkeit", 7, 16)),
      1,1,0,0,1,1,0
    );
   
    assertTokenStreamContents(wdf,
        new String[] { "übelkeit" },
View Full Code Here
226
227
228
229
230
231
232
233
234
235
236
 
  @Test
  public void testOffsetChange4() throws Exception
  {
    WordDelimiterFilter wdf = new WordDelimiterFilter(
      new SingleTokenTokenStream(new Token("(foo,bar)", 7, 16)),
      1,1,0,0,1,1,0
    );
   
    assertTokenStreamContents(wdf,
        new String[] { "foo", "bar", "foobar"},
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
    tokens.add(tokenFactory("earth", 0, 1f, 5, 10, ShingleMatrixFilter.TokenPositioner.newRow));
    tokens.add(tokenFactory("tellus", 0, 1f, 5, 10, ShingleMatrixFilter.TokenPositioner.newRow));

    tls = new TokenListStream(tokens);

    ts = new PrefixAndSuffixAwareTokenFilter(new SingleTokenTokenStream(tokenFactory("^", 1, 100f, 0, 0)), tls, new SingleTokenTokenStream(tokenFactory("$", 1, 50f, 0, 0)));
    tls = new TokenListStream(ts);

    // bi-grams, position incrememnt, weight, start offset, end offset

    ts = new ShingleMatrixFilter(tls, 2, 2, new Character('_'), false);
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
    tokens.add(tokenFactory("earth", 0, 1f, 5, 10, ShingleMatrixFilter.TokenPositioner.newRow));
    tokens.add(tokenFactory("tellus", 0, 1f, 5, 10, ShingleMatrixFilter.TokenPositioner.newRow));

    tls = new TokenListStream(tokens);

    ts = new PrefixAndSuffixAwareTokenFilter(new SingleTokenTokenStream(tokenFactory("^", 1, 100f, 0, 0)), tls, new SingleTokenTokenStream(tokenFactory("$", 1, 50f, 0, 0)));
    tls = new CachingTokenFilter(ts);

    // bi-grams, position incrememnt, weight, start offset, end offset

    ts = new ShingleMatrixFilter(tls, 2, 2, new Character('_'), false);
View Full Code Here
164
165
166
167
168
169
170
171
172
173
174
    tokens.add(tokenFactory("earth", 0, 1f, 5, 10, ShingleMatrixFilter.TokenPositioner.newRow));
    tokens.add(tokenFactory("tellus", 0, 1f, 5, 10, ShingleMatrixFilter.TokenPositioner.newRow));

    tls = new TokenListStream(tokens);

    ts = new PrefixAndSuffixAwareTokenFilter(new SingleTokenTokenStream(tokenFactory("^", 1, 100f, 0, 0)), tls, new SingleTokenTokenStream(tokenFactory("$", 1, 50f, 0, 0)));
    tls = new CachingTokenFilter(ts);

    // bi-grams, position incrememnt, weight, start offset, end offset

    ts = new ShingleMatrixFilter(tls, 2, 2, new Character('_'), false);
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.miscellaneous.SingleTokenTokenStream

  • org.apache.lucene.analysis.shingle.TestShingleMatrixFilter
  • org.apache.solr.analysis.TestWordDelimiterFilter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.