Examples of org.apache.lucene.analysis.CannedTokenStream

org.apache.lucene.analysis.CannedTokenStream
TokenStream from a canned list of Tokens.

    Document doc = new Document();
    Token t1 = new Token("foo", 0, 3);
    t1.setPositionIncrement(Integer.MAX_VALUE);
    Token t2 = new Token("bar", 4, 7);
    t2.setPositionIncrement(200);
    TokenStream overflowingTokenStream = new CannedTokenStream(
        new Token[] { t1, t2 }
    );
    Field field = new TextField("foo", overflowingTokenStream);
    doc.add(field);
    try {

View Full Code Here

    Token t1 = new Token("foo", 0, 3);
    t1.setPositionIncrement(Integer.MAX_VALUE-500);
    if (random().nextBoolean()) {
      t1.setPayload(new BytesRef(new byte[] { 0x1 } ));
    }
    TokenStream overflowingTokenStream = new CannedTokenStream(
        new Token[] { t1 }
    );
    Field field = new TextField("foo", overflowingTokenStream);
    doc.add(field);
    iw.addDocument(doc);

View Full Code Here

        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
        
        return new TokenStreamComponents(tokenizer) {
          int tokenStreamCounter = 0;
          final TokenStream[] tokenStreams = new TokenStream[] {
            new CannedTokenStream(new Token[] {
                token("wifi",1,1),
                token("hotspot",0,2),
                token("network",1,1),
                token("is",1,1),
                token("slow",1,1)
              }),
            new CannedTokenStream(new Token[] {
                token("wi",1,1),
                token("hotspot",0,3),
                token("fi",1,1),
                token("network",1,1),
                token("is",1,1),
                token("fast",1,1)


              }),
            new CannedTokenStream(new Token[] {
                token("wifi",1,1),
                token("hotspot",0,2),
                token("network",1,1)
              }),
          };

View Full Code Here

        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
        
        return new TokenStreamComponents(tokenizer) {
          int tokenStreamCounter = 0;
          final TokenStream[] tokenStreams = new TokenStream[] {
            new CannedTokenStream(new Token[] {
                token("ab",1,1),
                token("ba",0,1),
                token("xc",1,1)
              }),
            new CannedTokenStream(new Token[] {
                token("ba",1,1),          
                token("xd",1,1)
              }),
            new CannedTokenStream(new Token[] {
                token("ab",1,1),
                token("ba",0,1),
                token("x",1,1)
              })
          };

View Full Code Here

          @Override
          public TokenStream getTokenStream() {
            // 4th time we are called, return tokens a b,
            // else just a:
            if (count++ != 3) {
              return new CannedTokenStream(new Token[] {
                  token("a", 1, 1),
                });
            } else {
              // After that "a b":
              return new CannedTokenStream(new Token[] {
                  token("a", 1, 1),
                  token("b", 1, 1),
                });
            }
          }

View Full Code Here

      trySetFloatValue(field);
      trySetLongValue(field);
      trySetReaderValue(field);
      trySetShortValue(field);
      field.setStringValue("baz");
      field.setTokenStream(new CannedTokenStream(new Token("foo", 0, 3)));
      
      assertEquals("baz", field.stringValue());
      assertEquals(5f, field.boost(), 0f);
    }
  }

View Full Code Here

    trySetFloatValue(field);
    trySetLongValue(field);
    field.setReaderValue(new StringReader("foobar"));
    trySetShortValue(field);
    trySetStringValue(field);
    field.setTokenStream(new CannedTokenStream(new Token("foo", 0, 3)));
      
    assertNotNull(field.readerValue());
    assertEquals(5f, field.boost(), 0f);
  }

View Full Code Here

    }
  }
  
  private void trySetTokenStreamValue(Field f) {
    try {
      f.setTokenStream(new CannedTokenStream(new Token("foo", 0, 3)));
      fail();
    } catch (IllegalArgumentException expected) {
      // expected
    }
  }

View Full Code Here

  protected void shingleFilterTest(int maxSize, Token[] tokensToShingle, Token[] tokensToCompare,
                                   int[] positionIncrements, String[] types,
                                   boolean outputUnigrams)
    throws IOException {


    ShingleFilter filter = new ShingleFilter(new CannedTokenStream(tokensToShingle), maxSize);
    filter.setOutputUnigrams(outputUnigrams);
    shingleFilterTestCommon(filter, tokensToCompare, positionIncrements, types);
  }

View Full Code Here

  protected void shingleFilterTest(int minSize, int maxSize, Token[] tokensToShingle, 
                                   Token[] tokensToCompare, int[] positionIncrements,
                                   String[] types, boolean outputUnigrams)
    throws IOException {
    ShingleFilter filter 
      = new ShingleFilter(new CannedTokenStream(tokensToShingle), minSize, maxSize);
    filter.setOutputUnigrams(outputUnigrams);
    shingleFilterTestCommon(filter, tokensToCompare, positionIncrements, types);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.analysis.CannedTokenStream

org.apache.lucene.analysis.miscellaneous.TestRemoveDuplicatesTokenFilterFactory

org.apache.lucene.analysis.shingle.ShingleFilterTest

org.apache.lucene.document.TestField

org.apache.lucene.index.memory.MemoryIndexTest

org.apache.lucene.index.TestCheckIndex

org.apache.lucene.index.TestFieldReuse

org.apache.lucene.index.TestFieldReuse$MyField

org.apache.lucene.index.TestIndexWriterExceptions

org.apache.lucene.index.TestPayloadsOnVectors

org.apache.lucene.index.TestPostingsOffsets

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.