Examples of edu.buffalo.cse.ir.wikiindexer.tokenizer.TokenStream.mergeWithPrevious()

Package edu.buffalo.cse.ir.wikiindexer.tokenizer

Class edu.buffalo.cse.ir.wikiindexer.tokenizer.TokenStream

Examples of edu.buffalo.cse.ir.wikiindexer.tokenizer.TokenStream.mergeWithPrevious()

edu.buffalo.cse.ir.wikiindexer.tokenizer.TokenStream.mergeWithPrevious()
Method to merge the current token with the previous token, assumes whitespace separator between tokens when merged. The token iterator should now point to the newly merged token (i.e. the previous one) @return true if the merge succeeded, false otherwise

    
    //with merge with previous
    stream = new TokenStream("to");
    stream.append("be", "or", "not", "to", "be");
    stream.next(); //at be
    stream.mergeWithPrevious();
    stream.seekEnd();
    stream.previous();
    stream.mergeWithPrevious();
    smap = getSortedMap(stream.getTokenMap());
    assertEquals("[not, or, to be]", smap.keySet().toString());

View Full Code Here

    stream.append("be", "or", "not", "to", "be");
    stream.next(); //at be
    stream.mergeWithPrevious();
    stream.seekEnd();
    stream.previous();
    stream.mergeWithPrevious();
    smap = getSortedMap(stream.getTokenMap());
    assertEquals("[not, or, to be]", smap.keySet().toString());
    assertEquals("[1, 1, 2]", smap.values().toString());
    stream = null;

View Full Code Here

    
    //with merge with previous
    stream = new TokenStream("this");
    stream.append("is", "a", "test", "string");
    stream.next();
    stream.mergeWithPrevious();
    assertEquals(0, stream.query("this"));
    assertEquals(1, stream.query("this is"));
    stream = null;
    
    //with merge with next

View Full Code Here

    
    //with merge with previous
    stream = new TokenStream("this");
    stream.append("is","a","test","stream");
    stream.next();
    stream.mergeWithPrevious();
    assertTrue(stream.hasNext());
    stream.seekEnd();
    stream.previous();
    stream.mergeWithPrevious();
    assertTrue(stream.hasNext());

View Full Code Here

    stream.next();
    stream.mergeWithPrevious();
    assertTrue(stream.hasNext());
    stream.seekEnd();
    stream.previous();
    stream.mergeWithPrevious();
    assertTrue(stream.hasNext());
    stream = null;
    
    //with merge with next
    stream = new TokenStream("this");

View Full Code Here

    
    //with merge with previous
    stream = new TokenStream("this");
    stream.append("is","a","test","stream");
    stream.next();
    stream.mergeWithPrevious();
    assertFalse(stream.hasPrevious());
    stream = null;
    
    //with merge with next
    stream = new TokenStream("this");

View Full Code Here

    
    //with merge with previous
    stream = new TokenStream("this");
    stream.append("is","a","test","stream");
    stream.next();
    stream.mergeWithPrevious();
    assertEquals("this is", stream.next());
    stream = null;
    
    //with merge with next
    stream = new TokenStream("this");

View Full Code Here

    
    //with merge with previous
    stream = new TokenStream("this");
    stream.append("is","a","test","stream");
    stream.next();
    stream.mergeWithPrevious();
    assertNull(stream.previous());
    stream.next();
    assertEquals("this is", stream.previous());
    stream = null;

View Full Code Here

   */
  @Test
  public void testMergeWithPrevious() {
    //everything is null, empty
    TokenStream stream = new TokenStream((String) null);
    assertFalse(stream.mergeWithPrevious());
    stream = null;
    
    stream = new TokenStream("");
    assertFalse(stream.mergeWithPrevious());
    stream = null;

View Full Code Here

    TokenStream stream = new TokenStream((String) null);
    assertFalse(stream.mergeWithPrevious());
    stream = null;
    
    stream = new TokenStream("");
    assertFalse(stream.mergeWithPrevious());
    stream = null;
    
    //previous is null
    stream = new TokenStream("this");
    stream.append("is","a","test","stream");

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.