tokenHypernyms = new ArrayList<String>();
tokenHypernyms.add("token41");
tokenHypernyms.add("token42");
hypernyms.put("token4", tokenHypernyms);
HypernymFilter tokenFilter = new HypernymFilter(tokenStream, hypernyms);
Token nextToken = new Token();
tokenFilter.next(nextToken);
assertEquals("token1", nextToken.term());
assertEquals(0, nextToken.startOffset());
assertEquals(6, nextToken.endOffset());
assertEquals(1, nextToken.getPositionIncrement());
nextToken = tokenFilter.next(nextToken);
assertEquals("token2", nextToken.term());
assertEquals(6, nextToken.startOffset());
assertEquals(11, nextToken.endOffset());
assertEquals(1, nextToken.getPositionIncrement());
nextToken = tokenFilter.next(nextToken);
assertEquals("token21", nextToken.term());
assertEquals(6, nextToken.startOffset());
assertEquals(11, nextToken.endOffset());
assertEquals(0, nextToken.getPositionIncrement());
nextToken = tokenFilter.next(nextToken);
assertEquals("token22", nextToken.term());
assertEquals(6, nextToken.startOffset());
assertEquals(11, nextToken.endOffset());
assertEquals(0, nextToken.getPositionIncrement());
nextToken = tokenFilter.next(nextToken);
assertEquals("token23", nextToken.term());
assertEquals(6, nextToken.startOffset());
assertEquals(11, nextToken.endOffset());
assertEquals(0, nextToken.getPositionIncrement());
nextToken = tokenFilter.next(nextToken);
assertEquals("token3", nextToken.term());
assertEquals(11, nextToken.startOffset());
assertEquals(17, nextToken.endOffset());
assertEquals(1, nextToken.getPositionIncrement());
nextToken = tokenFilter.next(nextToken);
assertEquals("token4", nextToken.term());
assertEquals(17, nextToken.startOffset());
assertEquals(23, nextToken.endOffset());
assertEquals(1, nextToken.getPositionIncrement());
nextToken = tokenFilter.next(nextToken);
assertEquals("token41", nextToken.term());
assertEquals(17, nextToken.startOffset());
assertEquals(23, nextToken.endOffset());
assertEquals(0, nextToken.getPositionIncrement());
nextToken = tokenFilter.next(nextToken);
assertEquals("token42", nextToken.term());
assertEquals(17, nextToken.startOffset());
assertEquals(23, nextToken.endOffset());
assertEquals(0, nextToken.getPositionIncrement());