StringBuilder builder = new StringBuilder();
for (Pair<String, String> token : tokens) {
builder.append(token.getSecond());
}
System.out.println("'"+builder+"'");
Lexer lexer = newLexer();
lexer.setCharStream(new ANTLRStringStream(builder.toString()));
for (Pair<String, String> pair : tokens) {
CommonToken t = (CommonToken) lexer.nextToken();
String actual = tokenDefProvider().getTokenDefMap().get(t.getType());
System.out.println(actual+" :"+t.getText());
assertEquals(pair.toString(),pair.getFirst(), actual);
assertEquals(pair.toString(),pair.getSecond(), t.getText());
}