* Create a tokenizer that will read from the given string
*/
private BaseJavaCCParserImplTokenManager tokenizer(String testString) {
final Source source = new StringSource("TokenizerTest", testString);
return new BaseJavaCCParserImplTokenManager(new JavaCharStream(
source.createReader()));
}
/**
* Comments should be invisible in the output other than separating tokens
*/