protected void testTokenizing(String[] expectedTokens, BlockHeuristicsScannner scanner, boolean forward,
boolean bounce)
throws BadLocationException {
FnTokenAdvance advanceTokenFn = forward ? scanner.nextTokenFn : scanner.prevTokenFn;
if(forward) {
scanner.setScanRange(0, document.getLength());
} else {
scanner.setScanRange(document.getLength(), 0);
}
for (int i = 0; i < expectedTokens.length; i++) {
String expectedToken = expectedTokens[forward ? i : expectedTokens.length-i-1];
int oldPos = scanner.getPosition();
advanceTokenFn.advanceToken();
if(bounce) {
if(i != 0) {
int tokenSaved = scanner.token;
int positionSaved = scanner.pos;
advanceTokenFn.revertToken();
advanceTokenFn.advanceToken();
assertTrue(tokenSaved == scanner.token);
assertTrue(positionSaved == scanner.pos);
}
}
String tokenStr = forward ?