// For known, parsed statements, the terminator is consumed in the markEndOfStatement() method. So if we get
// here, we then we know we've got an unknown statement.
if (tokens.matches(getTerminator()) && sb.length() > 0) {
nextTokenValue = getTerminator();
// Let's call this a statement up until now
AstNode unknownNode = unknownTerminatedNode(getRootNode());
markEndOfStatement(tokens, unknownNode);
// We've determined that it's just an unknown node, which we determine is not a problem node.
problem = null;
} else {
// Just keep consuming, but check each token value and allow sub-classes to handle the token if they wish.
// ORACLE, for instance can terminator a complex statement with a backslash, '/'. Calling
// handleUnknownToken() allows that dialect to create it's own statement node that can be assessed and
// used during the rewrite() call at the end of parsing.
nextTokenValue = tokens.consume();
AstNode unknownNode = handleUnknownToken(tokens, nextTokenValue);
if (unknownNode != null) {
markEndOfStatement(tokens, unknownNode);
// We've determined that it's just an unknown node, which we determine is not a problem node.
problem = null;
}