* Get next token. If an exception is thrown by the underlying lexer,
* keep calling it, and append an invalid token at the very end.
*/
@Override
public Token nextToken() {
Token token = null;
CoffeeSymbol symbol = null;
try {
symbol = aptanaScanner.nextToken();
if (symbol == null || symbol.getId() < 0) {
logger.warn("Unexpected symbol " + symbol, new Exception());
token = CommonToken.INVALID_TOKEN;
}
else if (symbol.getId() == Terminals.EOF) {
token = CommonToken.EOF_TOKEN;
}
else {
token = new BeaverToken(symbol);
if (((CommonToken) token).getStopIndex() >= input.size()) {
assert false: "Token stop index overflows " + symbol + " in:\n<<<" + content + ">>>";
}
}
}
catch (Exception e) {
// Xtext wants token to be CommonToken, INVALID_TOKEN_TYPE, and HIDDEN_CHANNEL
String text = e.getLocalizedMessage();
if (text == null)
text = "simply " + e.getClass().getSimpleName();
CommonToken ct = new CommonToken(Token.INVALID_TOKEN_TYPE,
text);
ct.setChannel(Token.HIDDEN_CHANNEL);
if (prevToken != null) {
int start = prevToken.getStopIndex() + 1;
int stop = start + 1; // TODO: get more informative errors with length of token
ct.setStartIndex(start);
ct.setStopIndex(stop);
}
token = ct;
}
token.setTokenIndex(tokenIndex);
if (symbol != null && symbol.hidden)
token.setChannel(Token.HIDDEN_CHANNEL);
tokenIndex++;
if (token instanceof CommonToken) {
if (prevToken != null && token.getType() > 0) {
if (((CommonToken)token).getStartIndex() < prevToken.getStartIndex()) {
assert false: "Position not follows, prevToken: " + prevToken + ", token: " + token;
}
}
prevToken = (CommonToken)token;