{
// Remove any dummy tokens we created using the fake characters we appended to
// the end of the stream
for (int tokenIndex = tokenList.size() - 1; tokenIndex >= 0; tokenIndex--)
{
TokenBase token = (TokenBase)tokenList.get(tokenIndex);
if (token.getEnd() <= lastRealIndex)
break;
else if (token.getStart() < lastRealIndex &&
token.getEnd() > lastRealIndex)
{
// The end of this token is in the fake characters. Truncate it.
int actualLength = lastRealIndex - token.getStart();
if (actualLength > token.toString().length())
{
actualLength = token.toString().length();
}
tokenList.set(tokenIndex, buildToken(token.getType(), token.getStart(), lastRealIndex, -1, -1, token.toString().substring(0, actualLength)));
}
else
{
// This entire token is in the fake characters. Discard it.
tokenList.remove(tokenIndex);