int fakeTokenColumn = 0;
// Last token in always newline, see DocumentParserWorker.
final int l = tokens.size() - 1;
for (int i = 0; i < l; i++) {
Token token = tokens.get(i);
assert token != null;
// Ignore whitespaces
if (WHITESPACE == token.getType()) {
continue;
}
// Common precondition: context is not too short
if (context.predPred == null) {
context.push(token);
continue;
}
PyToken predPred = context.predPred;
PyToken pred = context.pred;
// In case we get ":" or "(" it may be a function definition
if (LITERAL_COLON.equals(token.getValue())
|| LITERAL_LEFT_PARANTHESIS.equals(token.getValue())) {
if ((KEYWORD == predPred.getType()) && "def".equals(predPred.getContent())) {
// getResultScope().addToken(lineNumber, fakeTokenColumn,
// new CodeToken(pred.getContent(), pred.getType(), true));
}
fakeTokenColumn++;
context.push(token);
continue;
}
// When we get ", id," construction,
// then do not reset accumulator (ids before first comma)
if (LITERAL_COMMA.equals(token.getValue())) {
if (LITERAL_COMMA.equals(predPred.getContent())) {
if (VARIABLE == pred.getType()) {
context.pushSavingAccumulator(token);
continue;
}
}
context.push(token);
continue;
}
// When we got "id =" then register all remembered ids as variables
if (LITERAL_EQUALS.equals(token.getValue())) {
if (VARIABLE == context.pred.getType()) {
context.accumulator.add(context.pred);
while (!context.accumulator.isEmpty()) {
PyToken nextVar = context.accumulator.pop();
// getResultScope().addToken(lineNumber, fakeTokenColumn,
// new CodeToken(nextVar.getContent(), nextVar.getType()));
fakeTokenColumn++;
}
}
context.push(token);
continue;
}
// When we get "id1, id2" construction
// then remember id1, because it is going to be pushed out of context.
if (VARIABLE == token.getType()) {
if (VARIABLE == context.predPred.getType()) {
if (LITERAL_COMMA.equals(context.pred.getContent())) {
context.pushAndAddToAccumulator(token);
continue;
}