//respond to problems from our file handler
includedFileSpec = includeHandler.getFileSpecificationForInclude(sourcePath, includeString);
//
if (includedFileSpec == null)
{
ICompilerProblem problem = new FileNotFoundProblem(token, filenameTokenText); //the text will be the path not found
problems.add(problem);
retVal = next();
return retVal;
}
if (includeHandler.isCyclicInclude(includedFileSpec.getPath()))
{
ICompilerProblem problem = new CyclicalIncludesProblem(token);
problems.add(problem);
retVal = next();
return retVal;
}
else
{
// Fork a tokenizer for the included file
try
{
forkIncludeTokenizer = createForIncludeFile(this, includedFileSpec, includeHandler);
retVal = forkIncludeTokenizer.next();
}
catch (FileNotFoundException fnfe)
{
includeHandler.handleFileNotFound(includedFileSpec);
ICompilerProblem problem = new FileNotFoundProblem(token, includedFileSpec.getPath());
problems.add(problem);
retVal = next();
return retVal;
}
}
}
// Recover from compiler problems and continue.
if (retVal == null)
{
// Included file is empty.
closeIncludeTokenizer();
// Fall back to main source.
retVal = this.next();
}
return retVal;
}
case TOKEN_RESERVED_WORD_CONFIG:
if (matches(LT(1), TOKEN_RESERVED_WORD_NAMESPACE))
{ //we config namespace
retVal.setType(TOKEN_RESERVED_WORD_CONFIG);
return retVal;
}
treatKeywordAsIdentifier(retVal); //identifier
processUserDefinedNamespace(retVal, 0);
return retVal;
case HIDDEN_TOKEN_BUILTIN_NS:
if (matches(LT(1), TOKEN_OPERATOR_NS_QUALIFIER))
{ //we have public:: and this structure is not an annotation but a name ref
retVal.setType(TOKEN_NAMESPACE_NAME);
return retVal;
}
retVal.setType(TOKEN_NAMESPACE_ANNOTATION);
return retVal;
case TOKEN_MODIFIER_DYNAMIC:
case TOKEN_MODIFIER_FINAL:
case TOKEN_MODIFIER_NATIVE:
case TOKEN_MODIFIER_OVERRIDE:
case TOKEN_MODIFIER_STATIC:
case TOKEN_MODIFIER_VIRTUAL:
{
// previous token is either a modifier or a namespace, or if
// null, assume keyword
// next token is from a definition or a modifier or a namespace
final ASToken nextToken = LT(1);
if (nextToken != null)
{
switch (nextToken.getType())
{
case TOKEN_KEYWORD_CLASS:
case TOKEN_KEYWORD_FUNCTION:
case TOKEN_KEYWORD_INTERFACE:
case TOKEN_RESERVED_WORD_NAMESPACE:
case TOKEN_KEYWORD_VAR:
case TOKEN_KEYWORD_CONST:
case TOKEN_MODIFIER_DYNAMIC:
case TOKEN_MODIFIER_FINAL:
case TOKEN_MODIFIER_NATIVE:
case TOKEN_MODIFIER_OVERRIDE:
case TOKEN_MODIFIER_STATIC:
case TOKEN_MODIFIER_VIRTUAL:
case TOKEN_NAMESPACE_ANNOTATION:
case TOKEN_NAMESPACE_NAME:
case HIDDEN_TOKEN_BUILTIN_NS:
return retVal;
case TOKEN_IDENTIFIER:
if (isUserDefinedNamespace(nextToken, 1)) // we're already looking ahead one so make sure we look ahead one further
return retVal;
default:
// Not applicable to other token types.
break;
}
}
treatKeywordAsIdentifier(retVal);
processUserDefinedNamespace(retVal, 0);
return retVal;
}
//we combine +/- for numeric literals here
case TOKEN_OPERATOR_MINUS:
case TOKEN_OPERATOR_PLUS:
{
if (lastToken == null || !lastToken.canPreceedSignedOperator())
{
final ASToken nextToken = LT(1);
if (nextToken != null)
{
switch (nextToken.getType())
{
case TOKEN_LITERAL_NUMBER:
case TOKEN_LITERAL_HEX_NUMBER:
retVal.setEnd(nextToken.getEnd());
final StringBuilder builder = new StringBuilder(retVal.getText());
builder.append(nextToken.getText());
retVal.setText(poolString(builder.toString()));
consume(1);
retVal.setType(nextToken.getType());
break;
default:
// ignore other tokens
break;
}
}
}
return retVal;
}
//RECOGNIZE: for each
case TOKEN_KEYWORD_FOR:
{
final ASToken token = LT(1);
if (matches(token, TOKEN_RESERVED_WORD_EACH))
{
retVal.setEnd(token.getEnd());
retVal.setText(FOR_EACH);
consume(1);
return retVal;
}
return retVal;
}
//RECOGNIZE: default xml namespace
//default xml namespace must exist on the same line
case TOKEN_KEYWORD_DEFAULT:
{
final ASToken maybeNS = LT(2);
final boolean foundTokenNamespace = maybeNS != null &&
maybeNS.getType() == TOKEN_RESERVED_WORD_NAMESPACE;
if (foundTokenNamespace)
{
final ASToken maybeXML = LT(1);
final boolean foundTokenXML = maybeXML != null &&
maybeXML.getType() == TOKEN_IDENTIFIER &&
XML.equals(maybeXML.getText());
if (!foundTokenXML)
{
final ICompilerProblem problem =
new ExpectXmlBeforeNamespaceProblem(maybeNS);
problems.add(problem);
}
//combine all of these tokens together
retVal.setEnd(maybeNS.getEnd());
retVal.setText(DEFAULT_XML_NAMESPACE);
retVal.setType(TOKEN_DIRECTIVE_DEFAULT_XML);
consume(2);
}
return retVal;
}
case TOKEN_KEYWORD_VOID:
{
//check for void 0
final ASToken token = LT(1);
if (matches(token, TOKEN_LITERAL_NUMBER) && ZERO.equals(token.getText()))
{
retVal.setType(TOKEN_VOID_0);
combineText(retVal, token);
consume(1);
}
//check for void(0)
else if (matches(token, TOKEN_PAREN_OPEN))
{
final ASToken zeroT = LT(2);
if (matches(zeroT, TOKEN_LITERAL_NUMBER) && ZERO.equals(zeroT.getText()))
{
final ASToken closeParenT = LT(3);
if (matches(closeParenT, TOKEN_PAREN_CLOSE))
{
combineText(retVal, token);
combineText(retVal, zeroT);
combineText(retVal, closeParenT);
retVal.setType(TOKEN_VOID_0);
consume(3);
}
}
}
return retVal;
}
case TOKEN_IDENTIFIER:
{
//check for user-defined namespace before we return anything
processUserDefinedNamespace(retVal, 0);
return retVal;
}
//this is for metadata processing
case TOKEN_SQUARE_OPEN:
{
retVal = tryParseMetadata(retVal);
return retVal;
}
case HIDDEN_TOKEN_STAR_ASSIGNMENT:
{
//this is to solve an ambiguous case, where we can't tell the difference between
//var foo:*=null and foo *= null;
retVal.setType(TOKEN_OPERATOR_STAR);
retVal.setEnd(retVal.getEnd() - 1);
retVal.setText("*");
//add the equals
final ASToken nextToken = tokenizer.buildToken(TOKEN_OPERATOR_ASSIGNMENT,
retVal.getEnd() + 1, retVal.getEnd() + 2,
retVal.getLine(), retVal.getColumn(), "=");
nextToken.setSourcePath(sourcePath);
addTokenToBuffer(nextToken);
return retVal;
}
case TOKEN_SEMICOLON:
if (consumeSemi)
{
return next();
}
return retVal;
case TOKEN_VOID_0:
case TOKEN_LITERAL_REGEXP:
case TOKEN_COMMA:
case TOKEN_COLON:
case TOKEN_PAREN_OPEN:
case TOKEN_PAREN_CLOSE:
case TOKEN_SQUARE_CLOSE:
case TOKEN_ELLIPSIS:
case TOKEN_OPERATOR_PLUS_ASSIGNMENT:
case TOKEN_OPERATOR_MINUS_ASSIGNMENT:
case TOKEN_OPERATOR_MULTIPLICATION_ASSIGNMENT:
case TOKEN_OPERATOR_DIVISION_ASSIGNMENT:
case TOKEN_OPERATOR_MODULO_ASSIGNMENT:
case TOKEN_OPERATOR_BITWISE_AND_ASSIGNMENT:
case TOKEN_OPERATOR_BITWISE_OR_ASSIGNMENT:
case TOKEN_OPERATOR_BITWISE_XOR_ASSIGNMENT:
case TOKEN_OPERATOR_BITWISE_LEFT_SHIFT_ASSIGNMENT:
case TOKEN_OPERATOR_BITWISE_RIGHT_SHIFT_ASSIGNMENT:
case TOKEN_OPERATOR_BITWISE_UNSIGNED_RIGHT_SHIFT_ASSIGNMENT:
case TOKEN_OPERATOR_STAR:
case TOKEN_OPERATOR_NS_QUALIFIER:
case TOKEN_ASDOC_COMMENT:
case TOKEN_OPERATOR_DIVISION:
case TOKEN_OPERATOR_MODULO:
case TOKEN_OPERATOR_BITWISE_LEFT_SHIFT:
case TOKEN_OPERATOR_BITWISE_RIGHT_SHIFT:
case TOKEN_OPERATOR_BITWISE_UNSIGNED_RIGHT_SHIFT:
case TOKEN_OPERATOR_LESS_THAN:
case TOKEN_OPERATOR_GREATER_THAN:
case TOKEN_OPERATOR_LESS_THAN_EQUALS:
case TOKEN_OPERATOR_GREATER_THAN_EQUALS:
case TOKEN_OPERATOR_EQUAL:
case TOKEN_OPERATOR_NOT_EQUAL:
case TOKEN_OPERATOR_STRICT_EQUAL:
case TOKEN_OPERATOR_STRICT_NOT_EQUAL:
case TOKEN_OPERATOR_BITWISE_AND:
case TOKEN_OPERATOR_BITWISE_XOR:
case TOKEN_OPERATOR_BITWISE_OR:
case TOKEN_OPERATOR_LOGICAL_AND:
case TOKEN_OPERATOR_LOGICAL_OR:
case TOKEN_OPERATOR_LOGICAL_AND_ASSIGNMENT:
case TOKEN_OPERATOR_LOGICAL_OR_ASSIGNMENT:
case TOKEN_TYPED_COLLECTION_OPEN:
case TOKEN_TYPED_COLLECTION_CLOSE:
case TOKEN_OPERATOR_MEMBER_ACCESS:
case TOKEN_KEYWORD_RETURN:
case TOKEN_KEYWORD_THROW:
case TOKEN_KEYWORD_NEW:
case TOKEN_RESERVED_WORD_NAMESPACE:
case TOKEN_RESERVED_WORD_GET:
case TOKEN_RESERVED_WORD_SET:
case TOKEN_OPERATOR_ASSIGNMENT:
case TOKEN_TYPED_LITERAL_CLOSE:
case TOKEN_TYPED_LITERAL_OPEN:
case TOKEN_OPERATOR_TERNARY:
case TOKEN_OPERATOR_DECREMENT:
case TOKEN_OPERATOR_INCREMENT:
case TOKEN_OPERATOR_ATSIGN:
case TOKEN_OPERATOR_BITWISE_NOT:
case TOKEN_OPERATOR_LOGICAL_NOT:
case TOKEN_E4X_BINDING_CLOSE:
case TOKEN_E4X_BINDING_OPEN:
case TOKEN_OPERATOR_DESCENDANT_ACCESS:
case TOKEN_NAMESPACE_ANNOTATION:
case TOKEN_NAMESPACE_NAME:
case TOKEN_BLOCK_OPEN:
case TOKEN_BLOCK_CLOSE:
case TOKEN_KEYWORD_FUNCTION:
return retVal;
case HIDDEN_TOKEN_MULTI_LINE_COMMENT:
case HIDDEN_TOKEN_SINGLE_LINE_COMMENT:
if (tokenizer.isCollectingComments())
{
return retVal;
}
assert (false);
return null;
default:
if (ASToken.isE4X(tokenType))
return retVal;
if (retVal.isKeywordOrContextualReservedWord() || retVal.isLiteral())
return retVal;
// If we reach here, the token fails to match any processing logic.
final UnexpectedTokenProblem problem = new UnexpectedTokenProblem(
retVal,
ASTokenKind.UNKNOWN);
problems.add(problem);
}
}
catch (final Exception e)
{
if (lastException != null)
{
if (lastException.getClass().isInstance(e))
{
ICompilerProblem problem = new InternalCompilerProblem2(sourcePath, e, "StreamingASTokenizer");
problems.add(problem);
return null;
}
}
else