// This file is part of PyANTLR. See LICENSE.txt for license
// details..........Copyright (C) Wolfgang Haefelinger, 2004.
//
// $Id:$
package antlr;
import java.util.Enumeration;
import java.util.Hashtable;
import antlr.collections.impl.BitSet;
import antlr.collections.impl.Vector;
import java.io.PrintWriter; //SAS: changed for proper text file io
import java.io.IOException;
import java.io.FileWriter;
/**Generate MyParser.java, MyLexer.java and MyParserTokenTypes.java */
public class PythonCodeGenerator extends CodeGenerator {
// non-zero if inside syntactic predicate generation
protected int syntacticPredLevel = 0;
// Are we generating ASTs (for parsers and tree parsers) right now?
protected boolean genAST = false;
// Are we saving the text consumed (for lexers) right now?
protected boolean saveText = false;
// Grammar parameters set up to handle different grammar classes.
// These are used to get instanceof tests out of code generation
String labeledElementType;
String labeledElementASTType;
String labeledElementInit;
String commonExtraArgs;
String commonExtraParams;
String commonLocalVars;
String lt1Value;
String exceptionThrown;
String throwNoViable;
public static final String initHeaderAction = "__init__";
public static final String mainHeaderAction = "__main__";
String lexerClassName;
String parserClassName;
String treeWalkerClassName;
/** Tracks the rule being generated. Used for mapTreeId */
RuleBlock currentRule;
/** Tracks the rule or labeled subrule being generated. Used for
AST generation. */
String currentASTResult;
/** Mapping between the ids used in the current alt, and the
* names of variables used to represent their AST values.
*/
Hashtable treeVariableMap = new Hashtable();
/** Used to keep track of which AST variables have been defined in a rule
* (except for the #rule_name and #rule_name_in var's
*/
Hashtable declaredASTVariables = new Hashtable();
/* Count of unnamed generated variables */
int astVarNumber = 1;
/** Special value used to mark duplicate in treeVariableMap */
protected static final String NONUNIQUE = new String();
public static final int caseSizeThreshold = 127; // ascii is max
private Vector semPreds;
/** Create a Java code-generator using the given Grammar.
* The caller must still call setTool, setBehavior, and setAnalyzer
* before generating code.
*/
protected void printTabs() {
for (int i = 0; i < tabs; i++) {
// don't print tabs ever - replace a tab by ' '
currentOutput.print(" ");
}
}
public PythonCodeGenerator() {
super();
charFormatter = new antlr.PythonCharFormatter();
DEBUG_CODE_GENERATOR = true;
}
/** Adds a semantic predicate string to the sem pred vector
These strings will be used to build an array of sem pred names
when building a debugging parser. This method should only be
called when the debug option is specified
*/
protected int addSemPred(String predicate) {
semPreds.appendElement(predicate);
return semPreds.size() - 1;
}
public void exitIfError() {
if (antlrTool.hasError()) {
antlrTool.fatalError("Exiting due to errors.");
}
}
protected void checkCurrentOutputStream() {
try
{
if(currentOutput == null)
throw new NullPointerException();
}
catch(Exception e)
{
System.err.println("error: current output is not set");
e.printStackTrace(System.err);
System.exit(1);
}
}
/** Get the identifier portion of an argument-action.
* For Python the ID of an action is assumed to be everything before
* the assignment, as Python does not support a type.
* @param s The action text
* @param line Line used for error reporting.
* @param column Line used for error reporting.
* @return A string containing the text of the identifier
*/
protected String extractIdOfAction(String s, int line, int column) {
s = removeAssignmentFromDeclaration(s);
//wh: removeAssignmentFromDeclaration returns an indentifier that
//wh: may start with whitespace.
s = s.trim();
// println("###ZZZZZ \""+s+"\"");
return s;
}
/** Get the type portion of an argument-action.
* Python does not have a type declaration before an identifier, so we
* just return the empty string.
* @param s The action text
* @param line Line used for error reporting.
* @return A string containing the text of the type
*/
protected String extractTypeOfAction(String s, int line, int column) {
return "";
}
protected void flushTokens() {
try
{
boolean generated = false;
checkCurrentOutputStream();
println("");
println("### import antlr.Token ");
println("from antlr import Token");
println("### >>>The Known Token Types <<<");
/* save current stream */
PrintWriter cout = currentOutput;
// Loop over all token managers (some of which are lexers)
Enumeration tmIter =
behavior.tokenManagers.elements();
while (tmIter.hasMoreElements())
{
TokenManager tm =
(TokenManager)tmIter.nextElement();
if (!tm.isReadOnly())
{
// Write the token manager tokens as Java
// this must appear before genTokenInterchange so that
// labels are set on string literals
if(! generated) {
genTokenTypes(tm);
generated = true;
}
/* restore stream */
currentOutput = cout;
// Write the token manager tokens as plain text
genTokenInterchange(tm);
currentOutput = cout;
}
exitIfError();
}
}
catch(Exception e) {
exitIfError();
}
checkCurrentOutputStream();
println("");
}
/**Generate the parser, lexer, treeparser, and token types in Java */
public void gen() {
// Do the code generation
try {
// Loop over all grammars
Enumeration grammarIter = behavior.grammars.elements();
while (grammarIter.hasMoreElements()) {
Grammar g = (Grammar)grammarIter.nextElement();
// Connect all the components to each other
g.setGrammarAnalyzer(analyzer);
g.setCodeGenerator(this);
analyzer.setGrammar(g);
// To get right overloading behavior across hetrogeneous grammars
setupGrammarParameters(g);
g.generate();
// print out the grammar with lookahead sets (and FOLLOWs)
// System.out.print(g.toString());
exitIfError();
}
}
catch (IOException e) {
antlrTool.reportException(e, null);
}
}
/** Generate code for the given grammar element.
* @param blk The {...} action to generate
*/
public void gen(ActionElement action) {
if (action.isSemPred) {
genSemPred(action.actionText, action.line);
}
else
{
if (grammar.hasSyntacticPredicate) {
println("if not self.inputState.guessing:");
tabs++;
}
// get the name of the followSet for the current rule so that we
// can replace $FOLLOW in the .g file.
ActionTransInfo tInfo = new ActionTransInfo();
String actionStr = processActionForSpecialSymbols(action.actionText,
action.getLine(),
currentRule,
tInfo);
if (tInfo.refRuleRoot != null) {
// Somebody referenced "#rule", make sure translated var is valid
// assignment to #rule is left as a ref also, meaning that assignments
// with no other refs like "#rule = foo();" still forces this code to be
// generated (unnecessarily).
println(tInfo.refRuleRoot + " = currentAST.root");
}
// dump the translated action
printAction(actionStr);
if (tInfo.assignToRoot) {
// Somebody did a "#rule=", reset internal currentAST.root
println("currentAST.root = " + tInfo.refRuleRoot + "");
println("if (" + tInfo.refRuleRoot + " != None) and (" + tInfo.refRuleRoot + ".getFirstChild() != None):");
tabs++;
println("currentAST.child = " + tInfo.refRuleRoot + ".getFirstChild()");
tabs--;
println("else:");
tabs++;
println("currentAST.child = " + tInfo.refRuleRoot);
tabs--;
println("currentAST.advanceChildToEnd()");
}
if (grammar.hasSyntacticPredicate) {
tabs--;
}
}
}
/** Generate code for the given grammar element.
* @param blk The "x|y|z|..." block to generate
*/
public void gen(AlternativeBlock blk) {
if (DEBUG_CODE_GENERATOR) System.out.println("gen(" + blk + ")");
genBlockPreamble(blk);
genBlockInitAction(blk);
// Tell AST generation to build subrule result
String saveCurrentASTResult = currentASTResult;
if (blk.getLabel() != null) {
currentASTResult = blk.getLabel();
}
boolean ok = grammar.theLLkAnalyzer.deterministic(blk);
{
int _tabs_ = tabs;
PythonBlockFinishingInfo howToFinish = genCommonBlock(blk, true);
genBlockFinish(howToFinish, throwNoViable);
tabs = _tabs_;
}
// Restore previous AST generation
currentASTResult = saveCurrentASTResult;
}
/** Generate code for the given grammar element.
* @param blk The block-end element to generate. Block-end
* elements are synthesized by the grammar parser to represent
* the end of a block.
*/
public void gen(BlockEndElement end) {
if (DEBUG_CODE_GENERATOR) System.out.println("genRuleEnd(" + end + ")");
}
/** Generate code for the given grammar element.
* @param blk The character literal reference to generate
*/
public void gen(CharLiteralElement atom) {
if (DEBUG_CODE_GENERATOR) System.out.println("genChar(" + atom + ")");
if (atom.getLabel() != null) {
println(atom.getLabel() + " = " + lt1Value );
}
boolean oldsaveText = saveText;
saveText = saveText && atom.getAutoGenType() == GrammarElement.AUTO_GEN_NONE;
genMatch(atom);
saveText = oldsaveText;
}
String toString(boolean v) {
String s;
if(v)
s = "True";
else
s = "False";
return s;
}
/** Generate code for the given grammar element.
* @param blk The character-range reference to generate
*/
public void gen(CharRangeElement r) {
if (r.getLabel() != null && syntacticPredLevel == 0) {
println(r.getLabel() + " = " + lt1Value);
}
boolean flag = ( grammar instanceof LexerGrammar &&
( !saveText ||
r.getAutoGenType() ==
GrammarElement.AUTO_GEN_BANG ) );
if (flag) {
println("_saveIndex = self.text.length()");
}
println("self.matchRange(u" + r.beginText + ", u" + r.endText + ")");
if (flag) {
println("self.text.setLength(_saveIndex)");
}
}
/** Generate the lexer Java file */
public void gen(LexerGrammar g) throws IOException
{
// If debugging, create a new sempred vector for this grammar
if (g.debuggingOutput)
semPreds = new Vector();
setGrammar(g);
if (!(grammar instanceof LexerGrammar)) {
antlrTool.panic("Internal error generating lexer");
}
// SAS: moved output creation to method so a subclass can change
// how the output is generated (for VAJ interface)
setupOutput(grammar.getClassName());
genAST = false; // no way to gen trees.
saveText = true; // save consumed characters.
tabs = 0;
// Generate header common to all Python output files
genHeader();
// Generate header specific to lexer Python file
println("### import antlr and other modules ..");
println("import sys");
println("import antlr");
println("");
println("version = sys.version.split()[0]");
println("if version < '2.2.1':");
tabs++;
println("False = 0");
tabs--;
println("if version < '2.3':");
tabs++;
println("True = not False");
tabs--;
println("### header action >>> ");
printActionCode(behavior.getHeaderAction(""),0);
println("### header action <<< ");
// Generate user-defined lexer file preamble
println("### preamble action >>> ");
printActionCode(grammar.preambleAction.getText(),0);
println("### preamble action <<< ");
// Generate lexer class definition
String sup = null;
if (grammar.superClass != null) {
sup = grammar.superClass;
}
else {
sup = "antlr." + grammar.getSuperClass();
}
// get prefix (replaces "public" and lets user specify)
String prefix = "";
Token tprefix = (Token)grammar.options.get("classHeaderPrefix");
if (tprefix != null) {
String p = StringUtils.stripFrontBack(tprefix.getText(), "\"", "\"");
if (p != null) {
prefix = p;
}
}
// print my literals
println("### >>>The Literals<<<");
println("literals = {}");
Enumeration keys = grammar.tokenManager.getTokenSymbolKeys();
while (keys.hasMoreElements()) {
String key = (String)keys.nextElement();
if (key.charAt(0) != '"') {
continue;
}
TokenSymbol sym = grammar.tokenManager.getTokenSymbol(key);
if (sym instanceof StringLiteralSymbol) {
StringLiteralSymbol s = (StringLiteralSymbol)sym;
println("literals[u" + s.getId() + "] = " + s.getTokenType());
}
}
println("");
flushTokens();
// print javadoc comment if any
genJavadocComment(grammar);
// class name remains the same, it's the module that changes in python.
println("class " + lexerClassName + "(" + sup + ") :");
tabs++;
printGrammarAction(grammar);
// Generate the constructor from InputStream, which in turn
// calls the ByteBuffer constructor
//
println("def __init__(self, *argv, **kwargs) :");
tabs++;
println(sup + ".__init__(self, *argv, **kwargs)");
// Generate the setting of various generated options.
// These need to be before the literals since ANTLRHashString depends on
// the casesensitive stuff.
println("self.caseSensitiveLiterals = " + toString(g.caseSensitiveLiterals));
println("self.setCaseSensitive(" + toString(g.caseSensitive) + ")" );
println("self.literals = literals");
Enumeration ids;
// generate the rule name array for debugging
if (grammar.debuggingOutput) {
println("ruleNames[] = [");
ids = grammar.rules.elements();
int ruleNum = 0;
tabs++;
while (ids.hasMoreElements()) {
GrammarSymbol sym = (GrammarSymbol)ids.nextElement();
if (sym instanceof RuleSymbol)
println("\"" + ((RuleSymbol)sym).getId() + "\",");
}
tabs--;
println("]");
}
genHeaderInit(grammar);
tabs--;
// wh: iterator moved to base class as proposed by mk.
// println("");
// Generate the __iter__ method for Python CharScanner (sub)classes.
// genIterator();
// Generate nextToken() rule.
// nextToken() is a synthetic lexer rule that is the implicit OR of all
// user-defined lexer rules.
genNextToken();
println("");
// Generate code for each rule in the lexer
ids = grammar.rules.elements();
int ruleNum = 0;
while (ids.hasMoreElements()) {
RuleSymbol sym = (RuleSymbol)ids.nextElement();
// Don't generate the synthetic rules
if (!sym.getId().equals("mnextToken")) {
genRule(sym, false, ruleNum++);
}
exitIfError();
}
// Generate the semantic predicate map for debugging
if (grammar.debuggingOutput)
genSemPredMap();
// Generate the bitsets used throughout the lexer
genBitsets(bitsetsUsed, ((LexerGrammar)grammar).charVocabulary.size());
println("");
genHeaderMain(grammar);
// Close the lexer output stream
currentOutput.close();
currentOutput = null;
}
protected void genHeaderMain(Grammar grammar)
{
String h = grammar.getClassName() + "." + mainHeaderAction;
String s = behavior.getHeaderAction(h);
if (isEmpty(s)) {
s = behavior.getHeaderAction(mainHeaderAction);
}
if(isEmpty(s)) {
if(grammar instanceof LexerGrammar) {
int _tabs = tabs;
tabs = 0;
println("### __main__ header action >>> ");
genLexerTest();
tabs = 0;
println("### __main__ header action <<< ");
tabs = _tabs;
}
} else {
int _tabs = tabs;
tabs = 0;
println("");
println("### __main__ header action >>> ");
printMainFunc(s);
tabs = 0;
println("### __main__ header action <<< ");
tabs = _tabs;
}
}
protected void genHeaderInit(Grammar grammar)
{
String h = grammar.getClassName() + "." + initHeaderAction;
String s = behavior.getHeaderAction(h);
if (isEmpty(s)) {
s = behavior.getHeaderAction(initHeaderAction);
}
if(isEmpty(s)) {
/* nothing gets generated by default */
} else {
int _tabs = tabs;
println("### __init__ header action >>> ");
printActionCode(s,0);
tabs = _tabs;
println("### __init__ header action <<< ");
}
}
protected void printMainFunc(String s) {
int _tabs = tabs;
tabs = 0;
println("if __name__ == '__main__':");
tabs++;
printActionCode(s,0);
tabs--;
tabs = _tabs;
}
/** Generate code for the given grammar element.
* @param blk The (...)+ block to generate
*/
public void gen(OneOrMoreBlock blk) {
String label;
String cnt;
/* save current tabs */
int _tabs_ = tabs;
genBlockPreamble(blk);
if (blk.getLabel() != null)
{
cnt = "_cnt_" + blk.getLabel();
}
else {
cnt = "_cnt" + blk.ID;
}
println("" + cnt + "= 0");
println("while True:");
tabs++;
_tabs_ = tabs;
// generate the init action for ()+ ()* inside the loop
// this allows us to do usefull EOF checking...
genBlockInitAction(blk);
// Tell AST generation to build subrule result
String saveCurrentASTResult = currentASTResult;
if (blk.getLabel() != null) {
currentASTResult = blk.getLabel();
}
boolean ok = grammar.theLLkAnalyzer.deterministic(blk);
// generate exit test if greedy set to false
// and an alt is ambiguous with exit branch
// or when lookahead derived purely from end-of-file
// Lookahead analysis stops when end-of-file is hit,
// returning set {epsilon}. Since {epsilon} is not
// ambig with any real tokens, no error is reported
// by deterministic() routines and we have to check
// for the case where the lookahead depth didn't get
// set to NONDETERMINISTIC (this only happens when the
// FOLLOW contains real atoms + epsilon).
boolean generateNonGreedyExitPath = false;
int nonGreedyExitDepth = grammar.maxk;
if (!blk.greedy &&
blk.exitLookaheadDepth <= grammar.maxk &&
blk.exitCache[blk.exitLookaheadDepth].containsEpsilon())
{
generateNonGreedyExitPath = true;
nonGreedyExitDepth = blk.exitLookaheadDepth;
}
else
{
if (!blk.greedy &&
blk.exitLookaheadDepth == LLkGrammarAnalyzer.NONDETERMINISTIC) {
generateNonGreedyExitPath = true;
}
}
// generate exit test if greedy set to false
// and an alt is ambiguous with exit branch
if (generateNonGreedyExitPath)
{
println("### nongreedy (...)+ loop; exit depth is " + blk.exitLookaheadDepth);
String predictExit =
getLookaheadTestExpression(
blk.exitCache,
nonGreedyExitDepth);
println("### nongreedy exit test");
println("if " + cnt + " >= 1 and " + predictExit + ":");
tabs++;
println("break");
tabs--;
}
{
int _tabs = tabs;
PythonBlockFinishingInfo howToFinish = genCommonBlock(blk, false);
genBlockFinish(howToFinish, "break");
tabs = _tabs;
}
/* no matter what previous block did, here we have to continue
** one the 'while block' level. Reseting tabs .. */
tabs = _tabs_;
println(cnt + " += 1");
tabs = _tabs_;
tabs--;
println("if " + cnt + " < 1:");
tabs++;
println(throwNoViable);
tabs--;
// Restore previous AST generation
currentASTResult = saveCurrentASTResult;
}
/** Generate the parser Java file */
public void gen(ParserGrammar g)
throws IOException {
// if debugging, set up a new vector to keep track of sempred
// strings for this grammar
if (g.debuggingOutput)
semPreds = new Vector();
setGrammar(g);
if (!(grammar instanceof ParserGrammar)) {
antlrTool.panic("Internal error generating parser");
}
// Open the output stream for the parser and set the currentOutput
// SAS: moved file setup so subclass could do it (for VAJ interface)
setupOutput(grammar.getClassName());
genAST = grammar.buildAST;
tabs = 0;
// Generate the header common to all output files.
genHeader();
// Generate header specific to lexer Java file
println("### import antlr and other modules ..");
println("import sys");
println("import antlr");
println("");
println("version = sys.version.split()[0]");
println("if version < '2.2.1':");
tabs++;
println("False = 0");
tabs--;
println("if version < '2.3':");
tabs++;
println("True = not False");
tabs--;
println("### header action >>> ");
printActionCode(behavior.getHeaderAction(""),0);
println("### header action <<< ");
println("### preamble action>>>");
// Output the user-defined parser preamble
printActionCode(grammar.preambleAction.getText(),0);
println("### preamble action <<<");
flushTokens();
// Generate parser class definition
String sup = null;
if (grammar.superClass != null)
sup = grammar.superClass;
else
sup = "antlr." + grammar.getSuperClass();
// print javadoc comment if any
genJavadocComment(grammar);
// get prefix (replaces "public" and lets user specify)
String prefix = "";
Token tprefix = (Token)grammar.options.get("classHeaderPrefix");
if (tprefix != null) {
String p = StringUtils.stripFrontBack(tprefix.getText(), "\"", "\"");
if (p != null) {
prefix = p;
}
}
print("class " + parserClassName + "(" + sup);
println("):");
tabs++;
// set up an array of all the rule names so the debugger can
// keep track of them only by number -- less to store in tree...
if (grammar.debuggingOutput) {
println("_ruleNames = [");
Enumeration ids = grammar.rules.elements();
int ruleNum = 0;
tabs++;
while (ids.hasMoreElements()) {
GrammarSymbol sym = (GrammarSymbol)ids.nextElement();
if (sym instanceof RuleSymbol)
println("\"" + ((RuleSymbol)sym).getId() + "\",");
}
tabs--;
println("]");
}
// Generate user-defined parser class members
printGrammarAction(grammar);
// Generate parser class constructor from TokenBuffer
println("");
println("def __init__(self, *args, **kwargs):");
tabs++;
println(sup + ".__init__(self, *args, **kwargs)");
println("self.tokenNames = _tokenNames");
// if debugging, set up arrays and call the user-overridable
// debugging setup method
if (grammar.debuggingOutput) {
println("self.ruleNames = _ruleNames");
println("self.semPredNames = _semPredNames");
println("self.setupDebugging(self.tokenBuf)");
}
if (grammar.buildAST) {
println("self.buildTokenTypeASTClassMap()");
println("self.astFactory = antlr.ASTFactory(self.getTokenTypeToASTClassMap())");
if(labeledElementASTType != null)
{
println("self.astFactory.setASTNodeClass("+
labeledElementASTType+")");
}
}
genHeaderInit(grammar);
println("");
// Generate code for each rule in the grammar
Enumeration ids = grammar.rules.elements();
int ruleNum = 0;
while (ids.hasMoreElements()) {
GrammarSymbol sym = (GrammarSymbol)ids.nextElement();
if (sym instanceof RuleSymbol) {
RuleSymbol rs = (RuleSymbol)sym;
genRule(rs, rs.references.size() == 0, ruleNum++);
}
exitIfError();
}
if ( grammar.buildAST ) {
genTokenASTNodeMap();
}
// Generate the token names
genTokenStrings();
// Generate the bitsets used throughout the grammar
genBitsets(bitsetsUsed, grammar.tokenManager.maxTokenType());
// Generate the semantic predicate map for debugging
if (grammar.debuggingOutput)
genSemPredMap();
// Close class definition
println("");
tabs = 0;
genHeaderMain(grammar);
// Close the parser output stream
currentOutput.close();
currentOutput = null;
}
/** Generate code for the given grammar element.
* @param blk The rule-reference to generate
*/
public void gen(RuleRefElement rr) {
if (DEBUG_CODE_GENERATOR) System.out.println("genRR(" + rr + ")");
RuleSymbol rs = (RuleSymbol)grammar.getSymbol(rr.targetRule);
if (rs == null || !rs.isDefined()) {
// Is this redundant???
antlrTool.error("Rule '" + rr.targetRule + "' is not defined", grammar.getFilename(), rr.getLine(), rr.getColumn());
return;
}
if (!(rs instanceof RuleSymbol)) {
// Is this redundant???
antlrTool.error("'" + rr.targetRule + "' does not name a grammar rule", grammar.getFilename(), rr.getLine(), rr.getColumn());
return;
}
genErrorTryForElement(rr);
// AST value for labeled rule refs in tree walker.
// This is not AST construction; it is just the input tree node value.
if (grammar instanceof TreeWalkerGrammar &&
rr.getLabel() != null &&
syntacticPredLevel == 0) {
println(rr.getLabel() + " = antlr.ifelse(_t == antlr.ASTNULL, None, " + lt1Value + ")");
}
// if in lexer and ! on rule ref or alt or rule, save buffer index to kill later
if (grammar instanceof LexerGrammar && (!saveText || rr.getAutoGenType() == GrammarElement.AUTO_GEN_BANG)) {
println("_saveIndex = self.text.length()");
}
// Process return value assignment if any
printTabs();
if (rr.idAssign != null) {
// Warn if the rule has no return type
if (rs.block.returnAction == null) {
antlrTool.warning("Rule '" + rr.targetRule + "' has no return type", grammar.getFilename(), rr.getLine(), rr.getColumn());
}
_print(rr.idAssign + "=");
}
else {
// Warn about return value if any, but not inside syntactic predicate
if (!(grammar instanceof LexerGrammar) && syntacticPredLevel == 0 && rs.block.returnAction != null) {
antlrTool.warning("Rule '" + rr.targetRule + "' returns a value", grammar.getFilename(), rr.getLine(), rr.getColumn());
}
}
// Call the rule
GenRuleInvocation(rr);
// if in lexer and ! on element or alt or rule, save buffer index to kill later
if (grammar instanceof LexerGrammar && (!saveText || rr.getAutoGenType() == GrammarElement.AUTO_GEN_BANG)) {
println("self.text.setLength(_saveIndex)");
}
// if not in a syntactic predicate
if (syntacticPredLevel == 0) {
boolean doNoGuessTest = (
grammar.hasSyntacticPredicate && (
grammar.buildAST && rr.getLabel() != null ||
(genAST && rr.getAutoGenType() == GrammarElement.AUTO_GEN_NONE)
)
);
if (doNoGuessTest) {
// println("if (inputState.guessing==0) {");
// tabs++;
}
if (grammar.buildAST && rr.getLabel() != null) {
// always gen variable for rule return on labeled rules
println(rr.getLabel() + "_AST = self.returnAST");
}
if (genAST) {
switch (rr.getAutoGenType()) {
case GrammarElement.AUTO_GEN_NONE:
println("self.addASTChild(currentAST, self.returnAST)");
break;
case GrammarElement.AUTO_GEN_CARET:
antlrTool.error("Internal: encountered ^ after rule reference");
break;
default:
break;
}
}
// if a lexer and labeled, Token label defined at rule level, just set it here
if (grammar instanceof LexerGrammar && rr.getLabel() != null) {
println(rr.getLabel() + " = self._returnToken");
}
if (doNoGuessTest) {
}
}
genErrorCatchForElement(rr);
}
/** Generate code for the given grammar element.
* @param blk The string-literal reference to generate
*/
public void gen(StringLiteralElement atom) {
if (DEBUG_CODE_GENERATOR) System.out.println("genString(" + atom + ")");
// Variable declarations for labeled elements
if (atom.getLabel() != null && syntacticPredLevel == 0) {
println(atom.getLabel() + " = " + lt1Value + "");
}
// AST
genElementAST(atom);
// is there a bang on the literal?
boolean oldsaveText = saveText;
saveText = saveText && atom.getAutoGenType() == GrammarElement.AUTO_GEN_NONE;
// matching
genMatch(atom);
saveText = oldsaveText;
// tack on tree cursor motion if doing a tree walker
if (grammar instanceof TreeWalkerGrammar) {
println("_t = _t.getNextSibling()");
}
}
/** Generate code for the given grammar element.
* @param blk The token-range reference to generate
*/
public void gen(TokenRangeElement r) {
genErrorTryForElement(r);
if (r.getLabel() != null && syntacticPredLevel == 0) {
println(r.getLabel() + " = " + lt1Value);
}
// AST
genElementAST(r);
// match
println("self.matchRange(u" + r.beginText + ", u" + r.endText + ")");
genErrorCatchForElement(r);
}
/** Generate code for the given grammar element.
* @param blk The token-reference to generate
*/
public void gen(TokenRefElement atom) {
if (DEBUG_CODE_GENERATOR) System.out.println("genTokenRef(" + atom + ")");
if (grammar instanceof LexerGrammar) {
antlrTool.panic("Token reference found in lexer");
}
genErrorTryForElement(atom);
// Assign Token value to token label variable
if (atom.getLabel() != null && syntacticPredLevel == 0) {
println(atom.getLabel() + " = " + lt1Value + "");
}
// AST
genElementAST(atom);
// matching
genMatch(atom);
genErrorCatchForElement(atom);
// tack on tree cursor motion if doing a tree walker
if (grammar instanceof TreeWalkerGrammar) {
println("_t = _t.getNextSibling()");
}
}
public void gen(TreeElement t) {
// save AST cursor
println("_t" + t.ID + " = _t");
// If there is a label on the root, then assign that to the variable
if (t.root.getLabel() != null) {
println(t.root.getLabel() + " = antlr.ifelse(_t == antlr.ASTNULL, None, _t)");
}
// check for invalid modifiers ! and ^ on tree element roots
if ( t.root.getAutoGenType() == GrammarElement.AUTO_GEN_BANG ) {
antlrTool.error("Suffixing a root node with '!' is not implemented",
grammar.getFilename(), t.getLine(), t.getColumn());
t.root.setAutoGenType(GrammarElement.AUTO_GEN_NONE);
}
if ( t.root.getAutoGenType() == GrammarElement.AUTO_GEN_CARET ) {
antlrTool.warning("Suffixing a root node with '^' is redundant; already a root",
grammar.getFilename(), t.getLine(), t.getColumn());
t.root.setAutoGenType(GrammarElement.AUTO_GEN_NONE);
}
// Generate AST variables
genElementAST(t.root);
if (grammar.buildAST) {
// Save the AST construction state
println("_currentAST" + t.ID + " = currentAST.copy()");
// Make the next item added a child of the TreeElement root
println("currentAST.root = currentAST.child");
println("currentAST.child = None");
}
// match root
if ( t.root instanceof WildcardElement ) {
println("if not _t: raise antlr.MismatchedTokenException()");
}
else {
genMatch(t.root);
}
// move to list of children
println("_t = _t.getFirstChild()");
// walk list of children, generating code for each
for (int i = 0; i < t.getAlternatives().size(); i++) {
Alternative a = t.getAlternativeAt(i);
AlternativeElement e = a.head;
while (e != null) {
e.generate();
e = e.next;
}
}
if (grammar.buildAST) {
// restore the AST construction state to that just after the
// tree root was added
println("currentAST = _currentAST" + t.ID + "");
}
// restore AST cursor
println("_t = _t" + t.ID + "");
// move cursor to sibling of tree just parsed
println("_t = _t.getNextSibling()");
}
/** Generate the tree-parser Java file */
public void gen(TreeWalkerGrammar g) throws IOException {
// SAS: debugging stuff removed for now...
setGrammar(g);
if (!(grammar instanceof TreeWalkerGrammar)) {
antlrTool.panic("Internal error generating tree-walker");
}
// Open the output stream for the parser and set the currentOutput
// SAS: move file open to method so subclass can override it
// (mainly for VAJ interface)
setupOutput(grammar.getClassName());
genAST = grammar.buildAST;
tabs = 0;
// Generate the header common to all output files.
genHeader();
// Generate header specific to lexer Java file
println("### import antlr and other modules ..");
println("import sys");
println("import antlr");
println("");
println("version = sys.version.split()[0]");
println("if version < '2.2.1':");
tabs++;
println("False = 0");
tabs--;
println("if version < '2.3':");
tabs++;
println("True = not False");
tabs--;
println("### header action >>> ");
printActionCode(behavior.getHeaderAction(""),0);
println("### header action <<< ");
flushTokens();
println("### user code>>>");
// Output the user-defined parser preamble
printActionCode(grammar.preambleAction.getText(),0);
println("### user code<<<");
// Generate parser class definition
String sup = null;
if (grammar.superClass != null) {
sup = grammar.superClass;
}
else {
sup = "antlr." + grammar.getSuperClass();
}
println("");
// get prefix (replaces "public" and lets user specify)
String prefix = "";
Token tprefix = (Token)grammar.options.get("classHeaderPrefix");
if (tprefix != null) {
String p = StringUtils.stripFrontBack(tprefix.getText(), "\"", "\"");
if (p != null) {
prefix = p;
}
}
// print javadoc comment if any
genJavadocComment(grammar);
println("class " + treeWalkerClassName + "(" + sup + "):");
tabs++;
// Generate default parser class constructor
println("");
println("# ctor ..");
println("def __init__(self, *args, **kwargs):");
tabs++;
println(sup + ".__init__(self, *args, **kwargs)");
println("self.tokenNames = _tokenNames");
genHeaderInit(grammar);
tabs--;
println("");
// print grammar specific action
printGrammarAction(grammar);
// Generate code for each rule in the grammar
Enumeration ids = grammar.rules.elements();
int ruleNum = 0;
String ruleNameInits = "";
while (ids.hasMoreElements()) {
GrammarSymbol sym = (GrammarSymbol)ids.nextElement();
if (sym instanceof RuleSymbol) {
RuleSymbol rs = (RuleSymbol)sym;
genRule(rs, rs.references.size() == 0, ruleNum++);
}
exitIfError();
}
// Generate the token names
genTokenStrings();
// Generate the bitsets used throughout the grammar
genBitsets(bitsetsUsed, grammar.tokenManager.maxTokenType());
tabs = 0;
genHeaderMain(grammar);
// Close the parser output stream
currentOutput.close();
currentOutput = null;
}
/** Generate code for the given grammar element.
* @param wc The wildcard element to generate
*/
public void gen(WildcardElement wc) {
// Variable assignment for labeled elements
if (wc.getLabel() != null && syntacticPredLevel == 0) {
println(wc.getLabel() + " = " + lt1Value + "");
}
// AST
genElementAST(wc);
// Match anything but EOF
if (grammar instanceof TreeWalkerGrammar) {
println("if not _t:");
tabs++;
println("raise MismatchedTokenException()");
tabs--;
}
else if (grammar instanceof LexerGrammar) {
if (grammar instanceof LexerGrammar &&
(!saveText || wc.getAutoGenType() == GrammarElement.AUTO_GEN_BANG)) {
println("_saveIndex = self.text.length()");
}
println("self.matchNot(antlr.EOF_CHAR)");
if (grammar instanceof LexerGrammar &&
(!saveText || wc.getAutoGenType() == GrammarElement.AUTO_GEN_BANG)) {
println("self.text.setLength(_saveIndex)"); // kill text atom put in buffer
}
}
else {
println("self.matchNot(" + getValueString(Token.EOF_TYPE,false) + ")");
}
// tack on tree cursor motion if doing a tree walker
if (grammar instanceof TreeWalkerGrammar) {
println("_t = _t.getNextSibling()");
}
}
/** Generate code for the given grammar element.
* @param blk The (...)* block to generate
*/
public void gen(ZeroOrMoreBlock blk) {
int _tabs_ = tabs;
genBlockPreamble(blk);
String label;
println("while True:");
tabs++;
_tabs_ = tabs;
// generate the init action for ()* inside the loop
// this allows us to do usefull EOF checking...
genBlockInitAction(blk);
// Tell AST generation to build subrule result
String saveCurrentASTResult = currentASTResult;
if (blk.getLabel() != null) {
currentASTResult = blk.getLabel();
}
boolean ok = grammar.theLLkAnalyzer.deterministic(blk);
// generate exit test if greedy set to false
// and an alt is ambiguous with exit branch
// or when lookahead derived purely from end-of-file
// Lookahead analysis stops when end-of-file is hit,
// returning set {epsilon}. Since {epsilon} is not
// ambig with any real tokens, no error is reported
// by deterministic() routines and we have to check
// for the case where the lookahead depth didn't get
// set to NONDETERMINISTIC (this only happens when the
// FOLLOW contains real atoms + epsilon).
boolean generateNonGreedyExitPath = false;
int nonGreedyExitDepth = grammar.maxk;
if (!blk.greedy &&
blk.exitLookaheadDepth <= grammar.maxk &&
blk.exitCache[blk.exitLookaheadDepth].containsEpsilon()) {
generateNonGreedyExitPath = true;
nonGreedyExitDepth = blk.exitLookaheadDepth;
}
else if (!blk.greedy &&
blk.exitLookaheadDepth == LLkGrammarAnalyzer.NONDETERMINISTIC) {
generateNonGreedyExitPath = true;
}
if (generateNonGreedyExitPath) {
if (DEBUG_CODE_GENERATOR) {
System.out.println("nongreedy (...)* loop; exit depth is " +
blk.exitLookaheadDepth);
}
String predictExit =
getLookaheadTestExpression(blk.exitCache,
nonGreedyExitDepth);
println("### nongreedy exit test");
println("if (" + predictExit + "):");
tabs++;
println("break");
tabs--;
}
{
int _tabs = tabs;
PythonBlockFinishingInfo howToFinish = genCommonBlock(blk, false);
genBlockFinish(howToFinish, "break");
tabs = _tabs;
}
tabs = _tabs_; /* no matter where we are */
tabs--;
// Restore previous AST generation
currentASTResult = saveCurrentASTResult;
}
/** Generate an alternative.
* @param alt The alternative to generate
* @param blk The block to which the alternative belongs
*/
protected void genAlt(Alternative alt, AlternativeBlock blk) {
// Save the AST generation state, and set it to that of the alt
boolean savegenAST = genAST;
genAST = genAST && alt.getAutoGen();
boolean oldsaveTest = saveText;
saveText = saveText && alt.getAutoGen();
// Reset the variable name map for the alternative
Hashtable saveMap = treeVariableMap;
treeVariableMap = new Hashtable();
// Generate try block around the alt for error handling
if (alt.exceptionSpec != null) {
println("try:");
tabs++;
}
println("pass"); // make sure that always something gets generated ..
AlternativeElement elem = alt.head;
while (!(elem instanceof BlockEndElement)) {
elem.generate(); // alt can begin with anything. Ask target to gen.
elem = elem.next;
}
if (genAST) {
if (blk instanceof RuleBlock) {
// Set the AST return value for the rule
RuleBlock rblk = (RuleBlock)blk;
if (grammar.hasSyntacticPredicate) {
}
println(rblk.getRuleName() + "_AST = currentAST.root");
if (grammar.hasSyntacticPredicate) {
}
}
else if (blk.getLabel() != null) {
antlrTool.warning(
"Labeled subrules not yet supported",
grammar.getFilename(), blk.getLine(), blk.getColumn());
}
}
if (alt.exceptionSpec != null) {
tabs--;
genErrorHandler(alt.exceptionSpec);
}
genAST = savegenAST;
saveText = oldsaveTest;
treeVariableMap = saveMap;
}
/** Generate all the bitsets to be used in the parser or lexer
* Generate the raw bitset data like "long _tokenSet1_data[] = {...}"
* and the BitSet object declarations like "BitSet _tokenSet1 = new BitSet(_tokenSet1_data)"
* Note that most languages do not support object initialization inside a
* class definition, so other code-generators may have to separate the
* bitset declarations from the initializations (e.g., put the initializations
* in the generated constructor instead).
* @param bitsetList The list of bitsets to generate.
* @param maxVocabulary Ensure that each generated bitset can contain at least this value.
*/
protected void genBitsets(Vector bitsetList,
int maxVocabulary
) {
println("");
for (int i = 0; i < bitsetList.size(); i++) {
BitSet p = (BitSet)bitsetList.elementAt(i);
// Ensure that generated BitSet is large enough for vocabulary
p.growToInclude(maxVocabulary);
genBitSet(p, i);
}
}
/** Do something simple like:
* private static final long[] mk_tokenSet_0() {
* long[] data = { -2305839160922996736L, 63L, 16777216L, 0L, 0L, 0L };
* return data;
* }
* public static final BitSet _tokenSet_0 = new BitSet(mk_tokenSet_0());
*
* Or, for large bitsets, optimize init so ranges are collapsed into loops.
* This is most useful for lexers using unicode.
*/
private void genBitSet(BitSet p, int id) {
int _tabs_ = tabs;
// wanna have bitsets on module scope, so they are available
// when module gets loaded.
tabs = 0;
println("");
println("### generate bit set");
println(
"def mk" + getBitsetName(id) + "(): " );
tabs++;
int n = p.lengthInLongWords();
if ( n<BITSET_OPTIMIZE_INIT_THRESHOLD )
{
println("### var1");
println("data = [ " + p.toStringOfWords() + "]");
}
else
{
// will init manually, allocate space then set values
println("data = [0L] * " + n + " ### init list");
long[] elems = p.toPackedArray();
for (int i = 0; i < elems.length;)
{
if ( elems[i]==0 )
{
// done automatically by Java, don't waste time/code
i++;
continue;
}
if ( (i+1)==elems.length || elems[i]!=elems[i+1] )
{
// last number or no run of numbers, just dump assignment
println("data["+ i + "] =" + elems[i] + "L");
i++;
continue;
}
// scan to find end of run
int j;
for (j = i + 1;j < elems.length && elems[j]==elems[i];j++)
{}
long e = elems[i];
// E0007: fixed
println("for x in xrange(" + i+", " + j + "):");
tabs++;
println("data[x] = " + e + "L");
tabs--;
i = j;
}
}
println("return data");
tabs--;
// BitSet object
println(
getBitsetName(id) + " = antlr.BitSet(mk" + getBitsetName(id) + "())" );
// restore tabs
tabs = _tabs_;
}
private void genBlockFinish(PythonBlockFinishingInfo howToFinish,
String noViableAction) {
if (howToFinish.needAnErrorClause &&
(howToFinish.generatedAnIf || howToFinish.generatedSwitch)) {
if (howToFinish.generatedAnIf)
{
println("else:" );
}
tabs++;
println(noViableAction);
tabs--;
}
if (howToFinish.postscript != null) {
println(howToFinish.postscript);
}
}
/* just to be called by nextToken */
private void genBlockFinish1(PythonBlockFinishingInfo howToFinish,
String noViableAction) {
if (howToFinish.needAnErrorClause &&
(howToFinish.generatedAnIf || howToFinish.generatedSwitch))
{
if (howToFinish.generatedAnIf)
{
// tabs++;
println("else:" );
}
tabs++;
println(noViableAction);
tabs--;
if (howToFinish.generatedAnIf)
{
// tabs--;
// println("### tabs--");
}
}
if (howToFinish.postscript != null) {
println(howToFinish.postscript);
}
}
/** Generate the init action for a block, which may be a RuleBlock or a
* plain AlternativeBLock.
* @blk The block for which the preamble is to be generated.
*/
protected void genBlockInitAction(AlternativeBlock blk) {
// dump out init action
if (blk.initAction != null) {
printAction(processActionForSpecialSymbols(blk.initAction, blk.getLine(), currentRule, null));
}
}
/** Generate the header for a block, which may be a RuleBlock or a
* plain AlternativeBLock. This generates any variable declarations
* and syntactic-predicate-testing variables.
* @blk The block for which the preamble is to be generated.
*/
protected void genBlockPreamble(AlternativeBlock blk) {
// define labels for rule blocks.
if (blk instanceof RuleBlock) {
RuleBlock rblk = (RuleBlock)blk;
if (rblk.labeledElements != null) {
for (int i = 0; i < rblk.labeledElements.size(); i++) {
AlternativeElement a = (AlternativeElement)rblk.labeledElements.elementAt(i);
// System.out.println("looking at labeled element: "+a);
// Variables for labeled rule refs and
// subrules are different than variables for
// grammar atoms. This test is a little tricky
// because we want to get all rule refs and ebnf,
// but not rule blocks or syntactic predicates
if (
a instanceof RuleRefElement ||
a instanceof AlternativeBlock &&
!(a instanceof RuleBlock) &&
!(a instanceof SynPredBlock)
) {
if (
!(a instanceof RuleRefElement) &&
((AlternativeBlock)a).not &&
analyzer.subruleCanBeInverted(((AlternativeBlock)a), grammar instanceof LexerGrammar)
) {
// Special case for inverted subrules that
// will be inlined. Treat these like
// token or char literal references
println(a.getLabel() + " = " + labeledElementInit);
if (grammar.buildAST) {
genASTDeclaration(a);
}
}
else {
if (grammar.buildAST) {
// Always gen AST variables for
// labeled elements, even if the
// element itself is marked with !
genASTDeclaration(a);
}
if (grammar instanceof LexerGrammar) {
println(a.getLabel() + " = None");
}
if (grammar instanceof TreeWalkerGrammar) {
// always generate rule-ref variables
// for tree walker
println(a.getLabel() + " = " + labeledElementInit);
}
}
}
else {
// It is a token or literal reference. Generate the
// correct variable type for this grammar
println(a.getLabel() + " = " + labeledElementInit);
// In addition, generate *_AST variables if
// building ASTs
if (grammar.buildAST) {
if (a instanceof GrammarAtom &&
((GrammarAtom)a).getASTNodeType() != null) {
GrammarAtom ga = (GrammarAtom)a;
genASTDeclaration(a, ga.getASTNodeType());
}
else {
genASTDeclaration(a);
}
}
}
}
}
}
}
/** Generate a series of case statements that implement a BitSet test.
* @param p The Bitset for which cases are to be generated
*/
protected void genCases(BitSet p) {
if (DEBUG_CODE_GENERATOR) System.out.println("genCases(" + p + ")");
int[] elems;
elems = p.toArray();
// Wrap cases four-per-line for lexer, one-per-line for parser
int wrap = (grammar instanceof LexerGrammar) ? 4 : 1;
int j = 1;
boolean startOfLine = true;
print("elif la1 and la1 in ");
if (grammar instanceof LexerGrammar)
{
_print("u'");
for (int i = 0; i < elems.length; i++) {
_print(getValueString(elems[i],false));
}
_print("':\n");
return;
}
// Parser or TreeParser ..
_print("[");
for (int i = 0; i < elems.length; i++) {
_print(getValueString(elems[i],false));
if(i+1<elems.length)
_print(",");
}
_print("]:\n");
}
/**Generate common code for a block of alternatives; return a
* postscript that needs to be generated at the end of the
* block. Other routines may append else-clauses and such for
* error checking before the postfix is generated. If the
* grammar is a lexer, then generate alternatives in an order
* where alternatives requiring deeper lookahead are generated
* first, and EOF in the lookahead set reduces the depth of
* the lookahead. @param blk The block to generate @param
* noTestForSingle If true, then it does not generate a test
* for a single alternative.
*/
public PythonBlockFinishingInfo genCommonBlock(AlternativeBlock blk,
boolean noTestForSingle) {
int _tabs_ = tabs; // remember where we are ..
int nIF = 0;
boolean createdLL1Switch = false;
int closingBracesOfIFSequence = 0;
PythonBlockFinishingInfo finishingInfo =
new PythonBlockFinishingInfo();
// Save the AST generation state, and set it to that of the block
boolean savegenAST = genAST;
genAST = genAST && blk.getAutoGen();
boolean oldsaveTest = saveText;
saveText = saveText && blk.getAutoGen();
// Is this block inverted? If so, generate special-case code
if (
blk.not &&
analyzer.subruleCanBeInverted(blk, grammar instanceof LexerGrammar)
)
{
if (DEBUG_CODE_GENERATOR) System.out.println("special case: ~(subrule)");
Lookahead p = analyzer.look(1, blk);
// Variable assignment for labeled elements
if (blk.getLabel() != null && syntacticPredLevel == 0)
{
println(blk.getLabel() + " = " + lt1Value);
}
// AST
genElementAST(blk);
String astArgs = "";
if (grammar instanceof TreeWalkerGrammar) {
astArgs = "_t, ";
}
// match the bitset for the alternative
println("self.match(" + astArgs + getBitsetName(markBitsetForGen(p.fset)) + ")");
// tack on tree cursor motion if doing a tree walker
if (grammar instanceof TreeWalkerGrammar) {
println("_t = _t.getNextSibling()");
}
return finishingInfo;
}
// Special handling for single alt
if (blk.getAlternatives().size() == 1)
{
Alternative alt = blk.getAlternativeAt(0);
// Generate a warning if there is a synPred for single alt.
if (alt.synPred != null) {
antlrTool.warning(
"Syntactic predicate superfluous for single alternative",
grammar.getFilename(),
blk.getAlternativeAt(0).synPred.getLine(),
blk.getAlternativeAt(0).synPred.getColumn()
);
}
if (noTestForSingle)
{
if (alt.semPred != null) {
// Generate validating predicate
genSemPred(alt.semPred, blk.line);
}
genAlt(alt, blk);
return finishingInfo;
}
}
// count number of simple LL(1) cases; only do switch for
// many LL(1) cases (no preds, no end of token refs)
// We don't care about exit paths for (...)*, (...)+
// because we don't explicitly have a test for them
// as an alt in the loop.
//
// Also, we now count how many unicode lookahead sets
// there are--they must be moved to DEFAULT or ELSE
// clause.
int nLL1 = 0;
for (int i = 0; i < blk.getAlternatives().size(); i++) {
Alternative a = blk.getAlternativeAt(i);
if (suitableForCaseExpression(a)) {
nLL1++;
}
}
// do LL(1) cases
if (nLL1 >= makeSwitchThreshold)
{
// Determine the name of the item to be compared
String testExpr = lookaheadString(1);
createdLL1Switch = true;
// when parsing trees, convert null to valid tree node with NULL lookahead
if (grammar instanceof TreeWalkerGrammar) {
println("if not _t:");
tabs++;
println("_t = antlr.ASTNULL");
tabs--;
}
println("la1 = " + testExpr);
// print dummy if to get a regular genCases ..
println("if False:");
tabs++;
println("pass");
//println("assert 0 # lunatic case");
tabs--;
for (int i = 0; i < blk.alternatives.size(); i++)
{
Alternative alt = blk.getAlternativeAt(i);
// ignore any non-LL(1) alts, predicated alts,
// or end-of-token alts for case expressions
if (!suitableForCaseExpression(alt)) {
continue;
}
Lookahead p = alt.cache[1];
if (p.fset.degree() == 0 && !p.containsEpsilon())
{
antlrTool.warning(
"Alternate omitted due to empty prediction set",
grammar.getFilename(),
alt.head.getLine(), alt.head.getColumn());
}
else
{
/* make the case statment, ie. if la1 in .. : */
genCases(p.fset);
tabs++;
genAlt(alt,blk);
tabs--;
}
}
/* does this else belong here? */
println("else:");
tabs++;
}
// do non-LL(1) and nondeterministic cases This is tricky in
// the lexer, because of cases like: STAR : '*' ; ASSIGN_STAR
// : "*="; Since nextToken is generated without a loop, then
// the STAR will have end-of-token as it's lookahead set for
// LA(2). So, we must generate the alternatives containing
// trailing end-of-token in their lookahead sets *after* the
// alternatives without end-of-token. This implements the
// usual lexer convention that longer matches come before
// shorter ones, e.g. "*=" matches ASSIGN_STAR not STAR
//
// For non-lexer grammars, this does not sort the alternates
// by depth Note that alts whose lookahead is purely
// end-of-token at k=1 end up as default or else clauses.
int startDepth = (grammar instanceof LexerGrammar) ? grammar.maxk : 0;
for (int altDepth = startDepth; altDepth >= 0; altDepth--)
{
for (int i = 0; i < blk.alternatives.size(); i++)
{
Alternative alt = blk.getAlternativeAt(i);
if (DEBUG_CODE_GENERATOR) System.out.println("genAlt: " + i);
// if we made a switch above, ignore what we already took care
// of. Specifically, LL(1) alts with no preds
// that do not have end-of-token in their prediction set
// and that are not giant unicode sets.
if (createdLL1Switch && suitableForCaseExpression(alt)) {
if (DEBUG_CODE_GENERATOR)
System.out.println("ignoring alt because it was in the switch");
continue;
}
String e;
boolean unpredicted = false;
if (grammar instanceof LexerGrammar)
{
// Calculate the "effective depth" of the alt,
// which is the max depth at which
// cache[depth]!=end-of-token
int effectiveDepth = alt.lookaheadDepth;
if (effectiveDepth == GrammarAnalyzer.NONDETERMINISTIC) {
// use maximum lookahead
effectiveDepth = grammar.maxk;
}
while (effectiveDepth >= 1 &&
alt.cache[effectiveDepth].containsEpsilon()) {
effectiveDepth--;
}
// Ignore alts whose effective depth is other than
// the ones we are generating for this iteration.
if (effectiveDepth != altDepth) {
if (DEBUG_CODE_GENERATOR)
System.out.println(
"ignoring alt because effectiveDepth!=altDepth"
+ effectiveDepth + "!=" + altDepth);
continue;
}
unpredicted = lookaheadIsEmpty(alt, effectiveDepth);
e = getLookaheadTestExpression(alt, effectiveDepth);
}
else
{
unpredicted = lookaheadIsEmpty(alt, grammar.maxk);
e = getLookaheadTestExpression(alt, grammar.maxk);
}
// Was it a big unicode range that forced unsuitability
// for a case expression?
if (alt.cache[1].fset.degree() > caseSizeThreshold &&
suitableForCaseExpression(alt)) {
if (nIF == 0) {
println("<m1> if " + e + ":");
}
else {
println("<m2> elif " + e + ":");
}
}
else
{
if (unpredicted &&
alt.semPred == null &&
alt.synPred == null)
{
// The alt has empty prediction set and no
// predicate to help out. if we have not
// generated a previous if, just put {...} around
// the end-of-token clause
if (nIF == 0) {
println("##<m3> <closing");
}
else
{
println("else: ## <m4>");
tabs++;
// to prevent an empty boyd
// println("pass");
}
finishingInfo.needAnErrorClause = false;
}
else
{
// check for sem and syn preds
// Add any semantic predicate expression to the
// lookahead test
if (alt.semPred != null)
{
// if debugging, wrap the evaluation of the
// predicate in a method translate $ and #
// references
ActionTransInfo tInfo = new ActionTransInfo();
String actionStr =
processActionForSpecialSymbols(
alt.semPred,
blk.line,
currentRule,
tInfo);
// ignore translation info...we don't need to
// do anything with it. call that will inform
// SemanticPredicateListeners of the result
if (((grammar instanceof ParserGrammar) ||
(grammar instanceof LexerGrammar)) &&
grammar.debuggingOutput)
{
e = "(" + e +
" and fireSemanticPredicateEvaluated(antlr.debug.SemanticPredicateEvent.PREDICTING, " +
addSemPred(charFormatter.escapeString(actionStr)) + ", " + actionStr + "))";
}
else
{
e = "(" + e + " and (" + actionStr + "))";
}
}
// Generate any syntactic predicates
if (nIF > 0)
{
if (alt.synPred != null)
{
println("else:");
tabs++; /* who's closing this one? */
genSynPred(alt.synPred, e);
closingBracesOfIFSequence++;
}
else
{
println("elif " + e + ":");
}
}
else
{
if (alt.synPred != null)
{
genSynPred(alt.synPred, e);
}
else
{
// when parsing trees, convert null to
// valid tree node with NULL lookahead.
if (grammar instanceof TreeWalkerGrammar) {
println("if not _t:");
tabs++;
println("_t = antlr.ASTNULL");
tabs--;
}
println("if " + e + ":");
}
}
}
}
nIF++;
tabs++;
genAlt(alt, blk); // this should have generated something. If not
// we could end up in an empty else:
tabs--;
}
}
String ps = "";
//for (int i = 1; i <= closingBracesOfIFSequence; i++) {
// ps += "";
//}
// Restore the AST generation state
genAST = savegenAST;
// restore save text state
saveText = oldsaveTest;
// Return the finishing info.
if (createdLL1Switch) {
finishingInfo.postscript = ps;
finishingInfo.generatedSwitch = true;
finishingInfo.generatedAnIf = nIF > 0;
}
else {
finishingInfo.postscript = ps;
finishingInfo.generatedSwitch = false;
finishingInfo.generatedAnIf = nIF > 0;
}
return finishingInfo;
}
private static boolean suitableForCaseExpression(Alternative a) {
return
a.lookaheadDepth == 1 &&
a.semPred == null &&
!a.cache[1].containsEpsilon() &&
a.cache[1].fset.degree() <= caseSizeThreshold;
}
/** Generate code to link an element reference into the AST */
private void genElementAST(AlternativeElement el) {
// handle case where you're not building trees, but are in tree walker.
// Just need to get labels set up.
if (grammar instanceof TreeWalkerGrammar && !grammar.buildAST) {
String elementRef;
String astName;
// Generate names and declarations of the AST variable(s)
if (el.getLabel() == null) {
elementRef = lt1Value;
// Generate AST variables for unlabeled stuff
astName = "tmp" + astVarNumber + "_AST";
astVarNumber++;
// Map the generated AST variable in the alternate
mapTreeVariable(el, astName);
// Generate an "input" AST variable also
println(astName + "_in = " + elementRef);
}
return;
}
if (grammar.buildAST && syntacticPredLevel == 0) {
boolean needASTDecl =
(genAST &&
(el.getLabel() != null ||
el.getAutoGenType() != GrammarElement.AUTO_GEN_BANG
)
);
// RK: if we have a grammar element always generate the decl
// since some guy can access it from an action and we can't
// peek ahead (well not without making a mess).
// I'd prefer taking this out.
if (el.getAutoGenType() != GrammarElement.AUTO_GEN_BANG &&
(el instanceof TokenRefElement))
{
needASTDecl = true;
}
boolean doNoGuessTest =
(grammar.hasSyntacticPredicate && needASTDecl);
String elementRef;
String astNameBase;
// Generate names and declarations of the AST variable(s)
if (el.getLabel() != null) {
elementRef = el.getLabel();
astNameBase = el.getLabel();
}
else {
elementRef = lt1Value;
// Generate AST variables for unlabeled stuff
astNameBase = "tmp" + astVarNumber;
;
astVarNumber++;
}
// Generate the declaration if required.
if (needASTDecl) {
// Generate the declaration
if (el instanceof GrammarAtom) {
GrammarAtom ga = (GrammarAtom)el;
if (ga.getASTNodeType() != null) {
genASTDeclaration(el, astNameBase, ga.getASTNodeType());
}
else {
genASTDeclaration(el, astNameBase, labeledElementASTType);
}
}
else {
genASTDeclaration(el, astNameBase, labeledElementASTType);
}
}
// for convenience..
String astName = astNameBase + "_AST";
// Map the generated AST variable in the alternate
mapTreeVariable(el, astName);
if (grammar instanceof TreeWalkerGrammar) {
// Generate an "input" AST variable also
println(astName + "_in = None");
}
// Enclose actions with !guessing
if (doNoGuessTest) {
// println("if (inputState.guessing==0) {");
// tabs++;
}
// if something has a label assume it will be used
// so we must initialize the RefAST
if (el.getLabel() != null) {
if (el instanceof GrammarAtom) {
println(astName + " = " + getASTCreateString((GrammarAtom)el, elementRef) + "");
}
else {
println(astName + " = " + getASTCreateString(elementRef) + "");
}
}
// if it has no label but a declaration exists initialize it.
if (el.getLabel() == null && needASTDecl) {
elementRef = lt1Value;
if (el instanceof GrammarAtom) {
println(astName + " = " + getASTCreateString((GrammarAtom)el, elementRef) + "");
}
else {
println(astName + " = " + getASTCreateString(elementRef) + "");
}
// Map the generated AST variable in the alternate
if (grammar instanceof TreeWalkerGrammar) {
// set "input" AST variable also
println(astName + "_in = " + elementRef + "");
}
}
if (genAST) {
switch (el.getAutoGenType()) {
case GrammarElement.AUTO_GEN_NONE:
println("self.addASTChild(currentAST, " + astName + ")");
break;
case GrammarElement.AUTO_GEN_CARET:
println("self.makeASTRoot(currentAST, " + astName + ")");
break;
default:
break;
}
}
if (doNoGuessTest) {
// tabs--;
}
}
}
/** Close the try block and generate catch phrases
* if the element has a labeled handler in the rule
*/
private void genErrorCatchForElement(AlternativeElement el) {
if (el.getLabel() == null) return;
String r = el.enclosingRuleName;
if (grammar instanceof LexerGrammar) {
r = CodeGenerator.encodeLexerRuleName(el.enclosingRuleName);
}
RuleSymbol rs = (RuleSymbol)grammar.getSymbol(r);
if (rs == null) {
antlrTool.panic("Enclosing rule not found!");
}
ExceptionSpec ex = rs.block.findExceptionSpec(el.getLabel());
if (ex != null) {
tabs--;
genErrorHandler(ex);
}
}
/** Generate the catch phrases for a user-specified error handler */
private void genErrorHandler(ExceptionSpec ex) {
// Each ExceptionHandler in the ExceptionSpec is a separate catch
for (int i = 0; i < ex.handlers.size(); i++) {
ExceptionHandler handler = (ExceptionHandler)ex.handlers.elementAt(i);
// Generate catch phrase
println("except " + handler.exceptionTypeAndName.getText() + ":");
tabs++;
if (grammar.hasSyntacticPredicate) {
println("if not inputState.guessing:");
tabs++;
}
// When not guessing, execute user handler action
ActionTransInfo tInfo = new ActionTransInfo();
printAction(
processActionForSpecialSymbols(handler.action.getText(),
handler.action.getLine(),
currentRule, tInfo)
);
if (grammar.hasSyntacticPredicate) {
tabs--;
println("else:");
tabs++;
// When guessing, rethrow exception
println(
"raise " +
extractIdOfAction(handler.exceptionTypeAndName));
tabs--;
}
// Close catch phrase
tabs--;
}
}
/** Generate a try { opening if the element has a labeled handler in the rule */
private void genErrorTryForElement(AlternativeElement el) {
if (el.getLabel() == null) return;
String r = el.enclosingRuleName;
if (grammar instanceof LexerGrammar) {
r = CodeGenerator.encodeLexerRuleName(el.enclosingRuleName);
}
RuleSymbol rs = (RuleSymbol)grammar.getSymbol(r);
if (rs == null) {
antlrTool.panic("Enclosing rule not found!");
}
ExceptionSpec ex = rs.block.findExceptionSpec(el.getLabel());
if (ex != null) {
println("try: # for error handling");
tabs++;
}
}
protected void genASTDeclaration(AlternativeElement el) {
genASTDeclaration(el, labeledElementASTType);
}
protected void genASTDeclaration(AlternativeElement el, String node_type) {
genASTDeclaration(el, el.getLabel(), node_type);
}
protected void genASTDeclaration(AlternativeElement el, String var_name, String node_type) {
// already declared?
if (declaredASTVariables.contains(el))
return;
// emit code
println(var_name + "_AST = None");
// mark as declared
declaredASTVariables.put(el,el);
}
/** Generate a header that is common to all Python files */
protected void genHeader() {
println("### $ANTLR " + Tool.version + ": " +
"\"" + antlrTool.fileMinusPath(antlrTool.grammarFile) + "\"" +
" -> " +
"\"" + grammar.getClassName() + ".py\"$");
}
/** Generate an iterator method for the Python CharScanner (sub)classes. */
// protected void genIterator() {
// println("def __iter__(self):");
// tabs++;
// println("return antlr.CharScannerIterator(self)");
// tabs--;
// }
/** Generate an automated test for Python CharScanner (sub)classes. */
protected void genLexerTest() {
String className = grammar.getClassName();
println("if __name__ == '__main__' :");
tabs++;
println("import sys");
println("import antlr");
println("import " + className);
println("");
println("### create lexer - shall read from stdin");
println("try:");
tabs++;
println("for token in " + className + ".Lexer():");
tabs++;
println("print token");
println("");
tabs--;
tabs--;
println("except antlr.TokenStreamException, e:");
tabs++;
println("print \"error: exception caught while lexing: \", e");
tabs--;
tabs--;
}
private void genLiteralsTest()
{
println("### option { testLiterals=true } ");
println("_ttype = self.testLiteralsTable(_ttype)");
}
private void genLiteralsTestForPartialToken() {
println("_ttype = self.testLiteralsTable(self.text.getBuffer(), _begin, self.text.length()-_begin), _ttype)");
}
protected void genMatch(BitSet b) {
}
protected void genMatch(GrammarAtom atom) {
if (atom instanceof StringLiteralElement) {
if (grammar instanceof LexerGrammar) {
genMatchUsingAtomText(atom);
}
else {
genMatchUsingAtomTokenType(atom);
}
}
else if (atom instanceof CharLiteralElement) {
if (grammar instanceof LexerGrammar) {
genMatchUsingAtomText(atom);
}
else {
antlrTool.error("cannot ref character literals in grammar: " + atom);
}
}
else if (atom instanceof TokenRefElement) {
genMatchUsingAtomText(atom);
}
else if (atom instanceof WildcardElement) {
gen((WildcardElement)atom);
}
}
protected void genMatchUsingAtomText(GrammarAtom atom) {
// match() for trees needs the _t cursor
String astArgs = "";
if (grammar instanceof TreeWalkerGrammar) {
astArgs = "_t,";
}
// if in lexer and ! on element, save buffer index to kill later
if (grammar instanceof LexerGrammar &&
(!saveText || atom.getAutoGenType() == GrammarElement.AUTO_GEN_BANG))
{
println("_saveIndex = self.text.length()");
}
print(atom.not ? "self.matchNot(" : "self.match(");
_print(astArgs);
// print out what to match
if (atom.atomText.equals("EOF")) {
// horrible hack to handle EOF case
_print("EOF_TYPE");
}
else {
_print(atom.atomText);
}
_println(")");
if (grammar instanceof LexerGrammar && (!saveText || atom.getAutoGenType() == GrammarElement.AUTO_GEN_BANG)) {
println("self.text.setLength(_saveIndex)"); // kill text atom put in buffer
}
}
protected void genMatchUsingAtomTokenType(GrammarAtom atom) {
// match() for trees needs the _t cursor
String astArgs = "";
if (grammar instanceof TreeWalkerGrammar) {
astArgs = "_t,";
}
// If the literal can be mangled, generate the symbolic constant instead
String mangledName = null;
String s = astArgs + getValueString(atom.getType(),true);
// matching
println((atom.not ? "self.matchNot(" : "self.match(") + s + ")");
}
/** Generate the nextToken() rule. nextToken() is a synthetic
* lexer rule that is the implicit OR of all user-defined
* lexer rules.
*/
public
void genNextToken() {
// Are there any public rules? If not, then just generate a
// fake nextToken().
boolean hasPublicRules = false;
for (int i = 0; i < grammar.rules.size(); i++) {
RuleSymbol rs = (RuleSymbol)grammar.rules.elementAt(i);
if (rs.isDefined() && rs.access.equals("public")) {
hasPublicRules = true;
break;
}
}
if (!hasPublicRules) {
println("");
println("def nextToken(self): ");
tabs++;
println("try:");
tabs++;
println("self.uponEOF()");
tabs--;
println("except antlr.CharStreamIOException, csioe:");
tabs++;
println("raise antlr.TokenStreamIOException(csioe.io)");
tabs--;
println("except antlr.CharStreamException, cse:");
tabs++;
println("raise antlr.TokenStreamException(str(cse))");
tabs--;
println("return antlr.CommonToken(type=EOF_TYPE, text=\"\")");
tabs--;
return;
}
// Create the synthesized nextToken() rule
RuleBlock nextTokenBlk =
MakeGrammar.createNextTokenRule(grammar, grammar.rules, "nextToken");
// Define the nextToken rule symbol
RuleSymbol nextTokenRs = new RuleSymbol("mnextToken");
nextTokenRs.setDefined();
nextTokenRs.setBlock(nextTokenBlk);
nextTokenRs.access = "private";
grammar.define(nextTokenRs);
// Analyze the nextToken rule
boolean ok = grammar.theLLkAnalyzer.deterministic(nextTokenBlk);
// Generate the next token rule
String filterRule = null;
if (((LexerGrammar)grammar).filterMode) {
filterRule = ((LexerGrammar)grammar).filterRule;
}
println("");
println("def nextToken(self):");
tabs++;
println("while True:");
tabs++;
println("try: ### try again ..");
tabs++;
println("while True:");
tabs++;
int _tabs_ = tabs; // while block
println("_token = None");
println("_ttype = INVALID_TYPE");
if (((LexerGrammar)grammar).filterMode)
{
println("self.setCommitToPath(False)");
if (filterRule != null)
{
// Here's a good place to ensure that the filter rule actually exists
if (!grammar.isDefined(CodeGenerator.encodeLexerRuleName(filterRule))) {
grammar.antlrTool.error(
"Filter rule " + filterRule + " does not exist in this lexer");
}
else
{
RuleSymbol rs = (RuleSymbol)grammar.getSymbol(
CodeGenerator.encodeLexerRuleName(filterRule));
if (!rs.isDefined()) {
grammar.antlrTool.error(
"Filter rule " + filterRule + " does not exist in this lexer");
}
else if (rs.access.equals("public")) {
grammar.antlrTool.error(
"Filter rule " + filterRule + " must be protected");
}
}
println("_m = self.mark()");
}
}
println("self.resetText()");
println("try: ## for char stream error handling");
tabs++;
_tabs_ = tabs; // inner try
// Generate try around whole thing to trap scanner errors
println("try: ##for lexical error handling");
tabs++;
_tabs_ = tabs; // inner try
// Test for public lexical rules with empty paths
for (int i = 0; i < nextTokenBlk.getAlternatives().size(); i++)
{
Alternative a = nextTokenBlk.getAlternativeAt(i);
if (a.cache[1].containsEpsilon())
{
//String r = a.head.toString();
RuleRefElement rr = (RuleRefElement)a.head;
String r = CodeGenerator.decodeLexerRuleName(rr.targetRule);
antlrTool.warning("public lexical rule "+r+" is optional (can match \"nothing\")");
}
}
// Generate the block
String newline = System.getProperty("line.separator");
/* generate the common block */
PythonBlockFinishingInfo howToFinish =
genCommonBlock(nextTokenBlk, false);
/* how to finish the block */
String errFinish = "";
// Is this a filter? if so we need to change the default handling.
// In non filter mode we generate EOF token on EOF and stop, other-
// wise an error gets generated. In filter mode we just continue
// by consuming the unknown character till EOF is seen.
if (((LexerGrammar)grammar).filterMode)
{
/* filter */
if (filterRule == null)
{
/* no specical filter rule has been given. */
errFinish += "self.filterdefault(self.LA(1))";
}
else
{
errFinish +=
"self.filterdefault(self.LA(1), self.m" + filterRule + ", False)";
}
}
else
{
/* non filter */
/* if an IF has been generated (in the default clause), indendation
** is not correct. In that case we need to close on level++.
**/
errFinish = "self.default(self.LA(1))" ;
}
/* finish the block */
genBlockFinish1(howToFinish, errFinish);
// alt block has finished .., reset tabs!
tabs = _tabs_;
// at this point a valid token has been matched, undo "mark" that was done
if (((LexerGrammar)grammar).filterMode && filterRule != null) {
println("self.commit()");
}
// Generate literals test if desired
// make sure _ttype is set first; note _returnToken must be
// non-null as the rule was required to create it.
println("if not self._returnToken:");
tabs++;
println("raise antlr.TryAgain ### found SKIP token");
tabs--;
// There's one literal test (in Lexer) after the large switch
// in 'nextToken'.
if (((LexerGrammar)grammar).getTestLiterals())
{
println("### option { testLiterals=true } ");
//genLiteralsTest();
println("self.testForLiteral(self._returnToken)");
}
// return token created by rule reference in switch
println("### return token to caller");
println("return self._returnToken");
// Close try block
tabs--;
println("### handle lexical errors ....");
println("except antlr.RecognitionException, e:");
tabs++;
if (((LexerGrammar)grammar).filterMode)
{
if (filterRule == null)
{
println("if not self.getCommitToPath():");
tabs++;
println("self.consume()");
println("raise antlr.TryAgain()");
tabs--;
}
else
{
println("if not self.getCommitToPath(): ");
tabs++;
println("self.rewind(_m)");
println("self.resetText()");
println("try:");
tabs++;
println("self.m" + filterRule + "(False)");
tabs--;
println("except antlr.RecognitionException, ee:");
tabs++;
println("### horrendous failure: error in filter rule");
println("self.reportError(ee)");
println("self.consume()");
tabs--;
println("raise antlr.TryAgain()");
tabs--;
}
}
if (nextTokenBlk.getDefaultErrorHandler()) {
println("self.reportError(e)");
println("self.consume()");
}
else {
// pass on to invoking routine
println("raise antlr.TokenStreamRecognitionException(e)");
}
tabs--;
//println("");
//println("### shall never be reached ");
//println("assert 0");
// close CharStreamException try
tabs--;
println("### handle char stream errors ...");
println("except antlr.CharStreamException,cse:");
tabs++;
println("if isinstance(cse, antlr.CharStreamIOException):");
tabs++;
println("raise antlr.TokenStreamIOException(cse.io)");
tabs--;
println("else:");
tabs++;
println("raise antlr.TokenStreamException(str(cse))");
tabs--;
tabs--;
//println("### shall never be reached ");
//println("assert 0");
// close for-loop
tabs--;
//println("### <end of inner while>");
//println("### shall never be reached ");
//println("assert 0");
tabs--;
//println("### <matching outer try>");
println("except antlr.TryAgain:");
tabs++;
println("pass");
tabs--;
// close method nextToken
tabs--;
//println("### <end of outer while>");
//println("### shall never be reached");
//println("assert 0");
//println("### <end of method nextToken>");
}
/** Gen a named rule block.
* ASTs are generated for each element of an alternative unless
* the rule or the alternative have a '!' modifier.
*
* If an alternative defeats the default tree construction, it
* must set <rule>_AST to the root of the returned AST.
*
* Each alternative that does automatic tree construction, builds
* up root and child list pointers in an ASTPair structure.
*
* A rule finishes by setting the returnAST variable from the
* ASTPair.
*
* @param rule The name of the rule to generate
* @param startSymbol true if the rule is a start symbol (i.e., not referenced elsewhere)
*/
public void genRule(RuleSymbol s, boolean startSymbol, int ruleNum) {
tabs=1;
if (!s.isDefined()) {
antlrTool.error("undefined rule: " + s.getId());
return;
}
// Generate rule return type, name, arguments
RuleBlock rblk = s.getBlock();
currentRule = rblk;
currentASTResult = s.getId();
// clear list of declared ast variables..
declaredASTVariables.clear();
// Save the AST generation state, and set it to that of the rule
boolean savegenAST = genAST;
genAST = genAST && rblk.getAutoGen();
// boolean oldsaveTest = saveText;
saveText = rblk.getAutoGen();
// print javadoc comment if any
genJavadocComment(s);
// Gen method name
print("def " + s.getId() + "(");
// Additional rule parameters common to all rules for this grammar
_print(commonExtraParams);
if (commonExtraParams.length() != 0 && rblk.argAction != null) {
_print(",");
}
// Gen arguments
if (rblk.argAction != null) {
// Has specified arguments
_println("");
tabs++;
println(rblk.argAction);
tabs--;
print("):");
}
else {
// No specified arguments
_print("):");
}
println("");
tabs++;
// Convert return action to variable declaration
if (rblk.returnAction != null) {
if (rblk.returnAction.indexOf('=') >= 0)
println(rblk.returnAction);
else {
// mx
println(extractIdOfAction(rblk.returnAction, rblk.getLine(), rblk.getColumn()) + " = None"); }
}
// print out definitions needed by rules for various grammar types
println(commonLocalVars);
if (grammar.traceRules) {
if (grammar instanceof TreeWalkerGrammar) {
println("self.traceIn(\"" + s.getId() + "\",_t)");
}
else {
println("self.traceIn(\"" + s.getId() + "\")");
}
}
if (grammar instanceof LexerGrammar) {
// lexer rule default return value is the rule's token name
// This is a horrible hack to support the built-in EOF lexer rule.
if (s.getId().equals("mEOF"))
println("_ttype = EOF_TYPE");
else
println("_ttype = " + s.getId().substring(1));
println("_saveIndex = 0"); // used for element! (so we can kill text matched for element)
}
// if debugging, write code to mark entry to the rule
if (grammar.debuggingOutput)
if (grammar instanceof ParserGrammar)
println("self.fireEnterRule(" + ruleNum + ", 0)");
else if (grammar instanceof LexerGrammar)
println("self.fireEnterRule(" + ruleNum + ", _ttype)");
// Generate trace code if desired
if (grammar.debuggingOutput || grammar.traceRules) {
println("try: ### debugging");
tabs++;
}
// Initialize AST variables
if (grammar instanceof TreeWalkerGrammar) {
// "Input" value for rule
println(s.getId() + "_AST_in = None");
println("if _t != antlr.ASTNULL:");
tabs++;
println(s.getId() + "_AST_in = _t");
tabs--;
}
if (grammar.buildAST)
{
// Parser member used to pass AST returns from rule invocations
println("self.returnAST = None");
println("currentAST = antlr.ASTPair()");
// User-settable return value for rule.
println(s.getId() + "_AST = None");
}
genBlockPreamble(rblk);
genBlockInitAction(rblk);
// Search for an unlabeled exception specification attached to the rule
ExceptionSpec unlabeledUserSpec = rblk.findExceptionSpec("");
// Generate try block around the entire rule for error handling
if (unlabeledUserSpec != null || rblk.getDefaultErrorHandler()) {
println("try: ## for error handling");
tabs++;
}
int _tabs_ = tabs;
// Generate the alternatives
if (rblk.alternatives.size() == 1)
{
// One alternative -- use simple form
Alternative alt = rblk.getAlternativeAt(0);
String pred = alt.semPred;
if (pred != null)
genSemPred(pred, currentRule.line);
if (alt.synPred != null)
{
antlrTool.warning(
"Syntactic predicate ignored for single alternative",
grammar.getFilename(),
alt.synPred.getLine(),
alt.synPred.getColumn()
);
}
genAlt(alt, rblk);
}
else
{
// Multiple alternatives -- generate complex form
boolean ok = grammar.theLLkAnalyzer.deterministic(rblk);
PythonBlockFinishingInfo howToFinish = genCommonBlock(rblk, false);
genBlockFinish(howToFinish, throwNoViable);
}
tabs = _tabs_;
// Generate catch phrase for error handling
if (unlabeledUserSpec != null || rblk.getDefaultErrorHandler()) {
// Close the try block
tabs--;
println("");
}
// Generate user-defined or default catch phrases
if (unlabeledUserSpec != null) {
genErrorHandler(unlabeledUserSpec);
}
else if (rblk.getDefaultErrorHandler()) {
// Generate default catch phrase
println("except " + exceptionThrown + ", ex:");
tabs++;
// Generate code to handle error if not guessing
if (grammar.hasSyntacticPredicate) {
println("if not self.inputState.guessing:");
tabs++;
}
println("self.reportError(ex)");
if (!(grammar instanceof TreeWalkerGrammar)) {
// Generate code to consume until token in k==1 follow set
Lookahead follow = grammar.theLLkAnalyzer.FOLLOW(1, rblk.endNode);
String followSetName = getBitsetName(markBitsetForGen(follow.fset));
println("self.consume()");
println("self.consumeUntil(" + followSetName + ")");
}
else {
// Just consume one token
println("if _t:");
tabs++;
println("_t = _t.getNextSibling()");
tabs--;
}
if (grammar.hasSyntacticPredicate) {
tabs--;
// When guessing, rethrow exception
println("else:");
tabs++;
println("raise ex");
tabs--;
}
// Close catch phrase
tabs--;
println("");
}
// Squirrel away the AST "return" value
if (grammar.buildAST) {
println("self.returnAST = " + s.getId() + "_AST");
}
// Set return tree value for tree walkers
if (grammar instanceof TreeWalkerGrammar) {
println("self._retTree = _t");
}
// Generate literals test for lexer rules so marked
if (rblk.getTestLiterals()) {
if (s.access.equals("protected")) {
genLiteralsTestForPartialToken();
}
else {
genLiteralsTest();
}
}
// if doing a lexer rule, dump code to create token if necessary
if (grammar instanceof LexerGrammar)
{
println("self.set_return_token(_createToken, _token, _ttype, _begin)");
}
if(rblk.returnAction != null)
{
// if(grammar instanceof LexerGrammar)
// {
println("return " +
extractIdOfAction(rblk.returnAction,
rblk.getLine(),
rblk.getColumn()) + "");
// }
// else
// {
// println("return r");
// }
}
if (grammar.debuggingOutput || grammar.traceRules) {
tabs--;
println("finally: ### debugging");
tabs++;
// If debugging, generate calls to mark exit of rule
if (grammar.debuggingOutput)
if (grammar instanceof ParserGrammar)
println("self.fireExitRule(" + ruleNum + ", 0)");
else if (grammar instanceof LexerGrammar)
println("self.fireExitRule(" + ruleNum + ", _ttype)");
if (grammar.traceRules) {
if (grammar instanceof TreeWalkerGrammar) {
println("self.traceOut(\"" + s.getId() + "\", _t)");
}
else {
println("self.traceOut(\"" + s.getId() + "\")");
}
}
tabs--;
}
tabs--;
println("");
// Restore the AST generation state
genAST = savegenAST;
// restore char save state
// saveText = oldsaveTest;
}
private void GenRuleInvocation(RuleRefElement rr) {
// dump rule name
_print("self." + rr.targetRule + "(");
// lexers must tell rule if it should set _returnToken
if (grammar instanceof LexerGrammar) {
// if labeled, could access Token, so tell rule to create
if (rr.getLabel() != null) {
_print("True");
}
else {
_print("False");
}
if (commonExtraArgs.length() != 0 || rr.args != null) {
_print(", ");
}
}
// Extra arguments common to all rules for this grammar
_print(commonExtraArgs);
if (commonExtraArgs.length() != 0 && rr.args != null) {
_print(", ");
}
// Process arguments to method, if any
RuleSymbol rs = (RuleSymbol)grammar.getSymbol(rr.targetRule);
if (rr.args != null) {
// When not guessing, execute user arg action
ActionTransInfo tInfo = new ActionTransInfo();
String args = processActionForSpecialSymbols(rr.args, 0, currentRule, tInfo);
if (tInfo.assignToRoot || tInfo.refRuleRoot != null) {
antlrTool.error("Arguments of rule reference '" + rr.targetRule + "' cannot set or ref #" +
currentRule.getRuleName(), grammar.getFilename(), rr.getLine(), rr.getColumn());
}
_print(args);
// Warn if the rule accepts no arguments
if (rs.block.argAction == null) {
antlrTool.warning("Rule '" + rr.targetRule + "' accepts no arguments", grammar.getFilename(), rr.getLine(), rr.getColumn());
}
}
else {
// For C++, no warning if rule has parameters, because there may be default
// values for all of the parameters
if (rs.block.argAction != null) {
antlrTool.warning("Missing parameters on reference to rule " + rr.targetRule, grammar.getFilename(), rr.getLine(), rr.getColumn());
}
}
_println(")");
// move down to the first child while parsing
if (grammar instanceof TreeWalkerGrammar) {
println("_t = self._retTree");
}
}
protected void genSemPred(String pred, int line) {
// translate $ and # references
ActionTransInfo tInfo = new ActionTransInfo();
pred = processActionForSpecialSymbols(pred, line, currentRule, tInfo);
// ignore translation info...we don't need to do anything with it.
String escapedPred = charFormatter.escapeString(pred);
// if debugging, wrap the semantic predicate evaluation in a method
// that can tell SemanticPredicateListeners the result
if (grammar.debuggingOutput &&
((grammar instanceof ParserGrammar) || (grammar instanceof LexerGrammar)))
pred = "fireSemanticPredicateEvaluated(antlr.debug.SemanticPredicateEvent.VALIDATING,"
+ addSemPred(escapedPred) + ", " + pred + ")";
/* always .. */
println("if not " + pred + ":");
tabs++;
println("raise SemanticException(\"" + escapedPred + "\")");
tabs--;
}
/** Write an array of Strings which are the semantic predicate
* expressions. The debugger will reference them by number only
*/
protected void genSemPredMap() {
Enumeration e = semPreds.elements();
println("_semPredNames = [");
tabs++;
while (e.hasMoreElements()) {
println("\"" + e.nextElement() + "\",");
}
tabs--;
println("]");
}
protected void genSynPred(SynPredBlock blk, String lookaheadExpr) {
if (DEBUG_CODE_GENERATOR) System.out.println("gen=>(" + blk + ")");
// Dump synpred result variable
println("synPredMatched" + blk.ID + " = False");
// Gen normal lookahead test
println("if " + lookaheadExpr + ":");
tabs++;
// Save input state
if (grammar instanceof TreeWalkerGrammar) {
println("_t" + blk.ID + " = _t");
}
else {
println("_m" + blk.ID + " = self.mark()");
}
// Once inside the try, assume synpred works unless exception caught
println("synPredMatched" + blk.ID + " = True");
println("self.inputState.guessing += 1");
// if debugging, tell listeners that a synpred has started
if (grammar.debuggingOutput && ((grammar instanceof ParserGrammar) ||
(grammar instanceof LexerGrammar))) {
println("self.fireSyntacticPredicateStarted()");
}
syntacticPredLevel++;
println("try:");
tabs++;
gen((AlternativeBlock)blk); // gen code to test predicate
tabs--;
println("except " + exceptionThrown + ", pe:");
tabs++;
println("synPredMatched" + blk.ID + " = False");
tabs--;
// Restore input state
if (grammar instanceof TreeWalkerGrammar) {
println("_t = _t" + blk.ID + "");
}
else {
println("self.rewind(_m" + blk.ID + ")");
}
println("self.inputState.guessing -= 1");
// if debugging, tell listeners how the synpred turned out
if (grammar.debuggingOutput && ((grammar instanceof ParserGrammar) ||
(grammar instanceof LexerGrammar))) {
println("if synPredMatched" + blk.ID + ":");
tabs++;
println("self.fireSyntacticPredicateSucceeded()");
tabs--;
println("else:");
tabs++;
println("self.fireSyntacticPredicateFailed()");
tabs--;
}
syntacticPredLevel--;
tabs--;
// Close lookahead test
// Test synred result
println("if synPredMatched" + blk.ID + ":");
}
/** Generate a static array containing the names of the tokens,
* indexed by the token type values. This static array is used
* to format error messages so that the token identifers or literal
* strings are displayed instead of the token numbers.
*
* If a lexical rule has a paraphrase, use it rather than the
* token label.
*/
public void genTokenStrings() {
// Generate a string for each token. This creates a static
// array of Strings indexed by token type.
int save_tabs = tabs;
tabs = 0;
println("");
println("_tokenNames = [");
tabs++;
// Walk the token vocabulary and generate a Vector of strings
// from the tokens.
Vector v = grammar.tokenManager.getVocabulary();
for (int i = 0; i < v.size(); i++) {
String s = (String)v.elementAt(i);
if (s == null) {
s = "<" + String.valueOf(i) + ">";
}
if (!s.startsWith("\"") && !s.startsWith("<")) {
TokenSymbol ts = (TokenSymbol)grammar.tokenManager.getTokenSymbol(s);
if (ts != null && ts.getParaphrase() != null) {
s = StringUtils.stripFrontBack(ts.getParaphrase(), "\"", "\"");
}
}
print(charFormatter.literalString(s));
if (i != v.size() - 1) {
_print(", ");
}
_println("");
}
// Close the string array initailizer
tabs--;
println("]");
tabs = save_tabs;
}
/** Create and set Integer token type objects that map
* to Java Class objects (which AST node to create).
*/
protected void genTokenASTNodeMap() {
println("");
println("def buildTokenTypeASTClassMap(self):");
// Generate a map.put("T","TNode") for each token
// if heterogeneous node known for that token T.
tabs++;
boolean generatedNewHashtable = false;
int n = 0;
// Walk the token vocabulary and generate puts.
Vector v = grammar.tokenManager.getVocabulary();
for (int i = 0; i < v.size(); i++) {
String s = (String)v.elementAt(i);
if (s != null) {
TokenSymbol ts = grammar.tokenManager.getTokenSymbol(s);
if (ts != null && ts.getASTNodeType() != null) {
n++;
if ( !generatedNewHashtable ) {
// only generate if we are going to add a mapping
println("self.tokenTypeToASTClassMap = {}");
generatedNewHashtable = true;
}
println(
"self.tokenTypeToASTClassMap["
+ ts.getTokenType()
+ "] = "
+ ts.getASTNodeType() );
}
}
}
if ( n==0 ) {
println("self.tokenTypeToASTClassMap = None");
}
tabs--;
}
/** Generate the token types Java file */
protected void genTokenTypes(TokenManager tm) throws IOException {
// Open the token output Python file and set the currentOutput
// stream
// SAS: file open was moved to a method so a subclass can override
// This was mainly for the VAJ interface
// setupOutput(tm.getName() + TokenTypesFileSuffix);
tabs = 0;
// Generate the header common to all Python files
// genHeader();
// Do not use printAction because we assume tabs==0
// println(behavior.getHeaderAction(""));
// Generate a definition for each token type
Vector v = tm.getVocabulary();
// Do special tokens manually
println("SKIP = antlr.SKIP");
println("INVALID_TYPE = antlr.INVALID_TYPE");
println("EOF_TYPE = antlr.EOF_TYPE");
println("EOF = antlr.EOF");
println("NULL_TREE_LOOKAHEAD = antlr.NULL_TREE_LOOKAHEAD");
println("MIN_USER_TYPE = antlr.MIN_USER_TYPE");
for (int i = Token.MIN_USER_TYPE; i < v.size(); i++)
{
String s = (String)v.elementAt(i);
if (s != null)
{
if (s.startsWith("\""))
{
// a string literal
StringLiteralSymbol sl = (StringLiteralSymbol)tm.getTokenSymbol(s);
if (sl == null)
antlrTool.panic("String literal " + s + " not in symbol table");
if (sl.label != null)
{
println(sl.label + " = " + i);
}
else
{
String mangledName = mangleLiteral(s);
if (mangledName != null) {
// We were able to create a meaningful mangled token name
println(mangledName + " = " + i);
// if no label specified, make the label equal to the mangled name
sl.label = mangledName;
}
else
{
println("### " + s + " = " + i);
}
}
}
else if (!s.startsWith("<")) {
println(s + " = " + i);
}
}
}
// Close the interface
tabs--;
exitIfError();
}
/** Get a string for an expression to generate creation of an AST subtree.
* @param v A Vector of String, where each element is an expression in the target language yielding an AST node.
*/
public String getASTCreateString(Vector v) {
if (v.size() == 0) {
return "";
}
StringBuffer buf = new StringBuffer();
buf.append("antlr.make(");
for (int i = 0; i < v.size(); i++) {
buf.append(v.elementAt(i));
if(i+1<v.size()) {
buf.append(", ");
}
}
buf.append(")");
return buf.toString();
}
/** Get a string for an expression to generate creating of an AST node
* @param atom The grammar node for which you are creating the node
* @param str The arguments to the AST constructor
*/
public String getASTCreateString(GrammarAtom atom, String astCtorArgs)
{
if (atom != null && atom.getASTNodeType() != null)
{
// they specified a type either on the reference or in tokens{} section
return
"self.astFactory.create(" + astCtorArgs + ", " + atom.getASTNodeType() + ")";
}
else {
// must be an action or something since not referencing an atom
return getASTCreateString(astCtorArgs);
}
}
/** Get a string for an expression to generate creating of an AST node.
* Parse the first (possibly only) argument looking for the token type.
* If the token type is a valid token symbol, ask for it's AST node type
* and add to the end if only 2 arguments. The forms are #[T], #[T,"t"],
* and as of 2.7.2 #[T,"t",ASTclassname].
*
* @param str The arguments to the AST constructor
*/
public String getASTCreateString(String astCtorArgs) {
if ( astCtorArgs==null ) {
astCtorArgs = "";
}
int nCommas = 0;
for (int i=0; i<astCtorArgs.length(); i++) {
if ( astCtorArgs.charAt(i)==',' ) {
nCommas++;
}
}
if ( nCommas<2 ) { // if 1 or 2 args
int firstComma = astCtorArgs.indexOf(',');
int lastComma = astCtorArgs.lastIndexOf(',');
String tokenName = astCtorArgs;
if ( nCommas>0 ) {
tokenName = astCtorArgs.substring(0,firstComma);
}
TokenSymbol ts = grammar.tokenManager.getTokenSymbol(tokenName);
if ( ts!=null ) {
String astNodeType = ts.getASTNodeType();
String emptyText = "";
if ( nCommas==0 ) {
// need to add 2nd arg of blank text for token text
emptyText = ", \"\"";
}
if ( astNodeType!=null ) {
return "self.astFactory.create(" + astCtorArgs + emptyText + ", " + astNodeType + ")";
}
// fall through and just do a regular create with cast on front
// if necessary (it differs from default "AST").
}
if ( labeledElementASTType.equals("AST") ) {
return "self.astFactory.create("+astCtorArgs+")";
}
return
"self.astFactory.create("+astCtorArgs+")";
}
// create default type or (since 2.7.2) 3rd arg is classname
return "self.astFactory.create(" + astCtorArgs + ")";
}
protected String getLookaheadTestExpression(Lookahead[] look, int k) {
StringBuffer e = new StringBuffer(100);
boolean first = true;
e.append("(");
for (int i = 1; i <= k; i++) {
BitSet p = look[i].fset;
if (!first) {
e.append(") and (");
}
first = false;
// Syn preds can yield <end-of-syn-pred> (epsilon) lookahead.
// There is no way to predict what that token would be. Just
// allow anything instead.
if (look[i].containsEpsilon()) {
e.append("True");
}
else {
e.append(getLookaheadTestTerm(i, p));
}
}
e.append(")");
String s = e.toString();
return s;
}
/**Generate a lookahead test expression for an alternate. This
* will be a series of tests joined by '&&' and enclosed by '()',
* the number of such tests being determined by the depth of the lookahead.
*/
protected String getLookaheadTestExpression(Alternative alt, int maxDepth) {
int depth = alt.lookaheadDepth;
if (depth == GrammarAnalyzer.NONDETERMINISTIC)
{
// if the decision is nondeterministic, do the best we can: LL(k)
// any predicates that are around will be generated later.
depth = grammar.maxk;
}
if (maxDepth == 0)
{
// empty lookahead can result from alt with sem pred
// that can see end of token. E.g., A : {pred}? ('a')? ;
return "True";
}
return getLookaheadTestExpression(alt.cache, depth);
}
/**Generate a depth==1 lookahead test expression given the BitSet.
* This may be one of:
* 1) a series of 'x==X||' tests
* 2) a range test using >= && <= where possible,
* 3) a bitset membership test for complex comparisons
* @param k The lookahead level
* @param p The lookahead set for level k
*/
protected String getLookaheadTestTerm(int k, BitSet p) {
// Determine the name of the item to be compared
String ts = lookaheadString(k);
// Generate a range expression if possible
int[] elems = p.toArray();
if (elementsAreRange(elems)) {
String s = getRangeExpression(k, elems);
return s;
}
// Generate a bitset membership test if possible
StringBuffer e;
int degree = p.degree();
if (degree == 0) {
return "True";
}
if (degree >= bitsetTestThreshold) {
int bitsetIdx = markBitsetForGen(p);
return getBitsetName(bitsetIdx) + ".member(" + ts + ")";
}
// Otherwise, generate the long-winded series of "x==X||" tests
e = new StringBuffer();
for (int i = 0; i < elems.length; i++) {
// Get the compared-to item (token or character value)
String cs = getValueString(elems[i],true);
// Generate the element comparison
if (i > 0) e.append(" or ");
e.append(ts);
e.append("==");
e.append(cs);
}
String x = e.toString();
return e.toString();
}
/** Return an expression for testing a contiguous renage of elements
* @param k The lookahead level
* @param elems The elements representing the set, usually from BitSet.toArray().
* @return String containing test expression.
*/
public String getRangeExpression(int k, int[] elems) {
if (!elementsAreRange(elems)) {
antlrTool.panic("getRangeExpression called with non-range");
}
int begin = elems[0];
int end = elems[elems.length - 1];
return
"(" + lookaheadString(k) + " >= " + getValueString(begin,true) + " and " +
lookaheadString(k) + " <= " + getValueString(end,true) + ")";
}
/** getValueString: get a string representation of a token or char value
* @param value The token or char value
*/
private String getValueString(int value,boolean wrap) {
String cs;
if (grammar instanceof LexerGrammar) {
cs = charFormatter.literalChar(value);
if(wrap)
cs = "u'" + cs + "'";
return cs;
}
// Parser or TreeParser => tokens ..
TokenSymbol ts =
grammar.tokenManager.getTokenSymbolAt(
value);
if (ts == null) {
cs = "" + value; // return token type as string
return cs;
}
String tId = ts.getId();
if (!(ts instanceof StringLiteralSymbol))
{
cs = tId;
return cs;
}
// if string literal, use predefined label if any
// if no predefined, try to mangle into LITERAL_xxx.
// if can't mangle, use int value as last resort
StringLiteralSymbol sl = (StringLiteralSymbol)ts;
String label = sl.getLabel();
if (label != null) {
cs = label;
}
else
{
cs = mangleLiteral(tId);
if (cs == null) {
cs = String.valueOf(value);
}
}
return cs;
}
/**Is the lookahead for this alt empty? */
protected boolean lookaheadIsEmpty(Alternative alt, int maxDepth) {
int depth = alt.lookaheadDepth;
if (depth == GrammarAnalyzer.NONDETERMINISTIC) {
depth = grammar.maxk;
}
for (int i = 1; i <= depth && i <= maxDepth; i++) {
BitSet p = alt.cache[i].fset;
if (p.degree() != 0) {
return false;
}
}
return true;
}
private String lookaheadString(int k) {
if (grammar instanceof TreeWalkerGrammar) {
return "_t.getType()";
}
return "self.LA(" + k + ")";
}
/** Mangle a string literal into a meaningful token name. This is
* only possible for literals that are all characters. The resulting
* mangled literal name is literalsPrefix with the text of the literal
* appended.
* @return A string representing the mangled literal, or null if not possible.
*/
private String mangleLiteral(String s) {
String mangled = antlrTool.literalsPrefix;
for (int i = 1; i < s.length() - 1; i++) {
if (!Character.isLetter(s.charAt(i)) &&
s.charAt(i) != '_') {
return null;
}
mangled += s.charAt(i);
}
if (antlrTool.upperCaseMangledLiterals) {
mangled = mangled.toUpperCase();
}
return mangled;
}
/** Map an identifier to it's corresponding tree-node variable.
* This is context-sensitive, depending on the rule and alternative
* being generated
* @param idParam The identifier name to map
* @return The mapped id (which may be the same as the input), or null if
* the mapping is invalid due to duplicates
*/
public String mapTreeId(String idParam, ActionTransInfo transInfo) {
// if not in an action of a rule, nothing to map.
if (currentRule == null) return idParam;
boolean in_var = false;
String id = idParam;
if (grammar instanceof TreeWalkerGrammar) {
if (!grammar.buildAST) {
in_var = true;
}
// If the id ends with "_in", then map it to the input variable
else if (id.length() > 3 && id.lastIndexOf("_in") == id.length() - 3) {
// Strip off the "_in"
id = id.substring(0, id.length() - 3);
in_var = true;
}
}
// Check the rule labels. If id is a label, then the output
// variable is label_AST, and the input variable is plain label.
for (int i = 0; i < currentRule.labeledElements.size(); i++) {
AlternativeElement elt = (AlternativeElement)currentRule.labeledElements.elementAt(i);
if (elt.getLabel().equals(id)) {
return in_var ? id : id + "_AST";
}
}
// Failing that, check the id-to-variable map for the alternative.
// If the id is in the map, then output variable is the name in the
// map, and input variable is name_in
String s = (String)treeVariableMap.get(id);
if (s != null) {
if (s == NONUNIQUE) {
// There is more than one element with this id
antlrTool.error("Ambiguous reference to AST element "+id+
" in rule "+currentRule.getRuleName());
return null;
}
else if (s.equals(currentRule.getRuleName())) {
// a recursive call to the enclosing rule is
// ambiguous with the rule itself.
antlrTool.error("Ambiguous reference to AST element "+id+
" in rule "+currentRule.getRuleName());
return null;
}
else {
return in_var ? s + "_in" : s;
}
}
// Failing that, check the rule name itself. Output variable
// is rule_AST; input variable is rule_AST_in (treeparsers).
if (id.equals(currentRule.getRuleName())) {
String r = in_var ? id + "_AST_in" : id + "_AST";
if (transInfo != null) {
if (!in_var) {
transInfo.refRuleRoot = r;
}
}
return r;
}
else {
// id does not map to anything -- return itself.
return id;
}
}
/** Given an element and the name of an associated AST variable,
* create a mapping between the element "name" and the variable name.
*/
private void mapTreeVariable(AlternativeElement e, String name) {
// For tree elements, defer to the root
if (e instanceof TreeElement) {
mapTreeVariable(((TreeElement)e).root, name);
return;
}
// Determine the name of the element, if any, for mapping purposes
String elName = null;
// Don't map labeled items
if (e.getLabel() == null) {
if (e instanceof TokenRefElement) {
// use the token id
elName = ((TokenRefElement)e).atomText;
}
else if (e instanceof RuleRefElement) {
// use the rule name
elName = ((RuleRefElement)e).targetRule;
}
}
// Add the element to the tree variable map if it has a name
if (elName != null) {
if (treeVariableMap.get(elName) != null) {
// Name is already in the map -- mark it as duplicate
treeVariableMap.remove(elName);
treeVariableMap.put(elName, NONUNIQUE);
}
else {
treeVariableMap.put(elName, name);
}
}
}
/** Lexically process $var and tree-specifiers in the action.
* This will replace #id and #(...) with the appropriate
* function calls and/or variables etc...
*/
protected String processActionForSpecialSymbols(String actionStr, int line,
RuleBlock currentRule,
ActionTransInfo tInfo) {
if (actionStr == null || actionStr.length() == 0)
return null;
if(isEmpty(actionStr))
return "";
// The action trans info tells us (at the moment) whether an
// assignment was done to the rule's tree root.
if (grammar == null) {
// to be processd by PyCodeFmt??
return actionStr;
}
// Create a lexer to read an action and return the translated version
antlr.actions.python.ActionLexer lexer =
new antlr.actions.python.ActionLexer(
actionStr,
currentRule,
this,
tInfo);
lexer.setLineOffset(line);
lexer.setFilename(grammar.getFilename());
lexer.setTool(antlrTool);
try {
lexer.mACTION(true);
actionStr = lexer.getTokenObject().getText();
}
catch (RecognitionException ex) {
lexer.reportError(ex);
}
catch (TokenStreamException tex) {
antlrTool.panic("Error reading action:" + actionStr);
}
catch (CharStreamException io) {
antlrTool.panic("Error reading action:" + actionStr);
}
return actionStr;
}
static boolean isEmpty(String s) {
char c;
boolean ws = true;
/* figure out whether there's something to be done */
for(int i=0;ws && i<s.length();++i) {
c = s.charAt(i);
switch(c) {
case '\n':
case '\r':
case ' ' :
case '\t':
case '\f':
{
break;
}
default: {
ws = false;
}
}
}
return ws;
}
protected String processActionCode(String actionStr,int line) {
/* shall process action code unconditionally */
if(actionStr == null || isEmpty(actionStr))
return "";
antlr.actions.python.CodeLexer lexer =
new antlr.actions.python.CodeLexer(
actionStr,
grammar.getFilename(),
line,
antlrTool
);
try {
lexer.mACTION(true);
actionStr = lexer.getTokenObject().getText();
}
catch (RecognitionException ex) {
lexer.reportError(ex);
}
catch (TokenStreamException tex) {
antlrTool.panic("Error reading action:" + actionStr);
}
catch (CharStreamException io) {
antlrTool.panic("Error reading action:" + actionStr);
}
return actionStr;
}
protected void printActionCode(String actionStr,int line) {
actionStr = processActionCode(actionStr,line);
printAction(actionStr);
}
private void setupGrammarParameters(Grammar g) {
if (g instanceof ParserGrammar)
{
labeledElementASTType = "";
if (g.hasOption("ASTLabelType"))
{
Token tsuffix = g.getOption("ASTLabelType");
if (tsuffix != null) {
String suffix = StringUtils.stripFrontBack(tsuffix.getText(), "\"", "\"");
if (suffix != null) {
labeledElementASTType = suffix;
}
}
}
labeledElementType = "";
labeledElementInit = "None";
commonExtraArgs = "";
commonExtraParams = "self";
commonLocalVars = "";
lt1Value = "self.LT(1)";
exceptionThrown = "antlr.RecognitionException";
throwNoViable = "raise antlr.NoViableAltException(self.LT(1), self.getFilename())";
parserClassName = "Parser";
if (g.hasOption("className"))
{
Token tcname = g.getOption("className");
if (tcname != null) {
String cname = StringUtils.stripFrontBack(tcname.getText(), "\"", "\"");
if (cname != null) {
parserClassName = cname;
}
}
}
return;
}
if (g instanceof LexerGrammar)
{
labeledElementType = "char ";
labeledElementInit = "'\\0'";
commonExtraArgs = "";
commonExtraParams = "self, _createToken";
commonLocalVars = "_ttype = 0\n _token = None\n _begin = self.text.length()";
lt1Value = "self.LA(1)";
exceptionThrown = "antlr.RecognitionException";
throwNoViable = "self.raise_NoViableAlt(self.LA(1))";
lexerClassName = "Lexer";
if (g.hasOption("className"))
{
Token tcname = g.getOption("className");
if (tcname != null) {
String cname = StringUtils.stripFrontBack(tcname.getText(), "\"", "\"");
if (cname != null) {
lexerClassName = cname;
}
}
}
return;
}
if (g instanceof TreeWalkerGrammar)
{
labeledElementASTType = "";
labeledElementType = "";
if (g.hasOption("ASTLabelType")) {
Token tsuffix = g.getOption("ASTLabelType");
if (tsuffix != null) {
String suffix = StringUtils.stripFrontBack(tsuffix.getText(), "\"", "\"");
if (suffix != null) {
labeledElementASTType = suffix;
labeledElementType = suffix;
}
}
}
if (!g.hasOption("ASTLabelType")) {
g.setOption("ASTLabelType", new Token(ANTLRTokenTypes.STRING_LITERAL, "<4>AST"));
}
labeledElementInit = "None";
commonExtraArgs = "_t";
commonExtraParams = "self, _t";
commonLocalVars = "";
lt1Value = "_t";
exceptionThrown = "antlr.RecognitionException";
throwNoViable = "raise antlr.NoViableAltException(_t)";
treeWalkerClassName = "Walker";
if (g.hasOption("className"))
{
Token tcname = g.getOption("className");
if (tcname != null) {
String cname = StringUtils.stripFrontBack(tcname.getText(), "\"", "\"");
if (cname != null) {
treeWalkerClassName = cname;
}
}
}
return;
}
/* serious error */
antlrTool.panic("Unknown grammar type");
}
/** This method exists so a subclass, namely VAJCodeGenerator,
* can open the file in its own evil way. JavaCodeGenerator
* simply opens a text file...
*/
public void setupOutput(String className) throws IOException {
currentOutput = antlrTool.openOutputFile(className + ".py");
}
protected boolean isspace(char c) {
boolean r = true;
switch (c) {
case '\n' :
case '\r' :
case ' ' :
case '\t' :
break;
default:
r = false;
break;
}
return r;
}
protected void _printAction(String s) {
if (s == null) {
return;
}
char c;
int offset; // shall be first no ws character in 's'. We are
// going to remove at most this number of ws chars after
// each newline. This will keep the block formatted as it is.
// When going to figure out the offset, we need to rese the
// counter after each newline has been seen.
// Skip leading newlines, tabs and spaces
int start = 0;
int end = s.length();
boolean ws;
offset = 0;
ws = true;
while (start < end && ws)
{
c = s.charAt(start++);
switch (c) {
case '\n' :
offset = start;
break;
case '\r':
if( (start)<=end && s.charAt(start) == '\n')
start++;
offset = start;
break;
case ' ' :
break;
case '\t':
default:
ws = false;
break;
}
}
if(ws == false) {
start--;
}
offset = start - offset;
// Skip leading newlines, tabs and spaces
end = end - 1;
while ((end > start) && isspace(s.charAt(end))) {
end--;
}
boolean newline = false;
int absorbed;
for (int i = start; i <= end; ++i)
{
c = s.charAt(i);
switch (c) {
case '\n':
newline = true;
break;
case '\r':
newline = true;
if ((i+1) <= end && s.charAt(i+1) == '\n') {
i++;
}
break;
case '\t':
System.err.println("warning: tab characters used in Python action");
currentOutput.print(" ");
break;
case ' ':
currentOutput.print(" ");
break;
default:
currentOutput.print(c);
break;
}
if (newline)
{
currentOutput.print("\n");
printTabs();
absorbed = 0;
newline = false;
// Absorb leading whitespace
for(i=i+1;i<=end;++i)
{
c = s.charAt(i);
if (!isspace(c)) {
i--;
break;
}
switch(c) {
case '\n' :
newline = true;
break;
case '\r':
if ((i+1) <= end && s.charAt(i+1) == '\n') {
i++;
}
newline = true;
break;
}
if(newline)
{
currentOutput.print("\n");
printTabs();
absorbed = 0;
newline = false;
continue;
}
if(absorbed<offset) {
absorbed++;
continue;
}
/* stop this loop */
break;
}
}
}
currentOutput.println();
}
protected void od(String s,int i,int end,String msg) {
System.out.println(msg);
char c;
for(int j=i;j<=end;++j)
{
c = s.charAt(j);
switch(c) {
case '\n':
System.out.print(" nl ");
break;
case '\t':
System.out.print(" ht ");
break;
case ' ':
System.out.print(" sp ");
break;
default:
System.out.print(" " + c + " ");
}
}
System.out.println("");
}
protected void printAction(String s) {
if (s != null) {
printTabs();
_printAction(s);
}
}
protected void printGrammarAction(Grammar grammar) {
println("### user action >>>");
printAction(
processActionForSpecialSymbols(
grammar.classMemberAction.getText(),
grammar.classMemberAction.getLine(),
currentRule,
null)
);
println("### user action <<<");
}
protected void _printJavadoc(String s) {
char c;
int end = s.length();
int start = 0;
boolean newline = false;
currentOutput.print("\n");
printTabs();
currentOutput.print("###");
for (int i = start; i < end; ++i)
{
c = s.charAt(i);
switch (c) {
case '\n':
newline = true;
break;
case '\r':
newline = true;
if ((i+1) <= end && s.charAt(i+1) == '\n') {
i++;
}
break;
case '\t':
currentOutput.print("\t");
break;
case ' ':
currentOutput.print(" ");
break;
default:
currentOutput.print(c);
break;
}
if (newline)
{
currentOutput.print("\n");
printTabs();
currentOutput.print("###");
newline = false;
}
}
currentOutput.println();
}
protected void genJavadocComment(Grammar g) {
// print javadoc comment if any
if (g.comment != null) {
_printJavadoc(g.comment);
}
}
protected void genJavadocComment(RuleSymbol g) {
// print javadoc comment if any
if (g.comment != null) {
_printJavadoc(g.comment);
}
}
}