Examples of org.antlr.runtime.CommonTokenStream

org.antlr.runtime.CommonTokenStream
The most common stream of tokens is one where every token is buffered up and tokens are prefiltered for a certain channel (the parser will only see these tokens and cannot change the filter channel number during the parse). TODO: how to access the full token stream? How to track all tokens matched per rule?

    return r;
  }


  public GrammarAST parseArtificialRule(String ruleText) {
    ANTLRLexer lexer = new ANTLRLexer(new ANTLRStringStream(ruleText));
    ANTLRParser parser = ANTLRParser.createParser(new CommonTokenStream(lexer));
    parser.setGrammar(this);
    parser.setGrammarType(this.type);
    try {
      ANTLRParser.rule_return result = parser.rule();
      return result.getTree();

View Full Code Here

      ErrorManager.internalError("unexpected stream error from parsing "+fileName, e);
      return;
    }


    lexer.setFileName(this.getFileName());
    tokenBuffer = new CommonTokenStream(lexer);
    ANTLRParser parser = ANTLRParser.createParser(tokenBuffer);
    parser.setFileName(this.getFileName());
    ANTLRParser.grammar__return result = null;
    try {
      result = parser.grammar_(this);

View Full Code Here

            throw new RuntimeException( "Unable to parser Reader", e );
        }
    }


    public static DRLParser buildParser(DRLLexer lexer, LanguageLevelOption languageLevel) {
        CommonTokenStream stream = new CommonTokenStream( lexer );
        return getDRLParser(stream, languageLevel);
    }

View Full Code Here

    Class<? extends TokenSource> lexerClass = Class.forName(lexerClassName).asSubclass(TokenSource.class);
    Constructor<? extends TokenSource> lexConstructor = lexerClass.getConstructor(CharStream.class);
    TokenSource lexer = lexConstructor.newInstance(is);
    is.setLine(scriptLine);


    CommonTokenStream tokens = new CommonTokenStream(lexer);


    Class<? extends Parser> parserClass = Class.forName(parserClassName).asSubclass(Parser.class);
    Constructor<? extends Parser> parConstructor = parserClass.getConstructor(TokenStream.class);
    Parser parser = parConstructor.newInstance(tokens);

View Full Code Here


  public GrammarRootAST parse(String fileName, CharStream in) {
    try {
      GrammarASTAdaptor adaptor = new GrammarASTAdaptor(in);
      ToolANTLRLexer lexer = new ToolANTLRLexer(in, this);
      CommonTokenStream tokens = new CommonTokenStream(lexer);
      lexer.tokens = tokens;
      ToolANTLRParser p = new ToolANTLRParser(tokens, this);
      p.setTreeAdaptor(adaptor);
      try {
        ParserRuleReturnScope r = p.grammarSpec();

View Full Code Here

  }


  public RuleAST parseArtificialRule(final Grammar g, String ruleText) {
    ANTLRLexer lexer = new ANTLRLexer(new ANTLRStringStream(ruleText));
    GrammarASTAdaptor adaptor = new GrammarASTAdaptor(lexer.getCharStream());
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    lexer.tokens = tokens;
    ToolANTLRParser p = new ToolANTLRParser(tokens, tool);
    p.setTreeAdaptor(adaptor);
    Token ruleStart = null;
    try {

View Full Code Here

  }


  private Tree getAntlrComputedExpressionTree() throws ParseException {
    CharStream input = new ANTLRStringStream(sourceText);
    JavascriptLexer lexer = new JavascriptLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    JavascriptParser parser = new JavascriptParser(tokens);


    try {
      return parser.expression().tree;

View Full Code Here

            throw new RuntimeException(ex);
        }


        PropertyExpressionLexer lexer = new PropertyExpressionLexer(ais);


        CommonTokenStream tokens = new CommonTokenStream(lexer);


        PropertyExpressionParser parser = new PropertyExpressionParser(tokens);


        try
        {

View Full Code Here

        Class<? extends TokenSource> lexerClass = Class.forName(lexerClassName).asSubclass(TokenSource.class);
        Constructor<? extends TokenSource> lexConstructor = lexerClass.getConstructor(CharStream.class);
        TokenSource lexer = lexConstructor.newInstance(is);
        is.setLine(scriptLine);


        CommonTokenStream tokens = new CommonTokenStream(lexer);


        Class<? extends Parser> parserClass = Class.forName(parserClassName).asSubclass(Parser.class);
        Constructor<? extends Parser> parConstructor = parserClass.getConstructor(TokenStream.class);
        Parser parser = parConstructor.newInstance(tokens);

View Full Code Here

    }


    private static boolean assembleSmaliFile(File smaliFile, DexBuilder dexBuilder, boolean verboseErrors,
                                             boolean printTokens, boolean allowOdex, int apiLevel)
            throws Exception {
        CommonTokenStream tokens;


        LexerErrorInterface lexer;


        FileInputStream fis = new FileInputStream(smaliFile.getAbsolutePath());
        InputStreamReader reader = new InputStreamReader(fis, "UTF-8");


        lexer = new smaliFlexLexer(reader);
        ((smaliFlexLexer)lexer).setSourceFile(smaliFile);
        tokens = new CommonTokenStream((TokenSource)lexer);


        if (printTokens) {
            tokens.getTokens();
            
            for (int i=0; i<tokens.size(); i++) {
                Token token = tokens.get(i);
                if (token.getChannel() == smaliParser.HIDDEN) {
                    continue;
                }


                System.out.println(smaliParser.tokenNames[token.getType()] + ": " + token.getText());

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.antlr.runtime.CommonTokenStream

aleGrammar.FrontendBeta

com.alvazan.orm.parser.antlr.ScannerSql

com.cloudera.flume.conf.FlumeBuilder

com.cloudera.flume.shell.CommandBuilder

com.dyuproject.protostuff.parser.ProtoUtil

com.espertech.esper.core.deploy.EPLModuleUtil

com.espertech.esper.epl.db.DatabasePollingViewableFactory

com.espertech.esper.event.property.PropertyParser

com.facebook.presto.sql.parser.SqlParser

com.github.sommeri.less4j.core.parser.ANTLRParser

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.