Examples of org.antlr.v4.runtime.CommonTokenStream

org.antlr.v4.runtime.CommonTokenStream
This class extends {@link BufferedTokenStream} with functionality to filtertoken streams to tokens on a particular channel (tokens where {@link Token#getChannel} returns a particular value).
This token stream provides access to all tokens by index or when calling methods like {@link #getText}. The channel filtering is only used for code accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and {@link #LB}.

By default, tokens are placed on the default channel ( {@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the {@code ->channel(HIDDEN)} lexer command, or by using an embedded action tocall {@link Lexer#setChannel}.

Note: lexer rules which use the {@code ->skip} lexer command or call{@link Lexer#skip} do not produce tokens at all, so input text matched bysuch a rule will not be available as part of the token stream, regardless of channel.

        int havingIndex = -1;
        int orderByIndex = -1;
        List<Integer> unionIndexes = new ArrayList<Integer>();


        EsperEPL2GrammarLexer lex = ParseHelper.newLexer(input);
        CommonTokenStream tokens = new CommonTokenStream(lex);
        tokens.fill();
        List tokenList = tokens.getTokens();


        for (int i = 0; i < tokenList.size(); i++)
        {
            Token token = (Token) tokenList.get(i);
            if ((token == null) || token.getText() == null)

View Full Code Here


    public static ParseNode getModule(EPLModuleParseItem item, String resourceName) throws ParseException, IOException {
        CharStream input = new NoCaseSensitiveStream(new StringReader(item.getExpression()));


        EsperEPL2GrammarLexer lex = ParseHelper.newLexer(input);
        CommonTokenStream tokenStream = new CommonTokenStream(lex);
        tokenStream.fill();


        List tokens = tokenStream.getTokens();
        int beginIndex = 0;
        boolean isMeta = false;
        boolean isModule = false;
        boolean isUses = false;
        boolean isExpression = false;

View Full Code Here

            log.error("Exception reading model expression: " + ex.getMessage(), ex);
            return null;
        }


        EsperEPL2GrammarLexer lex = ParseHelper.newLexer(input);
        CommonTokenStream tokens = new CommonTokenStream(lex);
        tokens.fill();


        List<EPLModuleParseItem> statements = new ArrayList<EPLModuleParseItem>();
        StringWriter current = new StringWriter();
        Integer lineNum = null;
        int charPosStart = 0;
        int charPos = 0;
        List<Token> tokenList = tokens.getTokens();
        Set<Integer> skippedSemicolonIndexes = getSkippedSemicolons(tokenList);
        int index = -1;
        for (Object token : tokenList) // Call getTokens first before invoking tokens.size! ANTLR problem
        {
            index++;

View Full Code Here

        catch (IOException ex) {
            throw new PropertyAccessException("IOException parsing property name '" + propertyName + '\'', ex);
        }


        EsperEPL2GrammarLexer lex = ParseHelper.newLexer(input);
        CommonTokenStream tokens = new CommonTokenStream(lex);
        tokens.fill();


        EsperEPL2GrammarParser g = ParseHelper.newParser(tokens);
        EsperEPL2GrammarParser.StartEventPropertyRuleContext r;


        try {

View Full Code Here

        catch (IOException ex) {
            throw new PropertyAccessException("IOException parsing property name '" + propertyName + '\'', ex);
        }


        EsperEPL2GrammarLexer lexEscaped = ParseHelper.newLexer(inputEscaped);
        CommonTokenStream tokensEscaped = new CommonTokenStream(lexEscaped);
        EsperEPL2GrammarParser gEscaped = ParseHelper.newParser(tokensEscaped);


        try {
            return gEscaped.startEventPropertyRule();
        }

View Full Code Here

   *            The input string to parse.
   */
  private void parse(String input) {
    CharStream charStream = new ANTLRInputStream(input);
    jcfLexer lexer = new jcfLexer(charStream);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    jcfParser parser = new jcfParser(tokens);
    parser.setErrorHandler(new ExceptionErrorStrategy());
    parser.query();
  }

View Full Code Here


  @Override
  protected AbstractExpression doParse(String query) throws ParsingException {
    CharStream charStream = new ANTLRInputStream(query);
    jcfLexer lexer = new jcfLexer(charStream);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    jcfParser parser = new jcfParser(tokens);
    parser.setErrorHandler(new ExceptionErrorStrategy());
    parser.query();


    return parser.root;

View Full Code Here

     */
    private <R extends SqlCallback> R run(R callback)
    {
        //create and prepare lexer, parser and grammar-listener which store the result
        SqlGrammarLexer lexer = new SqlGrammarLexer(script);
        SqlGrammarParser parser = new SqlGrammarParser(new CommonTokenStream(lexer));
        ParserGrammarListener pgl = new ParserGrammarListener(callback, interceptorChain);
        parser.addParseListener(pgl);


        //parsing the SQL step
        parser.sql();

View Full Code Here

      final Constructor<?> lexerConstructor = lexerClass.getConstructor(CharStream.class);
      final Constructor<?> parserConstructor = parserClass.getConstructor(TokenStream.class);
      System.out.println("Parsing :" + grammarFile.getAbsolutePath());
      ANTLRFileStream antlrFileStream = new ANTLRFileStream(grammarFile.getAbsolutePath(), "UTF-8");
      Lexer lexer = (Lexer) lexerConstructor.newInstance(antlrFileStream);
      final CommonTokenStream tokens = new CommonTokenStream(lexer);
      if (verbose) {
         tokens.fill();
         for (final Object tok : tokens.getTokens()) {
            System.out.println(tok);
         }
      }
      /*
       * get parser

View Full Code Here

    try {
      if ( !quiet ) System.err.println(f);
      // Create a scanner that reads from the input stream passed to us
      Lexer lexer = new JavaLexer(new ANTLRFileStream(f));


      CommonTokenStream tokens = new CommonTokenStream(lexer);
//      long start = System.currentTimeMillis();
//      tokens.fill(); // load all and check time
//      long stop = System.currentTimeMillis();
//      lexerTime += stop-start;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.antlr.v4.runtime.CommonTokenStream

com.espertech.esper.core.deploy.EPLModuleUtil

com.espertech.esper.event.property.PropertyParser

org.antlr.runtime.ANTLRStringStream

org.antlr.runtime.CharStream

org.antlr.runtime.CommonToken

org.antlr.runtime.Token

org.antlr.runtime.tree.CommonTree

org.antlr.runtime.tree.CommonTreeNodeStream

org.antlr.runtime.tree.Tree

org.antlr.runtime.tree.TreeVisitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.