Examples of org.antlr.v4.runtime.CommonTokenStream

org.antlr.v4.runtime.CommonTokenStream
This class extends {@link BufferedTokenStream} with functionality to filtertoken streams to tokens on a particular channel (tokens where {@link Token#getChannel} returns a particular value).
This token stream provides access to all tokens by index or when calling methods like {@link #getText}. The channel filtering is only used for code accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and {@link #LB}.

By default, tokens are placed on the default channel ( {@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the {@code ->channel(HIDDEN)} lexer command, or by using an embedded action tocall {@link Lexer#setChannel}.

Note: lexer rules which use the {@code ->skip} lexer command or call{@link Lexer#skip} do not produce tokens at all, so input text matched bysuch a rule will not be available as part of the token stream, regardless of channel.

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.insertBefore(2, "y");
    tokens.delete(2);
    String result = tokens.getText();
    String expecting = "aby";

View Full Code Here

                       "lexer grammar T;\n"+
                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream("abc"));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.insertBefore(0, "0");
    String result = tokens.getText();
    String expecting = "0abc";
    assertEquals(expecting, result);

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.insertAfter(2, "x");
    String result = tokens.getText();
    String expecting = "abcx";
    assertEquals(expecting, result);

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.insertBefore(1, "x");
    tokens.insertAfter(1, "x");
    String result = tokens.getText();
    String expecting = "axbxc";

View Full Code Here

        // Input:  x = 3 * 0 + 2 * 0;
    TestingUnbufferedCharStream input = createStream("x = 302 * 91 + 20234234 * 0;");
        LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    // copy text into tokens from char stream
    lexEngine.setTokenFactory(new CommonTokenFactory(true));
    CommonTokenStream tokens = new CommonTokenStream(lexEngine);
        String result = tokens.LT(1).getText();
        String expecting = "x";
        assertEquals(expecting, result);
    tokens.fill();
    expecting =
      "[[@0,0:0='x',<1>,1:0], [@1,1:1=' ',<7>,1:1], [@2,2:2='=',<4>,1:2]," +
      " [@3,3:3=' ',<7>,1:3], [@4,4:6='302',<2>,1:4], [@5,7:7=' ',<7>,1:7]," +
      " [@6,8:8='*',<6>,1:8], [@7,9:9=' ',<7>,1:9], [@8,10:11='91',<2>,1:10]," +
      " [@9,12:12=' ',<7>,1:12], [@10,13:13='+',<5>,1:13], [@11,14:14=' ',<7>,1:14]," +
      " [@12,15:22='20234234',<2>,1:15], [@13,23:23=' ',<7>,1:23]," +
      " [@14,24:24='*',<6>,1:24], [@15,25:25=' ',<7>,1:25], [@16,26:26='0',<2>,1:26]," +
      " [@17,27:27=';',<3>,1:27], [@18,28:27='',<-1>,1:28]]";
    assertEquals(expecting, tokens.getTokens().toString());
    }

View Full Code Here

   * {@link ParseTreePattern} using this method.
   */
  public ParseTreePattern compile(String pattern, int patternRuleIndex) {
    List<? extends Token> tokenList = tokenize(pattern);
    ListTokenSource tokenSrc = new ListTokenSource(tokenList);
    CommonTokenStream tokens = new CommonTokenStream(tokenSrc);


    ParserInterpreter parserInterp = new ParserInterpreter(parser.getGrammarFileName(),
                                 parser.getVocabulary(),
                                 Arrays.asList(parser.getRuleNames()),
                                 parser.getATNWithBypassAlts(),
                                 tokens);


    ParseTree tree = null;
    try {
      parserInterp.setErrorHandler(new BailErrorStrategy());
      tree = parserInterp.parse(patternRuleIndex);
//      System.out.println("pattern tree = "+tree.toStringTree(parserInterp));
    }
    catch (ParseCancellationException e) {
      throw (RecognitionException)e.getCause();
    }
    catch (RecognitionException re) {
      throw re;
    }
    catch (Exception e) {
      throw new CannotInvokeStartRule(e);
    }


    // Make sure tree pattern compilation checks for a complete parse
    if ( tokens.LA(1)!=Token.EOF ) {
      throw new StartRuleDoesNotConsumeFullPattern();
    }


    return new ParseTreePattern(this, pattern, patternRuleIndex, tree);
  }

View Full Code Here

    XPathLexer lexer = new XPathLexer(in) {
      public void recover(LexerNoViableAltException e) { throw e;  }
    };
    lexer.removeErrorListeners();
    lexer.addErrorListener(new XPathLexerErrorListener());
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    try {
      tokenStream.fill();
    }
    catch (LexerNoViableAltException e) {
      int pos = lexer.getCharPositionInLine();
      String msg = "Invalid tokens or characters at index "+pos+" in path '"+path+"'";
      throw new IllegalArgumentException(msg, e);
    }


    List<Token> tokens = tokenStream.getTokens();
//    System.out.println("path="+path+"=>"+tokens);
    List<XPathElement> elements = new ArrayList<XPathElement>();
    int n = tokens.size();
    int i=0;
loop:

View Full Code Here

    Constructor<? extends Lexer> ctor = lexerClass.getConstructor(CharStream.class);
    Lexer lexer = ctor.newInstance((CharStream) null);


    Class<? extends Parser> parserClass = loadParserClassFromTempDir(grammarName + "Parser");
    Constructor<? extends Parser> pctor = parserClass.getConstructor(TokenStream.class);
    Parser parser = pctor.newInstance(new CommonTokenStream(lexer));


    return parser.compileParseTreePattern(pattern, parser.getRuleIndex(ruleName));
  }

View Full Code Here

    Constructor<? extends Lexer> ctor = lexerClass.getConstructor(CharStream.class);
    Lexer lexer = ctor.newInstance((CharStream) null);


    Class<? extends Parser> parserClass = loadParserClassFromTempDir(grammarName + "Parser");
    Constructor<? extends Parser> pctor = parserClass.getConstructor(TokenStream.class);
    Parser parser = pctor.newInstance(new CommonTokenStream(lexer));


    return new ParseTreePatternMatcher(lexer, parser);
  }

View Full Code Here

  private static final Log LOG = LogFactory.getLog(HiveConverter.class.getName());
  private HiveParser parser;


  public Expr parse(String sql) {
    HiveLexer lexer = new HiveLexer(new ANTLRNoCaseStringStream(sql));
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    parser = new HiveParser(tokens);
    parser.setBuildParseTree(true);


    HiveParser.StatementContext context;
    try {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.antlr.v4.runtime.CommonTokenStream

com.espertech.esper.core.deploy.EPLModuleUtil

com.espertech.esper.event.property.PropertyParser

org.antlr.runtime.ANTLRStringStream

org.antlr.runtime.CharStream

org.antlr.runtime.CommonToken

org.antlr.runtime.Token

org.antlr.runtime.tree.CommonTree

org.antlr.runtime.tree.CommonTreeNodeStream

org.antlr.runtime.tree.Tree

org.antlr.runtime.tree.TreeVisitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.