Examples of org.antlr.v4.runtime.CommonTokenStream

org.antlr.v4.runtime.CommonTokenStream
This class extends {@link BufferedTokenStream} with functionality to filtertoken streams to tokens on a particular channel (tokens where {@link Token#getChannel} returns a particular value).
This token stream provides access to all tokens by index or when calling methods like {@link #getText}. The channel filtering is only used for code accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and {@link #LB}.

By default, tokens are placed on the default channel ( {@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the {@code ->channel(HIDDEN)} lexer command, or by using an embedded action tocall {@link Lexer#setChannel}.

Note: lexer rules which use the {@code ->skip} lexer command or call{@link Lexer#skip} do not produce tokens at all, so input text matched bysuch a rule will not be available as part of the token stream, regardless of channel.

    ClassUtils cu = new ClassUtils();
    setup(cu);
    registry.clearForType(JavaCompletionTypes.CUSTOM_TYPE);
    registry.clearForType(JavaCompletionTypes.FIELD);
    registry.clearForType(JavaCompletionTypes.NAME);
    Lexer lexer = new JavaLexer(new ANTLRInputStream(txt));
    CommonTokenStream tokens = new CommonTokenStream(lexer);


    // Create a parser that reads from the scanner
    JavaParser parser = new JavaParser(tokens);
    parser.removeErrorListeners();

View Full Code Here


  @Test
  public void Process() throws IOException {
    InputStream is = CheckSymbols.class.getResource("/Mson.js")
        .openStream();
    ANTLRInputStream input = new ANTLRInputStream(is);
    MsonLexer lexer = new MsonLexer(input);
//    SymbolASTFactory factory = new SymbolASTFactory();
//    lexer.setTokenFactory(factory);
    CommonTokenStream tokens = new CommonTokenStream(lexer);

View Full Code Here

    setup(cu);
    registry.clearForType(JavaCompletionTypes.CUSTOM_TYPE);
    registry.clearForType(JavaCompletionTypes.FIELD);
    registry.clearForType(JavaCompletionTypes.NAME);
    Lexer lexer = new JavaLexer(new ANTLRInputStream(txt));
    CommonTokenStream tokens = new CommonTokenStream(lexer);


    // Create a parser that reads from the scanner
    JavaParser parser = new JavaParser(tokens);
    parser.removeErrorListeners();

View Full Code Here

        .openStream();
    ANTLRInputStream input = new ANTLRInputStream(is);
    MsonLexer lexer = new MsonLexer(input);
//    SymbolASTFactory factory = new SymbolASTFactory();
//    lexer.setTokenFactory(factory);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    
    MsonParser parser = new MsonParser(tokens);
//    parser.setTokenFactory(factory);
    parser.addErrorListener(new DiagnosticErrorListener());
    parser.getInterpreter().setPredictionMode(

View Full Code Here


public class TestCommonTokenStream extends TestBufferedTokenStream {


  @Override
  protected TokenStream createTokenStream(TokenSource src) {
    return new CommonTokenStream(src);
  }

View Full Code Here

        public TokenFactory<?> getTokenFactory() {
          return null;
        }
      };


        CommonTokenStream tokens = new CommonTokenStream(lexer);


        assertEquals("x", tokens.LT(1).getText()); // must skip first off channel token
        tokens.consume();
        assertEquals("=", tokens.LT(1).getText());
        assertEquals("x", tokens.LT(-1).getText());


        tokens.consume();
        assertEquals("34", tokens.LT(1).getText());
        assertEquals("=", tokens.LT(-1).getText());


        tokens.consume();
        assertEquals(";", tokens.LT(1).getText());
        assertEquals("34", tokens.LT(-1).getText());


        tokens.consume();
        assertEquals(Token.EOF, tokens.LA(1));
        assertEquals(";", tokens.LT(-1).getText());


        assertEquals("34", tokens.LT(-2).getText());
        assertEquals("=", tokens.LT(-3).getText());
        assertEquals("x", tokens.LT(-4).getText());
    }

View Full Code Here

  void testInterp(LexerGrammar lg, Grammar g,
          String startRule, String input,
          String parseTree)
  {
    LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream tokens = new CommonTokenStream(lexEngine);
    ParserInterpreter parser = g.createParserInterpreter(tokens);
    ParseTree t = parser.parse(g.rules.get(startRule).index);
    System.out.println("parse tree: "+t.toStringTree(parser));
    assertEquals(parseTree, t.toStringTree(parser));
  }

View Full Code Here

        public TokenFactory<?> getTokenFactory() {
          return null;
        }
      };


    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    assertEquals(null, tokens.getHiddenTokensToLeft(0));
    assertEquals(null, tokens.getHiddenTokensToRight(0));


    assertEquals("[[@0,0:0=' ',<1>,channel=1,0:-1]]",
           tokens.getHiddenTokensToLeft(1).toString());
    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]",
           tokens.getHiddenTokensToRight(1).toString());


    assertEquals(null, tokens.getHiddenTokensToLeft(2));
    assertEquals(null, tokens.getHiddenTokensToRight(2));


    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]",
           tokens.getHiddenTokensToLeft(3).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(3));


    assertEquals(null, tokens.getHiddenTokensToLeft(4));
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]",
           tokens.getHiddenTokensToRight(4).toString());


    assertEquals(null, tokens.getHiddenTokensToLeft(5));
    assertEquals("[[@6,0:0=' ',<1>,channel=1,0:-1]]",
           tokens.getHiddenTokensToRight(5).toString());


    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1]]",
           tokens.getHiddenTokensToLeft(6).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(6));


    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]",
           tokens.getHiddenTokensToLeft(7).toString());
    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1], [@9,0:0='\\n',<1>,channel=1,0:-1]]",
           tokens.getHiddenTokensToRight(7).toString());


    assertEquals(null, tokens.getHiddenTokensToLeft(8));
    assertEquals("[[@9,0:0='\\n',<1>,channel=1,0:-1]]",
           tokens.getHiddenTokensToRight(8).toString());


    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1]]",
           tokens.getHiddenTokensToLeft(9).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(9));
  }

View Full Code Here

      public void setTokenFactory(TokenFactory<?> factory) {
        throw new UnsupportedOperationException("Not supported yet.");
      }
    };


    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();


    assertEquals(Token.EOF, tokens.LA(1));
    assertEquals(0, tokens.index());
    assertEquals(1, tokens.size());
  }

View Full Code Here

      public void setTokenFactory(TokenFactory<?> factory) {
        throw new UnsupportedOperationException("Not supported yet.");
      }
    };


    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();


    assertEquals(Token.EOF, tokens.LA(1));
    assertEquals(0, tokens.index());
    assertEquals(1, tokens.size());
    tokens.consume();
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.antlr.v4.runtime.CommonTokenStream

com.espertech.esper.core.deploy.EPLModuleUtil

com.espertech.esper.event.property.PropertyParser

org.antlr.runtime.ANTLRStringStream

org.antlr.runtime.CharStream

org.antlr.runtime.CommonToken

org.antlr.runtime.Token

org.antlr.runtime.tree.CommonTree

org.antlr.runtime.tree.CommonTreeNodeStream

org.antlr.runtime.tree.Tree

org.antlr.runtime.tree.TreeVisitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.