Examples of CommonTokenStream

com.dotcms.repackage.org.antlr.runtime.CommonTokenStream
org.antlr.runtime.CommonTokenStream
The most common stream of tokens is one where every token is buffered up and tokens are prefiltered for a certain channel (the parser will only see these tokens and cannot change the filter channel number during the parse). TODO: how to access the full token stream? How to track all tokens matched per rule?
org.antlr.v4.runtime.CommonTokenStream
This class extends {@link BufferedTokenStream} with functionality to filtertoken streams to tokens on a particular channel (tokens where {@link Token#getChannel} returns a particular value).
This token stream provides access to all tokens by index or when calling methods like {@link #getText}. The channel filtering is only used for code accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and {@link #LB}.

By default, tokens are placed on the default channel ( {@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the {@code ->channel(HIDDEN)} lexer command, or by using an embedded action tocall {@link Lexer#setChannel}.

Note: lexer rules which use the {@code ->skip} lexer command or call{@link Lexer#skip} do not produce tokens at all, so input text matched bysuch a rule will not be available as part of the token stream, regardless of channel.

Examples of org.antlr.runtime.CommonTokenStream


    private CommonTree parse(String sql) throws SqlJetException {
        try {
            CharStream chars = new ANTLRStringStream(sql);
            SqlLexer lexer = new SqlLexer(chars);
            CommonTokenStream tokens = new CommonTokenStream(lexer);
            SqlParser parser = new SqlParser(tokens);
            return (CommonTree) parser.create_table_stmt().getTree();
        } catch (RecognitionException re) {
            throw new SqlJetException(SqlJetErrorCode.ERROR, "Invalid sql statement: " + sql);
        }

View Full Code Here

Examples of org.antlr.runtime.CommonTokenStream

            throw new RuntimeException(ex);
        }


        PropertyExpressionLexer lexer = new PropertyExpressionLexer(ais);


        CommonTokenStream tokens = new CommonTokenStream(lexer);


        PropertyExpressionParser parser = new PropertyExpressionParser(tokens);


        try
        {

View Full Code Here

Examples of org.antlr.runtime.CommonTokenStream


  private RutaModule loadScript(String scriptLocation) throws IOException, RecognitionException {
    File scriptFile = new File(scriptLocation);
    CharStream st = new ANTLRFileStream(scriptLocation, scriptEncoding);
    RutaLexer lexer = new RutaLexer(st);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    RutaParser parser = new RutaParser(tokens);
    parser.setExternalFactory(factory);
    parser.setResourcePaths(resourcePaths);
    String name = scriptFile.getName();
    int lastIndexOf = name.lastIndexOf(SCRIPT_FILE_EXTENSION);

View Full Code Here

Examples of org.antlr.runtime.CommonTokenStream

    if (scriptInputStream == null) {
      throw new FileNotFoundException("No script found in location [" + scriptLocation + "]");
    }
    CharStream st = new ANTLRInputStream(scriptInputStream, scriptEncoding);
    RutaLexer lexer = new RutaLexer(st);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    RutaParser parser = new RutaParser(tokens);
    parser.setExternalFactory(factory);
    parser.setResourcePaths(resourcePaths);
    String name = scriptLocation;
    if (scriptLocation.indexOf("/") != -1) {

View Full Code Here

Examples of org.antlr.runtime.CommonTokenStream

  public ScriptFragment parse(String input) throws Exception {
    CharStream stream = new ANTLRStringStream(input);


    FsharpLexer lexer = new FsharpLexer(stream);


    CommonTokenStream tokens = new CommonTokenStream(lexer);


    FsharpParser parser = new FsharpParser(tokens);


    return parser.scriptFragment();
  }

View Full Code Here

Examples of org.antlr.runtime.CommonTokenStream

        new ANTLRInputStream(new StringBufferInputStream(message));


      CommandLexer commandLexer =
        new CommandLexer(input);


      CommonTokenStream tokens =
        new CommonTokenStream(commandLexer);


      CommandParser commandParser =
        new CommandParser(tokens);


      return commandParser.commands();

View Full Code Here

Examples of org.antlr.runtime.CommonTokenStream

        new ANTLRInputStream(new StringBufferInputStream(pushTrainString));


      PushTrainLexer pushTrainLexer =
        new PushTrainLexer(input);


      CommonTokenStream tokens = new CommonTokenStream(pushTrainLexer);


      PushTrainParser pushTrainParser =
        new PushTrainParser(tokens);


      return pushTrainParser.push_train();

View Full Code Here

Examples of org.antlr.runtime.CommonTokenStream

    public void testRuleParseLhs() throws Exception {
        final String text = "Person(age < 42, location==\"atlanta\") \nor\nPerson(name==\"bob\") \n";
        final AndDescr descrs = new AndDescr();
        final CharStream charStream = new ANTLRStringStream( text );
        final DRLLexer lexer = new DRLLexer( charStream );
        final TokenStream tokenStream = new CommonTokenStream( lexer );
        final DRLParser parser = new DRLParser( tokenStream );
        parser.setLineOffset( descrs.getLine() );
        parser.normal_lhs_block( descrs );
        if ( parser.hasErrors() ) {
            System.err.println( parser.getErrorMessages() );

View Full Code Here

Examples of org.antlr.runtime.CommonTokenStream

    public void testMemberof() throws Exception {
        final String text = "Country( $cities : city )\nPerson( city memberOf $cities )\n";
        final AndDescr descrs = new AndDescr();
        final CharStream charStream = new ANTLRStringStream( text );
        final DRLLexer lexer = new DRLLexer( charStream );
        final TokenStream tokenStream = new CommonTokenStream( lexer );
        final DRLParser parser = new DRLParser( tokenStream );
        parser.setLineOffset( descrs.getLine() );
        parser.normal_lhs_block( descrs );
        if ( parser.hasErrors() ) {
            System.err.println( parser.getErrorMessages() );

View Full Code Here

Examples of org.antlr.runtime.CommonTokenStream

    public void testNotMemberof() throws Exception {
        final String text = "Country( $cities : city )\nPerson( city not memberOf $cities )\n";
        final AndDescr descrs = new AndDescr();
        final CharStream charStream = new ANTLRStringStream( text );
        final DRLLexer lexer = new DRLLexer( charStream );
        final TokenStream tokenStream = new CommonTokenStream( lexer );
        final DRLParser parser = new DRLParser( tokenStream );
        parser.setLineOffset( descrs.getLine() );
        parser.normal_lhs_block( descrs );
        if ( parser.hasErrors() ) {
            System.err.println( parser.getErrorMessages() );

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.