Examples of org.antlr.runtime.TokenSource

org.antlr.runtime.TokenSource
A source of tokens must provide a sequence of tokens via nextToken() and also must reveal it's source of characters; CommonToken's text is computed from a CharStream; it only store indices into the char stream. Errors from the lexer are never passed to the parser. Either you want to keep going or you do not upon token recognition error. If you do not want to continue lexing then you do not want to continue parsing. Just throw an exception not under RecognitionException and Java will naturally toss you all the way out of the recognizers. If you want to continue lexing then you should not throw an exception to the parser--it has already requested a token. Keep lexing until you get a valid one. Just report errors and keep going, looking for a valid token.

    public abstract void onTextPhrase(String phrase);


    // convenience method because everybody needs this piece of code
    public static CmisQueryWalker getWalker(String statement) throws UnsupportedEncodingException, IOException, RecognitionException {
        CharStream input = new ANTLRInputStream(new ByteArrayInputStream(statement.getBytes("UTF-8")));
        TokenSource lexer = new CmisQlStrictLexer(input);
        TokenStream tokens = new CommonTokenStream(lexer);
        CmisQlStrictParser parser = new CmisQlStrictParser(tokens);
        CommonTree parserTree; // the ANTLR tree after parsing phase


        query_return parsedStatement = parser.query();

View Full Code Here

  throws Exception
  {
    ANTLRStringStream is = new ANTLRStringStream(input);
    Class<? extends TokenSource> lexerClass = Class.forName(lexerClassName).asSubclass(TokenSource.class);
    Constructor<? extends TokenSource> lexConstructor = lexerClass.getConstructor(CharStream.class);
    TokenSource lexer = lexConstructor.newInstance(is);
    is.setLine(scriptLine);


    CommonTokenStream tokens = new CommonTokenStream(lexer);


    Class<? extends Parser> parserClass = Class.forName(parserClassName).asSubclass(Parser.class);

View Full Code Here

        throws Exception
    {
        ANTLRStringStream is = new ANTLRStringStream(input);
        Class<? extends TokenSource> lexerClass = Class.forName(lexerClassName).asSubclass(TokenSource.class);
        Constructor<? extends TokenSource> lexConstructor = lexerClass.getConstructor(CharStream.class);
        TokenSource lexer = lexConstructor.newInstance(is);
        is.setLine(scriptLine);


        CommonTokenStream tokens = new CommonTokenStream(lexer);


        Class<? extends Parser> parserClass = Class.forName(parserClassName).asSubclass(Parser.class);

View Full Code Here

  @Inject
  private VanbGrammarAccess grammarAccess;
  
  @Override
  protected IParseResult parse(String ruleName, CharStream in) {
    TokenSource tokenSource = createLexer(in);
    XtextTokenStream tokenStream = createTokenStream(tokenSource);
    tokenStream.setInitialHiddenTokens("RULE_WS", "RULE_ML_COMMENT", "RULE_SL_COMMENT");
    org.parser.antlr.internal.InternalVanbParser parser = createParser(tokenStream);
    parser.setTokenTypeMap(getTokenDefProvider().getTokenDefMap());
    parser.setSyntaxErrorProvider(getSyntaxErrorProvider());

View Full Code Here

  @Inject
  private IDLGrammarAccess grammarAccess;
  
  @Override
  protected IParseResult parse(String ruleName, CharStream in) {
    TokenSource tokenSource = createLexer(in);
    XtextTokenStream tokenStream = createTokenStream(tokenSource);
    tokenStream.setInitialHiddenTokens("RULE_WS", "RULE_ML_COMMENT", "RULE_SL_COMMENT");
    org.csu.idl.xtext.parser.antlr.internal.InternalIDLParser parser = createParser(tokenStream);
    parser.setTokenTypeMap(getTokenDefProvider().getTokenDefMap());
    parser.setSyntaxErrorProvider(getSyntaxErrorProvider());

View Full Code Here

  @Inject
  private KappaGrammarAccess grammarAccess;
  
  @Override
  protected IParseResult parse(String ruleName, CharStream in) {
    TokenSource tokenSource = createLexer(in);
    XtextTokenStream tokenStream = createTokenStream(tokenSource);
    tokenStream.setInitialHiddenTokens("RULE_WS");
    urban.parser.antlr.internal.InternalKappaParser parser = createParser(tokenStream);
    parser.setTokenTypeMap(getTokenDefProvider().getTokenDefMap());
    parser.setSyntaxErrorProvider(getSyntaxErrorProvider());

View Full Code Here

  @Inject
  private UrbanGrammarAccess grammarAccess;
  
  @Override
  protected IParseResult parse(String ruleName, CharStream in) {
    TokenSource tokenSource = createLexer(in);
    XtextTokenStream tokenStream = createTokenStream(tokenSource);
    tokenStream.setInitialHiddenTokens("RULE_WS");
    urban.parser.antlr.internal.InternalUrbanParser parser = createParser(tokenStream);
    parser.setTokenTypeMap(getTokenDefProvider().getTokenDefMap());
    parser.setSyntaxErrorProvider(getSyntaxErrorProvider());

View Full Code Here

        this(sql, ImmutableSet.of(";"));
    }


    public StatementSplitter(String sql, Set<String> delimiters)
    {
        TokenSource tokens = getLexer(sql, delimiters);
        ImmutableList.Builder<Statement> list = ImmutableList.builder();
        StringBuilder sb = new StringBuilder();
        while (true) {
            Token token = tokens.nextToken();
            if (token.getType() == Token.EOF) {
                break;
            }
            if (token.getType() == StatementLexer.TERMINATOR) {
                String statement = sb.toString().trim();

View Full Code Here

        return partialStatement;
    }


    public static String squeezeStatement(String sql)
    {
        TokenSource tokens = getLexer(sql, ImmutableSet.<String>of());
        StringBuilder sb = new StringBuilder();
        int index = 0;
        while (true) {
            Token token;
            try {
                token = tokens.nextToken();
                index = ((CommonToken) token).getStopIndex() + 1;
            }
            catch (ParsingException e) {
                sb.append(sql.substring(index));
                break;

View Full Code Here

    
    // convenience method because everybody needs this piece of code
    public CmisQueryWalker getWalker(String statement) throws UnsupportedEncodingException, IOException, RecognitionException {
        
        CharStream input = new ANTLRInputStream(new ByteArrayInputStream(statement.getBytes("UTF-8")));
        TokenSource lexer = new CmisQlStrictLexer(input);
        TokenStream tokens = new CommonTokenStream(lexer);
        CmisQlStrictParser parser = new CmisQlStrictParser(tokens);
        CommonTree parserTree; // the ANTLR tree after parsing phase


        query_return parsedStatement = parser.query();

View Full Code Here

0 1 2

TOP

Related Classes of org.antlr.runtime.TokenSource

com.facebook.presto.sql.parser.StatementSplitter

io.crate.sql.parser.StatementSplitter

org.antlr.v4.test.TestASTStructure

org.apache.chemistry.opencmis.inmemory.query.AbstractQueryConditionProcessor

org.apache.chemistry.opencmis.inmemory.query.AbstractQueryTest

org.apache.chemistry.opencmis.server.support.query.AbstractQueryConditionProcessor

org.apache.chemistry.opencmis.server.support.query.QueryUtil

org.apache.chemistry.opencmis.server.support.query.QueryUtilStrict

org.csu.idl.xtext.parser.antlr.IDLParser

org.parser.antlr.VanbParser

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.