Examples of org.antlr.v4.runtime.Token

org.antlr.v4.runtime.Token
A token has properties: text, type, line, character position in the line (so we can ignore tabs), token channel, index, and source from which we obtained this token.

        }
        if (ctx.onSelectExpr() != null && ctx.onSelectExpr().onExprFrom() != null) {
            return getOnExprWindowName(ctx.onSelectExpr().onExprFrom());
        }
        if (ctx.onUpdateExpr() != null) {
            Token alias = ctx.onUpdateExpr().i;
            return new UniformPair<String>(ctx.onUpdateExpr().n.getText(), alias != null ? alias.getText() : null);
        }
        return null;
    }

View Full Code Here

        boolean isModule = false;
        boolean isUses = false;
        boolean isExpression = false;


        while (beginIndex < tokens.size()) {
            Token t = (Token) tokens.get(beginIndex);
            if (t.getType() == EsperEPL2GrammarParser.EOF) {
                break;
            }
            if ((t.getType() == EsperEPL2GrammarParser.WS) ||
                (t.getType() == EsperEPL2GrammarParser.SL_COMMENT) ||
                (t.getType() == EsperEPL2GrammarParser.ML_COMMENT)) {
                beginIndex++;
                continue;
            }
            String tokenText = t.getText().trim().toLowerCase();
            if (tokenText.equals("module")) {
                isModule = true; isMeta = true;
            }
            else if (tokenText.equals("uses")) {
                isUses = true; isMeta = true;
            }
            else if (tokenText.equals("import")) {
                isMeta = true;
            }
            else {
                isExpression = true;
                break;
            }
            beginIndex++;
            beginIndex++;   // skip space
            break;
        }


        if (isExpression) {
            return new ParseNodeExpression(item);
        }
        if (!isMeta) {
            return new ParseNodeComment(item);
        }


        // check meta tag (module, uses, import)
        StringWriter buffer = new StringWriter();
        for (int i = beginIndex; i < tokens.size(); i++)
        {
            Token t = (Token) tokens.get(i);
            if (t.getType() == EsperEPL2GrammarParser.EOF) {
                break;
            }
            if ((t.getType() != EsperEPL2GrammarParser.IDENT) &&
                (t.getType() != EsperEPL2GrammarParser.DOT) && 
                (t.getType() != EsperEPL2GrammarParser.STAR) &&
                (!t.getText().matches("[a-zA-Z]*"))) {
                throw getMessage(isModule, isUses, resourceName, t.getType());
            }
            buffer.append(t.getText().trim());
        }


        String result = buffer.toString().trim();
        if (result.length() == 0) {
            throw getMessage(isModule, isUses, resourceName, -1);

View Full Code Here

        Set<Integer> skippedSemicolonIndexes = getSkippedSemicolons(tokenList);
        int index = -1;
        for (Object token : tokenList) // Call getTokens first before invoking tokens.size! ANTLR problem
        {
            index++;
            Token t = (Token) token;
            boolean semi = t.getType() == EsperEPL2GrammarLexer.SEMI && !skippedSemicolonIndexes.contains(index);
            if (semi) {
                if (current.toString().trim().length() > 0) {
                    statements.add(new EPLModuleParseItem(current.toString().trim(), lineNum == null ? 0 : lineNum, charPosStart, charPos));
                    lineNum = null;
                }
                current = new StringWriter();
            }
            else {
                if ((lineNum == null) && (t.getType() != EsperEPL2GrammarParser.WS)) {
                    lineNum = t.getLine();
                    charPosStart = charPos;
                }
                if (t.getType() != EsperEPL2GrammarLexer.EOF) {
                    current.append(t.getText());
                    charPos += t.getText().length();
                }
            }
        }


        if (current.toString().trim().length() > 0) {

View Full Code Here


        int index = -1;
        for (Object token : tokens)
        {
            index++;
            Token t = (Token) token;
            if (t.getType() == EsperEPL2GrammarParser.EXPRESSIONDECL) {
                if (result == null) {
                    result = new HashSet<Integer>();
                }
                getSkippedSemicolonsBetweenSquareBrackets(index, tokens, result);
            }

View Full Code Here

            return;
        }


        int current = indexFirstSquare;
        while(current < indexCloseSquare) {
            Token t = tokens.get(current);
            if (t.getType() == EsperEPL2GrammarParser.SEMI) {
                result.add(current);
            }
            current++;
        }
    }

View Full Code Here


    private static int findEndSquareBrackets(int startIndex, List<Token> tokens) {
        int index = startIndex + 1;
        int squareBracketDepth = 0;
        while(index < tokens.size()) {
            Token t = tokens.get(index);
            if (t.getType() == EsperEPL2GrammarParser.RBRACK) {
                if (squareBracketDepth == 0) {
                    return index;
                }
                squareBracketDepth--;
            }
            if (t.getType() == EsperEPL2GrammarParser.LBRACK) {
                squareBracketDepth++;
            }
            index++;
        }
        return -1;

View Full Code Here

    }


    private static int indexFirstToken(int startIndex, List<Token> tokens, int tokenType) {
        int index = startIndex;
        while(index < tokens.size()) {
            Token t = tokens.get(index);
            if (t.getType() == tokenType) {
                return index;
            }
            index++;
        }
        return -1;

View Full Code Here

        }


        StringWriter writer = new StringWriter();
        for (Object token : tokens.getTokens()) // Call getTokens first before invoking tokens.size! ANTLR problem
        {
            Token t = (Token) token;
            if (t.getType() == EsperEPL2GrammarLexer.EOF) {
                break;
            }
            boolean isKeyword = keywordCache.contains(t.getText().toLowerCase());
            if (isKeyword) {
                writer.append('`');
                writer.append(t.getText());
                writer.append('`');
            }
            else {
                writer.append(t.getText());
            }
        }
        return writer.toString();
    }

View Full Code Here

  }


  @Override
  protected void reportMissingToken(@NotNull Parser recognizer) {
    beginErrorCondition(recognizer);
    Token t = recognizer.getCurrentToken();
    IntervalSet expecting = getExpectedTokens(recognizer);
    String msg = "missing "
        + expecting.toString(recognizer.getTokenNames()) + " at "
        + getTokenErrorDisplay(t);
    throw new RecognitionException(msg, recognizer,

View Full Code Here

  }


  @Override
  protected void reportUnwantedToken(@NotNull Parser recognizer) {
    beginErrorCondition(recognizer);
    Token t = recognizer.getCurrentToken();
    IntervalSet expecting = getExpectedTokens(recognizer);
    String msg = "unwanted token " + getTokenErrorDisplay(t) + " expected "
        + expecting.toString(recognizer.getTokenNames());
    throw new RecognitionException(msg, recognizer,
        recognizer.getInputStream(), recognizer.getContext());

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.antlr.v4.runtime.Token

com.espertech.esper.core.deploy.EPLModuleUtil

eu.bitfish.jcf.parser.antlr4.error.ExceptionErrorStrategy

jetbrick.template.parser.JetTemplateCodeVisitor

org.antlr.runtime.ANTLRFileStream

org.antlr.runtime.ANTLRStringStream

org.antlr.runtime.CharStream

org.antlr.runtime.CommonToken

org.antlr.runtime.Token

org.antlr.runtime.tree.CommonTree

org.antlr.runtime.tree.CommonTreeNodeStream

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.