Package org.antlr.runtime

Examples of org.antlr.runtime.CommonToken

69
70
71
72
73
74
75
76
77
    public GrammarAST(int type, Token t) {
    this(new CommonToken(t));
    token.setType(type);
  }
    public GrammarAST(int type, Token t, String text) {
    this(new CommonToken(t));
    token.setType(type);
    token.setText(text);
    }
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
    //System.out.println("actions="+chunks);
  }

  public Action(OutputModelFactory factory, StructDecl ctx, String action) {
    super(factory,null);
    ActionAST ast = new ActionAST(new CommonToken(ANTLRParser.ACTION, action));
    RuleFunction rf = factory.getCurrentRuleFunction();
    if ( rf!=null ) { // we can translate
      ast.resolver = rf.rule;
      chunks = ActionTranslator.translateActionChunk(factory, rf, action, ast);
    }
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
         */
        charPositionInLine += action.getToken().getCharPositionInLine() + 1;
      }

      int offset = ((CommonToken)action.getToken()).getStartIndex();
      attr.token = new CommonToken(action.getToken().getInputStream(), ANTLRParser.ID, BaseRecognizer.DEFAULT_TOKEN_CHANNEL, offset + declOffset + start + 1, offset + declOffset + stop);
      attr.token.setLine(line);
      attr.token.setCharPositionInLine(charPositionInLine);
      assert attr.name.equals(attr.token.getText()) : "Attribute text should match the pseudo-token text at this point.";
    }

View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
    /** Make sure even imaginary nodes know the input stream */
    public Object create(int tokenType, String text) {
    GrammarAST t;
    if ( tokenType==ANTLRParser.RULE ) {
      // needed by TreeWizard to make RULE tree
          t = new RuleAST(new CommonToken(tokenType, text));
    }
    else if ( tokenType==ANTLRParser.STRING_LITERAL ) {
      // implicit lexer construction done with wizard; needs this node type
      // whereas grammar ANTLRParser.g can use token option to spec node type
      t = new TerminalAST(new CommonToken(tokenType, text));
    }
    else {
      t = (GrammarAST)super.create(tokenType, text);
    }
        t.token.setInputStream(input);
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
  {
    String template = templateStream.substring(0, templateStream.size() - 1);
    String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
    String fullyQualifiedTemplateName = prefix + templateName;
    CompiledST impl = new Compiler(this).compile(fullyQualifiedTemplateName, template);
    CommonToken nameT = new CommonToken(STLexer.SEMI); // Seems like a hack, best I could come up with.
    nameT.setInputStream(templateStream);
    rawDefineTemplate(fullyQualifiedTemplateName, impl, nameT);
    impl.defineImplicitlyDefinedTemplates(this);
    return impl;
  }
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
147
            TreePath path = treeSelectionEvent.getNewLeadSelectionPath();
            if ( path==null ) return;
            CommonTree node = (CommonTree)treeSelectionEvent.getNewLeadSelectionPath().getLastPathComponent();
            //System.out.println("select AST: "+node);
            CommonToken a = (CommonToken)currentScope.st.impl.tokens.get(node.getTokenStartIndex());
            CommonToken b = (CommonToken)currentScope.st.impl.tokens.get(node.getTokenStopIndex());
            highlight(viewFrame.template, a.getStartIndex(), b.getStopIndex());
          }
          finally {
            updateDepth.decrementAndGet();
          }
        }
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
        CommonTokenStream tokenStream = new CommonTokenStream(lexer);
        tokenStream.fill();
        List tokens = tokenStream.getTokens();

        int expectedTokenIndex = 0;
        CommonToken token;
        for (int i=0; i<tokens.size()-1; i++) {
            token = (CommonToken)tokens.get(i);

            if (discardHiddenTokens && token.getChannel() == smaliParser.HIDDEN) {
                continue;
            }

            if (expectedTokenIndex >= expectedTokens.size()) {
                Assert.fail("Too many tokens");
            }

            if (token.getType() == smaliParser.INVALID_TOKEN) {
                Assert.assertTrue("Encountered an INVALID_TOKEN not on the error channel",
                        token.getChannel() == smaliParser.ERROR_CHANNEL);
            }

            ExpectedToken expectedToken = expectedTokens.get(expectedTokenIndex++);
            if (!tokenTypesByName.containsKey(expectedToken.tokenName)) {
                Assert.fail("Unknown token: " + expectedToken.tokenName);
            }
            int expectedTokenType = tokenTypesByName.get(expectedToken.tokenName);

            if (token.getType() != expectedTokenType) {
                Assert.fail(String.format("Invalid token at index %d. Expecting %s, got %s(%s)",
                        expectedTokenIndex-1, expectedToken.tokenName, getTokenName(token.getType()), token.getText()));
            }

            if (expectedToken.tokenText != null) {
                if (!expectedToken.tokenText.equals(token.getText())) {
                    Assert.fail(
                            String.format("Invalid token text at index %d. Expecting text \"%s\", got \"%s\"",
                                    expectedTokenIndex - 1, expectedToken.tokenText, token.getText()));
                }
            }
        }

        if (expectedTokenIndex < expectedTokens.size()) {
View Full Code Here
45
46
47
48
49
50
51
    this.token = newToken;
    this.entityNames = entityNames;
  }

  private Token createToken(Token fromToken) {
    return new CommonToken(fromToken);
  }
View Full Code Here
63
64
65
66
67
68
69
   *
   *  If you care what the token payload objects' type is, you should
   *  override this method and any other createToken variant.
   */
  public Token createToken(int tokenType, String text) {
    return new CommonToken(tokenType, text);
  }
View Full Code Here
81
82
83
84
85
86
87
   *
   *  If you care what the token payload objects' type is, you should
   *  override this method and any other createToken variant.
   */
  public Token createToken(Token fromToken) {
    return new CommonToken(fromToken);
  }
View Full Code Here
TOP

Related Classes of org.antlr.runtime.CommonToken

  • com.facebook.presto.sql.parser.StatementSplitter$ErrorHandlingLexer
  • com.github.sommeri.less4j.core.parser.CollectorTokenSource
  • com.github.sommeri.less4j.core.parser.HiddenTokenAwareTree
  • com.github.sommeri.less4j.core.parser.ListToTreeCombiner
  • com.github.sommeri.less4j.core.parser.ParsersSemanticPredicates
  • com.github.sommeri.less4j.grammar.CountNodesAction
  • com.redhat.ceylon.compiler.typechecker.analyzer.LiteralVisitor
  • csep.parser.FirstCommentIncludingXtextTokenStream
  • csep.parser.Lexer
  • imp.utils.ContentProposer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.