Examples of org.antlr.v4.runtime.TokenSource

org.antlr.v4.runtime.TokenSource
A source of tokens must provide a sequence of tokens via {@link #nextToken()}and also must reveal it's source of characters; {@link CommonToken}'s text is computed from a {@link CharStream}; it only store indices into the char stream.
Errors from the lexer are never passed to the parser. Either you want to keep going or you do not upon token recognition error. If you do not want to continue lexing then you do not want to continue parsing. Just throw an exception not under {@link RecognitionException} and Java will naturally tossyou all the way out of the recognizers. If you want to continue lexing then you should not throw an exception to the parser--it has already requested a token. Keep lexing until you get a valid one. Just report errors and keep going, looking for a valid token.

        .openStream();
    ANTLRInputStream input = new ANTLRInputStream(is);
    MsonLexer lexer = new MsonLexer(input);
//    SymbolASTFactory factory = new SymbolASTFactory();
//    lexer.setTokenFactory(factory);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    
    MsonParser parser = new MsonParser(tokens);
//    parser.setTokenFactory(factory);
    parser.addErrorListener(new DiagnosticErrorListener());
    parser.getInterpreter().setPredictionMode(

View Full Code Here

//    lexer.setTokenFactory(factory);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    
    MsonParser parser = new MsonParser(tokens);
//    parser.setTokenFactory(factory);
    parser.addErrorListener(new DiagnosticErrorListener());
    parser.getInterpreter().setPredictionMode(
        PredictionMode.LL_EXACT_AMBIG_DETECTION);
    parser.setBuildParseTree(true);
    ParseTree tree = parser.mson();
    // show tree in text form

View Full Code Here

    ClassUtils cu = new ClassUtils();
    setup(cu);
    registry.clearForType(JavaCompletionTypes.CUSTOM_TYPE);
    registry.clearForType(JavaCompletionTypes.FIELD);
    registry.clearForType(JavaCompletionTypes.NAME);
    Lexer lexer = new JavaLexer(new ANTLRInputStream(txt));
    CommonTokenStream tokens = new CommonTokenStream(lexer);


    // Create a parser that reads from the scanner
    JavaParser parser = new JavaParser(tokens);
    parser.removeErrorListeners();

View Full Code Here

    // Create a parser that reads from the scanner
    JavaParser parser = new JavaParser(tokens);
    parser.removeErrorListeners();
    
    // start parsing at the compilationUnit rule
    ParserRuleContext t = parser.compilationUnit();
    ParseTreeWalker walker = new ParseTreeWalker();
    List<AutocompleteCandidate> q = new ArrayList<AutocompleteCandidate>();
          
    ImportDeclarationCompletion extractor = new ImportDeclarationCompletion(txt,cur,registry,cps,cu);
    NameBuilder extractor2 = new NameBuilder(registry,cu );

View Full Code Here

    isKeywords = false;
  }


  @Override
  public void enterPackage(@NotNull PackageContext ctx) {
    ParserRuleContext prc = ctx.getParent();
    if (!(prc instanceof DomainArrayContext)) {
      return;
    }
    DomainArrayContext dac = (DomainArrayContext) prc;
    DomainHeaderContext dhctx = (DomainHeaderContext) dac.getChild(0);

View Full Code Here

  protected TokenStream createTokenStream(TokenSource src) {
    return new CommonTokenStream(src);
  }


  @Test public void testOffChannel() throws Exception {
        TokenSource lexer = // simulate input " x =34  ;\n"
            new TokenSource() {
                int i = 0;
                WritableToken[] tokens = {
                    new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},
                    new CommonToken(1,"x"),
                    new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},

View Full Code Here

        assertEquals("=", tokens.LT(-3).getText());
        assertEquals("x", tokens.LT(-4).getText());
    }


  @Test public void testFetchOffChannel() throws Exception {
    TokenSource lexer = // simulate input " x =34  ; \n"
                        // token indexes   01234 56789
      new TokenSource() {
        int i = 0;
        WritableToken[] tokens = {
        new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 0
        new CommonToken(1,"x"),                // 1
        new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},  // 2

View Full Code Here

    assertEquals(null, tokens.getHiddenTokensToRight(9));
  }


  @Test
  public void testSingleEOF() throws Exception {
    TokenSource lexer = new TokenSource() {


      @Override
      public Token nextToken() {
        return new CommonToken(Token.EOF);
      }

View Full Code Here

    assertEquals(1, tokens.size());
  }


  @Test(expected = IllegalStateException.class)
  public void testCannotConsumeEOF() throws Exception {
    TokenSource lexer = new TokenSource() {


      @Override
      public Token nextToken() {
        return new CommonToken(Token.EOF);
      }

View Full Code Here


            final Constructor<? extends Lexer> lexerCtor = lexerClass.getConstructor(CharStream.class);
            final Constructor<? extends Parser> parserCtor = parserClass.getConstructor(TokenStream.class);


            // construct initial instances of the lexer and parser to deserialize their ATNs
            TokenSource tokenSource = lexerCtor.newInstance(new ANTLRInputStream(""));
            parserCtor.newInstance(new CommonTokenStream(tokenSource));


            return new ParserFactory() {
        @Override
                public FileParseResult parseFile(CharStream input, int currentPass, int thread) {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.antlr.v4.runtime.TokenSource

org.antlr.runtime.ANTLRFileStream

org.antlr.runtime.ANTLRStringStream

org.antlr.runtime.CharStream

org.antlr.runtime.CommonToken

org.antlr.runtime.tree.CommonTree

org.antlr.runtime.tree.CommonTreeNodeStream

org.antlr.runtime.tree.Tree

org.antlr.runtime.tree.TreeVisitor

org.antlr.runtime.tree.TreeVisitorAction

org.antlr.v4.analysis.LeftRecursionDetector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.