Examples of org.antlr.v4.runtime.Lexer

org.antlr.v4.runtime.ParserRuleContext
A lexer is recognizer that draws input symbols from a character stream. lexer grammars result in a subclass of this object. A Lexer object uses simplified match() and error recovery mechanisms in the interest of speed.

              COMPUTE_CHECKSUM ? String.format(", checksum 0x%8X", checksum.getValue()) : "",
                          (double)(System.nanoTime() - startTime) / 1000000.0);


    if (sharedLexers.length > 0) {
      int index = FILE_GRANULARITY ? 0 : ((NumberedThread)Thread.currentThread()).getThreadNumber();
      Lexer lexer = sharedLexers[index];
      final LexerATNSimulator lexerInterpreter = lexer.getInterpreter();
      final DFA[] modeToDFA = lexerInterpreter.decisionToDFA;
      if (SHOW_DFA_STATE_STATS) {
        int states = 0;
        int configs = 0;
        Set<ATNConfig> uniqueConfigs = new HashSet<ATNConfig>();


        for (int i = 0; i < modeToDFA.length; i++) {
          DFA dfa = modeToDFA[i];
          if (dfa == null) {
            continue;
          }


          states += dfa.states.size();
          for (DFAState state : dfa.states.values()) {
            configs += state.configs.size();
            uniqueConfigs.addAll(state.configs);
          }
        }


        System.out.format("There are %d lexer DFAState instances, %d configs (%d unique).%n", states, configs, uniqueConfigs.size());


        if (DETAILED_DFA_STATE_STATS) {
          System.out.format("\tMode\tStates\tConfigs\tMode%n");
          for (int i = 0; i < modeToDFA.length; i++) {
            DFA dfa = modeToDFA[i];
            if (dfa == null || dfa.states.isEmpty()) {
              continue;
            }


            int modeConfigs = 0;
            for (DFAState state : dfa.states.values()) {
              modeConfigs += state.configs.size();
            }


            String modeName = lexer.getModeNames()[i];
            System.out.format("\t%d\t%d\t%d\t%s%n", dfa.decision, dfa.states.size(), modeConfigs, modeName);
          }
        }
      }
    }

View Full Code Here

            if (listener == null) {
              listener = listenerClass.newInstance();
              sharedListeners[thread] = listener;
            }


            Lexer lexer = sharedLexers[thread];
                        if (REUSE_LEXER && lexer != null) {
                            lexer.setInputStream(input);
                        } else {
              Lexer previousLexer = lexer;
                            lexer = lexerCtor.newInstance(input);
              DFA[] decisionToDFA = (FILE_GRANULARITY || previousLexer == null ? lexer : previousLexer).getInterpreter().decisionToDFA;
              if (!REUSE_LEXER_DFA || (!FILE_GRANULARITY && previousLexer == null)) {
                decisionToDFA = new DFA[decisionToDFA.length];
              }

View Full Code Here


    ANTLRInputStream in = new ANTLRInputStream(new StringReader(input));


    Class<? extends Lexer> c = lexerClass.asSubclass(Lexer.class);
    Constructor<? extends Lexer> ctor = c.getConstructor(CharStream.class);
    Lexer lexer = ctor.newInstance(in);


    Class<? extends Parser> pc = parserClass.asSubclass(Parser.class);
    Constructor<? extends Parser> pctor = pc.getConstructor(TokenStream.class);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    Parser parser = pctor.newInstance(tokens);

View Full Code Here

  public ParseTreePattern getPattern(String grammarName, String pattern, String ruleName)
    throws Exception
  {
    Class<? extends Lexer> lexerClass = loadLexerClassFromTempDir(grammarName + "Lexer");
    Constructor<? extends Lexer> ctor = lexerClass.getConstructor(CharStream.class);
    Lexer lexer = ctor.newInstance((CharStream) null);


    Class<? extends Parser> parserClass = loadParserClassFromTempDir(grammarName + "Parser");
    Constructor<? extends Parser> pctor = parserClass.getConstructor(TokenStream.class);
    Parser parser = pctor.newInstance(new CommonTokenStream(lexer));

View Full Code Here

  public ParseTreePatternMatcher getPatternMatcher(String grammarName)
    throws Exception
  {
    Class<? extends Lexer> lexerClass = loadLexerClassFromTempDir(grammarName + "Lexer");
    Constructor<? extends Lexer> ctor = lexerClass.getConstructor(CharStream.class);
    Lexer lexer = ctor.newInstance((CharStream) null);


    Class<? extends Parser> parserClass = loadParserClassFromTempDir(grammarName + "Parser");
    Constructor<? extends Parser> pctor = parserClass.getConstructor(TokenStream.class);
    Parser parser = pctor.newInstance(new CommonTokenStream(lexer));

View Full Code Here

       */
      final Constructor<?> lexerConstructor = lexerClass.getConstructor(CharStream.class);
      final Constructor<?> parserConstructor = parserClass.getConstructor(TokenStream.class);
      System.out.println("Parsing :" + grammarFile.getAbsolutePath());
      ANTLRFileStream antlrFileStream = new ANTLRFileStream(grammarFile.getAbsolutePath(), "UTF-8");
      Lexer lexer = (Lexer) lexerConstructor.newInstance(antlrFileStream);
      final CommonTokenStream tokens = new CommonTokenStream(lexer);
      if (verbose) {
         tokens.fill();
         for (final Object tok : tokens.getTokens()) {
            System.out.println(tok);

View Full Code Here


  public static void parseFile(String f) {
    try {
      if ( !quiet ) System.err.println(f);
      // Create a scanner that reads from the input stream passed to us
      Lexer lexer = new JavaLexer(new ANTLRFileStream(f));


      CommonTokenStream tokens = new CommonTokenStream(lexer);
//      long start = System.currentTimeMillis();
//      tokens.fill(); // load all and check time
//      long stop = System.currentTimeMillis();

View Full Code Here


  public static void parseFile(String f) {
    try {
      if ( !quiet ) System.err.println(f);
      // Create a scanner that reads from the input stream passed to us
      Lexer lexer = new Java8Lexer(new ANTLRFileStream(f));


      CommonTokenStream tokens = new CommonTokenStream(lexer);
//      long start = System.currentTimeMillis();
//      tokens.fill(); // load all and check time
//      long stop = System.currentTimeMillis();

View Full Code Here

    // Create a parser that reads from the scanner
    JavaParser parser = new JavaParser(tokens);
    parser.removeErrorListeners();
    
    // start parsing at the compilationUnit rule
    ParserRuleContext t = parser.compilationUnit();
    ParseTreeWalker walker = new ParseTreeWalker();
    List<AutocompleteCandidate> q = new ArrayList<AutocompleteCandidate>();
          
    ImportDeclarationCompletion extractor = new ImportDeclarationCompletion(txt,cur,registry,cps,cu);
    NameBuilder extractor2 = new NameBuilder(registry,cu );

View Full Code Here

    isKeywords = false;
  }


  @Override
  public void enterPackage(@NotNull PackageContext ctx) {
    ParserRuleContext prc = ctx.getParent();
    if (!(prc instanceof DomainArrayContext)) {
      return;
    }
    DomainArrayContext dac = (DomainArrayContext) prc;
    DomainHeaderContext dhctx = (DomainHeaderContext) dac.getChild(0);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.antlr.v4.runtime.Lexer

org.antlr.runtime.ANTLRFileStream

org.antlr.runtime.ANTLRStringStream

org.antlr.runtime.CharStream

org.antlr.runtime.CommonToken

org.antlr.runtime.tree.CommonTree

org.antlr.runtime.tree.CommonTreeNodeStream

org.antlr.runtime.tree.Tree

org.antlr.runtime.tree.TreeVisitorAction

org.antlr.v4.analysis.LeftRecursionDetector

org.antlr.v4.analysis.LeftRecursiveRuleAltInfo

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.