Examples of fri.patterns.interpreter.parsergenerator.Lexer

Package fri.patterns.interpreter.parsergenerator

Examples of fri.patterns.interpreter.parsergenerator.Lexer

fri.patterns.interpreter.parsergenerator.Lexer

A Lexer reads input bytes (InputStream) or characters (Reader), until one of its terminals is fulfilled. This happens when the Parser calls getNextToken(). The terminals will be set by the Parser on init.

Usage:

 SyntaxSeparation separation = new SyntaxSeparation(new Syntax(myRules)); LexerBuilder builder = new LexerBuilder(separation.getLexerSyntax(), separation.getIgnoredSymbols()); Lexer lexer = builder.getLexer(inputStream); lexer.setTerminals(separation.getTokenSymbols()); Token token; do	{ token = lexer.getNextToken(null); System.err.println("token.symbol="+token.symbol+", text >"+token.text+"<"); } while (token.symbol != null && Token.isEpsilon(token) == false); boolean error = token.symbol == null;

    
  /** Returns the built Lexer, loaded with passed input (file, stream, string, ...). */
  public Lexer getLexer(Object input)
    throws IOException
  {
    Lexer lexer = getLexer();
    lexer.setInput(input);
    return lexer;
  }

View Full Code Here

    @return deserialized Lexer, or one built from scratch that gets written to filesystem.
  */
  public Lexer get(Object syntaxInput, String baseName, List tokenSymbols, List ignoredSymbols)
    throws Exception
  {
    Lexer lexer = readLexer(syntaxInput, baseName);
    if (lexer == null)
      lexer = buildAndStoreLexer(syntaxInput, baseName, tokenSymbols, ignoredSymbols);
    return lexer;
  }

View Full Code Here

    }
    // else: assume that syntaxInput is a prebuilt lexer Syntax and a list of ignored tokens
    
    TimeStopper ts = new TimeStopper();
    LexerBuilder builder = newLexerBuilder(syntax, ignoredSymbols);
    Lexer lexer = builder.getLexer();
    lexer.setTerminals(tokenSymbols);
    System.err.println("Lexer scratch construction took "+ts.getTimeMillis()+" millis");
    
    if (PRODUCTION)
      write(ensureFileName(syntaxInput, baseName), lexer);

View Full Code Here


  /** Test main. Building serialized Lexer takes 330, building from scratch takes 130 millis. */
  public static void main(String [] args)  {
    try  {
      TimeStopper ts = new TimeStopper();
      Lexer lexer = new SerializedLexer().get(StandardLexerRules.lexerSyntax, "SyntaxBuilder");
      System.err.println("Lexer was built in "+ts.getTimeMillis()+" millis");
    }
    catch (Exception e)  {
      e.printStackTrace();
    }

View Full Code Here

    try  {
      fri.util.TimeStopper ts = new fri.util.TimeStopper();


      SyntaxSeparation separation = new SyntaxSeparation(syntax);
      LexerBuilder builder = new LexerBuilder(separation.getLexerSyntax(), separation.getIgnoredSymbols());
      Lexer lexer = builder.getLexer();
      lexer.setDebug(true);
      lexer.setTerminals(separation.getTokenSymbols());


      System.err.println("time to build lexer was: "+ts.getInterval());


      InputStream in = ExampleLexer.class.getResourceAsStream("ExampleLexer.java");  //Reader in = new FileReader(args[0]);
      lexer.setInput(in);


      System.err.println("time to build input was: "+ts.getInterval());
      
      Token t;
      do  {
        t = lexer.getNextToken(null);
        //System.err.println(ts.getInterval()+" was time to read token "+t.symbol+" -> "+t.text);
        
        if (t.symbol == null)  {
          lexer.dump(System.err);
          throw new LexerException("Uninterpretable input!");
        }
        System.out.println(t.symbol+" "+">"+t.text+"<");
      }
      while (Token.isEpsilon(t) == false);

View Full Code Here

  public static void main(String [] args)
    throws Exception
  {
    SyntaxSeparation separation = new SyntaxSeparation(new Syntax(syntax));  // separate lexer and parser syntax
    LexerBuilder builder = new LexerBuilder(separation.getLexerSyntax(), separation.getIgnoredSymbols());  // build a Lexer
    Lexer lexer = builder.getLexer();
    lexer.setInput("\tHello \r\n\tWorld\n");  // give the lexer some very complex input :-)
    ParserTables parserTables = new SLRParserTables(separation.getParserSyntax());
    Parser parser = new Parser(parserTables);
    parser.parse(lexer, new PrintSemantic());  // start parsing with a print-semantic
  }

View Full Code Here

TOP

Related Classes of fri.patterns.interpreter.parsergenerator.Lexer

fri.patterns.interpreter.parsergenerator.builder.SerializedLexer

fri.patterns.interpreter.parsergenerator.examples.HelloWorldParser

fri.patterns.interpreter.parsergenerator.lexer.examples.ExampleLexer

fri.patterns.interpreter.parsergenerator.lexer.LexerBuilder

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.