Examples of org.jruby.lexer.yacc.LexerSource

org.jruby.lexer.yacc.LexerSource
This class is what feeds the lexer. It is primarily a wrapper around a Reader that can unread() data back onto the source. Originally, I thought about using the PushBackReader to handle read/unread, but I realized that some extremely pathological case could overflow the pushback buffer. Better safe than sorry. I could have combined this implementation with a PushbackBuffer, but the added complexity did not seem worth it.

      return;
    }


    List<String> lines = new ArrayList<String>(canvas.getLineCount());


    LexerSource lexerSource = new ByteArrayLexerSource(title, utf8Script, lines, 0, true);


    lexer.reset();
    lexer.setSource(lexerSource);
    lexer.setState(RubyYaccLexer.LexState.EXPR_BEG);


    // remember bounds of current token
    int leftTokenBorder = 0;
    int rightTokenBorder = 0;
    int t = 0;
    int prevt = 0;
    int lastCommentEnd = 0;


    ArrayList<StyleRange> ranges = new ArrayList<StyleRange>(200);
    ArrayList<Integer> intRanges = new ArrayList<Integer>(400);


    try {
      
      boolean keepParsing = true;
      
      while (keepParsing) {
        
        /* take care of comments, which are stripped out by the lexer */
        int[] upcomingComment = null;
        while ((rightTokenBorder >= lastCommentEnd || rightTokenBorder == 0 ) && (upcomingComment = getUpcomingCommentPos(utf8Script, rightTokenBorder)) != null){
          leftTokenBorder = upcomingComment[0];
          rightTokenBorder = leftTokenBorder + upcomingComment[1];
          lastCommentEnd = rightTokenBorder;
          //System.out.println("Found comment -> [" + leftTokenBorder + "," + rightTokenBorder + "]");
          ranges.add(tokenToStyleRange(TOKEN_COMMENT, null, prevt));


          int left = leftTokenBorder - encodingBytes[leftTokenBorder];
          int right = rightTokenBorder-encodingBytes[rightTokenBorder]- left;
          
          intRanges.add(left);
          intRanges.add(right);
        }
        
        /* read language syntax */
        int oldOffset = lexerSource.getOffset();
        keepParsing = lexer.advance();
        prevt = t;
        t = lexer.token();
        Object v = lexer.value();


        leftTokenBorder = oldOffset;
        if (leftTokenBorder < lastCommentEnd && lexerSource.getOffset() > lastCommentEnd){
          leftTokenBorder = lastCommentEnd;
        }
        rightTokenBorder = lexerSource.getOffset();        
        
        //System.out.println("Found token " + t + " -> " + lexer.value() + " [" + leftTokenBorder + "," + rightTokenBorder + "]");


        // skip whitespace and error formatting
        if (t != '\n' && t != -1){

View Full Code Here

    public Node parseRewriter(String file, InputStream content, 
            ParserConfiguration configuration) throws SyntaxException {
        long startTime = System.nanoTime();
        DefaultRubyParser parser = RubyParserPool.getInstance().borrowParser();
        parser.setWarnings(new NullWarnings());
        LexerSource lexerSource = LexerSource.getSource(file, content, null, configuration);
        
        try {
            return parser.parse(configuration, lexerSource).getAST();
        } finally {
            RubyParserPool.getInstance().returnParser(parser);
            totalTime += System.nanoTime() - startTime;
            totalBytes += lexerSource.getOffset();
        }
    }

View Full Code Here

        
        DefaultRubyParser parser = null;
        RubyParserResult result = null;
        parser = RubyParserPool.getInstance().borrowParser();
        parser.setWarnings(runtime.getWarnings());
        LexerSource lexerSource = LexerSource.getSource(file, content, list, configuration);
        try {
            result = parser.parse(configuration, lexerSource);
            if (result.getEndOffset() >= 0) {
                IRubyObject verbose = runtime.getVerbose();
                runtime.setVerbose(runtime.getNil());
              runtime.defineGlobalConstant("DATA", new RubyFile(runtime, file, content));
                runtime.setVerbose(verbose);
              result.setEndOffset(-1);
            }
        } catch (SyntaxException e) {
            StringBuilder buffer = new StringBuilder(100);
            buffer.append(e.getPosition().getFile()).append(':');
            buffer.append(e.getPosition().getEndLine() + 1).append(": ");
            buffer.append(e.getMessage());
            throw runtime.newSyntaxError(buffer.toString());
        } finally {
            RubyParserPool.getInstance().returnParser(parser);
        }
        
        // If variables were added then we may need to grow the dynamic scope to match the static
        // one.
        // FIXME: Make this so we only need to check this for blockScope != null.  We cannot
        // currently since we create the DynamicScope for a LocalStaticScope before parse begins.
        // Refactoring should make this fixable.
        if (result.getScope() != null) {
            result.getScope().growIfNeeded();
        }


        Node ast = result.getAST();
        
        totalTime += System.nanoTime() - startTime;
        totalBytes += lexerSource.getOffset();


        return ast;
    }

View Full Code Here


    @SuppressWarnings("unchecked")
    public Node parse(String file, byte[] content, DynamicScope blockScope,
            ParserConfiguration configuration) {
        RubyArray list = getLines(configuration, runtime, file);
        LexerSource lexerSource = LexerSource.getSource(file, content, list, configuration);
        return parse(file, lexerSource, blockScope, configuration);
    }

View Full Code Here

            ParserConfiguration configuration) {
        RubyArray list = getLines(configuration, runtime, file);
        if (content instanceof LoadServiceResourceInputStream) {
            return parse(file, ((LoadServiceResourceInputStream) content).getBytes(), blockScope, configuration);
        } else {
            LexerSource lexerSource = LexerSource.getSource(file, content, list, configuration);
            return parse(file, lexerSource, blockScope, configuration);
        }
    }

View Full Code Here


    @SuppressWarnings("unchecked")
    public Node parse(String file, byte[] content, DynamicScope blockScope,
            ParserConfiguration configuration) {
        RubyArray list = getLines(configuration, runtime, file);
        LexerSource lexerSource = LexerSource.getSource(file, content, list, configuration);
        return parse(file, lexerSource, blockScope, configuration);
    }

View Full Code Here

            ParserConfiguration configuration) {
        RubyArray list = getLines(configuration, runtime, file);
        if (content instanceof LoadServiceResourceInputStream) {
            return parse(file, ((LoadServiceResourceInputStream) content).getBytes(), blockScope, configuration);
        } else {
            LexerSource lexerSource = LexerSource.getSource(file, content, list, configuration);
            return parse(file, lexerSource, blockScope, configuration);
        }
    }

View Full Code Here

      setUp();
    
    // Set up input 
    final List<String> lexerCapture = null; // do not want to record the read text lines
    FileInputStream input = new FileInputStream(file);
    LexerSource source = new InputStreamLexerSource(file.getPath(), 
        input, lexerCapture, startLine, extraPositionInfo);


    // Get a parser
    RubyParser parser = RubyParserPool.getInstance().borrowParser(rubyVersion);

View Full Code Here

TOP

Related Classes of org.jruby.lexer.yacc.LexerSource

com.puppetlabs.geppetto.ruby.jruby.JRubyServices

org.jruby.parser.Parser

org.typeexit.kettle.plugin.steps.ruby.RubyStepSyntaxHighlighter

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.