Examples of org.python.pydev.parser.jython.FastCharStream

org.python.pydev.parser.jython.FastCharStream
An implementation of interface CharStream, where the data is read from a Reader. Completely recreated so that we can read data directly from a String, as the initial implementation was highly inefficient when working only with a string (actually, if it was small, there would be no noticeable delays, but if it became big, then the improvement would be HUGE). It keeps the same semantics for line and column stuff (and shares the previous approach of keeping a buffer for this info). If we wanted we could optimize it for also taking less memory, but as there is usually not so many concurrent parses, this is probably not worth it -- and it would probably be a little slower)

            }


            //create a 'synthetic token' in the place we were expecting it.
            if (currentToken != null) {
                AbstractTokenManager tokenManager = grammar.getTokenManager();
                FastCharStream inputStream = tokenManager.getInputStream();


                final int created = tokenManager.addCustom(currentToken, token);
                if (created != AbstractTokenManager.CUSTOM_NOT_CREATED) {
                    if (created == AbstractTokenManager.CUSTOM_CREATED_WAS_PARENS) {
                        //if we had a parens, let's clear the tokens we iterated because we can have skipped indentations!
                        currentToken.next.next = null;


                        //EOF was already found... let's restore the previous indentation level!
                        if (tokenManager.levelBeforeEof != -1) {
                            tokenManager.level = tokenManager.levelBeforeEof;
                            tokenManager.levelBeforeEof = -1; //mark it as not found again.
                        }
                        inputStream.restoreLineColPos(currentToken.endLine, currentToken.endColumn);
                    }
                    grammar.addAndReport(e, "Created custom token: " + token);
                    return new SpecialStr(token, currentToken.beginLine, currentToken.beginColumn);
                }
            }

View Full Code Here

     * Actually creates the grammar.
     * @param generateTree whether we should generate the AST or not.
     */
    private static IGrammar createGrammar(boolean generateTree, int grammarVersion, char[] charArray) {
        IGrammar grammar;
        FastCharStream in = new FastCharStream(charArray);
        switch (grammarVersion) {
            case IPythonNature.GRAMMAR_PYTHON_VERSION_2_4:
                grammar = new PythonGrammar24(generateTree, in);
                break;
            case IPythonNature.GRAMMAR_PYTHON_VERSION_2_5:

View Full Code Here

     * Searches for a new line in the input stream. If found, it'll stop right after it, otherwise, the stream will be
     * backed up the number of chars that've been read.
     */
    protected static boolean searchNewLine(ITokenManager tokenManager, boolean breakOnFirstNotWhitespace) {
        boolean foundNewLine = false;
        FastCharStream inputStream = tokenManager.getInputStream();
        int currentPos = inputStream.getCurrentPos();


        try {
            while (true) {
                try {
                    char c = inputStream.readChar();
                    if (c == '\r' || c == '\n') {
                        if (c == '\r') {
                            c = inputStream.readChar();
                            if (c != '\n') {
                                inputStream.backup(1);
                            }
                        }
                        foundNewLine = true;
                        break;
                    }
                    if (breakOnFirstNotWhitespace && !Character.isWhitespace(c)) {
                        break;
                    }
                } catch (IOException e) {
                    break;
                }
            }
        } finally {
            if (!foundNewLine) {
                inputStream.restorePos(currentPos);
            }
        }
        return foundNewLine;
    }

View Full Code Here

        assertFalse(iterator.hasNext()); //break on indent
    }


    private ITokenManager createTokenManager() {


        final FastCharStream stream = new FastCharStream("if True:\n    pass".toCharArray());


        final Integer[] curr = new Integer[] { 0 };


        return new ITokenManager() {

View Full Code Here

TOP

Related Classes of org.python.pydev.parser.jython.FastCharStream

org.python.pydev.parser.grammarcommon.AbstractGrammarWalkHelpers

org.python.pydev.parser.grammarcommon.DefaultPythonGrammarActions

org.python.pydev.parser.grammarcommon.TokensIteratorTest

org.python.pydev.parser.PyParser

java.io.IOException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.