Package de.fosd.typechef.lexer

Examples of de.fosd.typechef.lexer.Token


    }

    protected final Token parse_main() throws IOException, LexerException {

        for (; ; ) {
            Token tok;
            if (!isActive()) {
                try {
                    /* XXX Tell lexer to ignore warnings. */
                    sourceManager.getSource().setActive(false);
                    tok = retrieveTokenFromSource();
                } finally {
                    /* XXX Tell lexer to stop ignoring warnings. */
                    if (sourceManager.getSource() != null)
                        sourceManager.getSource().setActive(true);
                }
                switch (tok.getType()) {
                    case HASH:
                    case NL:
                        /* The preprocessor has to take action here. */
                        break;
                    case EOF:
                    case WHITESPACE:
                        return tok;
                    case CCOMMENT:
                    case CPPCOMMENT:
                        // Patch up to preserve whitespace.
                        if (getFeature(Feature.KEEPALLCOMMENTS))
                            return tok;
                        if (!isActive())
                            return toWhitespace(tok);
                        if (getFeature(Feature.KEEPCOMMENTS))
                            return tok;
                        return toWhitespace(tok);
                    default:
                        // Return NL to preserve whitespace.
                        /* XXX This might lose a comment. */
                        return source_skipline(false);
                }
            } else {
                tok = retrieveTokenFromSource();
            }

            LEX:
            switch (tok.getType()) {
                case EOF:
                    /* Pop the stacks. */
                    return tok;

                case WHITESPACE:
                case NL:
                    return tok;

                case CCOMMENT:
                case CPPCOMMENT:
                    return tok;

                case '!':
                case '%':
                case '&':
                case '(':
                case ')':
                case '*':
                case '+':
                case ',':
                case '-':
                case '/':
                case ':':
                case ';':
                case '<':
                case '=':
                case '>':
                case '?':
                case '[':
                case ']':
                case '^':
                case '{':
                case '|':
                case '}':
                case '~':
                case '.':

                    /* From Olivier Chafik for Objective C? */
                case '@':
                    /* The one remaining ASCII, might as well. */
                case '`':

                    // case '#':

                case AND_EQ:
                case ARROW:
                case CHARACTER:
                case DEC:
                case DIV_EQ:
                case ELLIPSIS:
                case EQ:
                case GE:
                case HEADER: /* Should only arise from include() */
                case INC:
                case LAND:
                case LE:
                case LOR:
                case LSH:
                case LSH_EQ:
                case SUB_EQ:
                case MOD_EQ:
                case MULT_EQ:
                case NE:
                case OR_EQ:
                case PLUS_EQ:
                case RANGE:
                case RSH:
                case RSH_EQ:
                case STRING:
                case XOR_EQ:
                    return tok;

                case INTEGER:
                    return tok;

                case IDENTIFIER:
                    // apply macro (in visible code)
                    MacroExpansion<MacroData>[] m = macros.getApplicableMacroExpansions(tok
                            .getText(), state.getFullPresenceCondition());
                    if (m.length == 0)
                        return tok;
                    if (!sourceManager.getSource().mayExpand(tok.getText())
                            || !tok.mayExpand())
                        return tok;
                    if (macro_expandToken(tok.getText(), m, tok, processing_include))
                        break;
                    return tok;

                case P_LINE:
                    if (getFeature(Feature.LINEMARKERS))
                        return tok;
                    break;

                case INVALID:
                    if (getFeature(Feature.CSYNTAX))
                        error(tok, String.valueOf(tok.getValue()));
                    return tok;

                case HASH:
                    tok = source_token_nonwhite();
                    // (new Exception("here")).printStackTrace();
                    switch (tok.getType()) {
                        case NL:
                            break LEX; /* Some code has #\n */
                        case IDENTIFIER:
                            break;
                        default:
                            //no warning, interpreted as comment
//                            warning(tok, "Preprocessor directive not a word "
//                                    + tok.getText() + ", skipping line");
                            return source_skipline(false);
                    }
                    // System.out.println(previousToken);
                    Integer _ppcmd = ppcmds.get(tok.getText());
                    if (_ppcmd == null) {
                        warning(tok, "Unknown preprocessor directive "
                                + tok.getText() + ", skipping line");
                        return source_skipline(false);
                    }
                    int ppcmd = _ppcmd.intValue();

                    // PP:
                    switch (ppcmd) {

                        case PP_DEFINE:
                            if (!isActive())
                                return source_skipline(false);
                            else
                                return parse_define();
                            // break;

                        case PP_UNDEF:
                            if (!isActive())
                                return source_skipline(false);
                            else
                                return parse_undef();
                            // break;

                        case PP_INCLUDE:
                            if (!isActive())
                                return source_skipline(false);
                            else {
                                Token ret_tok = parse_include(false);
                                assert !processing_include;
                                return ret_tok;
                            }
                            // break;
                        case PP_INCLUDE_NEXT:
                            if (!isActive())
                                return source_skipline(false);
                            if (!getFeature(Feature.INCLUDENEXT)) {
                                error(tok, "Directive include_next not enabled");
                                return source_skipline(false);
                            }
                            Token ret_tok = parse_include(true);
                            assert !processing_include;
                            return ret_tok;
                        // break;

                        /**
                         * error tokens are not processed by the jcpp but left for
                         * the type system
                         */
                        case PP_WARNING:
                        case PP_ERROR:
                            if (!isActive())
                                return source_skipline(false);
                            else
                                // cppError(tok, ppcmd == PP_ERROR);
                                return parseErrorToken(tok, ppcmd == PP_ERROR);

                        case PP_IF:
                            push_state();
                            expr_token = null;
                            if (isParentActive()) {
                                FeatureExpr localFeatureExpr = parse_featureExpr();
                                state.putLocalFeature(localFeatureExpr, macros);
                                tok = expr_token(true); /* unget */
                                if (tok.getType() != NL)
                                    source_skipline(isParentActive());
                            } else {
                                state.putLocalFeature(FeatureExprLib.False(), macros);
                                source_skipline(false);
                            }


                            return ifdefPrinter.startIf(tok, isParentActive(), state);

                        // break;

                        case PP_ELIF:
                            if (state.sawElse()) {
                                error(tok, "#elif after #" + "else");
                                return source_skipline(false);
                            } else {
                                expr_token = null;
                                // parse with parents state to allow macro expansion
                                State oldState = state;
                                state = state.parent;
                                FeatureExpr localFeaturExpr = isActive() ? parse_featureExpr() : FeatureExprLib.False();
                                state = oldState;
                                state.processElIf();
                                state.putLocalFeature(localFeaturExpr, macros);
                                tok = expr_token(true); /* unget */

                                if (tok.getType() != NL)
                                    source_skipline(isParentActive());

                                return ifdefPrinter.startElIf(tok, isParentActive(),
                                        state);

                            }
                            // break;

                        case PP_ELSE:
                            if (state.sawElse()) {
                                error(tok, "#" + "else after #" + "else");
                                return source_skipline(false);
                            } else {
                                state.setSawElse();
                                source_skipline(warnings.contains(Warning.ENDIF_LABELS));

                                return ifdefPrinter.startElIf(tok, isParentActive(),
                                        state);
                            }
                            // break;

                        case PP_IFDEF:
                            push_state();
                            tok = source_token_nonwhite();
                            // System.out.println("ifdef " + tok);
                            if (tok.getType() != IDENTIFIER) {
                                error(tok, "Expected identifier, not " + tok.getText());
                                return source_skipline(false);
                            } else {
                                FeatureExpr localFeatureExpr2 = parse_ifdefExpr(tok
                                        .getText());
                                state.putLocalFeature(
                                        isParentActive() ? localFeatureExpr2
                                                : FeatureExprLib.False(), macros);
                                // return

                                if (tok.getType() != NL)
                                    source_skipline(isParentActive());

                                return ifdefPrinter.startIf(tok, isParentActive(),
                                        state);
                            }
                            // break;

                        case PP_IFNDEF:
                            push_state();
                            tok = source_token_nonwhite();
                            if (tok.getType() != IDENTIFIER) {
                                error(tok, "Expected identifier, not " + tok.getText());
                                return source_skipline(false);
                            } else {
                                FeatureExpr localFeatureExpr3 = parse_ifndefExpr(tok
                                        .getText());
                                state.putLocalFeature(
                                        isParentActive() ? localFeatureExpr3
                                                : FeatureExprLib.False(), macros);
                                if (tok.getType() != NL)
                                    source_skipline(isParentActive());

                                return ifdefPrinter.startIf(tok, isParentActive(),
                                        state);

                            }
                            // break;

                        case PP_ENDIF:
                            pop_state();

                            Token l = source_skipline(warnings
                                    .contains(Warning.ENDIF_LABELS));
                            return ifdefPrinter.endIf(l);
                        // break;

                        case PP_LINE:
View Full Code Here


    private FeatureExpr parse_ifdefExpr(String feature) {
        return FeatureExprLib.l().createDefinedMacro(feature, macros);
    }

    private Token getNextNonwhiteToken() throws IOException, LexerException {
        Token tok;
        do {
            tok = parse_main();
        } while (tok.isWhite());
        return tok;
    }
View Full Code Here

     * @see Token
     */
    public Token getNextToken() throws IOException, LexerException {
//        FeatureExpr lastPC = state.getFullPresenceCondition();
        try {
            Token tok = parse_main();
            tok = tok.clone();
            tok.setFeature(state.getFullPresenceCondition());
            if (getFeature(Feature.DEBUG_VERBOSE))
                System.err.println("pp: Returning " + tok);
            return tok;
        } catch (de.fosd.typechef.featureexpr.FeatureException e) {
//            error(0,0,e.getMessage(),lastPC);
View Full Code Here

        sourceManager.close();
    }

    private Token retrieveTokenFromSource() throws IOException, LexerException {
        if (source_token != null) {
            Token tok = source_token;
            source_token = null;
            if (getFeature(Feature.DEBUG_VERBOSE))
                System.err.println("Returning unget token " + tok);
            return tok;
        }
View Full Code Here

        unread(d);
        return new SimpleToken(no, this);
    }

    public Token token() throws IOException, LexerException {
        Token tok = null;

        int _l = line;
        int _c = column;

        int c = read();
        int d;

        switch (c) {
            case '\n':
                if (ppvalid) {
                    bol = true;
                    if (include) {
                        tok = new SimpleToken(NL, _l, _c, "\n", this);
                    } else {
                        int nls = 0;
                        do {
                            nls++;
                            d = read();
                        } while (d == '\n');
                        unread(d);
                        char[] text = new char[nls];
                        for (int i = 0; i < text.length; i++)
                            text[i] = '\n';
                        // Skip the bol = false below.
                        tok = new SimpleToken(NL, _l, _c, new String(text), this);
                    }
                    if (DEBUG)
                        System.out.println("lx: Returning NL: " + tok);
                    return tok;
                }
                /* Let it be handled as whitespace. */
                break;

            case '!':
                tok = cond('=', NE, '!');
                break;

            case '#':
                if (bol)
                    tok = new SimpleToken(HASH, this);
                else
                    tok = cond('#', PASTE, '#');
                break;

            case '+':
                d = read();
                if (d == '+')
                    tok = new SimpleToken(INC, this);
                else if (d == '=')
                    tok = new SimpleToken(PLUS_EQ, this);
                else
                    unread(d);
                break;
            case '-':
                d = read();
                if (d == '-')
                    tok = new SimpleToken(DEC, this);
                else if (d == '=')
                    tok = new SimpleToken(SUB_EQ, this);
                else if (d == '>')
                    tok = new SimpleToken(ARROW, this);
                else
                    unread(d);
                break;

            case '*':
                tok = cond('=', MULT_EQ, '*');
                break;
            case '/':
                d = read();
                if (d == '*')
                    tok = ccomment();
                else if (d == '/')
                    tok = cppcomment();
                else if (d == '=')
                    tok = new SimpleToken(DIV_EQ, this);
                else
                    unread(d);
                break;

            case '%':
                d = read();
                if (d == '=')
                    tok = new SimpleToken(MOD_EQ, this);
                else if (digraphs && d == '>')
                    tok = new SimpleToken('}', this); // digraph
                else if (digraphs && d == ':')
                    PASTE:{
                        d = read();
                        if (d != '%') {
                            unread(d);
                            tok = new SimpleToken('#', this); // digraph
                            break PASTE;
                        }
                        d = read();
                        if (d != ':') {
                            unread(d); // Unread 2 chars here.
                            unread('%');
                            tok = new SimpleToken('#', this); // digraph
                            break PASTE;
                        }
                        tok = new SimpleToken(PASTE, this); // digraph
                    }
                else
                    unread(d);
                break;

            case ':':
                /* :: */
                d = read();
                if (digraphs && d == '>')
                    tok = new SimpleToken(']', this); // digraph
                else
                    unread(d);
                break;

            case '<':
                if (include) {
                    tok = string(false, '<', '>');
                } else {
                    d = read();
                    if (d == '=')
                        tok = new SimpleToken(LE, this);
                    else if (d == '<')
                        tok = cond('=', LSH_EQ, LSH);
                    else if (digraphs && d == ':')
                        tok = new SimpleToken('[', this); // digraph
                    else if (digraphs && d == '%')
                        tok = new SimpleToken('{', this); // digraph
                    else
                        unread(d);
                }
                break;

            case '=':
                tok = cond('=', EQ, '=');
                break;

            case '>':
                d = read();
                if (d == '=')
                    tok = new SimpleToken(GE, this);
                else if (d == '>')
                    tok = cond('=', RSH_EQ, RSH);
                else
                    unread(d);
                break;

            case '^':
                tok = cond('=', XOR_EQ, '^');
                break;

            case '|':
                d = read();
                if (d == '=')
                    tok = new SimpleToken(OR_EQ, this);
                else if (d == '|')
                    tok = cond('=', LOR_EQ, LOR);
                else
                    unread(d);
                break;
            case '&':
                d = read();
                if (d == '&')
                    tok = cond('=', LAND_EQ, LAND);
                else if (d == '=')
                    tok = new SimpleToken(AND_EQ, this);
                else
                    unread(d);
                break;

            case '.':
                d = read();
                if (d == '.')
                    tok = cond('.', ELLIPSIS, RANGE);
                else if (Character.isDigit(d))
                    tok = number_decimal(d, true);
                else
                    unread(d);
                break;

            case '0':
                /* octal, hex or 0.decimal */
                d = read();
                if (d == 'x' || d == 'X')
                    tok = number_hex((char) d);
                else if (d == '.') {
                    unread(d);
                    tok = number_decimal('0', false);
                } else {
                    unread(d);
                    tok = number_octal();
                }
                break;

            case '\'':
                tok = character(false);
                break;

            case 'L': // WIDE strings
                d = read();
                if (d == '"')
                    tok = string(true, '"', '"');
                else if (d == '\'')
                    tok = character(true);
                else
                    unread(d);
                break;

            case '"':
                tok = string(false, '"', '"');
                break;

            case -1:
                close();
                tok = new SimpleToken(EOF, _l, _c, "<eof>", this);
                break;
        }

        if (tok == null) {
            if (Character.isWhitespace(c)) {
                tok = whitespace(c);
            } else if (Character.isDigit(c)) {
                tok = number_decimal(c, false);
            } else if (Character.isJavaIdentifierStart(c)) {
                tok = identifier(c);
            } else {
                tok = new SimpleToken(c, this);
            }
        }

        if (bol) {
            switch (tok.getType()) {
                case WHITESPACE:
                case CCOMMENT:
                    break;
                default:
                    bol = false;
                    break;
            }
        }

        tok.setLocation(_l, _c);
        if (DEBUG)
            System.out.println("lx: Returning " + tok);
        // (new Exception("here")).printStackTrace(System.out);
        lastTokens.add(tok);
        return tok;
View Full Code Here

     *              line.
     * @return the NL token.
     */
    public Token skipline(boolean white) throws IOException, LexerException {
        for (; ;) {
            Token tok = token();
            switch (tok.getType()) {
                case EOF:
                    /*
                      * There ought to be a newline before EOF. At least, in any
                      * skipline context.
                      */
                    /* XXX Are we sure about this? */
                    // warning(tok.getLine(), tok.getColumn(),
                    // "No newline before end of file");
                    return new SimpleToken(NL, tok.getLine(), tok.getColumn(), "\n", this);
                // return tok;
                case NL:
                    /* This may contain one or more newlines. */
                    return tok;
                case CCOMMENT:
                case CPPCOMMENT:
                case WHITESPACE:
                    break;
                default:
                    /* XXX Check white, if required. */
                    if (white)
                        warning(tok.getLine(), tok.getColumn(),
                                "Unexpected nonwhite token (" + tok.getText()
                                        + ") while skipping line");
                    break;
            }
        }
    }
View Full Code Here

        try {
            assert cpp != null : "cpp is null : was it closed?";
            if (token == null)
                return false;
            while (idx >= token.length()) {
                Token tok = cpp.getNextToken();
                switch (tok.getType()) {
                    case EOF:
                        token = null;
                        return false;
                    case CCOMMENT:
                    case CPPCOMMENT:
                        if (!cpp.getFeature(Feature.KEEPCOMMENTS)) {
                            token = " ";
                            break;
                        }
                    default:
                        token = tok.getText();
                        break;
                }
                idx = 0;
            }
            return true;
View Full Code Here

TOP

Related Classes of de.fosd.typechef.lexer.Token

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.