Package de.fosd.typechef.lexer

Examples of de.fosd.typechef.lexer.Token


    @After
    public void tearDown() throws Exception {
        writer.close();

        Token t;
        do {
            t = p.getNextToken();
            System.out.println("Remaining token " + t);
        } while (t.getType() != EOF);
    }
View Full Code Here


    private void testInput(String in, Object... out) throws Exception {
        System.out.print("Input: " + in);
        writer.write(in);
        writer.flush();
        for (int i = 0; i < out.length; i++) {
            Token t = p.getNextToken();
            System.out.println(t);
            Object v = out[i];
            if (v instanceof String) {
                if (t.getType() != STRING)
                    fail("Expected STRING, but got " + t);
                assertEquals((String) v, (String) t.getValue());
            } else if (v instanceof I) {
                if (t.getType() != IDENTIFIER)
                    fail("Expected IDENTIFIER " + v + ", but got " + t);
                assertEquals(((I) v).getText(), (String) t.getText());
            } else if (v instanceof Character)
                assertEquals((int) ((Character) v).charValue(), t.getType());
            else if (v instanceof Integer)
                assertEquals(((Integer) v).intValue(), t.getType());
            else
                fail("Bad object " + v.getClass());
        }
    }
View Full Code Here

        System.out.println("Testing '" + in + "' => " +
                Arrays.toString(out));
        StringLexerSource s = new StringLexerSource(in);

        for (int i = 0; i < out.length; i++) {
            Token tok = s.token();
            System.out.println("Token is " + tok);
            assertEquals(out[i], tok.getType());
        }
        assertEquals(EOF, s.token().getType());
    }
View Full Code Here

            throws LexerException {
        try {
            MacroData m = new MacroData(null);
            StringLexerSource s = new StringLexerSource(value);
            for (; ; ) {
                Token tok = s.token();
                if (tok.getType() == EOF)
                    break;
                m.addToken(tok);
            }
            addMacro(name, feature, m);
        } catch (IOException e) {
View Full Code Here

            throw new IllegalStateException("Cannot return two tokens");
        this.source_token = tok;
    }

    private Token source_token_nonwhite() throws IOException, LexerException {
        Token tok;
        do {
            tok = retrieveTokenFromSource();
        } while (tok.isWhite());
        return tok;
    }
View Full Code Here

     */
    private Token source_skipline(boolean white) throws IOException,
            LexerException {
        // (new Exception("skipping line")).printStackTrace(System.out);
        Source s = getSource();
        Token tok = s.skipline(white);
        /* XXX Refactor with source_token() */
        if (tok.getType() == EOF && s.isAutopop()) {
            // System.out.println("Autopop " + s);
            sourceManager.pop_source();
            Source t = getSource();
            if (getFeature(Feature.LINEMARKERS) && s.isNumbered() && t != null) {
                /*
 
View Full Code Here

    private List<Argument> parse_macroParameters(String macroName,
                                                 boolean inlineCppExpression, List<Token> originalTokens,
                                                 MacroData firstMacro) throws IOException, LexerException,
            ParseParamException {
        Token tok;
        List<Argument> args;
        // attempt to parse all alternative macros in parallel (when all have
        // the same parameters)
        if (firstMacro.isFunctionLike()) {
            OPEN:
            for (; ; ) {
                tok = retrieveTokenFromSource();
                originalTokens.add(tok);
                // System.out.println("pp: open: token is " + tok);
                switch (tok.getType()) {
                    case WHITESPACE:
                    case CCOMMENT:
                    case CPPCOMMENT:
                    case NL:
                        break; /* continue */
                    // ChK TODO whitespace will be removed in case a bracket is not
                    // found eventually
                    case '(':
                        break OPEN;
                    default:
                        //ChK: do not use source_untoken here, instead push back  originalTokens to the source
                        //push back swallowed whitespace from parsing parameters!
                        sourceManager.push_source(new FixedTokenSource(originalTokens), true);
//                        source_untoken(tok);
//                        originalTokens.remove(originalTokens.size() - 1);
                        return null;
                }
            }

            // tok = expanded_token_nonwhite();
            tok = source_token_nonwhite();
            //Note: tok might be a NL, and we would like to strip
            //that away. It makes a difference if the argument is stringified!
            originalTokens.add(tok);

            args = new ArrayList<Argument>();
            /*
                * We either have, or we should have args. This deals elegantly with
                * the case that we have one empty arg.
                */
            if (tok.getType() != ')' || firstMacro.getArgCount() > 0) {
                List<Token> argTokens = new ArrayList<Token>();
                int depth = 0;
                boolean space = false;

                ARGS:
                for (; ; ) {
                    // System.out.println("pp: arg: token is " + tok);
                    switch (tok.getType()) {
                        case EOF:
                            throw new ParseParamException(tok, "EOF in macro args");

                        case ',':
                            if (depth == 0) {
View Full Code Here

        sourceManager.push_source(new FixedTokenSource(arg), false);

        EXPANSION:
        for (; ; ) {
            Token tok = expanded_token(inlineCppExpression, true);
            switch (tok.getType()) {
                case EOF:
                    break EXPANSION;

                case WHITESPACE:
                case CCOMMENT:
View Full Code Here

        return expansion;
    }

    /* processes a #define directive */
    private Token parse_define() throws IOException, LexerException {
        Token tok = source_token_nonwhite();
        if (tok.getType() != IDENTIFIER) {
            error(tok, "Expected identifier");
            return source_skipline(false);
        }
        /* if predefined */

        String name = tok.getText();
        if ("defined".equals(name)) {
            error(tok, "Cannot redefine name 'defined'");
            return source_skipline(false);
        }

        MacroData m = new MacroData(getSource());
        List<String> args;

        tok = retrieveTokenFromSource();
        if (tok.getType() == '(') {
            tok = source_token_nonwhite();
            boolean seenParamName = false;
            if (tok.getType() != ')') {
                args = new ArrayList<String>();
                ARGS:
                for (; ; ) {
                    switch (tok.getType()) {
                        case IDENTIFIER:
                            args.add(tok.getText());
                            seenParamName = true;
                            break;
                        case ELLIPSIS:
                            assert !seenParamName; // PG: let's check this.
                            if (!seenParamName) {
                                // This trick correctly implements the semantics of
                                // C99 variadic macros as described by the standard
                                // (6.3.10.1).
                                args.add("__VA_ARGS__");
                            }
                            tok = source_token_nonwhite();
                            if (tok.getType() != ')')
                                error(tok, "ellipsis must be on last argument");
                            m.setVariadic(true);
                            break ARGS;
                        case NL:
                        case EOF:
                            error(tok, "Unterminated macro parameter list");
                            return tok;
                        default:
                            error(tok, "error in macro parameters: "
                                    + getTextOrDefault(tok, "<Feature Expression>"));
                            return source_skipline(false);
                    }
                    tok = source_token_nonwhite();
                    switch (tok.getType()) {
                        case ',':
                            seenParamName = false;
                            break;
                        case ELLIPSIS:
                            assert seenParamName; // PG: verify if this is true
                            tok = source_token_nonwhite();
                            if (tok.getType() != ')')
                                error(tok, "ellipsis must be on last argument");
                            if (m.isVariadic()) {
                                error(tok, "ellipsis must not be repeated");
                            }
                            m.setVariadic(true);
                            break ARGS;
                        case ')':
                            break ARGS;

                        case NL:
                        case EOF:
                            /* Do not skip line. */
                            error(tok, "Unterminated macro parameters");
                            return tok;
                        default:
                            error(tok, "Bad token in macro parameters: "
                                    + getTextOrDefault(tok, "<Feature Expression>"));
                            return source_skipline(false);
                    }
                    tok = source_token_nonwhite();
                }
            } else {
                assert tok.getType() == ')' : "Expected ')'";
                args = Collections.emptyList();
            }

            m.setArgs(args);
        } else {
View Full Code Here

        return tok; /* NL or EOF. */
    }

    private Token parse_macroBody(MacroData m, List<String> args)
            throws IOException, LexerException {
        Token tok;
        /* Get an expansion for the macro, using indexOf. */
        boolean space = false;
        boolean paste = false;
        int idx;

        /* Ensure no space at start. */
        tok = source_token_nonwhite();
        EXPANSION:
        for (; ; ) {
            switch (tok.getType()) {
                case EOF:
                    break EXPANSION;
                case NL:
                    break EXPANSION;

                case CCOMMENT:
                case CPPCOMMENT:
                    /* XXX This is where we implement GNU's cpp -CC. */
                    // break;
                case WHITESPACE:
                    if (!paste)
                        space = true;
                    break;

                /* Paste. */
                case PASTE:
                    space = false;
                    paste = true;
                    try {
                        m.addPaste(new SimpleToken(M_PASTE, tok.getLine(), tok
                                .getColumn(), "#" + "#", null));
                    } catch (LexerException le) {
                        error(tok, le.getMessage());
                    }
                    break;

                /* Stringify. */
                case '#':
                    if (space)
                        m.addToken(Token.space);
                    space = false;
                    Token la = source_token_nonwhite();
                    if (la.getType() == IDENTIFIER
                            && ((idx = args.indexOf(la.getText())) != -1)) {
                        m.addToken(new SimpleToken(M_STRING, la.getLine(), la
                                .getColumn(), "#" + la.getText(), Integer
                                .valueOf(idx), null));
                    } else {
                        m.addToken(tok);
                        /* Allow for special processing. */
                        source_untoken(la);
View Full Code Here

TOP

Related Classes of de.fosd.typechef.lexer.Token

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.