Examples of net.percederberg.grammatica.parser.Token

Package net.percederberg.grammatica.parser

Examples of net.percederberg.grammatica.parser.Token

net.percederberg.grammatica.parser.Token
A token node. This class represents a token (i.e. a set of adjacent characters) in a parse tree. The tokens are created by a tokenizer, that groups characters together into tokens according to a set of token patterns. @author Per Cederberg, @version 1.4


        TokenPattern  pattern;
        String        name;
        int           type;
        String        str;
        Token         token;
        Node          child;


        // Create token pattern
        name = getIdentifier((Token) getChildAt(node, 0));
        child = getChildAt(node, 2);
        type = getIntValue(child, 0);
        str = getStringValue(child, 1);
        pattern = new TokenPattern(nextTokenId++, name, type, str);


        // Process optional ignore or error
        if (node.getChildCount() == 4) {
            child = getChildAt(node, 3);
            token = (Token) getValue(child, 0);
            str = null;
            if (child.getValueCount() == 2) {
                str = getStringValue(child, 1);
            }
            switch (token.getId()) {
            case GrammarConstants.IGNORE:
                if (str == null) {
                    pattern.setIgnore();
                } else {
                    pattern.setIgnore(str);

View Full Code Here

     * @param grammar        the grammar to use
     * @param file           the file to parse
     */
    private static void tokenize(Grammar grammar, File file) {
        Tokenizer  tokenizer;
        Token      token;


        try {
            tokenizer = grammar.createTokenizer(new FileReader(file));
            System.out.println("Tokens from " + file + ":");
            while ((token = tokenizer.next()) != null) {

View Full Code Here

     * @throws RuntimeException if the input file couldn't be tokenized
     *             correctly
     */
    private void tokenize(Grammar grammar) throws RuntimeException {
        Tokenizer  tokenizer;
        Token      token;


        try {
            tokenizer = grammar.createTokenizer(new FileReader(file));
            if (!quiet) {
                System.out.println("Tokens from " + file + ":");

View Full Code Here

     *         null if no comments were found
     */
    static String getComments(Node node) {
        String  comment = "";
        String  str;
        Token   token;


        str = getCommentsBefore(node);
        if (str != null) {
            comment = str;
        }
        str = getCommentsInside(node);
        if (str != null) {
            if (comment.length() > 0) {
                comment += "\n\n";
            }
            comment += str;
        }
        token = getCommentTokenSameLine(node);
        if (token != null) {
            if (comment.length() > 0) {
                comment += "\n\n";
            }
            token = token.getPreviousToken();
            comment += getCommentsAfter(token);
        }
        return comment.length() <= 0 ? null : comment;
    }

View Full Code Here

     *
     * @return the comment string, or
     *         null if no comments were found
     */
    private static String getCommentsBefore(Node node) {
        Token         token = getFirstToken(node);
        ArrayList     comments = new ArrayList();
        StringBuffer  buffer = new StringBuffer();
        String        res = "";


        if (token != null) {
            token = token.getPreviousToken();
        }
        while (token != null) {
            if (token.getId() == Asn1Constants.WHITESPACE) {
                comments.add(getLineBreaks(token.getImage()));
            } else if (token.getId() == Asn1Constants.COMMENT &&
                       !commentTokens.containsKey(token)) {


                commentTokens.put(token, null);
                comments.add(token.getImage().substring(2).trim());
            } else {
                break;
            }
            token = token.getPreviousToken();
        }
        for (int i = comments.size() - 1; i >= 0; i--) {
            buffer.append(comments.get(i));
        }
        res = buffer.toString().trim();

View Full Code Here

     *
     * @return the comment string, or
     *         null if no comments were found
     */
    private static String getCommentsAfter(Node node) {
        Token         token = getLastToken(node);
        StringBuffer  comment = new StringBuffer();
        String        res;


        if (token != null) {
            token = token.getNextToken();
        }
        while (token != null) {
            if (token.getId() == Asn1Constants.WHITESPACE) {
                comment.append(getLineBreaks(token.getImage()));
            } else if (token.getId() == Asn1Constants.COMMENT &&
                       !commentTokens.containsKey(token)) {


                commentTokens.put(token, null);
                comment.append(token.getImage().substring(2).trim());
            } else {
                break;
            }
            token = token.getNextToken();
        }
        res = comment.toString().trim();
        return res.length() <= 0 ? null : res;
    }

View Full Code Here

     *
     * @return the comment string, or
     *         null if no comments were found
     */
    private static String getCommentsInside(Node node) {
        Token         token = getFirstToken(node);
        Token         last = getLastToken(node);
        StringBuffer  comment = new StringBuffer();
        String        res;


        while (token != null && token != last) {
            if (token.getId() == Asn1Constants.COMMENT &&

View Full Code Here

     * @param node           the production node
     *
     * @return the first comment token on the same line
     */
    private static Token getCommentTokenSameLine(Node node) {
        Token  last = getLastToken(node);
        Token  token;


        if (last == null) {
            return null;
        }
        token = last.getNextToken();
        while (token != null) {
            switch (token.getId()) {
            case Asn1Constants.WHITESPACE:
            case Asn1Constants.COMMA:
                // Skip to next
                break;
            case Asn1Constants.COMMENT:
                if (last.getEndLine() == token.getStartLine()) {
                    return token;
                } else {
                    return null;
                }
            default:
                return null;
            }
            token = token.getNextToken();
        }
        return null;
    }

View Full Code Here

TOP

Related Classes of net.percederberg.grammatica.parser.Token

net.percederberg.grammatica.ant.ValidationElement

net.percederberg.grammatica.FirstPassAnalyzer

net.percederberg.grammatica.Grammatica

net.percederberg.mibble.MibAnalyzerUtil

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.