Examples of net.percederberg.grammatica.parser.Tokenizer$TokenMatcher

Package net.percederberg.grammatica.parser

Examples of net.percederberg.grammatica.parser.Tokenizer$TokenMatcher

net.percederberg.grammatica.parser.Tokenizer
A token pattern matcher. This class is the base class for the various types of token matchers that exist. The token matcher checks for matches with the tokenizer buffer, and maintains the state of the last match.

     *             or initialized correctly
     */
    public Tokenizer createTokenizer(Reader in)
        throws GrammarException {


        Tokenizer  tokenizer;


        try {
            tokenizer = new Tokenizer(in, !getCaseSensitive());
            for (int i = 0; i < tokens.size(); i++) {
                tokenizer.addPattern((TokenPattern) tokens.get(i));
            }
        } catch (ParserCreationException e) {
            if (e.getName() == null) {
                throw new GrammarException(fileName, e.getMessage());
            } else {

View Full Code Here

     * Debugs a grammar by printing the internal representation.
     *
     * @param grammar        the grammar to use
     */
    private static void debug(Grammar grammar) {
        Tokenizer  tokenizer = null;
        Parser     parser = null;


        // Create tokenizer and parser
        try {
            tokenizer = grammar.createTokenizer(null);

View Full Code Here

     *
     * @param grammar        the grammar to use
     * @param file           the file to parse
     */
    private static void tokenize(Grammar grammar, File file) {
        Tokenizer  tokenizer;
        Token      token;


        try {
            tokenizer = grammar.createTokenizer(new FileReader(file));
            System.out.println("Tokens from " + file + ":");
            while ((token = tokenizer.next()) != null) {
                System.out.println(token);
            }
        } catch (FileNotFoundException e) {
            printError(file.toString(), e);
            System.exit(1);

View Full Code Here

     *
     * @param grammar        the grammar to use
     * @param file           the file to parse
     */
    private static void parse(Grammar grammar, File file) {
        Tokenizer  tokenizer;
        Analyzer   analyzer;
        Parser     parser;


        try {
            tokenizer = grammar.createTokenizer(new FileReader(file));

View Full Code Here

     * @param files          the files to parse
     * @param first          the index of the first file
     */
    private static void profile(Grammar grammar, String[] files, int first) {
        File       file = new File(files[first]);
        Tokenizer  tokenizer;
        Parser     parser;
        Node       node;
        int        fileCount = files.length - first;
        long       time;
        int        counter;


        // Profile tokenizer
        try {
            System.out.println("Tokenizing " + fileCount + " file(s)...");
            tokenizer = grammar.createTokenizer(new FileReader(file));
            time = System.currentTimeMillis();
            counter = 0;
            for (int i = first; i < files.length; i++) {
                if (i > first) {
                    file = new File(files[i]);
                    tokenizer.reset(new FileReader(file));
                }
                while (tokenizer.next() != null) {
                    counter++;
                }
            }
            time = System.currentTimeMillis() - time + 1;
            System.out.println("  Time elapsed:  " + time + " millisec");

View Full Code Here

     * @param grammar        the grammar to use
     *
     * @throws RuntimeException if a parser couldn't be created
     */
    private void debug(Grammar grammar) throws RuntimeException {
        Tokenizer  tokenizer = null;
        Parser     parser = null;


        // Create tokenizer and parser
        try {
            tokenizer = grammar.createTokenizer(null);

View Full Code Here

     *
     * @throws RuntimeException if the input file couldn't be tokenized
     *             correctly
     */
    private void tokenize(Grammar grammar) throws RuntimeException {
        Tokenizer  tokenizer;
        Token      token;


        try {
            tokenizer = grammar.createTokenizer(new FileReader(file));
            if (!quiet) {
                System.out.println("Tokens from " + file + ":");
            }
            while ((token = tokenizer.next()) != null) {
                if (!quiet) {
                    System.out.println(token);
                }
            }
        } catch (FileNotFoundException e) {

View Full Code Here

     *
     * @throws RuntimeException if the input file couldn't be parsed
     *             correctly
     */
    private void parse(Grammar grammar) throws RuntimeException {
        Tokenizer  tokenizer;
        Analyzer   analyzer;
        Parser     parser;


        try {
            tokenizer = grammar.createTokenizer(new FileReader(file));

View Full Code Here

     *
     * @throws RuntimeException if the input file couldn't be profiled
     *             correctly
     */
    private void profile(Grammar grammar) throws RuntimeException {
        Tokenizer  tokenizer;
        Parser     parser;
        Node       node;
        long       time;
        int        counter;


        // Profile tokenizer
        try {
            tokenizer = grammar.createTokenizer(new FileReader(file));
            System.out.println("Tokenizing " + file);
            time = System.currentTimeMillis();
            counter = 0;
            while (tokenizer.next() != null) {
                counter++;
            }
            time = System.currentTimeMillis() - time;
            System.out.println("  Time elapsed:  " + time + " millisec");
            System.out.println("  Tokens found:  " + counter);

View Full Code Here

TOP

Related Classes of net.percederberg.grammatica.parser.Tokenizer$TokenMatcher

net.percederberg.grammatica.ant.ValidationElement

net.percederberg.grammatica.Grammar

net.percederberg.grammatica.Grammatica

net.percederberg.grammatica.parser.re.RegExp

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.