Examples of net.percederberg.grammatica.parser.Tokenizer

Package net.percederberg.grammatica.parser

Examples of net.percederberg.grammatica.parser.Tokenizer

net.percederberg.grammatica.parser.Tokenizer
A character stream tokenizer. This class groups the characters read from the stream together into tokens ("words"). The grouping is controlled by token patterns that contain either a fixed string to search for, or a regular expression. If the stream of characters don't match any of the token patterns, a parse exception is thrown. @author Per Cederberg, @version 1.5

     *             or initialized correctly
     */
    public Tokenizer createTokenizer(Reader in)
        throws GrammarException {


        Tokenizer  tokenizer;


        try {
            tokenizer = new Tokenizer(in, !getCaseSensitive());
            for (int i = 0; i < tokens.size(); i++) {
                tokenizer.addPattern((TokenPattern) tokens.get(i));
            }
        } catch (ParserCreationException e) {
            if (e.getName() == null) {
                throw new GrammarException(fileName, e.getMessage());
            } else {

View Full Code Here

     * Debugs a grammar by printing the internal representation.
     *
     * @param grammar        the grammar to use
     */
    private static void debug(Grammar grammar) {
        Tokenizer  tokenizer = null;
        Parser     parser = null;


        // Create tokenizer and parser
        try {
            tokenizer = grammar.createTokenizer(null);

View Full Code Here

     *
     * @param grammar        the grammar to use
     * @param file           the file to parse
     */
    private static void tokenize(Grammar grammar, File file) {
        Tokenizer  tokenizer;
        Token      token;


        try {
            tokenizer = grammar.createTokenizer(new FileReader(file));
            System.out.println("Tokens from " + file + ":");
            while ((token = tokenizer.next()) != null) {
                System.out.println(token);
            }
        } catch (FileNotFoundException e) {
            printError(file.toString(), e);
            System.exit(1);

View Full Code Here

     *
     * @param grammar        the grammar to use
     * @param file           the file to parse
     */
    private static void parse(Grammar grammar, File file) {
        Tokenizer  tokenizer;
        Analyzer   analyzer;
        Parser     parser;


        try {
            tokenizer = grammar.createTokenizer(new FileReader(file));

View Full Code Here

     * @param files          the files to parse
     * @param first          the index of the first file
     */
    private static void profile(Grammar grammar, String[] files, int first) {
        File       file = new File(files[first]);
        Tokenizer  tokenizer;
        Parser     parser;
        Node       node;
        int        fileCount = files.length - first;
        long       time;
        int        counter;


        // Profile tokenizer
        try {
            System.out.println("Tokenizing " + fileCount + " file(s)...");
            tokenizer = grammar.createTokenizer(new FileReader(file));
            time = System.currentTimeMillis();
            counter = 0;
            for (int i = first; i < files.length; i++) {
                if (i > first) {
                    file = new File(files[i]);
                    tokenizer.reset(new FileReader(file));
                }
                while (tokenizer.next() != null) {
                    counter++;
                }
            }
            time = System.currentTimeMillis() - time + 1;
            System.out.println("  Time elapsed:  " + time + " millisec");

View Full Code Here

     * @param grammar        the grammar to use
     *
     * @throws RuntimeException if a parser couldn't be created
     */
    private void debug(Grammar grammar) throws RuntimeException {
        Tokenizer  tokenizer = null;
        Parser     parser = null;


        // Create tokenizer and parser
        try {
            tokenizer = grammar.createTokenizer(null);

View Full Code Here

     *
     * @throws RuntimeException if the input file couldn't be tokenized
     *             correctly
     */
    private void tokenize(Grammar grammar) throws RuntimeException {
        Tokenizer  tokenizer;
        Token      token;


        try {
            tokenizer = grammar.createTokenizer(new FileReader(file));
            if (!quiet) {
                System.out.println("Tokens from " + file + ":");
            }
            while ((token = tokenizer.next()) != null) {
                if (!quiet) {
                    System.out.println(token);
                }
            }
        } catch (FileNotFoundException e) {

View Full Code Here

     *
     * @throws RuntimeException if the input file couldn't be parsed
     *             correctly
     */
    private void parse(Grammar grammar) throws RuntimeException {
        Tokenizer  tokenizer;
        Analyzer   analyzer;
        Parser     parser;


        try {
            tokenizer = grammar.createTokenizer(new FileReader(file));

View Full Code Here

     *
     * @throws RuntimeException if the input file couldn't be profiled
     *             correctly
     */
    private void profile(Grammar grammar) throws RuntimeException {
        Tokenizer  tokenizer;
        Parser     parser;
        Node       node;
        long       time;
        int        counter;


        // Profile tokenizer
        try {
            tokenizer = grammar.createTokenizer(new FileReader(file));
            System.out.println("Tokenizing " + file);
            time = System.currentTimeMillis();
            counter = 0;
            while (tokenizer.next() != null) {
                counter++;
            }
            time = System.currentTimeMillis() - time;
            System.out.println("  Time elapsed:  " + time + " millisec");
            System.out.println("  Tokens found:  " + counter);

View Full Code Here

TOP

Related Classes of net.percederberg.grammatica.parser.Tokenizer

net.percederberg.grammatica.ant.ValidationElement

net.percederberg.grammatica.Grammar

net.percederberg.grammatica.Grammatica

net.percederberg.grammatica.parser.re.RegExp

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.