Package fasp.parser.state

Examples of fasp.parser.state.LookaheadParseState


*
* @author Jeroen Janssen <Jeroen.Janssen@vub.ac.be>
*/
public class Tokenizer {
  public static ArrayList<Token> tokenize(String str) throws ParseException {
    LookaheadParseState st = new LookaheadParseState(str);
    ArrayList<Token> tokenList = new ArrayList<Token>();
    Character c = st.getNext();
    while(!c.equals(ParseState.EOF)) {
      if (Character.isLetter(c))
        tokenList.add(readSymbolToken(st,c));
      else if (Character.isDigit(c))
        tokenList.add(readLattToken(st,c));
      else if (c.equals('('))
        tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.OB));
      else if (c.equals(')'))
        tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.CB));
      else if (c.equals(':'))
        tokenList.add(readRuleToken(st,c));
      else if (c.equals('.'))
        tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.DOT));
      else if (c.equals(','))
        tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.COMMA));
      else if (c.equals('-'))
        tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.NEG));
      else if (c.equals('~'))
        tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.NAF));
      else if (c.equals('%'))
        while(!st.lookahead().equals(ParseState.EOF) && !st.lookahead().equals('\n'))
          st.eat();
      else if (c.equals('/'))
        tokenList.add(readInEqExp(st,c));
      else if (c.equals('='))
        tokenList.add(readEqExp(st,c));
      else if(!(c.equals(' ') || c.equals('\n') || c.equals('\r'))) // spaces and newlines are ignored
          throw new ParseException(st,"recognizable input",c.toString());
      // Proceed to the next input char
      c = st.getNext();
    }
    tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.EOF));
    return tokenList;
  }
View Full Code Here

TOP

Related Classes of fasp.parser.state.LookaheadParseState

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.