*
* @author Jeroen Janssen <Jeroen.Janssen@vub.ac.be>
*/
public class Tokenizer {
public static ArrayList<Token> tokenize(String str) throws ParseException {
LookaheadParseState st = new LookaheadParseState(str);
ArrayList<Token> tokenList = new ArrayList<Token>();
Character c = st.getNext();
while(!c.equals(ParseState.EOF)) {
if (Character.isLetter(c))
tokenList.add(readSymbolToken(st,c));
else if (Character.isDigit(c))
tokenList.add(readLattToken(st,c));
else if (c.equals('('))
tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.OB));
else if (c.equals(')'))
tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.CB));
else if (c.equals(':'))
tokenList.add(readRuleToken(st,c));
else if (c.equals('.'))
tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.DOT));
else if (c.equals(','))
tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.COMMA));
else if (c.equals('-'))
tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.NEG));
else if (c.equals('~'))
tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.NAF));
else if (c.equals('%'))
while(!st.lookahead().equals(ParseState.EOF) && !st.lookahead().equals('\n'))
st.eat();
else if (c.equals('/'))
tokenList.add(readInEqExp(st,c));
else if (c.equals('='))
tokenList.add(readEqExp(st,c));
else if(!(c.equals(' ') || c.equals('\n') || c.equals('\r'))) // spaces and newlines are ignored
throw new ParseException(st,"recognizable input",c.toString());
// Proceed to the next input char
c = st.getNext();
}
tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.EOF));
return tokenList;
}