/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package fasp.parser.tokenizer;
import fasp.parser.state.ParseState;
import fasp.parser.*;
import fasp.parser.state.LookaheadParseState;
import java.util.ArrayList;
/**
* Tokenizer class.
*
* @author Jeroen Janssen <Jeroen.Janssen@vub.ac.be>
*/
public class Tokenizer {
public static ArrayList<Token> tokenize(String str) throws ParseException {
LookaheadParseState st = new LookaheadParseState(str);
ArrayList<Token> tokenList = new ArrayList<Token>();
Character c = st.getNext();
while(!c.equals(ParseState.EOF)) {
if (Character.isLetter(c))
tokenList.add(readSymbolToken(st,c));
else if (Character.isDigit(c))
tokenList.add(readLattToken(st,c));
else if (c.equals('('))
tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.OB));
else if (c.equals(')'))
tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.CB));
else if (c.equals(':'))
tokenList.add(readRuleToken(st,c));
else if (c.equals('.'))
tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.DOT));
else if (c.equals(','))
tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.COMMA));
else if (c.equals('-'))
tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.NEG));
else if (c.equals('~'))
tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.NAF));
else if (c.equals('%'))
while(!st.lookahead().equals(ParseState.EOF) && !st.lookahead().equals('\n'))
st.eat();
else if (c.equals('/'))
tokenList.add(readInEqExp(st,c));
else if (c.equals('='))
tokenList.add(readEqExp(st,c));
else if(!(c.equals(' ') || c.equals('\n') || c.equals('\r'))) // spaces and newlines are ignored
throw new ParseException(st,"recognizable input",c.toString());
// Proceed to the next input char
c = st.getNext();
}
tokenList.add(new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.EOF));
return tokenList;
}
/**
* Helper functions
*/
private static Token readSymbolToken (LookaheadParseState st, Character c) throws ParseException {
boolean upper = Character.isUpperCase(c);
int line = st.getLineNumber();
int col = st.getColumnNumber()-1;
StringBuffer buff = new StringBuffer();
buff.append(c);
while(!st.lookahead().equals(ParseState.EOF) && Character.isLetterOrDigit(st.lookahead())) {
buff.append(st.getNext());
}
return new SymbolToken(st.getLocation(),buff.toString(),upper);
}
private static Token readLattToken(LookaheadParseState st, Character c) throws ParseException {
ParseLocation loc = st.getLocation();
StringBuffer buff = new StringBuffer();
buff.append(c);
while(!st.lookahead().equals(ParseState.EOF) && Character.isDigit(st.lookahead())) {
buff.append(st.getNext());
}
if(st.lookahead().equals('.')) {
st.consume('.');
buff.append('.');
while(!st.lookahead().equals(ParseState.EOF) && Character.isDigit(st.lookahead())) {
buff.append(st.getNext());
}
}
Double d = Double.parseDouble(buff.toString());
if(d < 0.0 || d > 1.0)
throw new ParseException(loc,"value between [0,1]",d.toString());
else
return new LattToken(st.getLocation(),Double.parseDouble(buff.toString()));
}
private static Token readRuleToken(LookaheadParseState st, Character c) throws ParseException {
int line = st.getLineNumber();
int col = st.getColumnNumber()-1;
c = st.getNext();
if(c.equals('-'))
return new Token(line,col,Token.Type.REGRULE);
else if(c.equals('/'))
return new Token(line,col,Token.Type.CHOICERULE);
else if(c.equals('~'))
return new Token(line,col,Token.Type.FUZZYRULE);
else
throw new ParseException(st,":-, :/ or :~",c.toString());
}
private static Token readInEqExp(LookaheadParseState st, Character c) throws ParseException {
c = st.getNext();
if(c.equals('='))
return new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.NEQOP);
else
throw new ParseException(st,"/=",c.toString());
}
private static Token readEqExp(LookaheadParseState st, Character c) throws ParseException {
c = st.getNext();
if(c.equals('='))
return new Token(st.getLineNumber(),st.getColumnNumber()-1,Token.Type.EQOP);
else
throw new ParseException(st,"==",c.toString());
}
}