boolean after_def = false;
boolean value_expected = true;
TokenType kind = null;
while (source.hasMore()) {
MatchResult m = null;
switch (state) {
case initial:
if ((m = source.scan(SPACE)) != null) {
encoder.textToken(m.group(), TokenType.space);
if (m.group().indexOf("\n") != -1) {
import_clause = after_def = false;
if (!value_expected) {
value_expected = true;
}
}
continue;
}
else if ((m = source.scan(COMMENT)) != null) {
value_expected = true;
after_def = false;
encoder.textToken(m.group(), TokenType.comment);
}
else if ((m = source.scan(DOCTYPE)) != null) {
encoder.textToken(m.group(), TokenType.doctype);
}
else if (import_clause && (m = source.scan(INCLUDE)) != null) {
after_def = value_expected = false;
encoder.textToken(m.group(), TokenType.include);
}
else if ((m = source.scan(IDENT)) != null) {
kind = IDENT_KIND.lookup(m.group());
value_expected = (kind == TokenType.keyword) && KEYWORDS_EXPECTING_VALUE.lookup(m.group());
if (".".equals(last_token)) {
kind = TokenType.ident;
}
else if (class_name_follows) {
kind = TokenType.class_;
class_name_follows = false;
}
else if (after_def && source.check(AFTER_DEF) != null) {
kind = TokenType.method;
after_def = false;
}
else if (kind == TokenType.ident && !"?".equals(last_token) && source.check(":") != null) {
kind = TokenType.key;
}
else {
if (m.group().equals("class") || (import_clause && m.group().equals("as"))) {
class_name_follows = true;
}
import_clause = (m.group().equals("import"));
if (m.group().equals("def")) {
after_def = true;
}
}
encoder.textToken(m.group(), kind);
}
else if ((m = source.scan(SEMI_COLON)) != null) {
import_clause = after_def = false;
value_expected = true;
encoder.textToken(m.group(), TokenType.operator);
}
else if ((m = source.scan(START_BRACKET)) != null) {
class_name_follows = after_def = false;
value_expected = true;
encoder.textToken(m.group(), TokenType.operator);
if (!inlineBlockStack.isEmpty()) {
inlineBlockParenDepth += 1;
}
}
else if ((m = source.scan(OPERATOR)) != null) {
value_expected = true;
//value_expected = :regexp if match == '~';
after_def = false;
encoder.textToken(m.group(), TokenType.operator);
}
else if ((m = source.scan(END_BRACKET)) != null) {
value_expected = after_def = false;
if (!inlineBlockStack.isEmpty() && m.group().equals("}")) {
inlineBlockParenDepth -= 1;
if (inlineBlockParenDepth == 0) // # closing brace of inline block reached
{
encoder.textToken(m.group(), TokenType.inline_delimiter);
encoder.endGroup(TokenType.inline);
Object[] inlineBlock = inlineBlockStack.pop();
state = (State) inlineBlock[0];
stringDelimiter = (String) inlineBlock[1];
inlineBlockParenDepth = (int) inlineBlock[2];
continue;
}
}
encoder.textToken(m.group(), TokenType.operator);
}
else if (source.check(NUMBER) != null) {
after_def = value_expected = false;
if ((m = source.scan(HEX)) != null) {
encoder.textToken(m.group(), TokenType.hex);
}
else if ((m = source.scan(OCTAL)) != null) {
encoder.textToken(m.group(), TokenType.octal);
}
else if ((m = source.scan(FLOAT)) != null) {
encoder.textToken(m.group(), TokenType.float_);
}
else if ((m = source.scan(INTEGER)) != null) {
encoder.textToken(m.group(), TokenType.integer);
}
}
else if ((m = source.scan(MULTI_LINE_DELIMITER)) != null) {
after_def = value_expected = false;
state = State.multiline_string;
encoder.beginGroup(TokenType.string);
stringDelimiter = m.group();
encoder.textToken(m.group(), TokenType.delimiter);
}
else if ((m = source.scan(STRING_DELIMITER)) != null) {
after_def = value_expected = false;
state = m.group().equals("/") ? State.regexp : State.string;
encoder.beginGroup(TokenType.valueOf(state.name()));
stringDelimiter = m.group();
encoder.textToken(m.group(), TokenType.delimiter);
}
else if (value_expected && (m = source.scan(START_REGEXP)) != null) {
after_def = value_expected = false;
encoder.beginGroup(TokenType.regexp);
state = State.regexp;
stringDelimiter = "/";
encoder.textToken(m.group(), TokenType.delimiter);
}
else if ((m = source.scan(ANNOTATION)) != null) {
after_def = value_expected = false;
encoder.textToken(m.group(), TokenType.annotation);
}
else if ((m = source.scan(END_OPERATOR)) != null) {
after_def = false;
value_expected = true;
encoder.textToken(m.group(), TokenType.operator);
}
else {
encoder.textToken(source.next(), TokenType.error);
}
break;
case string:
case regexp:
case multiline_string:
if ((m = source.scan(STRING_CONTENT_PATTERN.get(stringDelimiter))) != null) {
encoder.textToken(m.group(), TokenType.content);
}
else if ((m = source.scan(state == State.multiline_string ? "'''|\"\"\"" : "[\"'\\/]")) != null) {
encoder.textToken(m.group(), TokenType.delimiter);
if (state == State.regexp) {
MatchResult modifiers = source.scan("[ix]+");
if (modifiers != null && !modifiers.group().equals("")) {
encoder.textToken(modifiers.group(), TokenType.modifier);
}
}
if (state == State.multiline_string) {
state = State.string;
}