/**
* <a href="http://java.sun.com/docs/books/jls/third_edition/html/lexical.html#3.8">Identifiers</a>
*/
@Test
public void shouldPreserveIdentifiers() {
assertThat(chunk("String"), isTokens(new Token("String", 1, 0)));
assertThat(chunk("i3"), isTokens(new Token("i3", 1, 0)));
assertThat(chunk("MAX_VALUE"), isTokens(new Token("MAX_VALUE", 1, 0)));
assertThat(chunk("isLetterOrDigit"), isTokens(new Token("isLetterOrDigit", 1, 0)));
assertThat(chunk("_"), isTokens(new Token("_", 1, 0)));
assertThat(chunk("_123_"), isTokens(new Token("_123_", 1, 0)));
assertThat(chunk("_Field"), isTokens(new Token("_Field", 1, 0)));
assertThat(chunk("_Field5"), isTokens(new Token("_Field5", 1, 0)));
assertThat(chunk("$"), isTokens(new Token("$", 1, 0)));
assertThat(chunk("$field"), isTokens(new Token("$field", 1, 0)));
assertThat(chunk("i2j"), isTokens(new Token("i2j", 1, 0)));
assertThat(chunk("from1to4"), isTokens(new Token("from1to4", 1, 0)));
assertThat("identifier with unicode", chunk("αβγ"), isTokens(new Token("αβγ", 1, 0)));
}