tokenStartIndex++;
}
BaseToken startToken = (BaseToken) tokens
.get(tokenStartIndex);
BaseToken endToken = token;
SuffixFrequencyToken fractionToken = new SuffixFrequencyToken(
startToken.getStartOffset(), endToken
.getEndOffset());
fractionSet.add(fractionToken);
fsm.reset();
}