}
String word = token.substring(0, split);
wordEnd = wordStart + word.length();
// Consider creating a token similar to the way
// TokenConverter.convert method creates BaseToken's
BaseToken baseToken = new BaseToken(jCas, wordStart, wordEnd);
if (!loadWordsOnly) {
String tag = token.substring(split + 1);
baseToken.setPartOfSpeech(tag);
}
baseToken.setTokenNumber(wordNumber++);
baseToken.addToIndexes();
documentText.append(word + " ");
wordStart = wordEnd + 1;
}
Sentence sentence = new Sentence(jCas, 0, wordEnd);