formattedToken = new AnalyzedTokenReadings(new AnalyzedToken(
matchedToken.getToken(), posTag, formattedToken.getToken()),
matchedToken.getStartPos());
formattedToken.setWhitespaceBefore(matchedToken.isWhitespaceBefore());
*/
matchedToken.leaveReading(new AnalyzedToken(
matchedToken.getToken(), posTag, formattedToken.getToken()));
formattedToken = matchedToken;
}
String token = formattedToken.getToken();
if (pRegexMatch != null && regexReplace != null) {
/* only replace if it is something to replace*/
token = pRegexMatch.matcher(token).replaceAll(regexReplace);
}
token = convertCase(token, token);
if (posTag != null) {
final int numRead = formattedToken.getReadingsLength();
if (postagRegexp) {
String targetPosTag = posTag;
for (int i = 0; i < numRead; i++) {
final String tst = formattedToken.getAnalyzedToken(i).getPOSTag();
if (tst != null && pPosRegexMatch.matcher(tst).matches()) {
targetPosTag = formattedToken.getAnalyzedToken(i).getPOSTag();
if (posTagReplace != null) {
targetPosTag = pPosRegexMatch.matcher(targetPosTag).replaceAll(
posTagReplace);
}
l.add(new AnalyzedToken(token, targetPosTag, formattedToken
.getAnalyzedToken(i).getLemma()));
l.get(l.size() - 1).setWhitespaceBefore(formattedToken.isWhitespaceBefore());
}
}
if (l.isEmpty()) {
for (final AnalyzedToken anaTok : getNewToken(numRead, token)) {
l.add(anaTok);
}
}
} else {
for (final AnalyzedToken anaTok : getNewToken(numRead, token)) {
l.add(anaTok);
}
}
if (formattedToken.isSentEnd()) {
l.add(new AnalyzedToken(formattedToken.getToken(),
JLanguageTool.SENTENCE_END_TAGNAME,
formattedToken.getAnalyzedToken(0).getLemma()));
}
if (formattedToken.isParaEnd()) {
l.add(new AnalyzedToken(formattedToken.getToken(),
JLanguageTool.PARAGRAPH_END_TAGNAME,
formattedToken.getAnalyzedToken(0).getLemma()));
}
}
}