}
Token oldToken = tokenList.get(i);
// if it's a negative scan (NOT 1* Noun), then the target of the next token becomes the barrier + SENT_END
if (oldToken.negate) {
CgSet newTarget = oldToken.barrier;
CgTag sentEndTag = new CgTag();
sentEndTag.tag = SENT_END;
newTarget.single_tags.add(sentEndTag);
oldToken.target = newTarget;
oldToken.postags = oldToken.target.getPostagsString();
oldToken.baseforms = oldToken.target.getSingleTagBaseformsString();
oldToken.surfaceforms = oldToken.target.getSingleTagSurfaceformsString();
oldToken.compositeTags = oldToken.target.getCompositeTags();
tokenList.set(0, oldToken);
}
tokenList.add(0, newToken);
} else {
int index = i-1;
String exceptionString = null;
if (!tokenList.get(i).barrier.isEmpty() || tokenList.get(i).negate) {
exceptionString = getBarrierExceptionStringFromToken(tokenList.get(i));
}
int prevOffset = tokenList.get(index).offset;
while (index >= 0 && tokenList.get(index).offset == prevOffset) {
Token prevToken = tokenList.get(index);
prevToken.skip = -1;
prevToken.exceptionString = exceptionString;
tokenList.set(index, prevToken);
index--;
}
Token oldToken = tokenList.get(i);
if (oldToken.negate) {
CgSet newTarget = oldToken.barrier;
CgTag sentEndTag = new CgTag();
sentEndTag.tag = SENT_END;
newTarget.single_tags.add(sentEndTag);
oldToken.target = newTarget;
oldToken.postags = oldToken.target.getPostagsString();
oldToken.baseforms = oldToken.target.getSingleTagBaseformsString();
oldToken.surfaceforms = oldToken.target.getSingleTagSurfaceformsString();
oldToken.compositeTags = oldToken.target.getCompositeTags();
oldToken.negate = false;
tokenList.set(i,oldToken);
}
}
}
// reverse scans (-1* Verb)
else if (tokenList.get(i).scanbehind) {
Token newToken = new Token(new CgSet(), false, tokenList.get(i).offset - 1, false, false, new CgSet(), false, -1, false);
String exceptionString = null;
if (!tokenList.get(i).barrier.isEmpty() || tokenList.get(i).negate) {
exceptionString = getBarrierExceptionStringFromToken(tokenList.get(i));
}
CgSet newTarget = newToken.target;
CgTag sentStartTag = new CgTag();
sentStartTag.tag = SENT_START;
newTarget.single_tags.add(sentStartTag);
newToken.target = newTarget;
newToken.postags = newToken.target.getPostagsString();
newToken.baseforms = newToken.target.getSingleTagBaseformsString();