log.error(message,e);
throw new EngineException(this, ci, message, e);
}
}
//now the tokens
TokenStream tokens = new WordTokenFilter(new AnalyzedTextSentenceTokenizer(at));
try {
while(tokens.incrementToken()){
OffsetAttribute offset = tokens.addAttribute(OffsetAttribute.class);
Token t = at.addToken(offset.startOffset(), offset.endOffset());
log.trace("detected {}",t);
}
} catch (IOException e) {
String message = String.format("IOException while reading from "