ObjectStream<TokenSample> sampleStream =
TokenizerTrainerTool.openSampleData("Training Data",
trainingDataInFile, encoding);
TokenizerCrossValidator validator;
if (mlParams == null)
mlParams = TokenizerTrainerTool.createTrainingParameters(
params.getIterations(), params.getCutoff());
TokenizerEvaluationMonitor listener = null;
if (params.getMisclassified()) {
listener = new TokenEvaluationErrorListener();
}
try {
Dictionary dict = TokenizerTrainerTool.loadDict(params.getAbbDict());
validator = new opennlp.tools.tokenize.TokenizerCrossValidator(
params.getLang(), dict, params.getAlphaNumOpt(), mlParams, listener);
validator.evaluate(sampleStream, params.getFolds());
}
catch (IOException e) {
CmdLineUtil.printTrainingIoError(e);
throw new TerminateToolException(-1);
}
finally {
try {
sampleStream.close();
} catch (IOException e) {
// sorry that this can fail
}
}
FMeasure result = validator.getFMeasure();
System.out.println(result.toString());
}