return null;
}
@SuppressWarnings("PMD.AvoidCatchingThrowable")
private void runEvaluation() throws IOException, RecommenderBuildException {
EventBus bus = task.getProject().getEventBus();
bus.post(JobEvents.started(this));
Closer closer = Closer.create();
try {
outputs = task.getOutputs().getPrefixed(algorithmInfo, dataSet);
TableWriter userResults = outputs.getUserWriter();
List<Object> outputRow = Lists.newArrayList();
logger.info("Building {} on {}", algorithmInfo, dataSet);
StopWatch buildTimer = new StopWatch();
buildTimer.start();
buildRecommender();
buildTimer.stop();
logger.info("Built {} in {}", algorithmInfo.getName(), buildTimer);
logger.info("Measuring {} on {}", algorithmInfo.getName(), dataSet.getName());
StopWatch testTimer = new StopWatch();
testTimer.start();
List<Object> userRow = Lists.newArrayList();
List<MetricWithAccumulator<?>> accumulators = Lists.newArrayList();
for (Metric<?> eval: outputs.getMetrics()) {
accumulators.add(makeMetricAccumulator(eval));
}
LongSet testUsers = dataSet.getTestData().getUserDAO().getUserIds();
final NumberFormat pctFormat = NumberFormat.getPercentInstance();
pctFormat.setMaximumFractionDigits(2);
pctFormat.setMinimumFractionDigits(2);
final int nusers = testUsers.size();
logger.info("Testing {} on {} ({} users)", algorithmInfo, dataSet, nusers);
int ndone = 0;
for (LongIterator iter = testUsers.iterator(); iter.hasNext();) {
if (Thread.interrupted()) {
throw new InterruptedException("eval job interrupted");
}
long uid = iter.nextLong();
userRow.add(uid);
userRow.add(null); // placeholder for the per-user time
assert userRow.size() == 2;
Stopwatch userTimer = Stopwatch.createStarted();
TestUser test = getUserResults(uid);
userRow.add(test.getTrainHistory().size());
userRow.add(test.getTestHistory().size());
for (MetricWithAccumulator<?> accum : accumulators) {
List<Object> ures = accum.measureUser(test);
if (ures != null) {
userRow.addAll(ures);
}
}
userTimer.stop();
userRow.set(1, userTimer.elapsed(TimeUnit.MILLISECONDS) * 0.001);
if (userResults != null) {
try {
userResults.writeRow(userRow);
} catch (IOException e) {
throw new RuntimeException("error writing user row", e);
}
}
userRow.clear();
ndone += 1;
if (ndone % 100 == 0) {
testTimer.split();
double time = testTimer.getSplitTime();
double tpu = time / ndone;
double tleft = (nusers - ndone) * tpu;
logger.info("tested {} of {} users ({}), ETA {}",
ndone, nusers, pctFormat.format(((double) ndone) / nusers),
DurationFormatUtils.formatDurationHMS((long) tleft));
}
}
testTimer.stop();
logger.info("Tested {} in {}", algorithmInfo.getName(), testTimer);
writeMetricValues(buildTimer, testTimer, outputRow, accumulators);
bus.post(JobEvents.finished(this));
} catch (Throwable th) {
bus.post(JobEvents.failed(this, th));
throw closer.rethrow(th, RecommenderBuildException.class);
} finally {
try {
cleanup();
} finally {