writer.addDocument(buildDocument(key, value));
}
@Override
public void close(final TaskAttemptContext context) throws IOException, InterruptedException {
TaskHeartbeatThread heartBeat = new TaskHeartbeatThread(context) {
@Override
public void progress() {
String[] filesLeft = tmpDirFile.list();
if (filesLeft != null) {
int remaining = filesLeft.length - 2;
LOG.info("Optimizing " + remaining + " segments");
} else {
LOG.info("Done optimizing segments, heartbeat thread still alive");
}
}
};
try {
LOG.info("Starting heartbeat thread");
heartBeat.start();
Path work = committer.getWorkPath();
Path output = new Path(work, "index-"
+ String.valueOf(HadoopCompat.getTaskAttemptID(context).getTaskID().getId()));
writer.forceMerge(1);
writer.close();
FileSystem fs = FileSystem.get(HadoopCompat.getConfiguration(context));
LOG.info("Copying index to HDFS...");
if (!FileUtil.copy(tmpDirFile, fs, output, true, HadoopCompat.getConfiguration(context))) {
throw new IOException("Failed to copy local index to HDFS!");
}
LOG.info("Index written to: " + output);
} catch (IOException e) {
LOG.error("Error committing index", e);
throw e;
} finally {
// all things must die, eventually
LOG.info("Stopping heartbeat thread");
heartBeat.stop();
}
}