// the following is a race condition
fileSys.delete(outputPath.suffix("/_logs"), true);
SequenceFile.Reader[] readers =
SequenceFileOutputFormat.getReaders(xjob, outputPath);
FileReaderZip z = new FileReaderZip(readers);
TTable tt = new TTable_monolithic_IFAs(fileSys, ttablePath, false);
boolean emittedATable = false;
while (z.next(k,t)) {
if (t.getType() == PartialCountContainer.CONTENT_ARRAY) {
tt.set(k.get(), (IndexedFloatArray)t.getContent());
if (k.get() % 1000 == 0) reporter.progress();
reporter.incrCounter(MergeCounters.EWORDS, 1);
reporter.incrCounter(MergeCounters.STATISTICS, ((IndexedFloatArray)t.getContent()).size() + 1);
} else {
if (emittedATable)
throw new RuntimeException("Should only have a single ATable!");
ATable at = (ATable)t.getContent();
fileSys.delete(atablePath, true);
DataOutputStream dos = new DataOutputStream(
new BufferedOutputStream(fileSys.create(atablePath)));
at.write(dos);
dos.close();
emittedATable = true;
}
}
fileSys.delete(ttablePath, true); // delete old ttable
tt.write(); // write new one to same location
output.collect(key, value);
}