input.start();
}
for (LogicalOutput output : outputs.values()) {
output.start();
}
MROutput out = (MROutput) outputs.get("union");
MROutput allParts = (MROutput) outputs.get("all-parts");
KeyValueWriter kvWriter = out.getWriter();
KeyValueWriter partsWriter = allParts.getWriter();
Map<String, AtomicInteger> unionKv = Maps.newHashMap();
LogicalInput union = inputs.get("union");
KeyValuesReader kvReader = (KeyValuesReader) union.getReader();
while (kvReader.next()) {
String word = ((Text) kvReader.getCurrentKey()).toString();
IntWritable intVal = (IntWritable) kvReader.getCurrentValues().iterator().next();
for (int i=0; i<intVal.get(); ++i) {
partsWriter.write(word, one);
}
AtomicInteger value = unionKv.get(word);
if (value == null) {
unionKv.put(word, new AtomicInteger(intVal.get()));
} else {
value.addAndGet(intVal.get());
}
}
LogicalInput map3 = inputs.get("map3");
kvReader = (KeyValuesReader) map3.getReader();
while (kvReader.next()) {
String word = ((Text) kvReader.getCurrentKey()).toString();
IntWritable intVal = (IntWritable) kvReader.getCurrentValues().iterator().next();
AtomicInteger value = unionKv.get(word);
if (value == null) {
throw new TezUncheckedException("Expected to exist: " + word);
} else {
value.getAndAdd(intVal.get()*-2);
}
}
for (AtomicInteger value : unionKv.values()) {
if (value.get() != 0) {
throw new TezUncheckedException("Unexpected non-zero value");
}
}
kvWriter.write("Union", new IntWritable(unionKv.size()));
if (out.isCommitRequired()) {
while (!context.canCommit()) {
Thread.sleep(100);
}
out.commit();
}
if (allParts.isCommitRequired()) {
while (!context.canCommit()) {
Thread.sleep(100);
}
allParts.commit();
}
}