throw new TezUncheckedException("Unexpected event type: " + event.getClass().getName());
}
}
private void processDataMovementEvent(DataMovementEvent dme) throws IOException {
DataMovementEventPayloadProto shufflePayload;
try {
shufflePayload = DataMovementEventPayloadProto.parseFrom(dme.getUserPayload());
} catch (InvalidProtocolBufferException e) {
throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e);
}
int srcIndex = dme.getSourceIndex();
LOG.info("Processing DataMovementEvent with srcIndex: "
+ srcIndex + ", targetIndex: " + dme.getTargetIndex()
+ ", attemptNum: " + dme.getVersion() + ", payload: "
+ stringify(shufflePayload));
if (shufflePayload.hasEmptyPartitions()) {
byte[] emptyPartitions = TezUtils.decompressByteStringToByteArray(shufflePayload
.getEmptyPartitions());
BitSet emptyPartionsBitSet = TezUtils.fromByteArray(emptyPartitions);
if (emptyPartionsBitSet.get(srcIndex)) {
InputAttemptIdentifier srcAttemptIdentifier = new InputAttemptIdentifier(dme.getTargetIndex(),
dme.getVersion());
LOG.info("Source partition: " + srcIndex + " did not generate any data. Not fetching.");
shuffleManager.addCompletedInputWithNoData(srcAttemptIdentifier);
return;
}
} else {
InputAttemptIdentifier srcAttemptIdentifier = new InputAttemptIdentifier(dme.getTargetIndex(),
dme.getVersion(), shufflePayload.getPathComponent());
if (shufflePayload.hasData()) {
DataProto dataProto = shufflePayload.getData();
FetchedInput fetchedInput = inputAllocator.allocate(dataProto.getRawLength(),
dataProto.getCompressedLength(), srcAttemptIdentifier);
moveDataToFetchedInput(dataProto, fetchedInput);
shuffleManager.addCompletedInputWithData(srcAttemptIdentifier, fetchedInput);
} else {
shuffleManager.addKnownInput(shufflePayload.getHost(), shufflePayload.getPort(),
srcAttemptIdentifier, srcIndex);
}
}
}