mapInputPayload = MRHelpers.createMRInputPayload(
mapUserPayload, inputSplitInfo.getSplitsProto());
}
int numTasks = generateSplitsInAM ? -1 : numMapper;
Vertex mapVertex = new Vertex("map", new ProcessorDescriptor(
MapProcessor.class.getName()).setUserPayload(mapUserPayload),
numTasks, MRHelpers.getMapResource(mapStageConf));
mapVertex.setJavaOpts(MRHelpers.getMapJavaOpts(mapStageConf));
if (!generateSplitsInAM) {
mapVertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
}
if (writeSplitsToDFS) {
Map<String, LocalResource> mapLocalResources = new HashMap<String, LocalResource>();
mapLocalResources.putAll(commonLocalResources);
MRHelpers.updateLocalResourcesForInputSplits(remoteFs, inputSplitInfo,
mapLocalResources);
mapVertex.setTaskLocalResources(mapLocalResources);
} else {
mapVertex.setTaskLocalResources(commonLocalResources);
}
Map<String, String> mapEnv = new HashMap<String, String>();
MRHelpers.updateEnvironmentForMRTasks(mapStageConf, mapEnv, true);
mapVertex.setTaskEnvironment(mapEnv);
if (generateSplitsInAM) {
MRHelpers.addMRInput(mapVertex, mapInputPayload, MRInputAMSplitGenerator.class);
} else {
if (writeSplitsToDFS) {
MRHelpers.addMRInput(mapVertex, mapInputPayload, null);
} else {
MRHelpers.addMRInput(mapVertex, mapInputPayload, MRInputSplitDistributor.class);
}
}
vertices.add(mapVertex);
if (iReduceStagesCount > 0
&& numIReducer > 0) {
for (int i = 0; i < iReduceStagesCount; ++i) {
Configuration iconf =
intermediateReduceStageConfs[i];
byte[] iReduceUserPayload = MRHelpers.createUserPayloadFromConf(iconf);
Vertex ivertex = new Vertex("ireduce" + (i+1),
new ProcessorDescriptor(ReduceProcessor.class.getName()).
setUserPayload(iReduceUserPayload), numIReducer,
MRHelpers.getReduceResource(iconf));
ivertex.setJavaOpts(MRHelpers.getReduceJavaOpts(iconf));
ivertex.setTaskLocalResources(commonLocalResources);
Map<String, String> reduceEnv = new HashMap<String, String>();
MRHelpers.updateEnvironmentForMRTasks(iconf, reduceEnv, false);
ivertex.setTaskEnvironment(reduceEnv);
vertices.add(ivertex);
}
}
Vertex finalReduceVertex = null;
if (numReducer > 0) {
byte[] reducePayload = MRHelpers.createUserPayloadFromConf(finalReduceConf);
finalReduceVertex = new Vertex("reduce", new ProcessorDescriptor(
ReduceProcessor.class.getName()).setUserPayload(reducePayload),
numReducer, MRHelpers.getReduceResource(finalReduceConf));
finalReduceVertex.setJavaOpts(
MRHelpers.getReduceJavaOpts(finalReduceConf));
finalReduceVertex.setTaskLocalResources(commonLocalResources);
Map<String, String> reduceEnv = new HashMap<String, String>();
MRHelpers.updateEnvironmentForMRTasks(finalReduceConf, reduceEnv, false);
finalReduceVertex.setTaskEnvironment(reduceEnv);
MRHelpers.addMROutputLegacy(finalReduceVertex, reducePayload);
vertices.add(finalReduceVertex);
} else {
// Map only job
MRHelpers.addMROutputLegacy(mapVertex, mapUserPayload);