if (inputSplitInfo.getCredentials() != null) {
this.credentials.addAll(inputSplitInfo.getCredentials());
}
}
DAG dag = new DAG("MRRSleepJob");
String jarPath = ClassUtil.findContainingJar(getClass());
if (jarPath == null) {
throw new TezUncheckedException("Could not find any jar containing"
+ " MRRSleepJob.class in the classpath");
}
Path remoteJarPath = remoteFs.makeQualified(
new Path(remoteStagingDir, "dag_job.jar"));
remoteFs.copyFromLocalFile(new Path(jarPath), remoteJarPath);
FileStatus jarFileStatus = remoteFs.getFileStatus(remoteJarPath);
TokenCache.obtainTokensForNamenodes(this.credentials, new Path[] { remoteJarPath },
mapStageConf);
Map<String, LocalResource> commonLocalResources =
new HashMap<String, LocalResource>();
LocalResource dagJarLocalRsrc = LocalResource.newInstance(
ConverterUtils.getYarnUrlFromPath(remoteJarPath),
LocalResourceType.FILE,
LocalResourceVisibility.APPLICATION,
jarFileStatus.getLen(),
jarFileStatus.getModificationTime());
commonLocalResources.put("dag_job.jar", dagJarLocalRsrc);
List<Vertex> vertices = new ArrayList<Vertex>();
byte[] mapInputPayload = null;
byte[] mapUserPayload = MRHelpers.createUserPayloadFromConf(mapStageConf);
if (writeSplitsToDFS || generateSplitsInAM) {
mapInputPayload = MRHelpers.createMRInputPayload(mapUserPayload, null);
} else {
mapInputPayload = MRHelpers.createMRInputPayload(
mapUserPayload, inputSplitInfo.getSplitsProto());
}
int numTasks = generateSplitsInAM ? -1 : numMapper;
Vertex mapVertex = new Vertex("map", new ProcessorDescriptor(
MapProcessor.class.getName()).setUserPayload(mapUserPayload),
numTasks, MRHelpers.getMapResource(mapStageConf));
mapVertex.setJavaOpts(MRHelpers.getMapJavaOpts(mapStageConf));
if (!generateSplitsInAM) {
mapVertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
}
if (writeSplitsToDFS) {
Map<String, LocalResource> mapLocalResources = new HashMap<String, LocalResource>();
mapLocalResources.putAll(commonLocalResources);
MRHelpers.updateLocalResourcesForInputSplits(remoteFs, inputSplitInfo,
mapLocalResources);
mapVertex.setTaskLocalResources(mapLocalResources);
} else {
mapVertex.setTaskLocalResources(commonLocalResources);
}
Map<String, String> mapEnv = new HashMap<String, String>();
MRHelpers.updateEnvironmentForMRTasks(mapStageConf, mapEnv, true);
mapVertex.setTaskEnvironment(mapEnv);
if (generateSplitsInAM) {
MRHelpers.addMRInput(mapVertex, mapInputPayload, MRInputAMSplitGenerator.class);
} else {
if (writeSplitsToDFS) {
MRHelpers.addMRInput(mapVertex, mapInputPayload, null);
} else {
MRHelpers.addMRInput(mapVertex, mapInputPayload, MRInputSplitDistributor.class);
}
}
vertices.add(mapVertex);
if (iReduceStagesCount > 0
&& numIReducer > 0) {
for (int i = 0; i < iReduceStagesCount; ++i) {
Configuration iconf =
intermediateReduceStageConfs[i];
byte[] iReduceUserPayload = MRHelpers.createUserPayloadFromConf(iconf);
Vertex ivertex = new Vertex("ireduce" + (i+1),
new ProcessorDescriptor(ReduceProcessor.class.getName()).
setUserPayload(iReduceUserPayload), numIReducer,
MRHelpers.getReduceResource(iconf));
ivertex.setJavaOpts(MRHelpers.getReduceJavaOpts(iconf));
ivertex.setTaskLocalResources(commonLocalResources);
Map<String, String> reduceEnv = new HashMap<String, String>();
MRHelpers.updateEnvironmentForMRTasks(iconf, reduceEnv, false);
ivertex.setTaskEnvironment(reduceEnv);
vertices.add(ivertex);
}
}
Vertex finalReduceVertex = null;
if (numReducer > 0) {
byte[] reducePayload = MRHelpers.createUserPayloadFromConf(finalReduceConf);
finalReduceVertex = new Vertex("reduce", new ProcessorDescriptor(
ReduceProcessor.class.getName()).setUserPayload(reducePayload),
numReducer, MRHelpers.getReduceResource(finalReduceConf));
finalReduceVertex.setJavaOpts(
MRHelpers.getReduceJavaOpts(finalReduceConf));
finalReduceVertex.setTaskLocalResources(commonLocalResources);
Map<String, String> reduceEnv = new HashMap<String, String>();
MRHelpers.updateEnvironmentForMRTasks(finalReduceConf, reduceEnv, false);
finalReduceVertex.setTaskEnvironment(reduceEnv);
MRHelpers.addMROutputLegacy(finalReduceVertex, reducePayload);
vertices.add(finalReduceVertex);
} else {
// Map only job
MRHelpers.addMROutput(mapVertex, mapUserPayload);
}
for (int i = 0; i < vertices.size(); ++i) {
dag.addVertex(vertices.get(i));
if (i != 0) {
dag.addEdge(new Edge(vertices.get(i-1),
vertices.get(i), new EdgeProperty(
DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
SchedulingType.SEQUENTIAL,
new OutputDescriptor(
OnFileSortedOutput.class.getName()),