new InputDescriptor(MRInputLegacy.class.getName())
.setUserPayload(MRHelpers.createMRInputPayload(mapConf, null)),
0);
OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", new OutputDescriptor(LocalOnFileSorterOutput.class.getName()), 1);
// Run a map
LogicalIOProcessorRuntimeTask mapTask = MapUtils.createLogicalTask(localFs, workDir, mapConf, 0,
mapInput, new TestUmbilical(), dagName, mapVertexName,
Collections.singletonList(mapInputSpec),
Collections.singletonList(mapOutputSpec));
mapTask.initialize();
mapTask.run();
mapTask.close();
LOG.info("Starting reduce...");
Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>();
Configuration reduceStageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
reduceVertexName);
JobConf reduceConf = new JobConf(reduceStageConf);
reduceConf.setOutputFormat(SequenceFileOutputFormat.class);
reduceConf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
"localized-resources").toUri().toString());
FileOutputFormat.setOutputPath(reduceConf, new Path(workDir, "output"));
ProcessorDescriptor reduceProcessorDesc = new ProcessorDescriptor(
ReduceProcessor.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(reduceConf));
InputSpec reduceInputSpec = new InputSpec(mapVertexName,
new InputDescriptor(LocalMergedInput.class.getName()), 1);
OutputSpec reduceOutputSpec = new OutputSpec("NullDestinationVertex",
new OutputDescriptor(MROutputLegacy.class.getName()), 1);
// Now run a reduce
TaskSpec taskSpec = new TaskSpec(
TezTestUtils.getMockTaskAttemptId(0, 1, 0, 0),
dagName,
reduceVertexName,
reduceProcessorDesc,
Collections.singletonList(reduceInputSpec),
Collections.singletonList(reduceOutputSpec), null);
Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<String, ByteBuffer>();
serviceConsumerMetadata.put(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID,
ShuffleUtils.convertJobTokenToBytes(shuffleToken));
LogicalIOProcessorRuntimeTask task = new LogicalIOProcessorRuntimeTask(
taskSpec,
0,
reduceConf,
new TestUmbilical(),
serviceConsumerMetadata,
HashMultimap.<String, String>create());
task.initialize();
task.run();
task.close();
// MRTask mrTask = (MRTask)t.getProcessor();
// TODO NEWTEZ Verify the partitioner has not been created
// Likely not applicable anymore.
// Assert.assertNull(mrTask.getPartitioner());