Examples of org.apache.tez.runtime.api.impl.TaskSpec

org.apache.tez.runtime.api.impl.TaskSpec

        MapProcessor.class.getName()).setUserPayload(
        TezUtils.createUserPayloadFromConf(jobConf));
    
    Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>();


    TaskSpec taskSpec = new TaskSpec(
        TezTestUtils.getMockTaskAttemptId(0, 0, mapId, 0),
        dagName, vertexName, -1,
        mapProcessorDesc,
        inputSpecs,
        outputSpecs, null);

View Full Code Here

    OutputSpec reduceOutputSpec = new OutputSpec("NullDestinationVertex",
        OutputDescriptor.create(MROutputLegacy.class.getName())
            .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1);


    // Now run a reduce
    TaskSpec taskSpec = new TaskSpec(
        TezTestUtils.getMockTaskAttemptId(0, 1, 0, 0),
        dagName,
        reduceVertexName, -1,
        reduceProcessorDesc,
        Collections.singletonList(reduceInputSpec),

View Full Code Here

    TezVertexID vertexId = TezVertexID.getInstance(dagId, 1);
    TezTaskID taskId = TezTaskID.getInstance(vertexId, 1);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 1);
    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create(processorClass)
        .setUserPayload(UserPayload.create(ByteBuffer.wrap(processorConf)));
    TaskSpec taskSpec = new TaskSpec(taskAttemptId, "dagName", "vertexName", -1, processorDescriptor,
        new ArrayList<InputSpec>(), new ArrayList<OutputSpec>(), null);


    TezTaskRunner taskRunner = new TezTaskRunner(tezConf, ugi, localDirs, taskSpec, umbilical, 1,
        new HashMap<String, ByteBuffer>(), HashMultimap.<String, String> create(), taskReporter,
        executor, null);

View Full Code Here

      Set<String> hostsSet = Sets.newHashSet(hosts);
      Set<String> racksSet = Sets.newHashSet(racks);
      locationHint = TaskLocationHint.createTaskLocationHint(hostsSet, racksSet);
    }
    AMSchedulerEventTALaunchRequest lr = new AMSchedulerEventTALaunchRequest(
      taID, capability, new TaskSpec(taID, "dagName", "vertexName", -1,
        ProcessorDescriptor.create("processorClassName"),
      Collections.singletonList(new InputSpec("vertexName",
          InputDescriptor.create("inputClassName"), 1)),
      Collections.singletonList(new OutputSpec("vertexName",
          OutputDescriptor.create("outputClassName"), 1)), null), ta, locationHint,

View Full Code Here


  TaskSpec createRemoteTaskSpec() {
    Vertex vertex = getVertex();
    ProcessorDescriptor procDesc = vertex.getProcessorDescriptor();
    int taskId = getTaskID().getId();
    return new TaskSpec(getID(),
        vertex.getDAG().getName(),
        vertex.getName(), vertex.getTotalTasks(), procDesc,
        vertex.getInputSpecList(taskId), vertex.getOutputSpecList(taskId), 
        vertex.getGroupInputSpecList(taskId));
  }

View Full Code Here


      // TODO Creating the remote task here may not be required in case of
      // recovery.


      // Create the remote task.
      TaskSpec remoteTaskSpec = ta.createRemoteTaskSpec();
      // Create startTaskRequest


      String[] requestHosts = new String[0];


      // Compute node/rack location request even if re-scheduled.

View Full Code Here

    TezConfiguration tezConf = new TezConfiguration();
    tezConf.set(TezConfiguration.TEZ_TASK_SCALE_TASK_MEMORY_ALLOCATOR_CLASS,
        ScalingAllocator.class.getName());


    TezTaskAttemptID taId1 = createTaskAttemptID(vertexId, 1);
    TaskSpec task1 = createTaskSpec(taId1, "dag1", "vertex1", 30);


    TezTaskAttemptID taId2 = createTaskAttemptID(vertexId, 2);
    TaskSpec task2 = createTaskSpec(taId2, "dag2", "vertex1", 10);


    LogicalIOProcessorRuntimeTask lio1 = new LogicalIOProcessorRuntimeTask(task1, 0, tezConf, null,
        umbilical, serviceConsumerMetadata, startedInputsMap, null);


    lio1.initialize();

View Full Code Here

  }


  private TaskSpec createTaskSpec(TezTaskAttemptID taskAttemptID,
      String dagName, String vertexName, int parallelism) {
    ProcessorDescriptor processorDesc = createProcessorDescriptor();
    TaskSpec taskSpec = new TaskSpec(taskAttemptID,
        dagName, vertexName, parallelism, processorDesc,
        createInputSpecList(), createOutputSpecList(), null);
    return taskSpec;
  }

View Full Code Here

          }
        }), defaultConf);
    RelocalizationUtils.addUrlsToClassPath(downloadedUrls);


    LOG.info("Done localizing additional resources");
    final TaskSpec taskSpec = containerTask.getTaskSpec();
    if (LOG.isDebugEnabled()) {
      LOG.debug("New container task context:" + taskSpec.toString());
    }
  }

View Full Code Here

    Multimap<String, String> startedInputsMap = HashMultimap.create();
    TezUmbilical umbilical = mock(TezUmbilical.class);
    TezConfiguration tezConf = new TezConfiguration();


    TezTaskAttemptID taId1 = createTaskAttemptID(vertexId, 1);
    TaskSpec task1 = createTaskSpec(taId1, "vertex1");


    TezTaskAttemptID taId2 = createTaskAttemptID(vertexId, 2);
    TaskSpec task2 = createTaskSpec(taId2, "vertex1");


    LogicalIOProcessorRuntimeTask lio1 = new LogicalIOProcessorRuntimeTask(task1, 0, tezConf,
        umbilical, serviceConsumerMetadata, startedInputsMap);


    lio1.initialize();

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.tez.runtime.api.impl.TaskSpec

org.apache.hadoop.mapred.YarnTezDagChild

org.apache.tez.common.ContainerTask

org.apache.tez.dag.api.ProcessorDescriptor

org.apache.tez.dag.app.dag.impl.TaskAttemptImpl

org.apache.tez.dag.app.dag.impl.TaskAttemptImpl$ScheduleTaskattemptTransition

org.apache.tez.dag.app.rm.TestContainerReuse

org.apache.tez.dag.records.TezTaskAttemptID

org.apache.tez.mapreduce.processor.MapUtils

org.apache.tez.mapreduce.processor.reduce.TestReduceProcessor

org.apache.tez.runtime.task.TestTaskExecution

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.