Package org.apache.tez.runtime.api.impl

Examples of org.apache.tez.runtime.api.impl.TaskSpec

96
97
98
99
100
101
102
103
104
  }

  private TaskSpec createTaskSpec(TezTaskAttemptID taskAttemptID, String vertexName) {
    ProcessorDescriptor processorDesc = createProcessorDescriptor();
    TaskSpec taskSpec = new TaskSpec(taskAttemptID, vertexName, processorDesc,
        createInputSpecList(), createOutputSpecList(), null);
    return taskSpec;
  }
View Full Code Here
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
        Map<String, TezLocalResource> additionalResources = containerTask.getAdditionalResources();
        if (LOG.isDebugEnabled()) {
          LOG.debug("Additional Resources added to container: " + additionalResources);
        }
        processAdditionalResources(additionalResources, defaultConf);
        final TaskSpec taskSpec = containerTask.getTaskSpec();
        if (LOG.isDebugEnabled()) {
          LOG.debug("New container task context:"
              + taskSpec.toString());
        }

        try {
          taskLock.writeLock().lock();
          currentTaskAttemptID = taskSpec.getTaskAttemptID();
          TezVertexID newVertexId =
              currentTaskAttemptID.getTaskID().getVertexID();
          currentTaskComplete.set(false);

          if (lastVertexId != null) {
            if (!lastVertexId.equals(newVertexId)) {
              objectRegistry.clearCache(ObjectLifeCycle.VERTEX);
            }
            if (!lastVertexId.getDAGId().equals(newVertexId.getDAGId())) {
              objectRegistry.clearCache(ObjectLifeCycle.DAG);
              startedInputsMap = HashMultimap.create();
            }
          }
          lastVertexId = newVertexId;
          updateLoggers(currentTaskAttemptID);

          currentTask = createLogicalTask(attemptNumber, taskSpec,
              defaultConf, tezUmbilical, serviceConsumerMetadata);
        } finally {
          taskLock.writeLock().unlock();
        }

        final EventMetaData sourceInfo = new EventMetaData(
            EventProducerConsumerType.SYSTEM,
            taskSpec.getVertexName(), "", currentTaskAttemptID);
        currentSourceInfo = sourceInfo;

        // TODO Initiate Java VM metrics
        // JvmMetrics.initSingleton(containerId.toString(), job.getSessionId());
View Full Code Here
168
169
170
171
172
173
174
175
176
177
178
        new InputDescriptor(LocalMergedInput.class.getName()), 1);
    OutputSpec reduceOutputSpec = new OutputSpec("NullDestinationVertex",
        new OutputDescriptor(MROutputLegacy.class.getName()), 1);

    // Now run a reduce
    TaskSpec taskSpec = new TaskSpec(
        TezTestUtils.getMockTaskAttemptId(0, 1, 0, 0),
        reduceVertexName,
        reduceProcessorDesc,
        Collections.singletonList(reduceInputSpec),
        Collections.singletonList(reduceOutputSpec), null);
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
    ProcessorDescriptor mapProcessorDesc = new ProcessorDescriptor(
        MapProcessor.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(jobConf));
   
    Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>();

    TaskSpec taskSpec = new TaskSpec(
        TezTestUtils.getMockTaskAttemptId(0, 0, mapId, 0),
        vertexName,
        mapProcessorDesc,
        inputSpecs,
        outputSpecs, null);
View Full Code Here
513
514
515
516
517
518
519
520
521
  TaskSpec createRemoteTaskSpec() {
    Vertex vertex = getVertex();
    ProcessorDescriptor procDesc = vertex.getProcessorDescriptor();
    int taskId = getTaskID().getId();
    return new TaskSpec(getID(), vertex.getName(), procDesc,
        vertex.getInputSpecList(taskId), vertex.getOutputSpecList(taskId),
        vertex.getGroupInputSpecList(taskId));
  }
View Full Code Here
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
      // TODO Creating the remote task here may not be required in case of
      // recovery.

      // Create the remote task.
      TaskSpec remoteTaskSpec = ta.createRemoteTaskSpec();
      // Create startTaskRequest

      String[] requestHosts = new String[0];
      String[] requestRacks = new String[0];
View Full Code Here
893
894
895
896
897
898
899
900
901
902
903
  private AMSchedulerEventTALaunchRequest createLaunchRequestEvent(
    TezTaskAttemptID taID, TaskAttempt ta, Resource capability,
    String[] hosts, String[] racks, Priority priority,
    ContainerContext containerContext) {
    AMSchedulerEventTALaunchRequest lr = new AMSchedulerEventTALaunchRequest(
      taID, capability, new TaskSpec(taID, "vertexName",
      new ProcessorDescriptor("processorClassName"),
      Collections.singletonList(new InputSpec("vertexName",
        new InputDescriptor("inputClassName"), 1)),
      Collections.singletonList(new OutputSpec("vertexName",
        new OutputDescriptor("outputClassName"), 1)), null), ta, hosts, racks,
View Full Code Here
198
199
200
201
202
203
204
205
206
207
208
    ProcessorDescriptor mapProcessorDesc = new ProcessorDescriptor(
        MapProcessor.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(jobConf));
   
    Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>();

    TaskSpec taskSpec = new TaskSpec(
        TezTestUtils.getMockTaskAttemptId(0, 0, mapId, 0),
        "testuser",
        vertexName,
        mapProcessorDesc,
        inputSpecs,
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
   
    InputSpec reduceInputSpec = new InputSpec(mapVertexName, new InputDescriptor(LocalMergedInput.class.getName()), 1);
    OutputSpec reduceOutputSpec = new OutputSpec("NullDestinationVertex", new OutputDescriptor(MROutput.class.getName()), 1);
   
    // Now run a reduce
    TaskSpec taskSpec = new TaskSpec(
        TezTestUtils.getMockTaskAttemptId(0, 1, 0, 0),
        "testUser",
        reduceVertexName,
        reduceProcessorDesc,
        Collections.singletonList(reduceInputSpec),
View Full Code Here
59
60
61
62
63
64
65
66
67
  @Override
  public void readFields(DataInput in) throws IOException {
    shouldDie = in.readBoolean();
    boolean taskComing = in.readBoolean();
    if (taskComing) {
      taskSpec = new TaskSpec();
      taskSpec.readFields(in);
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.api.impl.TaskSpec

  • org.apache.hadoop.mapred.YarnTezDagChild
  • org.apache.tez.common.ContainerTask
  • org.apache.tez.dag.api.ProcessorDescriptor
  • org.apache.tez.dag.app.dag.impl.TaskAttemptImpl
  • org.apache.tez.dag.app.dag.impl.TaskAttemptImpl$ScheduleTaskattemptTransition
  • org.apache.tez.dag.app.rm.TestContainerReuse
  • org.apache.tez.dag.records.TezTaskAttemptID
  • org.apache.tez.mapreduce.processor.MapUtils
  • org.apache.tez.mapreduce.processor.reduce.TestReduceProcessor
  • org.apache.tez.runtime.task.TestTaskExecution
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.