Package org.apache.tez.runtime.api.impl

Examples of org.apache.tez.runtime.api.impl.TaskSpec

726
727
728
729
730
731
732
733
734
735
736
  private AMSchedulerEventTALaunchRequest createLaunchRequestEvent(
    TezTaskAttemptID taID, TaskAttempt ta, Resource capability,
    String[] hosts, String[] racks, Priority priority,
    ContainerContext containerContext) {
    AMSchedulerEventTALaunchRequest lr = new AMSchedulerEventTALaunchRequest(
      taID, capability, new TaskSpec(taID, "user", "vertexName",
      new ProcessorDescriptor("processorClassName"),
      Collections.singletonList(new InputSpec("vertexName",
        new InputDescriptor("inputClassName"), 1)),
      Collections.singletonList(new OutputSpec("vertexName",
        new OutputDescriptor("outputClassName"), 1))), ta, hosts, racks,
View Full Code Here
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
                + containerTask.getTaskSpec().getTaskAttemptID()));
        if (containerTask.shouldDie()) {
          return;
        }
        taskCount++;
        final TaskSpec taskSpec = containerTask.getTaskSpec();
        if (LOG.isDebugEnabled()) {
          LOG.debug("New container task context:"
              + taskSpec.toString());
        }

        try {
          taskLock.writeLock().lock();
          currentTaskAttemptID = taskSpec.getTaskAttemptID();
          TezVertexID newVertexId =
              currentTaskAttemptID.getTaskID().getVertexID();

          if (lastVertexId != null) {
            if (!lastVertexId.equals(newVertexId)) {
              objectRegistry.clearCache(ObjectLifeCycle.VERTEX);
            }
            if (!lastVertexId.getDAGId().equals(newVertexId.getDAGId())) {
              objectRegistry.clearCache(ObjectLifeCycle.DAG);
            }
          }
          lastVertexId = newVertexId;
          updateLoggers(currentTaskAttemptID);

          currentTask = createLogicalTask(attemptNumber, taskSpec,
              defaultConf, tezUmbilical, serviceConsumerMetadata);
        } finally {
          taskLock.writeLock().unlock();
        }

        final EventMetaData sourceInfo = new EventMetaData(
            EventProducerConsumerType.SYSTEM,
            taskSpec.getVertexName(), "", currentTaskAttemptID);
        currentSourceInfo = sourceInfo;

        // TODO Initiate Java VM metrics
        // JvmMetrics.initSingleton(containerId.toString(), job.getSessionId());
        childUGI = UserGroupInformation.createRemoteUser(System
View Full Code Here
284
285
286
287
288
289
290
291
292
  TaskSpec createRemoteTaskSpec() {
    Vertex vertex = getVertex();
    ProcessorDescriptor procDesc = vertex.getProcessorDescriptor();
    DAG dag = vertex.getDAG();
    int taskId = getTaskID().getId();
    return new TaskSpec(getID(), dag.getUserName(),
        vertex.getName(), procDesc, vertex.getInputSpecList(taskId),
        vertex.getOutputSpecList(taskId));
  }
View Full Code Here
814
815
816
817
818
819
820
821
822
823
824
      // TODO Creating the remote task here may not be required in case of
      // recovery.

      // Create the remote task.
      TaskSpec remoteTaskSpec = ta.createRemoteTaskSpec();
      // Create startTaskRequest

      String[] requestHosts = new String[0];
      String[] requestRacks = new String[0];
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
  @Override
  public void readFields(DataInput in) throws IOException {
    shouldDie = in.readBoolean();
    boolean taskComing = in.readBoolean();
    if (taskComing) {
      taskSpec = new TaskSpec();
      taskSpec.readFields(in);
    }
    int numAdditionalResources = in.readInt();
    additionalResources = Maps.newHashMap();
    if (numAdditionalResources != -1) {
View Full Code Here
168
169
170
171
172
173
174
175
176
177
178
        new InputDescriptor(LocalMergedInput.class.getName()), 1);
    OutputSpec reduceOutputSpec = new OutputSpec("NullDestinationVertex",
        new OutputDescriptor(MROutputLegacy.class.getName()), 1);

    // Now run a reduce
    TaskSpec taskSpec = new TaskSpec(
        TezTestUtils.getMockTaskAttemptId(0, 1, 0, 0),
        dagName,
        reduceVertexName,
        reduceProcessorDesc,
        Collections.singletonList(reduceInputSpec),
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
    Multimap<String, String> startedInputsMap = HashMultimap.create();
    TezUmbilical umbilical = mock(TezUmbilical.class);
    TezConfiguration tezConf = new TezConfiguration();

    TezTaskAttemptID taId1 = createTaskAttemptID(vertexId, 1);
    TaskSpec task1 = createTaskSpec(taId1, "dag1", "vertex1");

    TezTaskAttemptID taId2 = createTaskAttemptID(vertexId, 2);
    TaskSpec task2 = createTaskSpec(taId2, "dag2", "vertex1");

    LogicalIOProcessorRuntimeTask lio1 = new LogicalIOProcessorRuntimeTask(task1, 0, tezConf,
        umbilical, serviceConsumerMetadata, startedInputsMap);

    lio1.initialize();
View Full Code Here
97
98
99
100
101
102
103
104
105
106
  }

  private TaskSpec createTaskSpec(TezTaskAttemptID taskAttemptID,
      String dagName, String vertexName) {
    ProcessorDescriptor processorDesc = createProcessorDescriptor();
    TaskSpec taskSpec = new TaskSpec(taskAttemptID,
        dagName, vertexName, processorDesc,
        createInputSpecList(), createOutputSpecList(), null);
    return taskSpec;
  }
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
    ProcessorDescriptor mapProcessorDesc = new ProcessorDescriptor(
        MapProcessor.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(jobConf));
   
    Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>();

    TaskSpec taskSpec = new TaskSpec(
        TezTestUtils.getMockTaskAttemptId(0, 0, mapId, 0),
        dagName, vertexName,
        mapProcessorDesc,
        inputSpecs,
        outputSpecs, null);
View Full Code Here
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
              }
            }), defaultConf);
        RelocalizationUtils.addUrlsToClassPath(downloadedUrls);

        LOG.info("Done localizing additional resources");
        final TaskSpec taskSpec = containerTask.getTaskSpec();
        if (LOG.isDebugEnabled()) {
          LOG.debug("New container task context:"
              + taskSpec.toString());
        }

        try {
          taskLock.writeLock().lock();
          currentTaskAttemptID = taskSpec.getTaskAttemptID();
          TezVertexID newVertexId =
              currentTaskAttemptID.getTaskID().getVertexID();
          currentTaskComplete.set(false);

          if (lastVertexId != null) {
            if (!lastVertexId.equals(newVertexId)) {
              objectRegistry.clearCache(ObjectLifeCycle.VERTEX);
            }
            if (!lastVertexId.getDAGId().equals(newVertexId.getDAGId())) {
              objectRegistry.clearCache(ObjectLifeCycle.DAG);
              startedInputsMap = HashMultimap.create();
            }
          }
          lastVertexId = newVertexId;
          TezUtils.updateLoggers(currentTaskAttemptID.toString());
          loggerAddend = currentTaskAttemptID.toString() + "_post";
         
          currentTask = createLogicalTask(attemptNumber, taskSpec,
              defaultConf, tezUmbilical, serviceConsumerMetadata);
         
          taskNonOobHeartbeatCounter = 0;
          nextHeartbeatNumToLog = (Math.max(1,
              (int) (LOG_COUNTER_START_INTERVAL / (amPollInterval == 0 ? 0.000001f
                  : (float) amPollInterval))));
        } finally {
          taskLock.writeLock().unlock();
        }

        final EventMetaData sourceInfo = new EventMetaData(
            EventProducerConsumerType.SYSTEM,
            taskSpec.getVertexName(), "", currentTaskAttemptID);
        currentSourceInfo = sourceInfo;

        // TODO Initiate Java VM metrics
        // JvmMetrics.initSingleton(containerId.toString(), job.getSessionId());
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.api.impl.TaskSpec

  • org.apache.hadoop.mapred.YarnTezDagChild
  • org.apache.tez.common.ContainerTask
  • org.apache.tez.dag.api.ProcessorDescriptor
  • org.apache.tez.dag.app.dag.impl.TaskAttemptImpl
  • org.apache.tez.dag.app.dag.impl.TaskAttemptImpl$ScheduleTaskattemptTransition
  • org.apache.tez.dag.app.rm.TestContainerReuse
  • org.apache.tez.dag.records.TezTaskAttemptID
  • org.apache.tez.mapreduce.processor.MapUtils
  • org.apache.tez.mapreduce.processor.reduce.TestReduceProcessor
  • org.apache.tez.runtime.task.TestTaskExecution
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.