Examples of TaskSpec

Examples of org.apache.tez.runtime.api.impl.TaskSpec

1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
      // TODO Creating the remote task here may not be required in case of
      // recovery.

      // Create the remote task.
      TaskSpec remoteTaskSpec = ta.createRemoteTaskSpec();
      // Create startTaskRequest

      String[] requestHosts = new String[0];

      // Compute node/rack location request even if re-scheduled.
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

68
69
70
71
72
73
74
75
76
77
78
79
80
81
    TezConfiguration tezConf = new TezConfiguration();
    tezConf.set(TezConfiguration.TEZ_TASK_SCALE_TASK_MEMORY_ALLOCATOR_CLASS,
        ScalingAllocator.class.getName());

    TezTaskAttemptID taId1 = createTaskAttemptID(vertexId, 1);
    TaskSpec task1 = createTaskSpec(taId1, "dag1", "vertex1", 30);

    TezTaskAttemptID taId2 = createTaskAttemptID(vertexId, 2);
    TaskSpec task2 = createTaskSpec(taId2, "dag2", "vertex1", 10);

    LogicalIOProcessorRuntimeTask lio1 = new LogicalIOProcessorRuntimeTask(task1, 0, tezConf, null,
        umbilical, serviceConsumerMetadata, startedInputsMap, null);

    lio1.initialize();
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

113
114
115
116
117
118
119
120
121
122
  }

  private TaskSpec createTaskSpec(TezTaskAttemptID taskAttemptID,
      String dagName, String vertexName, int parallelism) {
    ProcessorDescriptor processorDesc = createProcessorDescriptor();
    TaskSpec taskSpec = new TaskSpec(taskAttemptID,
        dagName, vertexName, parallelism, processorDesc,
        createInputSpecList(), createOutputSpecList(), null);
    return taskSpec;
  }
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

292
293
294
295
296
297
298
299
300
301
          }
        }), defaultConf);
    RelocalizationUtils.addUrlsToClassPath(downloadedUrls);

    LOG.info("Done localizing additional resources");
    final TaskSpec taskSpec = containerTask.getTaskSpec();
    if (LOG.isDebugEnabled()) {
      LOG.debug("New container task context:" + taskSpec.toString());
    }
  }
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

63
64
65
66
67
68
69
70
71
72
73
74
75
76
    Multimap<String, String> startedInputsMap = HashMultimap.create();
    TezUmbilical umbilical = mock(TezUmbilical.class);
    TezConfiguration tezConf = new TezConfiguration();

    TezTaskAttemptID taId1 = createTaskAttemptID(vertexId, 1);
    TaskSpec task1 = createTaskSpec(taId1, "vertex1");

    TezTaskAttemptID taId2 = createTaskAttemptID(vertexId, 2);
    TaskSpec task2 = createTaskSpec(taId2, "vertex1");

    LogicalIOProcessorRuntimeTask lio1 = new LogicalIOProcessorRuntimeTask(task1, 0, tezConf,
        umbilical, serviceConsumerMetadata, startedInputsMap);

    lio1.initialize();
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

96
97
98
99
100
101
102
103
104
  }

  private TaskSpec createTaskSpec(TezTaskAttemptID taskAttemptID, String vertexName) {
    ProcessorDescriptor processorDesc = createProcessorDescriptor();
    TaskSpec taskSpec = new TaskSpec(taskAttemptID, vertexName, processorDesc,
        createInputSpecList(), createOutputSpecList(), null);
    return taskSpec;
  }
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
        Map<String, TezLocalResource> additionalResources = containerTask.getAdditionalResources();
        if (LOG.isDebugEnabled()) {
          LOG.debug("Additional Resources added to container: " + additionalResources);
        }
        processAdditionalResources(additionalResources, defaultConf);
        final TaskSpec taskSpec = containerTask.getTaskSpec();
        if (LOG.isDebugEnabled()) {
          LOG.debug("New container task context:"
              + taskSpec.toString());
        }

        try {
          taskLock.writeLock().lock();
          currentTaskAttemptID = taskSpec.getTaskAttemptID();
          TezVertexID newVertexId =
              currentTaskAttemptID.getTaskID().getVertexID();
          currentTaskComplete.set(false);

          if (lastVertexId != null) {
            if (!lastVertexId.equals(newVertexId)) {
              objectRegistry.clearCache(ObjectLifeCycle.VERTEX);
            }
            if (!lastVertexId.getDAGId().equals(newVertexId.getDAGId())) {
              objectRegistry.clearCache(ObjectLifeCycle.DAG);
              startedInputsMap = HashMultimap.create();
            }
          }
          lastVertexId = newVertexId;
          updateLoggers(currentTaskAttemptID);

          currentTask = createLogicalTask(attemptNumber, taskSpec,
              defaultConf, tezUmbilical, serviceConsumerMetadata);
        } finally {
          taskLock.writeLock().unlock();
        }

        final EventMetaData sourceInfo = new EventMetaData(
            EventProducerConsumerType.SYSTEM,
            taskSpec.getVertexName(), "", currentTaskAttemptID);
        currentSourceInfo = sourceInfo;

        // TODO Initiate Java VM metrics
        // JvmMetrics.initSingleton(containerId.toString(), job.getSessionId());
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

168
169
170
171
172
173
174
175
176
177
178
        new InputDescriptor(LocalMergedInput.class.getName()), 1);
    OutputSpec reduceOutputSpec = new OutputSpec("NullDestinationVertex",
        new OutputDescriptor(MROutputLegacy.class.getName()), 1);

    // Now run a reduce
    TaskSpec taskSpec = new TaskSpec(
        TezTestUtils.getMockTaskAttemptId(0, 1, 0, 0),
        reduceVertexName,
        reduceProcessorDesc,
        Collections.singletonList(reduceInputSpec),
        Collections.singletonList(reduceOutputSpec), null);
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

201
202
203
204
205
206
207
208
209
210
211
    ProcessorDescriptor mapProcessorDesc = new ProcessorDescriptor(
        MapProcessor.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(jobConf));
   
    Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>();

    TaskSpec taskSpec = new TaskSpec(
        TezTestUtils.getMockTaskAttemptId(0, 0, mapId, 0),
        vertexName,
        mapProcessorDesc,
        inputSpecs,
        outputSpecs, null);
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

513
514
515
516
517
518
519
520
521
  TaskSpec createRemoteTaskSpec() {
    Vertex vertex = getVertex();
    ProcessorDescriptor procDesc = vertex.getProcessorDescriptor();
    int taskId = getTaskID().getId();
    return new TaskSpec(getID(), vertex.getName(), procDesc,
        vertex.getInputSpecList(taskId), vertex.getOutputSpecList(taskId),
        vertex.getGroupInputSpecList(taskId));
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.