Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.VertexLocationHint

256
257
258
259
260
261
262
263
264
265
266
    }

    // Replace the Edge Managers
    context.setVertexParallelism(
        taskCount,
        new VertexLocationHint(createTaskLocationHintsFromSplits(finalSplits
            .toArray(new InputSplit[finalSplits.size()]))), emMap);

    // Set the actual events for the tasks.
    context.addRootInputEvents(inputName, taskEvents);
  }
View Full Code Here
596
597
598
599
600
601
602
603
604
605
606
    ProcessorDescriptor prewarmProcDescriptor = new ProcessorDescriptor(HivePreWarmProcessor.class.getName());
    prewarmProcDescriptor.setUserPayload(MRHelpers.createUserPayloadFromConf(conf));

    PreWarmContext context = new PreWarmContext(prewarmProcDescriptor, getContainerResource(conf),
        numContainers, new VertexLocationHint(null));

    Map<String, LocalResource> combinedResources = new HashMap<String, LocalResource>();

    combinedResources.putAll(sessionConfig.getSessionResources());
    if (localResources != null) {
View Full Code Here
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
    for (int i = 0; i < vCnt; ++i) {
      VertexPlan vPlan = dagPlan.getVertex(i);
      String vName = vPlan.getName();
      TezVertexID vertexId = TezVertexID.getInstance(dagId, i+1);
      VertexImpl v = null;
      VertexLocationHint locationHint = DagTypeConverters.convertFromDAGPlan(
          vPlan.getTaskLocationHintList());
      if (useCustomInitializer) {
        if (customInitializer == null) {
          v = new VertexImplWithControlledInitializerManager(vertexId, vPlan, vPlan.getName(), conf,
              dispatcher.getEventHandler(), taskAttemptListener,
View Full Code Here
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
  private static VertexImpl createVertex(DAGImpl dag, String vertexName, int vId) {
    TezVertexID vertexId = TezBuilderUtils.newVertexID(dag.getID(), vId);

    VertexPlan vertexPlan = dag.getJobPlan().getVertex(vId);
    VertexLocationHint vertexLocationHint = DagTypeConverters
        .convertFromDAGPlan(vertexPlan.getTaskLocationHintList());

    VertexImpl v = new VertexImpl(
        vertexId, vertexPlan, vertexName, dag.conf,
        dag.eventHandler, dag.taskAttemptListener,
View Full Code Here
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
    for (int i = 0; i < vCnt; ++i) {
      VertexPlan vPlan = dagPlan.getVertex(i);
      String vName = vPlan.getName();
      TezVertexID vertexId = TezVertexID.getInstance(dagId, i+1);
      VertexImpl v = null;
      VertexLocationHint locationHint = DagTypeConverters.convertFromDAGPlan(
          vPlan.getTaskLocationHintList());
      if (useCustomInitializer) {
        v = new VertexImplWithCustomInitializer(vertexId, vPlan, vPlan.getName(), conf,
          dispatcher.getEventHandler(), taskAttemptListener,
          clock, thh, appContext, locationHint, dispatcher);
View Full Code Here
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
    private VertexImpl createVertex(DAGImpl dag, String vertexName, int vId) {
      TezVertexID vertexId = TezBuilderUtils.newVertexID(dag.getID(), vId);

      VertexPlan vertexPlan = dag.getJobPlan().getVertex(vId);
      VertexLocationHint vertexLocationHint = DagTypeConverters
          .convertFromDAGPlan(vertexPlan.getTaskLocationHintList());

      VertexImpl v = new VertexImpl(
          vertexId, vertexPlan, vertexName, dag.conf,
          dag.eventHandler, dag.taskAttemptListener,
View Full Code Here
423
424
425
426
427
428
429
430
431
432
433
434
            doPreWarm = false;
          }
        }
        if (doPreWarm) {
          LOG.info("Pre-warming Session");
          VertexLocationHint vertexLocationHint =
              new VertexLocationHint(null);
          ProcessorDescriptor sleepProcDescriptor =
            new ProcessorDescriptor(SleepProcessor.class.getName());
          SleepProcessor.SleepProcessorConfig sleepProcessorConfig =
            new SleepProcessor.SleepProcessorConfig(4000);
          sleepProcDescriptor.setUserPayload(
View Full Code Here
886
887
888
889
890
891
892
893
894
895
896
    private VertexImpl createVertex(DAGImpl dag, String vertexName, int vId) {
      TezVertexID vertexId = TezBuilderUtils.newVertexID(dag.getID(), vId);

      VertexPlan vertexPlan = dag.getJobPlan().getVertex(vId);
      VertexLocationHint vertexLocationHint = DagTypeConverters
          .convertFromDAGPlan(vertexPlan.getTaskLocationHintList());

      VertexImpl v = new VertexImpl(
          vertexId, vertexPlan, vertexName, dag.conf,
          dag.eventHandler, dag.taskAttemptListener,
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
        Preconditions
            .checkState(
                managedVertex.getTotalTasks() == -1,
                "Parallelism for the vertex should be set to -1 if the InputInitializer is setting parallelism");
        RootInputConfigureVertexTasksEvent cEvent = (RootInputConfigureVertexTasksEvent) event;
        managedVertex.setVertexLocationHint(new VertexLocationHint(cEvent
            .getNumTasks(), cEvent.getTaskLocationHints()));
        managedVertex.setParallelism(cEvent.getNumTasks(), null);
      }
      if (event instanceof RootInputUpdatePayloadEvent) {
        // No tasks should have been started yet. Checked by initial state check.
View Full Code Here
426
427
428
429
430
431
432
433
434
435
436
437
            doPreWarm = false;
          }
        }
        if (doPreWarm) {
          LOG.info("Pre-warming Session");
          VertexLocationHint vertexLocationHint =
              new VertexLocationHint(null);
          ProcessorDescriptor sleepProcDescriptor =
            new ProcessorDescriptor(SleepProcessor.class.getName());
          SleepProcessor.SleepProcessorConfig sleepProcessorConfig =
            new SleepProcessor.SleepProcessorConfig(4000);
          sleepProcDescriptor.setUserPayload(
View Full Code Here
TOP

Related Classes of org.apache.tez.dag.api.VertexLocationHint

  • org.apache.hadoop.hive.ql.exec.tez.CustomPartitionVertex
  • org.apache.hadoop.hive.ql.exec.tez.DagUtils
  • org.apache.tez.dag.app.dag.impl.DAGImpl
  • org.apache.tez.dag.app.dag.impl.DAGImpl$InitTransition
  • org.apache.tez.dag.app.dag.impl.RootInputVertexManager
  • org.apache.tez.dag.app.dag.impl.TestVertexImpl
  • org.apache.tez.dag.history.events.TestHistoryEventsProtoConversion
  • org.apache.tez.mapreduce.examples.OrderedWordCount
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.