Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.ProcessorDescriptor

467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
    int stage1NumTasks = genSplitsInAM ? -1 : inputSplitInfo.getNumTasks();
    Class<? extends TezRootInputInitializer> inputInitializerClazz =
        genSplitsInAM ? (initializerClass == null ? MRInputAMSplitGenerator.class : initializerClass)
        : null;
    LOG.info("Using initializer class: " + initializerClass);
    Vertex stage1Vertex = new Vertex("map", new ProcessorDescriptor(
        MapProcessor.class.getName()).setUserPayload(stage1Payload),
        stage1NumTasks, Resource.newInstance(256, 1));
    MRHelpers.addMRInput(stage1Vertex, stage1InputPayload, inputInitializerClazz);
    Vertex stage2Vertex = new Vertex("ireduce", new ProcessorDescriptor(
        ReduceProcessor.class.getName()).setUserPayload(
        MRHelpers.createUserPayloadFromConf(stage2Conf)),
        1, Resource.newInstance(256, 1));
    Vertex stage3Vertex = new Vertex("reduce", new ProcessorDescriptor(
        ReduceProcessor.class.getName()).setUserPayload(stage3Payload),
        1, Resource.newInstance(256, 1));
    MRHelpers.addMROutputLegacy(stage3Vertex, stage3Payload);

    Map<String, LocalResource> commonLocalResources = new HashMap<String, LocalResource>();
View Full Code Here
522
523
524
525
526
527
528
529
530
531
532
    return getVertexID().getDAGId();
  }

  TaskSpec createRemoteTaskSpec() {
    Vertex vertex = getVertex();
    ProcessorDescriptor procDesc = vertex.getProcessorDescriptor();
    int taskId = getTaskID().getId();
    return new TaskSpec(getID(),
        vertex.getDAG().getName(),
        vertex.getName(), procDesc,
        vertex.getInputSpecList(taskId), vertex.getOutputSpecList(taskId),
View Full Code Here
750
751
752
753
754
755
756
757
758
759
760
761
762
  // TODO remove hacky name lookup
  @Override
  public boolean needsWaitAfterOutputConsumable() {
    Vertex vertex = getVertex();
    ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor();
    if (processorDescriptor != null &&
        processorDescriptor.getClassName().contains("InitialTaskWithInMemSort")) {
      return true;
    } else {
      return false;
    }
  }
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
    dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc1);
   
    // Fourth request - processor
    MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
    TezProcessorContext e4ProcessorContext1 = createTestProcessortContext();
    ProcessorDescriptor e4ProcessorDesc1 = createTestProcessorDescriptor();
    dist.requestMemory(5000, e4Callback, e4ProcessorContext1, e4ProcessorDesc1);
   
   
    dist.makeInitialAllocations();
   
View Full Code Here
232
233
234
235
236
237
238
239
240
    doReturn("OutputClass").when(desc).getClassName();
    return desc;
  }

  protected ProcessorDescriptor createTestProcessorDescriptor() {
    ProcessorDescriptor desc = mock(ProcessorDescriptor.class);
    doReturn("ProcessorClass").when(desc).getClassName();
    return desc;
  }
View Full Code Here
648
649
650
651
652
653
654
655
656
657
658
659
660
  private Vertex newVertex( FlowNode flowNode, Configuration conf, int parallelism )
    {
    conf.set( FlowNode.CASCADING_FLOW_NODE, pack( flowNode, conf ) ); // todo: pack into payload directly

    ProcessorDescriptor descriptor = ProcessorDescriptor.create( FlowProcessor.class.getName() );

    descriptor.setUserPayload( getPayload( conf ) );

    Vertex vertex = Vertex.create( flowNode.getID(), descriptor, parallelism );

    if( environment != null )
      vertex.setTaskEnvironment( environment );
View Full Code Here
TOP

Related Classes of org.apache.tez.dag.api.ProcessorDescriptor

  • cascading.flow.tez.Hadoop2TezFlowStep
  • org.apache.hadoop.hive.ql.exec.tez.DagUtils
  • org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder
  • org.apache.tez.dag.app.dag.impl.TaskAttemptImpl
  • org.apache.tez.dag.app.dag.impl.TaskImpl
  • org.apache.tez.dag.app.dag.impl.TestDAGImpl
  • org.apache.tez.dag.app.rm.TestContainerReuse
  • org.apache.tez.mapreduce.client.YARNRunner
  • org.apache.tez.mapreduce.examples.FilterLinesByWord
  • org.apache.tez.mapreduce.examples.FilterLinesByWordOneToOne
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.