Examples of org.apache.tez.dag.api.InputDescriptor

org.apache.tez.dag.api.InputDescriptor
Describes the {@link Input}


    // Configure the Input for stage1
    Class<? extends TezRootInputInitializer> initializerClazz = generateSplitsInClient ? null
        : MRInputAMSplitGenerator.class;
    stage1Vertex.addInput("MRInput",
        new InputDescriptor(MRInputLegacy.class.getName())
            .setUserPayload(MRHelpers.createMRInputPayload(stage1Payload, null)),
        initializerClazz);


    // Setup stage2 Vertex
    Vertex stage2Vertex = new Vertex("stage2", new ProcessorDescriptor(
        FilterByWordOutputProcessor.class.getName()).setUserPayload(MRHelpers
        .createUserPayloadFromConf(stage2Conf)), stage1NumTasks,
        MRHelpers.getMapResource(stage2Conf));
    stage2Vertex.setJavaOpts(MRHelpers.getReduceJavaOpts(stage2Conf)).setTaskLocalResources(commonLocalResources);
    Map<String, String> stage2Env = new HashMap<String, String>();
    MRHelpers.updateEnvironmentForMRTasks(stage2Conf, stage2Env, false);
    stage2Vertex.setTaskEnvironment(stage2Env);


    // Configure the Output for stage2
    stage2Vertex.addOutput("MROutput",
        new OutputDescriptor(MROutput.class.getName()).setUserPayload(MRHelpers
            .createUserPayloadFromConf(stage2Conf)));


    DAG dag = new DAG("FilterLinesByWord");
    Edge edge = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
        DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(
            OnFileUnorderedKVOutput.class.getName()), new InputDescriptor(
            ShuffledUnorderedKVInput.class.getName())));
    dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge);


    LOG.info("Submitting DAG to Tez Session");
    DAGClient dagClient = tezSession.submitDAG(dag);

View Full Code Here


  @Override
  public void readFields(DataInput in) throws IOException {
    sourceVertexName = in.readUTF();
    physicalEdgeCount = in.readInt();
    inputDescriptor = new InputDescriptor();
    inputDescriptor.readFields(in);
  }

View Full Code Here

            vertices.get(i), new EdgeProperty(
                DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                new OutputDescriptor(
                    OnFileSortedOutput.class.getName()),
                new InputDescriptor(
                    ShuffledMergedInputLegacy.class.getName()))));
      }
    }
    return dag;
  }

View Full Code Here

    return processorContext;
  }


  private LogicalInput createInput(InputSpec inputSpec, InputContext inputContext) {
    LOG.info("Creating Input");
    InputDescriptor inputDesc = inputSpec.getInputDescriptor();
    Input input = ReflectionUtils.createClazzInstance(inputDesc.getClassName(),
        new Class[]{InputContext.class, Integer.TYPE},
        new Object[]{inputContext, inputSpec.getPhysicalEdgeCount()});
    if (!(input instanceof LogicalInput)) {
      throw new TezUncheckedException(inputDesc.getClass().getName()
          + " is not a sub-type of LogicalInput."
          + " Only LogicalInput sub-types supported by LogicalIOProcessor.");
    }
    return (LogicalInput) input;
  }

View Full Code Here


  @Override
  public void readFields(DataInput in) throws IOException {
    sourceVertexName = StringInterner.weakIntern(in.readUTF());
    physicalEdgeCount = in.readInt();
    inputDescriptor = new InputDescriptor();
    inputDescriptor.readFields(in);
  }

View Full Code Here


    if (dagVertexGroups != null && !dagVertexGroups.isEmpty()) {
      List<GroupInputSpec> groupSpecList = Lists.newLinkedList();
      for (VertexGroupInfo groupInfo : dagVertexGroups.values()) {
        if (groupInfo.edgeMergedInputs.containsKey(getName())) {
          InputDescriptor mergedInput = groupInfo.edgeMergedInputs.get(getName());
          groupSpecList.add(new GroupInputSpec(groupInfo.groupName,
              Lists.newLinkedList(groupInfo.groupMembers), mergedInput));
        }
      }
      if (!groupSpecList.isEmpty()) {

View Full Code Here

  @Override
  public void setAdditionalInputs(List<RootInputLeafOutputProto> inputs) {
    this.rootInputDescriptors = Maps.newHashMapWithExpectedSize(inputs.size());
    for (RootInputLeafOutputProto input : inputs) {


      InputDescriptor id = DagTypeConverters
          .convertInputDescriptorFromDAGPlan(input.getIODescriptor());


      this.rootInputDescriptors
          .put(
              input.getName(),

View Full Code Here

    int numMembers = in.readInt();
    groupVertices = Lists.newArrayListWithCapacity(numMembers);
    for (int i=0; i<numMembers; ++i) {
      groupVertices.add(StringInterner.weakIntern(Text.readString(in)));
    }
    mergedInputDescriptor = new InputDescriptor();
    mergedInputDescriptor.readFields(in);
  }

View Full Code Here

    dist.setJvmMemory(10000l);


    // First request - ScatterGatherShuffleInput
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    TezInputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor(ShuffledMergedInput.class);
    dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);


    // Second request - BroadcastInput
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    TezInputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor(ShuffledUnorderedKVInput.class);
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);


    // Third request - randomOutput (simulates MROutput)
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    TezOutputContext e3OutputContext1 = createTestOutputContext();

View Full Code Here

    dist.setJvmMemory(10000l);


    // First request - ScatterGatherShuffleInput [weight 6]
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    TezInputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor(ShuffledMergedInput.class);
    dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);


    // Second request - BroadcastInput [weight 2]
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    TezInputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor(ShuffledUnorderedKVInput.class);
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);


    // Third request - randomOutput (simulates MROutput) [weight 1]
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    TezOutputContext e3OutputContext1 = createTestOutputContext();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.tez.dag.api.InputDescriptor

cascading.flow.tez.Hadoop2TezFlowStep

org.apache.hadoop.hive.ql.exec.tez.DagUtils

org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder

org.apache.tez.dag.app.dag.impl.TestDAGImpl

org.apache.tez.dag.app.dag.impl.TestRootInputVertexManager

org.apache.tez.dag.app.dag.impl.TestVertexScheduler

org.apache.tez.dag.app.dag.impl.VertexImpl

org.apache.tez.dag.app.dag.impl.VertexImpl$InitTransition

org.apache.tez.dag.app.rm.TestContainerReuse

org.apache.tez.dag.library.vertexmanager.TestShuffleVertexManager

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.