Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.OutputDescriptor

943
944
945
946
947
948
949
950
951
952
953

    // final vertices need to have at least one output
    if (!hasChildren) {
      v.addOutput("out_"+work.getName(),
          new OutputDescriptor(MROutput.class.getName())
          .setUserPayload(MRHelpers.createUserPayloadFromConf(conf)));
    }

    return v;
  }
View Full Code Here
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
  public void setAdditionalOutputs(List<RootInputLeafOutputProto> outputs) {
    LOG.info("setting additional outputs for vertex " + this.vertexName);
    this.additionalOutputs = Maps.newHashMapWithExpectedSize(outputs.size());
    this.outputCommitters = Maps.newHashMapWithExpectedSize(outputs.size());
    for (RootInputLeafOutputProto output : outputs) {
      OutputDescriptor od = DagTypeConverters
          .convertOutputDescriptorFromDAGPlan(output.getEntityDescriptor());

      this.additionalOutputs.put(
          output.getName(),
          new RootInputLeafOutputDescriptor<OutputDescriptor>(output.getName(), od,
View Full Code Here
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
        dummyTaskCount, dummyTaskResource);
   
    DAG dag = new DAG("testDag");
    String groupName1 = "uv12";
    org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
    OutputDescriptor outDesc = new OutputDescriptor("output.class");
    uv12.addOutput("uvOut", outDesc, TotalCountingOutputCommitter.class);
    v3.addOutput("uvOut", outDesc, TotalCountingOutputCommitter.class);
   
    GroupInputEdge e1 = new GroupInputEdge(uv12, v3,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
            new OutputDescriptor("dummy output class"),
            new InputDescriptor("dummy input class")),
            new InputDescriptor("merge.class"));
   
    dag.addVertex(v1);
    dag.addVertex(v2);
View Full Code Here
278
279
280
281
282
283
284
285
286
287
288
      if (i != 0) {
        dag.addEdge(new Edge(vertices.get(i-1),
            vertices.get(i), new EdgeProperty(
                DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                new OutputDescriptor(
                    OnFileSortedOutput.class.getName()),
                new InputDescriptor(
                    ShuffledMergedInputLegacy.class.getName()))));
      }
    }
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
140
141
  }

  @Test(timeout=120000)
  public void testBasicRecovery() throws Exception {
    DAG dag = SimpleVTestDAG.createDAG("FailingCommitterDAG", null);
    OutputDescriptor od =
        new OutputDescriptor(MultiAttemptDAG.NoOpOutput.class.getName());
    od.setUserPayload(new
        MultiAttemptDAG.FailingOutputCommitter.FailingOutputCommitterConfig(true)
            .toUserPayload());
    dag.getVertex("v3").addOutput("FailingOutput", od,
        MultiAttemptDAG.FailingOutputCommitter.class);
    runDAGAndVerify(dag, State.FAILED);
View Full Code Here
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
    dag.addVertex(stage2Vertex);
    dag.addVertex(stage3Vertex);

    Edge edge1 = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(
        OnFileSortedOutput.class.getName()), new InputDescriptor(
                ShuffledMergedInputLegacy.class.getName())));
    Edge edge2 = new Edge(stage2Vertex, stage3Vertex, new EdgeProperty(
        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(
        OnFileSortedOutput.class.getName()), new InputDescriptor(
                ShuffledMergedInputLegacy.class.getName())));

    dag.addEdge(edge1);
    dag.addEdge(edge2);
View Full Code Here
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
  public void setAdditionalOutputs(List<RootInputLeafOutputProto> outputs) {
    LOG.info("setting additional outputs for vertex " + this.vertexName);
    this.additionalOutputs = Maps.newHashMapWithExpectedSize(outputs.size());
    this.outputCommitters = Maps.newHashMapWithExpectedSize(outputs.size());
    for (RootInputLeafOutputProto output : outputs) {
      OutputDescriptor od = DagTypeConverters
          .convertOutputDescriptorFromDAGPlan(output.getEntityDescriptor());

      this.additionalOutputs.put(
          output.getName(),
          new RootInputLeafOutputDescriptor<OutputDescriptor>(output.getName(), od,
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
   
    // Third request - output
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    TezOutputContext e3OutputContext1 = createTestOutputContext();
    OutputDescriptor e3OutDesc2 = createTestOutputDescriptor();
    dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2);
   
    dist.makeInitialAllocations();
   
    // Total available: 70% of 10K = 7000
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
   
    // Third request - output
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    TezOutputContext e3OutputContext1 = createTestOutputContext();
    OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
    dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc1);
   
    // Fourth request - processor
    MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
    TezProcessorContext e4ProcessorContext1 = createTestProcessortContext();
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
   
    // Third request - output
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    TezOutputContext e3OutputContext1 = createTestOutputContext();
    OutputDescriptor e3OutDesc2 = createTestOutputDescriptor();
    dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2);
   
    dist.makeInitialAllocations();
   
    // Total available: 50% of 10K = 7000
View Full Code Here
TOP

Related Classes of org.apache.tez.dag.api.OutputDescriptor

  • org.apache.hadoop.hive.ql.exec.tez.DagUtils
  • org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder
  • org.apache.tez.dag.app.dag.impl.TestDAGImpl
  • org.apache.tez.dag.app.dag.impl.TestVertexScheduler
  • org.apache.tez.dag.app.dag.impl.VertexImpl
  • org.apache.tez.dag.app.rm.TestContainerReuse
  • org.apache.tez.dag.history.utils.TestDAGUtils
  • org.apache.tez.dag.library.vertexmanager.TestShuffleVertexManager
  • org.apache.tez.mapreduce.client.YARNRunner
  • org.apache.tez.mapreduce.examples.FilterLinesByWord
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.