Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.EdgeProperty

659
660
661
662
663
664
665
666
667
668
669
    for (int i = 0; i < vertices.size(); ++i) {
      dag.addVertex(vertices.get(i));
      if (i != 0) {
        dag.addEdge(new Edge(vertices.get(i-1),
            vertices.get(i), new EdgeProperty(
                DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                new OutputDescriptor(
                    OnFileSortedOutput.class.getName()),
                new InputDescriptor(
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
    }
    DAG dag = new DAG(name);
    Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
    return dag;
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
    Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    dag.addVertex(v1).addVertex(v2).addVertex(v3);
    dag.addEdge(new Edge(v1, v3,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
    dag.addEdge(new Edge(v2, v3,
            new EdgeProperty(DataMovementType.SCATTER_GATHER,
                DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                TestOutput.getOutputDesc(payload),
                TestInput.getInputDesc(payload))));
    return dag;
View Full Code Here
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
    v3.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
        FailOnAttemptVertexManagerPlugin.class.getName())
        .setUserPayload(new String("3").getBytes()));
    dag.addVertex(v1).addVertex(v2).addVertex(v3);
    dag.addEdge(new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
    dag.addEdge(new Edge(v2, v3,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
    return dag;
View Full Code Here
354
355
356
357
358
359
360
361
362
363
364
    OutputDescriptor outDesc = new OutputDescriptor("output.class");
    uv12.addOutput("uvOut", outDesc, TotalCountingOutputCommitter.class);
    v3.addOutput("uvOut", outDesc, TotalCountingOutputCommitter.class);
   
    GroupInputEdge e1 = new GroupInputEdge(uv12, v3,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
            new OutputDescriptor("dummy output class"),
            new InputDescriptor("dummy input class")),
            new InputDescriptor("merge.class"));
   
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
    DAG dag = new DAG(name);
    Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
    dag.addVertex(v3).addEdge(new Edge(v2, v3,
            new EdgeProperty(DataMovementType.SCATTER_GATHER,
                DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                TestOutput.getOutputDesc(payload),
                TestInput.getInputDesc(payload))));
    return dag;
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
     * @param v2 vertice 2
     * @param dataMovementType Data movement type
     */
    protected static void addEdge(Vertex v1, Vertex v2, DataMovementType dataMovementType) {
        dag.addEdge(new Edge(v1, v2,
            new EdgeProperty(dataMovementType,
                DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                TestOutput.getOutputDesc(payload),
                TestInput.getInputDesc(payload))));
    }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
    Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    dag.addVertex(v1).addVertex(v2).addVertex(v3);
    dag.addEdge(new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
    dag.addEdge(new Edge(v1, v3,
            new EdgeProperty(DataMovementType.SCATTER_GATHER,
                DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                TestOutput.getOutputDesc(payload),
                TestInput.getInputDesc(payload))));
    return dag;
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
  @SuppressWarnings({ "rawtypes", "unchecked" })
  @Test (timeout = 5000)
  public void testCompositeEventHandling() {
    EventHandler eventHandler = mock(EventHandler.class);
    EdgeProperty edgeProp = new EdgeProperty(DataMovementType.SCATTER_GATHER,
        DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, mock(OutputDescriptor.class),
        mock(InputDescriptor.class));
    Edge edge = new Edge(edgeProp, eventHandler);
   
    TezVertexID srcVertexID = createVertexID(1);
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
        new OutputDescriptor(MROutput.class.getName()).setUserPayload(MRHelpers
            .createUserPayloadFromConf(stage2Conf)),
        MROutputCommitter.class);

    DAG dag = new DAG("FilterLinesByWord");
    Edge edge = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
        DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(
            OnFileUnorderedKVOutput.class.getName()), new InputDescriptor(
            ShuffledUnorderedKVInput.class.getName())));
    dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge);
View Full Code Here
TOP

Related Classes of org.apache.tez.dag.api.EdgeProperty

  • cascading.flow.tez.Hadoop2TezFlowStep
  • org.apache.hadoop.hive.ql.exec.tez.DagUtils
  • org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder
  • org.apache.tez.dag.app.dag.impl.DAGImpl
  • org.apache.tez.dag.app.dag.impl.DAGImpl$InitTransition
  • org.apache.tez.dag.app.dag.impl.Edge
  • org.apache.tez.dag.app.dag.impl.TestDAGImpl
  • org.apache.tez.dag.app.dag.impl.TestEdge
  • org.apache.tez.dag.app.dag.impl.TestVertexImpl
  • org.apache.tez.dag.app.dag.impl.TestVertexScheduler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.