Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.Edge

322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
        .setUserPayload(new String("2").getBytes()));
    v3.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
        FailOnAttemptVertexManagerPlugin.class.getName())
        .setUserPayload(new String("3").getBytes()));
    dag.addVertex(v1).addVertex(v2).addVertex(v3);
    dag.addEdge(new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
    dag.addEdge(new Edge(v2, v3,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
    }
    DAG dag = new DAG(name);
    Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
    dag.addVertex(v3).addEdge(new Edge(v2, v3,
            new EdgeProperty(DataMovementType.SCATTER_GATHER,
                DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                TestOutput.getOutputDesc(payload),
                TestInput.getInputDesc(payload))));
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
     * @param v1 vertice 1
     * @param v2 vertice 2
     * @param dataMovementType Data movement type
     */
    protected static void addEdge(Vertex v1, Vertex v2, DataMovementType dataMovementType) {
        dag.addEdge(new Edge(v1, v2,
            new EdgeProperty(dataMovementType,
                DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                TestOutput.getOutputDesc(payload),
                TestInput.getInputDesc(payload))));
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
    DAG dag = new DAG(name);
    Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    dag.addVertex(v1).addVertex(v2).addVertex(v3);
    dag.addEdge(new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
    dag.addEdge(new Edge(v1, v3,
            new EdgeProperty(DataMovementType.SCATTER_GATHER,
                DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                TestOutput.getOutputDesc(payload),
                TestInput.getInputDesc(payload))));
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
        new OutputDescriptor(MROutput.class.getName()).setUserPayload(MRHelpers
            .createUserPayloadFromConf(stage2Conf)),
        MROutputCommitter.class);

    DAG dag = new DAG("FilterLinesByWord");
    Edge edge = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
        DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(
            OnFileUnorderedKVOutput.class.getName()), new InputDescriptor(
            ShuffledUnorderedKVInput.class.getName())));
    dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge);
View Full Code Here
280
281
282
283
284
285
286
287
288
289
290
        workToConf.put(w, wxConf);

        // add all dependencies (i.e.: edges) to the graph
        for (BaseWork v: work.getChildren(w)) {
          assert workToVertex.containsKey(v);
          Edge e = null;

          TezEdgeProperty edgeProp = work.getEdgeProperty(w, v);

          e = utils.createEdge(wxConf, wx, workToConf.get(v), workToVertex.get(v), edgeProp);
          dag.addEdge(e);
View Full Code Here
289
290
291
292
293
294
295
          CustomPartitionVertex.class.getName());
      desc.setUserPayload(userPayload);
      w.setVertexManagerPlugin(desc);
    }

    return new Edge(v, w, createEdgeProperty(edgeProp));
  }
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
        any(Vertex.class), any(TezEdgeProperty.class))).thenAnswer(new Answer<Edge>() {

          @Override
          public Edge answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            return new Edge((Vertex)args[1], (Vertex)args[3], mock(EdgeProperty.class));
          }
        });

    work = new TezWork("");
View Full Code Here
336
337
338
339
340
341
342
343
344
345
346
        workToConf.put(w, wxConf);

        // add all dependencies (i.e.: edges) to the graph
        for (BaseWork v: work.getChildren(w)) {
          assert workToVertex.containsKey(v);
          Edge e = null;

          TezEdgeProperty edgeProp = work.getEdgeProperty(w, v);

          e = utils.createEdge(wxConf, wx, workToVertex.get(v), edgeProp, work.getVertexType(v));
          dag.addEdge(e);
View Full Code Here
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
        .setUserPayload(new String("2").getBytes()));
    v3.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
        FailOnAttemptVertexManagerPlugin.class.getName())
        .setUserPayload(new String("3").getBytes()));
    dag.addVertex(v1).addVertex(v2).addVertex(v3);
    dag.addEdge(new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
    dag.addEdge(new Edge(v2, v3,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
View Full Code Here
TOP

Related Classes of org.apache.tez.dag.api.Edge

  • org.apache.hadoop.hive.ql.exec.tez.DagUtils
  • org.apache.hadoop.hive.ql.exec.tez.TestTezTask
  • org.apache.hadoop.hive.ql.exec.tez.TezTask
  • org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder
  • org.apache.tez.examples.JoinExample
  • org.apache.tez.examples.JoinValidate
  • org.apache.tez.mapreduce.client.YARNRunner
  • org.apache.tez.mapreduce.examples.FilterLinesByWord
  • org.apache.tez.mapreduce.examples.FilterLinesByWordOneToOne
  • org.apache.tez.mapreduce.examples.MRRSleepJob
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.