Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.EdgeProperty

63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
    ShuffleVertexManager manager = null;
   
    HashMap<String, EdgeProperty> mockInputVertices =
        new HashMap<String, EdgeProperty>();
    String mockSrcVertexId1 = "Vertex1";
    EdgeProperty eProp1 = new EdgeProperty(
        EdgeProperty.DataMovementType.SCATTER_GATHER,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL,
        new OutputDescriptor("out"),
        new InputDescriptor("in"));
    String mockSrcVertexId2 = "Vertex2";
    EdgeProperty eProp2 = new EdgeProperty(
        EdgeProperty.DataMovementType.SCATTER_GATHER,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL,
        new OutputDescriptor("out"),
        new InputDescriptor("in"));
    String mockSrcVertexId3 = "Vertex3";
    EdgeProperty eProp3 = new EdgeProperty(
        EdgeProperty.DataMovementType.BROADCAST,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL,
        new OutputDescriptor("out"),
        new InputDescriptor("in"));
View Full Code Here
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
    Configuration conf = new Configuration();
    ShuffleVertexManager manager = null;
    HashMap<String, EdgeProperty> mockInputVertices =
        new HashMap<String, EdgeProperty>();
    String mockSrcVertexId1 = "Vertex1";
    EdgeProperty eProp1 = new EdgeProperty(
        EdgeProperty.DataMovementType.SCATTER_GATHER,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL,
        new OutputDescriptor("out"),
        new InputDescriptor("in"));
    String mockSrcVertexId2 = "Vertex2";
    EdgeProperty eProp2 = new EdgeProperty(
        EdgeProperty.DataMovementType.SCATTER_GATHER,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL,
        new OutputDescriptor("out"),
        new InputDescriptor("in"));
    String mockSrcVertexId3 = "Vertex3";
    EdgeProperty eProp3 = new EdgeProperty(
        EdgeProperty.DataMovementType.BROADCAST,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL,
        new OutputDescriptor("out"),
        new InputDescriptor("in"));
View Full Code Here
911
912
913
914
915
916
917
918
919
920
921
          for(Map.Entry<String, EdgeManagerDescriptor> entry : sourceEdgeManagers.entrySet()) {
            LOG.info("Replacing edge manager for source:"
                + entry.getKey() + " destination: " + getVertexId());
            Vertex sourceVertex = appContext.getCurrentDAG().getVertex(entry.getKey());
            Edge edge = sourceVertices.get(sourceVertex);
            EdgeProperty edgeProperty = edge.getEdgeProperty();
            try {
              edge.setCustomEdgeManager(entry.getValue());
            } catch (Exception e) {
              LOG.warn("Failed to initialize edge manager for edge"
                  + ", sourceVertexName=" + sourceVertex.getName()
View Full Code Here
659
660
661
662
663
664
665
666
667
668
669
    for (int i = 0; i < vertices.size(); ++i) {
      dag.addVertex(vertices.get(i));
      if (i != 0) {
        dag.addEdge(new Edge(vertices.get(i-1),
            vertices.get(i), new EdgeProperty(
                DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                new OutputDescriptor(
                    OnFileSortedOutput.class.getName()),
                new InputDescriptor(
View Full Code Here
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
    });

    // TODO - this test logic is tightly linked to impl DAGImpl code.
    edges = new HashMap<String, Edge>();
    for (EdgePlan edgePlan : dagPlan.getEdgeList()) {
      EdgeProperty edgeProperty = DagTypeConverters
          .createEdgePropertyMapFromDAGPlan(edgePlan);
      edges.put(edgePlan.getId(), new Edge(edgeProperty, dispatcher.getEventHandler()));
    }

    parseVertexEdges();
View Full Code Here
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
    }

    private void createDAGEdges(DAGImpl dag) {
      for (EdgePlan edgePlan : dag.getJobPlan().getEdgeList()) {
        EdgeProperty edgeProperty = DagTypeConverters
            .createEdgePropertyMapFromDAGPlan(edgePlan);
       
        // If CUSTOM without an edge manager, setup a fake edge manager. Avoid
        // referencing the fake edge manager within the API module.
        if (edgeProperty.getDataMovementType() == DataMovementType.CUSTOM
            && edgeProperty.getEdgeManagerDescriptor() == null) {
          EdgeManagerDescriptor edgeDesc = new EdgeManagerDescriptor(
              NullEdgeManager.class.getName());
          EdgeProperty ep = new EdgeProperty(edgeDesc, edgeProperty.getDataSourceType(),
              edgeProperty.getSchedulingType(), edgeProperty.getEdgeSource(),
              edgeProperty.getEdgeDestination());
          edgeProperty = ep;
        }
       
View Full Code Here
342
343
344
345
346
347
348
349
350
351
352
    OutputDescriptor outDesc = new OutputDescriptor("output.class");
    uv12.addOutput("uvOut", outDesc, TotalCountingOutputCommitter.class);
    v3.addOutput("uvOut", outDesc, TotalCountingOutputCommitter.class);
   
    GroupInputEdge e1 = new GroupInputEdge(uv12, v3,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
            new OutputDescriptor("dummy output class"),
            new InputDescriptor("dummy input class")),
            new InputDescriptor("merge.class"));
   
View Full Code Here
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
    dag.addVertex(stage1Vertex);
    dag.addVertex(stage2Vertex);
    dag.addVertex(stage3Vertex);

    Edge edge1 = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(
        OnFileSortedOutput.class.getName()), new InputDescriptor(
                ShuffledMergedInputLegacy.class.getName())));
    Edge edge2 = new Edge(stage2Vertex, stage3Vertex, new EdgeProperty(
        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(
        OnFileSortedOutput.class.getName()), new InputDescriptor(
                ShuffledMergedInputLegacy.class.getName())));
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
  public void testBasicSuccessBroadcast() throws Exception {
    DAG dag = new DAG("testBasicSuccessBroadcast");
    Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
    Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
    dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2,
        new EdgeProperty(DataMovementType.BROADCAST,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(null),
            TestInput.getInputDesc(null))));
    runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
View Full Code Here
275
276
277
278
279
280
281
282
283
284
285
    DAG dag = new DAG("OrderedWordCount" + dagIndex);
    for (int i = 0; i < vertices.size(); ++i) {
      dag.addVertex(vertices.get(i));
      if (i != 0) {
        dag.addEdge(new Edge(vertices.get(i-1),
            vertices.get(i), new EdgeProperty(
                DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                new OutputDescriptor(
                    OnFileSortedOutput.class.getName()),
                new InputDescriptor(
View Full Code Here
TOP

Related Classes of org.apache.tez.dag.api.EdgeProperty

  • cascading.flow.tez.Hadoop2TezFlowStep
  • org.apache.hadoop.hive.ql.exec.tez.DagUtils
  • org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder
  • org.apache.tez.dag.app.dag.impl.DAGImpl
  • org.apache.tez.dag.app.dag.impl.DAGImpl$InitTransition
  • org.apache.tez.dag.app.dag.impl.Edge
  • org.apache.tez.dag.app.dag.impl.TestDAGImpl
  • org.apache.tez.dag.app.dag.impl.TestEdge
  • org.apache.tez.dag.app.dag.impl.TestVertexImpl
  • org.apache.tez.dag.app.dag.impl.TestVertexScheduler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.