Examples of org.apache.tez.dag.api.GroupInputEdge

org.apache.tez.dag.api.GroupInputEdge
A composite edge that represents a common connection between a group of input vertices and a single output vertex. This can be used to perform e.g. an efficient union of the data produced by the input vertices. The output vertex tasks see a unified/merged view of the data from all the input vertices.

            utils.updateConfigurationForEdge(workToConf.get(part), workToVertex.get(part), 
                 workToConf.get(v), workToVertex.get(v));
          }
          
          // finally we can create the grouped edge
          GroupInputEdge e = utils.createEdge(group, workToConf.get(v),
               workToVertex.get(v), work.getEdgeProperty(w, v));


          dag.addEdge(e);
        }
      } else {

View Full Code Here

      default:
        mergeInputClass = TezMergedLogicalInput.class;
        break;
    }


    return new GroupInputEdge(group, w, createEdgeProperty(edgeProp),
        new InputDescriptor(mergeInputClass.getName()));
  }

View Full Code Here

                        VertexGroup from = pred.getVertexGroupInfo().getVertexGroup();
                        // The plan of vertex group is empty. Since we create the Edge based on
                        // some of the operators in the plan refer to one of the vertex group members.
                        // Both the vertex group and its members reference same EdgeDescriptor object to the
                        // the successor
                        GroupInputEdge edge = newGroupInputEdge(
                                getPlan().getOperator(pred.getVertexGroupMembers().get(0)), tezOp, from, to);
                        dag.addEdge(edge);
                    } else {
                        Vertex from = dag.getVertex(pred.getOperatorKey().toString());
                        if (tezOp.isVertexGroup()) {

View Full Code Here

    org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
    OutputDescriptor outDesc = new OutputDescriptor("output.class");
    uv12.addOutput("uvOut", outDesc, TotalCountingOutputCommitter.class);
    v3.addOutput("uvOut", outDesc, TotalCountingOutputCommitter.class);
    
    GroupInputEdge e1 = new GroupInputEdge(uv12, v3,
        new EdgeProperty(DataMovementType.SCATTER_GATHER, 
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
            new OutputDescriptor("dummy output class"),
            new InputDescriptor("dummy input class")), 
            new InputDescriptor("merge.class"));

View Full Code Here

    OutputCommitterDescriptor ocd =
        OutputCommitterDescriptor.create(OutputCommitter.class.getName());
    uv12.addDataSink("uvOut", new DataSinkDescriptor(outDesc, ocd, null));
    v3.addDataSink("uvOut", new DataSinkDescriptor(outDesc, ocd, null));


    GroupInputEdge e1 = GroupInputEdge.create(uv12, v3,
        EdgeProperty.create(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
            OutputDescriptor.create("dummy output class").setHistoryText("Dummy History Text"),
            InputDescriptor.create("dummy input class").setHistoryText("Dummy History Text")),
        InputDescriptor.create("merge.class").setHistoryText("Merge HistoryText"));

View Full Code Here

    org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
    OutputDescriptor outDesc = OutputDescriptor.create("output.class");
    uv12.addDataSink("uvOut", new DataSinkDescriptor(outDesc, ocd, null));
    v3.addDataSink("uvOut", new DataSinkDescriptor(outDesc, ocd, null));


    GroupInputEdge e1 = GroupInputEdge.create(uv12, v3,
        EdgeProperty.create(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
            OutputDescriptor.create("dummy output class"),
            InputDescriptor.create("dummy input class")),
        InputDescriptor.create("merge.class"));

View Full Code Here

                new OutputDescriptor(OnFileSortedOutput.class.getName())
                        .setUserPayload(mapPayload), 
                new InputDescriptor(ShuffledMergedInput.class.getName())
                        .setUserPayload(finalReducePayload))))
        .addEdge(
            new GroupInputEdge(unionVertex, checkerVertex, new EdgeProperty(
                DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                new OutputDescriptor(OnFileSortedOutput.class.getName())
                    .setUserPayload(mapPayload), 
                new InputDescriptor(ShuffledMergedInput.class.getName())

View Full Code Here

    org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
    OutputDescriptor outDesc = new OutputDescriptor("output.class");
    uv12.addOutput("uvOut", outDesc, TotalCountingOutputCommitter.class);
    v3.addOutput("uvOut", outDesc, TotalCountingOutputCommitter.class);
    
    GroupInputEdge e1 = new GroupInputEdge(uv12, v3,
        new EdgeProperty(DataMovementType.SCATTER_GATHER, 
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
            new OutputDescriptor("dummy output class"),
            new InputDescriptor("dummy input class")), 
            new InputDescriptor("merge.class"));

View Full Code Here

            utils.updateConfigurationForEdge(workToConf.get(part), workToVertex.get(part),
                 workToConf.get(v), workToVertex.get(v));
          }


          // finally we can create the grouped edge
          GroupInputEdge e = utils.createEdge(group, workToConf.get(v),
               workToVertex.get(v), work.getEdgeProperty(w, v));


          dag.addEdge(e);
        }
      } else {

View Full Code Here

      default:
        mergeInputClass = TezMergedLogicalInput.class;
        break;
    }


    return new GroupInputEdge(group, w, createEdgeProperty(edgeProp),
        new InputDescriptor(mergeInputClass.getName()));
  }

View Full Code Here

0 1

TOP

Related Classes of org.apache.tez.dag.api.GroupInputEdge

org.apache.hadoop.hive.ql.exec.tez.DagUtils

org.apache.hadoop.hive.ql.exec.tez.TezTask

org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder

org.apache.tez.dag.app.dag.impl.TestDAGImpl

org.apache.tez.dag.history.utils.TestDAGUtils

org.apache.tez.mapreduce.examples.UnionExample

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.