Examples of org.apache.tez.dag.api.EdgeProperty

org.apache.tez.dag.api.EdgeProperty
An @link {@link EdgeProperty} defines the relation between the source anddestination vertices of an edge. The relation consists of defining their communication pattern and dependencies. It also defines the user code that actually does the job of writing out data at the source and reading that data at the destination via the @link {@link InputDescriptor} and @link{@link OutputDescriptor}

    DAG dag = new DAG("OrderedWordCount" + dagIndex);
    for (int i = 0; i < vertices.size(); ++i) {
      dag.addVertex(vertices.get(i));
      if (i != 0) {
        dag.addEdge(new Edge(vertices.get(i-1),
            vertices.get(i), new EdgeProperty(
                DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                new OutputDescriptor(
                    OnFileSortedOutput.class.getName()),
                new InputDescriptor(

View Full Code Here

        sourceVertex.getName(), 
        null);
  }


  public synchronized void setCustomEdgeManager(EdgeManagerPluginDescriptor descriptor) {
    EdgeProperty modifiedEdgeProperty =
        EdgeProperty.create(descriptor,
            edgeProperty.getDataSourceType(),
            edgeProperty.getSchedulingType(),
            edgeProperty.getEdgeSource(),
            edgeProperty.getEdgeDestination());

View Full Code Here


  @SuppressWarnings({ "rawtypes", "unchecked" })
  @Test (timeout = 5000)
  public void testCompositeEventHandling() {
    EventHandler eventHandler = mock(EventHandler.class);
    EdgeProperty edgeProp = EdgeProperty.create(DataMovementType.SCATTER_GATHER,
        DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, mock(OutputDescriptor.class),
        mock(InputDescriptor.class));
    Edge edge = new Edge(edgeProp, eventHandler);
    
    TezVertexID srcVertexID = createVertexID(1);

View Full Code Here

  public void testBasicSuccessBroadcast() throws Exception {
    DAG dag = new DAG("testBasicSuccessBroadcast");
    Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
    Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
    dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
        new EdgeProperty(DataMovementType.BROADCAST, 
            DataSourceType.PERSISTED, 
            SchedulingType.SEQUENTIAL, 
            TestOutput.getOutputDesc(null), 
            TestInput.getInputDesc(null))));
    runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);

View Full Code Here


    dag.addVertex(stage1Vertex);
    dag.addVertex(stage2Vertex);
    dag.addVertex(stage3Vertex);


    Edge edge1 = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(
        OnFileSortedOutput.class.getName()), new InputDescriptor(
                ShuffledMergedInputLegacy.class.getName())));
    Edge edge2 = new Edge(stage2Vertex, stage3Vertex, new EdgeProperty(
        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(
        OnFileSortedOutput.class.getName()), new InputDescriptor(
                ShuffledMergedInputLegacy.class.getName())));

View Full Code Here

    }


    for (int i = 0; i < vertices.length; i++) {
      dag.addVertex(vertices[i]);
      if (i > 0) {
        EdgeProperty edgeProperty = new EdgeProperty(
            DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL, 
            new OutputDescriptor(OnFileSortedOutput.class.getName()),
            new InputDescriptor(ShuffledMergedInputLegacy.class.getName()));

View Full Code Here

    OutputDescriptor od = new OutputDescriptor(MROutput.class.getName())
        .setUserPayload(MRHelpers.createUserPayloadFromConf(stage2Conf));
    stage2Vertex.addOutput("MROutput", od, MROutputCommitter.class);


    DAG dag = new DAG("FilterLinesByWord");
    Edge edge = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
        DataMovementType.BROADCAST, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(
            OnFileUnorderedKVOutput.class.getName()), new InputDescriptor(
            ShuffledUnorderedKVInput.class.getName())));
    dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge);

View Full Code Here

    
    DAG dag = new DAG("WordCount");
    dag.addVertex(tokenizerVertex)
        .addVertex(summerVertex)
        .addEdge(
            new Edge(tokenizerVertex, summerVertex, new EdgeProperty(
                DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL, 
                new OutputDescriptor(OnFileSortedOutput.class.getName())
                        .setUserPayload(mapPayload), 
                new InputDescriptor(ShuffledMergedInput.class.getName())

View Full Code Here

    dag.addVertex(mapVertex1)
        .addVertex(mapVertex2)
        .addVertex(mapVertex3)
        .addVertex(checkerVertex)
        .addEdge(
            new Edge(mapVertex3, checkerVertex, new EdgeProperty(
                DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL, 
                new OutputDescriptor(OnFileSortedOutput.class.getName())
                        .setUserPayload(mapPayload), 
                new InputDescriptor(ShuffledMergedInput.class.getName())
                        .setUserPayload(finalReducePayload))))
        .addEdge(
            new GroupInputEdge(unionVertex, checkerVertex, new EdgeProperty(
                DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                new OutputDescriptor(OnFileSortedOutput.class.getName())
                    .setUserPayload(mapPayload), 
                new InputDescriptor(ShuffledMergedInput.class.getName())

View Full Code Here

    DAG dag = new DAG("OrderedWordCount" + dagIndex);
    for (int i = 0; i < vertices.size(); ++i) {
      dag.addVertex(vertices.get(i));
      if (i != 0) {
        dag.addEdge(new Edge(vertices.get(i-1),
            vertices.get(i), new EdgeProperty(
                DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                new OutputDescriptor(
                    OnFileSortedOutput.class.getName()),
                new InputDescriptor(

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.tez.dag.api.EdgeProperty

cascading.flow.tez.Hadoop2TezFlowStep

org.apache.hadoop.hive.ql.exec.tez.DagUtils

org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder

org.apache.tez.dag.app.dag.impl.DAGImpl

org.apache.tez.dag.app.dag.impl.DAGImpl$InitTransition

org.apache.tez.dag.app.dag.impl.Edge

org.apache.tez.dag.app.dag.impl.TestDAGImpl

org.apache.tez.dag.app.dag.impl.TestEdge

org.apache.tez.dag.app.dag.impl.TestVertexImpl

org.apache.tez.dag.app.dag.impl.TestVertexScheduler

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.