Examples of EdgeProperty

org.apache.tez.dag.api.EdgeProperty
An @link {@link EdgeProperty} defines the relation between the source anddestination vertices of an edge. The relation consists of defining their communication pattern and dependencies. It also defines the user code that actually does the job of writing out data at the source and reading that data at the destination via the @link {@link InputDescriptor} and @link{@link OutputDescriptor}

Examples of org.apache.tez.dag.api.EdgeProperty

    ShuffleVertexManager manager = null;
    
    HashMap<String, EdgeProperty> mockInputVertices = 
        new HashMap<String, EdgeProperty>();
    String mockSrcVertexId1 = "Vertex1";
    EdgeProperty eProp1 = new EdgeProperty(
        EdgeProperty.DataMovementType.SCATTER_GATHER,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, 
        new OutputDescriptor("out"),
        new InputDescriptor("in"));
    String mockSrcVertexId2 = "Vertex2";
    EdgeProperty eProp2 = new EdgeProperty(
        EdgeProperty.DataMovementType.SCATTER_GATHER,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, 
        new OutputDescriptor("out"),
        new InputDescriptor("in"));
    String mockSrcVertexId3 = "Vertex3";
    EdgeProperty eProp3 = new EdgeProperty(
        EdgeProperty.DataMovementType.BROADCAST,
        EdgeProperty.DataSourceType.PERSISTED, 
        SchedulingType.SEQUENTIAL, 
        new OutputDescriptor("out"),
        new InputDescriptor("in"));

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeProperty

    Configuration conf = new Configuration();
    ShuffleVertexManager manager = null;
    HashMap<String, EdgeProperty> mockInputVertices = 
        new HashMap<String, EdgeProperty>();
    String mockSrcVertexId1 = "Vertex1";
    EdgeProperty eProp1 = new EdgeProperty(
        EdgeProperty.DataMovementType.SCATTER_GATHER,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, 
        new OutputDescriptor("out"),
        new InputDescriptor("in"));
    String mockSrcVertexId2 = "Vertex2";
    EdgeProperty eProp2 = new EdgeProperty(
        EdgeProperty.DataMovementType.SCATTER_GATHER,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, 
        new OutputDescriptor("out"),
        new InputDescriptor("in"));
    String mockSrcVertexId3 = "Vertex3";
    EdgeProperty eProp3 = new EdgeProperty(
        EdgeProperty.DataMovementType.BROADCAST,
        EdgeProperty.DataSourceType.PERSISTED, 
        SchedulingType.SEQUENTIAL, 
        new OutputDescriptor("out"),
        new InputDescriptor("in"));

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeProperty

          for(Map.Entry<String, EdgeManagerDescriptor> entry : sourceEdgeManagers.entrySet()) {
            LOG.info("Replacing edge manager for source:"
                + entry.getKey() + " destination: " + getVertexId());
            Vertex sourceVertex = appContext.getCurrentDAG().getVertex(entry.getKey());
            Edge edge = sourceVertices.get(sourceVertex);
            EdgeProperty edgeProperty = edge.getEdgeProperty();
            try {
              edge.setCustomEdgeManager(entry.getValue());
            } catch (Exception e) {
              LOG.warn("Failed to initialize edge manager for edge"
                  + ", sourceVertexName=" + sourceVertex.getName()

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeProperty


    for (int i = 0; i < vertices.size(); ++i) {
      dag.addVertex(vertices.get(i));
      if (i != 0) {
        dag.addEdge(new Edge(vertices.get(i-1),
            vertices.get(i), new EdgeProperty(
                DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL, 
                new OutputDescriptor(
                    OnFileSortedOutput.class.getName()),
                new InputDescriptor(

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeProperty

    });


    // TODO - this test logic is tightly linked to impl DAGImpl code.
    edges = new HashMap<String, Edge>();
    for (EdgePlan edgePlan : dagPlan.getEdgeList()) {
      EdgeProperty edgeProperty = DagTypeConverters
          .createEdgePropertyMapFromDAGPlan(edgePlan);
      edges.put(edgePlan.getId(), new Edge(edgeProperty, dispatcher.getEventHandler()));
    }


    parseVertexEdges();

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeProperty


    }


    private void createDAGEdges(DAGImpl dag) {
      for (EdgePlan edgePlan : dag.getJobPlan().getEdgeList()) {
        EdgeProperty edgeProperty = DagTypeConverters
            .createEdgePropertyMapFromDAGPlan(edgePlan);
        
        // If CUSTOM without an edge manager, setup a fake edge manager. Avoid
        // referencing the fake edge manager within the API module.
        if (edgeProperty.getDataMovementType() == DataMovementType.CUSTOM
            && edgeProperty.getEdgeManagerDescriptor() == null) {
          EdgeManagerDescriptor edgeDesc = new EdgeManagerDescriptor(
              NullEdgeManager.class.getName());
          EdgeProperty ep = new EdgeProperty(edgeDesc, edgeProperty.getDataSourceType(),
              edgeProperty.getSchedulingType(), edgeProperty.getEdgeSource(),
              edgeProperty.getEdgeDestination());
          edgeProperty = ep;
        }

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeProperty

    OutputDescriptor outDesc = new OutputDescriptor("output.class");
    uv12.addOutput("uvOut", outDesc, TotalCountingOutputCommitter.class);
    v3.addOutput("uvOut", outDesc, TotalCountingOutputCommitter.class);
    
    GroupInputEdge e1 = new GroupInputEdge(uv12, v3,
        new EdgeProperty(DataMovementType.SCATTER_GATHER, 
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
            new OutputDescriptor("dummy output class"),
            new InputDescriptor("dummy input class")), 
            new InputDescriptor("merge.class"));

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeProperty


    dag.addVertex(stage1Vertex);
    dag.addVertex(stage2Vertex);
    dag.addVertex(stage3Vertex);


    Edge edge1 = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(
        OnFileSortedOutput.class.getName()), new InputDescriptor(
                ShuffledMergedInputLegacy.class.getName())));
    Edge edge2 = new Edge(stage2Vertex, stage3Vertex, new EdgeProperty(
        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(
        OnFileSortedOutput.class.getName()), new InputDescriptor(
                ShuffledMergedInputLegacy.class.getName())));

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeProperty

  public void testBasicSuccessBroadcast() throws Exception {
    DAG dag = new DAG("testBasicSuccessBroadcast");
    Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
    Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
    dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
        new EdgeProperty(DataMovementType.BROADCAST, 
            DataSourceType.PERSISTED, 
            SchedulingType.SEQUENTIAL, 
            TestOutput.getOutputDesc(null), 
            TestInput.getInputDesc(null))));
    runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeProperty

    DAG dag = new DAG("OrderedWordCount" + dagIndex);
    for (int i = 0; i < vertices.size(); ++i) {
      dag.addVertex(vertices.get(i));
      if (i != 0) {
        dag.addEdge(new Edge(vertices.get(i-1),
            vertices.get(i), new EdgeProperty(
                DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                new OutputDescriptor(
                    OnFileSortedOutput.class.getName()),
                new InputDescriptor(

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.