Examples of org.apache.tez.dag.api.EdgeProperty

org.apache.tez.dag.api.EdgeProperty
An @link {@link EdgeProperty} defines the relation between the source anddestination vertices of an edge. The relation consists of defining their communication pattern and dependencies. It also defines the user code that actually does the job of writing out data at the source and reading that data at the destination via the @link {@link InputDescriptor} and @link{@link OutputDescriptor}

  private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp) throws IOException {
    DataMovementType dataMovementType;
    Class logicalInputClass;
    Class logicalOutputClass;


    EdgeProperty edgeProperty = null;
    EdgeType edgeType = edgeProp.getEdgeType();
    switch (edgeType) {
      case BROADCAST_EDGE:
        dataMovementType = DataMovementType.BROADCAST;
        logicalOutputClass = OnFileUnorderedKVOutput.class;
        logicalInputClass = ShuffledUnorderedKVInput.class;
        break;


      case CUSTOM_EDGE:
        
        dataMovementType = DataMovementType.CUSTOM;
        logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class;
        logicalInputClass = ShuffledUnorderedKVInput.class;
        EdgeManagerDescriptor edgeDesc = new EdgeManagerDescriptor(
            CustomPartitionEdge.class.getName());
        CustomEdgeConfiguration edgeConf = 
            new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null);
          DataOutputBuffer dob = new DataOutputBuffer();
          edgeConf.write(dob);
          byte[] userPayload = dob.getData();
        edgeDesc.setUserPayload(userPayload);
        edgeProperty =
          new EdgeProperty(edgeDesc,
              DataSourceType.PERSISTED,
              SchedulingType.SEQUENTIAL,
              new OutputDescriptor(logicalOutputClass.getName()),
              new InputDescriptor(logicalInputClass.getName()));
        break;


      case CUSTOM_SIMPLE_EDGE:
        dataMovementType = DataMovementType.SCATTER_GATHER;
        logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class;
        logicalInputClass = ShuffledUnorderedKVInput.class;
        break;


      case SIMPLE_EDGE:
      default:
        dataMovementType = DataMovementType.SCATTER_GATHER;
        logicalOutputClass = OnFileSortedOutput.class;
        logicalInputClass = ShuffledMergedInputLegacy.class;
        break;
    }


    if (edgeProperty == null) {
      edgeProperty =
        new EdgeProperty(dataMovementType,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            new OutputDescriptor(logicalOutputClass.getName()),
            new InputDescriptor(logicalInputClass.getName()));
    }

View Full Code Here

                    } else {
                        Vertex from = dag.getVertex(pred.getOperatorKey().toString());
                        if (tezOp.isVertexGroup()) {
                            groupMembers[i] = from;
                        } else {
                            EdgeProperty prop = newEdge(pred, tezOp);
                            Edge edge = Edge.create(from, to, prop);
                            dag.addEdge(edge);
                        }
                    }
                } catch (IOException e) {

View Full Code Here

    }


    private GroupInputEdge newGroupInputEdge(TezOperator fromOp,
            TezOperator toOp, VertexGroup from, Vertex to) throws IOException {


        EdgeProperty edgeProperty = newEdge(fromOp, toOp);


        String groupInputClass = ConcatenatedMergedKeyValueInput.class.getName();


        // In case of SCATTER_GATHER and UnorderedKVInput it will still be
        // ConcatenatedMergedKeyValueInput
        if(edgeProperty.getDataMovementType().equals(DataMovementType.SCATTER_GATHER)
                && edgeProperty.getEdgeDestination().getClassName().equals(OrderedGroupedKVInput.class.getName())) {
            groupInputClass = OrderedGroupedMergedKVInput.class.getName();
        }


        return GroupInputEdge.create(from, to, edgeProperty,
                InputDescriptor.create(groupInputClass).setUserPayload(edgeProperty.getEdgeDestination().getUserPayload()));
    }

View Full Code Here

    });


    // TODO - this test logic is tightly linked to impl DAGImpl code.
    edges = new HashMap<String, Edge>();
    for (EdgePlan edgePlan : dagPlan.getEdgeList()) {
      EdgeProperty edgeProperty = DagTypeConverters
          .createEdgePropertyMapFromDAGPlan(edgePlan);
      edges.put(edgePlan.getId(), new Edge(edgeProperty, dispatcher.getEventHandler()));
    }


    parseVertexEdges();

View Full Code Here

    return DAGState.INITED;
  }


  private void createDAGEdges(DAGImpl dag) {
    for (EdgePlan edgePlan : dag.getJobPlan().getEdgeList()) {
      EdgeProperty edgeProperty = DagTypeConverters
          .createEdgePropertyMapFromDAGPlan(edgePlan);


      // edge manager may be also set via API when using custom edge type
      dag.edges.put(edgePlan.getId(),
          new Edge(edgeProperty, dag.getEventHandler()));

View Full Code Here

    // TODO Until TEZ-1013 we cannot handle custom input formats
    Map<String, EdgeProperty> edges = getContext().getInputVertexEdgeProperties();
    int oneToOneSrcTaskCount = 0;
    numOneToOneEdges = 0;
    for (Map.Entry<String, EdgeProperty> entry : edges.entrySet()) {
      EdgeProperty edgeProp = entry.getValue();
      String srcVertex = entry.getKey();
      int numSrcTasks = getContext().getVertexNumTasks(srcVertex);
      switch (edgeProp.getDataMovementType()) {
      case CUSTOM:
        throw new TezUncheckedException("Cannot handle custom edge");
      case ONE_TO_ONE:
        numOneToOneEdges++;
        if (oneToOneSrcTaskCount == 0) {
          oneToOneSrcTaskCount = numSrcTasks;
        } else if (oneToOneSrcTaskCount != numSrcTasks) {
          throw new TezUncheckedException(
              "All 1-1 source vertices must have identical concurrency");
        }
        break;
      case SCATTER_GATHER:
      case BROADCAST:
        break;
      default:
        throw new TezUncheckedException(
            "Unknown edge type: " + edgeProp.getDataMovementType());
      }
      srcVertexInfo.put(srcVertex, new SourceVertexInfo(numSrcTasks, edgeProp));
    }
    
    if (numOneToOneEdges > 0) {

View Full Code Here

    /**
     * Connect the join vertex with the stream side
     */
    Edge e1 = Edge.create(streamFileVertex, joinVertex, streamConf.createDefaultEdgeProperty());
    
    EdgeProperty hashSideEdgeProperty = null;
    if (doBroadcast) {
      /**
       * This option can be used when the hash side is small. We can broadcast the entire data to 
       * all fragments of the stream side. This avoids re-partitioning the fragments of the stream 
       * side to match the partitioning scheme of the hash side and avoids costly network data

View Full Code Here

    ShuffleVertexManager manager = null;
    
    HashMap<String, EdgeProperty> mockInputVertices = 
        new HashMap<String, EdgeProperty>();
    String mockSrcVertexId1 = "Vertex1";
    EdgeProperty eProp1 = EdgeProperty.create(
        EdgeProperty.DataMovementType.SCATTER_GATHER,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL,
        OutputDescriptor.create("out"),
        InputDescriptor.create("in"));
    String mockSrcVertexId2 = "Vertex2";
    EdgeProperty eProp2 = EdgeProperty.create(
        EdgeProperty.DataMovementType.SCATTER_GATHER,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL,
        OutputDescriptor.create("out"),
        InputDescriptor.create("in"));
    String mockSrcVertexId3 = "Vertex3";
    EdgeProperty eProp3 = EdgeProperty.create(
        EdgeProperty.DataMovementType.BROADCAST,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL,
        OutputDescriptor.create("out"),
        InputDescriptor.create("in"));

View Full Code Here

    Configuration conf = new Configuration();
    ShuffleVertexManager manager = null;
    HashMap<String, EdgeProperty> mockInputVertices = 
        new HashMap<String, EdgeProperty>();
    String mockSrcVertexId1 = "Vertex1";
    EdgeProperty eProp1 = EdgeProperty.create(
        EdgeProperty.DataMovementType.SCATTER_GATHER,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL,
        OutputDescriptor.create("out"),
        InputDescriptor.create("in"));
    String mockSrcVertexId2 = "Vertex2";
    EdgeProperty eProp2 = EdgeProperty.create(
        EdgeProperty.DataMovementType.SCATTER_GATHER,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL,
        OutputDescriptor.create("out"),
        InputDescriptor.create("in"));
    String mockSrcVertexId3 = "Vertex3";
    EdgeProperty eProp3 = EdgeProperty.create(
        EdgeProperty.DataMovementType.BROADCAST,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL,
        OutputDescriptor.create("out"),
        InputDescriptor.create("in"));

View Full Code Here

  @Test (timeout=5000)
  public void testBasicScatterGather() {
    HashMap<String, EdgeProperty> mockInputVertices = 
        new HashMap<String, EdgeProperty>();
    String mockSrcVertexId1 = "Vertex1";
    EdgeProperty eProp1 = EdgeProperty.create(
        EdgeProperty.DataMovementType.SCATTER_GATHER,
        EdgeProperty.DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL,
        OutputDescriptor.create("out"),
        InputDescriptor.create("in"));

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.tez.dag.api.EdgeProperty

cascading.flow.tez.Hadoop2TezFlowStep

org.apache.hadoop.hive.ql.exec.tez.DagUtils

org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder

org.apache.tez.dag.app.dag.impl.DAGImpl

org.apache.tez.dag.app.dag.impl.DAGImpl$InitTransition

org.apache.tez.dag.app.dag.impl.Edge

org.apache.tez.dag.app.dag.impl.TestDAGImpl

org.apache.tez.dag.app.dag.impl.TestEdge

org.apache.tez.dag.app.dag.impl.TestVertexImpl

org.apache.tez.dag.app.dag.impl.TestVertexScheduler

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.