Examples of org.apache.tez.dag.api.UserPayload

org.apache.tez.dag.api.UserPayload
Wrapper class to hold user payloads Provides a version to help in evolving the payloads

    }
  }


  private OutputContext createTezOutputContext() throws IOException {
    String[] workingDirs = { workingDir.toString() };
    UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
    DataOutputBuffer serviceProviderMetaData = new DataOutputBuffer();
    serviceProviderMetaData.writeInt(PORT);


    TezCounters counters = new TezCounters();

View Full Code Here

        edgeManagerContext = new EdgeManagerPluginContextImpl(UserPayload.create(null));
        edgeManager = new ScatterGatherEdgeManager(edgeManagerContext);
        break;
      case CUSTOM:
        if (edgeProperty.getEdgeManagerDescriptor() != null) {
          UserPayload payload = null;
          if (edgeProperty.getEdgeManagerDescriptor().getUserPayload() != null &&
              edgeProperty.getEdgeManagerDescriptor().getUserPayload().hasPayload()) {
            payload = edgeProperty.getEdgeManagerDescriptor().getUserPayload();
          } else {
            payload = UserPayload.create(null);

View Full Code Here

    super(context);
  }


  @Override
  public void initialize() throws Exception {
    UserPayload userPayload = getContext().getUserPayload();
    this.conf = TezUtils.createConfFromUserPayload(userPayload);
  }

View Full Code Here

    // the routing table.
    EdgeManagerPluginDescriptor hiveEdgeManagerDesc = null;
    if ((vertexType == VertexType.MULTI_INPUT_INITIALIZED_EDGES)
        || (vertexType == VertexType.INITIALIZED_EDGES)) {
      hiveEdgeManagerDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName());
      UserPayload payload = getBytePayload(bucketToTaskMap);
      hiveEdgeManagerDesc.setUserPayload(payload);
    }


    // Replace the edge manager for all vertices which have routing type custom.
    for (Entry<String, EdgeProperty> edgeEntry : context.getInputVertexEdgeProperties().entrySet()) {

View Full Code Here


      // set up the operator plan. (after generating splits - that changes configs)
      Utilities.setMapWork(conf, mapWork, mrScratchDir, false);
    }


    UserPayload serializedConf = TezUtils.createUserPayloadFromConf(conf);
    String procClassName = MapTezProcessor.class.getName();
    if (mapWork instanceof MergeFileWork) {
      procClassName = MergeFileTezProcessor.class.getName();
    }
    map = Vertex.create(mapWork.getName(), ProcessorDescriptor.create(procClassName)

View Full Code Here

      pluginConf.setInt(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM,
          edgeProp.getMinReducer());
      pluginConf.setLong(
          ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE,
          edgeProp.getInputSizePerReducer());
      UserPayload payload = TezUtils.createUserPayloadFromConf(pluginConf);
      desc.setUserPayload(payload);
      v.setVertexManagerPlugin(desc);
    }
  }

View Full Code Here

  private EdgeProperty createEdgeProperty( EdgeValues edgeValues )
    {
    TezConfiguration outputConfig = new TezConfiguration( edgeValues.getConfig() );
    outputConfig.set( "cascading.node.sink", FlowElements.id( edgeValues.getFlowElement() ) );


    UserPayload outputPayload = createIntermediatePayloadOutput( outputConfig, edgeValues );


    TezConfiguration inputConfig = new TezConfiguration( edgeValues.getConfig() );
    inputConfig.set( "cascading.node.source", FlowElements.id( edgeValues.getFlowElement() ) );
    inputConfig.set( "cascading.node.source.ordinals", Util.join( edgeValues.getOrdinals(), "," ) );


    UserPayload inputPayload = createIntermediatePayloadInput( inputConfig, edgeValues );


    return EdgeProperty.create(
      edgeValues.getMovementType(),
      edgeValues.getSourceType(),
      edgeValues.getSchedulingType(),

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.tez.dag.api.UserPayload

cascading.flow.tez.Hadoop2TezFlowStep

org.apache.hadoop.hive.ql.exec.tez.CustomPartitionVertex

org.apache.hadoop.hive.ql.exec.tez.DagUtils

org.apache.hadoop.hive.ql.exec.tez.HivePreWarmProcessor

org.apache.pig.backend.hadoop.executionengine.tez.runtime.PigProcessor

org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder

org.apache.pig.tools.pigstats.tez.TezDAGStats

org.apache.tez.common.TestTezUtils

org.apache.tez.dag.app.dag.impl.Edge

org.apache.tez.dag.app.dag.impl.TestVertexImpl

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.