Examples of org.apache.tez.dag.api.UserPayload

org.apache.tez.dag.api.UserPayload
Wrapper class to hold user payloads Provides a version to help in evolving the payloads


        TezScriptState ss = TezScriptState.get();
        ss.addVertexSettingsToConf(dag.getName(), tezOp, payloadConf);


        // Take our assembled configuration and create a vertex
        UserPayload userPayload = TezUtils.createUserPayloadFromConf(payloadConf);
        procDesc.setUserPayload(userPayload);


        Vertex vertex = Vertex.create(tezOp.getOperatorKey().toString(), procDesc, tezOp.getVertexParallelism(),
                tezOp.isUseMRMapSettings() ? MRHelpers.getResourceForMRMapper(globalConf) : MRHelpers.getResourceForMRReducer(globalConf));

View Full Code Here

        sampleMap = null;


        // Reset static variables cleared for avoiding OOM.
        new JVMReuseImpl().cleanupStaticData();


        UserPayload payload = getContext().getUserPayload();
        conf = TezUtils.createConfFromUserPayload(payload);
        PigContext.setPackageImportList((ArrayList<String>) ObjectSerializer
                .deserialize(conf.get("udf.import.list")));
        PigContext pc = (PigContext) ObjectSerializer.deserialize(conf.get("pig.pigContext"));

View Full Code Here


        for (Entry<String, TezVertexStats> entry : tezVertexStatsMap.entrySet()) {
            Vertex v = dag.getVertex(entry.getKey());
            if (v != null && tezVertexStatsMap.containsKey(v.getName())) {
                TezVertexStats vertexStats = entry.getValue();
                UserPayload payload = v.getProcessorDescriptor().getUserPayload();
                Configuration conf = TezUtils.createConfFromUserPayload(payload);
                vertexStats.setConf(conf);


                VertexStatus status = tezJob.getVertexStatus(v.getName());
                vertexStats.accumulateStats(status, v.getParallelism());

View Full Code Here

    Resource taskResource = isMap ? MRHelpers.getResourceForMRMapper(stageConf)
        : MRHelpers.getResourceForMRReducer(stageConf);
    
    stageConf.set(MRJobConfig.MROUTPUT_FILE_NAME_PREFIX, "part");
    
    UserPayload vertexUserPayload = TezUtils.createUserPayloadFromConf(stageConf);
    Vertex vertex = Vertex.create(vertexName,
        ProcessorDescriptor.create(processorName).setUserPayload(vertexUserPayload),
        numTasks, taskResource);
    if (isMap) {
      vertex.addDataSource("MRInput",

View Full Code Here

    super(committerContext);
  }


  @Override
  public void initialize() throws IOException {
    UserPayload userPayload = getContext().getOutputUserPayload();
    if (!userPayload.hasPayload()) {
      jobConf = new JobConf();
    } else {
      jobConf = new JobConf(
          TezUtils.createConfFromUserPayload(userPayload));
    }

View Full Code Here

      } catch (Exception e) {
        throw new TezUncheckedException(e);
      }
      MRHelpers.translateMRConfToTez(conf);


      UserPayload payload = MRInputHelpersInternal.createMRInputPayload(conf, inputSplitInfo.getSplitsProto());
      Credentials credentials = null;
      if (getCredentialsForSourceFilesystem && inputSplitInfo.getCredentials() != null) {
        credentials = inputSplitInfo.getCredentials();
      }
      return DataSourceDescriptor.create(

View Full Code Here


      MRHelpers.translateMRConfToTez(conf);


      Credentials credentials = maybeGetCredentials();


      UserPayload payload = null;
      if (groupSplitsInAM) {
        payload = MRInputHelpersInternal.createMRInputPayloadWithGrouping(conf);
      } else {
        payload = MRInputHelpersInternal.createMRInputPayload(conf, null);
      }

View Full Code Here

      setupBasicConf(conf);
      MRHelpers.translateMRConfToTez(conf);
      
      Credentials credentials = maybeGetCredentials();


      UserPayload payload = null;
      if (groupSplitsInAM) {
        payload = MRInputHelpersInternal.createMRInputPayloadWithGrouping(conf);
      } else {
        payload = MRInputHelpersInternal.createMRInputPayload(conf, null);
      }

View Full Code Here

            processorContext.getApplicationId().getId(),
            (isMap ? TaskType.MAP : TaskType.REDUCE),
            processorContext.getTaskIndex()),
        processorContext.getTaskAttemptNumber());


    UserPayload userPayload = processorContext.getUserPayload();
    Configuration conf = TezUtils.createConfFromUserPayload(userPayload);
    if (conf instanceof JobConf) {
      this.jobConf = (JobConf)conf;
    } else {
      this.jobConf = new JobConf(conf);

View Full Code Here

    splitsProtoBuilder.addSplits(proto1);
    splitsProtoBuilder.addSplits(proto2);
    MRInputUserPayloadProto.Builder payloadProto = MRInputUserPayloadProto.newBuilder();
    payloadProto.setSplits(splitsProtoBuilder.build());
    payloadProto.setConfigurationBytes(confByteString);
    UserPayload userPayload =
        UserPayload.create(payloadProto.build().toByteString().asReadOnlyByteBuffer());


    InputInitializerContext context = new TezRootInputInitializerContextForTest(userPayload);
    MRInputSplitDistributor splitDist = new MRInputSplitDistributor(context);

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.tez.dag.api.UserPayload

cascading.flow.tez.Hadoop2TezFlowStep

org.apache.hadoop.hive.ql.exec.tez.CustomPartitionVertex

org.apache.hadoop.hive.ql.exec.tez.DagUtils

org.apache.hadoop.hive.ql.exec.tez.HivePreWarmProcessor

org.apache.pig.backend.hadoop.executionengine.tez.runtime.PigProcessor

org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder

org.apache.pig.tools.pigstats.tez.TezDAGStats

org.apache.tez.common.TestTezUtils

org.apache.tez.dag.app.dag.impl.Edge

org.apache.tez.dag.app.dag.impl.TestVertexImpl

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.