Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.UserPayload

115
116
117
118
119
120
121
122
123
124
125
    splitsProtoBuilder.addSplits(proto1);
    splitsProtoBuilder.addSplits(proto2);
    MRInputUserPayloadProto.Builder payloadProto = MRInputUserPayloadProto.newBuilder();
    payloadProto.setSplits(splitsProtoBuilder.build());
    payloadProto.setConfigurationBytes(confByteString);
    UserPayload userPayload =
        UserPayload.create(payloadProto.build().toByteString().asReadOnlyByteBuffer());

    InputInitializerContext context = new TezRootInputInitializerContextForTest(userPayload);
    MRInputSplitDistributor splitDist = new MRInputSplitDistributor(context);
View Full Code Here
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
    EdgeManagerPlugin em = edge.getEdgeManager();
    EdgeManagerForTest originalEm = (EdgeManagerForTest) em;
    Assert.assertTrue(Arrays.equals(edgePayload, originalEm.getEdgeManagerContext()
        .getUserPayload().deepCopyAsArray()));

    UserPayload userPayload = UserPayload.create(ByteBuffer.wrap(new String("foo").getBytes()));
    EdgeManagerPluginDescriptor edgeManagerDescriptor =
        EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
    edgeManagerDescriptor.setUserPayload(userPayload);

    Vertex v3 = vertices.get("vertex3");
    Vertex v5 = vertices.get("vertex5"); // Vertex5 linked to v3 (v3 src, v5
                                         // dest)

    Map<String, EdgeManagerPluginDescriptor> edgeManagerDescriptors =
        Collections.singletonMap(v3.getName(), edgeManagerDescriptor);
    Assert.assertTrue(v5.setParallelism(v5.getTotalTasks() - 1, null,
        edgeManagerDescriptors, null)); // Must decrease.

    VertexImpl v5Impl = (VertexImpl) v5;

    EdgeManagerPlugin modifiedEdgeManager = v5Impl.sourceVertices.get(v3)
        .getEdgeManager();
    Assert.assertNotNull(modifiedEdgeManager);
    Assert.assertTrue(modifiedEdgeManager instanceof EdgeManagerForTest);

    // Ensure initialize() is called with the correct payload
    Assert.assertTrue(Arrays.equals(userPayload.deepCopyAsArray(),
        ((EdgeManagerForTest) modifiedEdgeManager).getUserPayload().deepCopyAsArray()));
  }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
      "tez.simple-v-test-dag.num-tasks";
  public static int TEZ_SIMPLE_V_DAG_NUM_TASKS_DEFAULT = 2;
 
  public static DAG createDAG(String name,
      Configuration conf) throws Exception {
    UserPayload payload = UserPayload.create(null);
    int taskCount = TEZ_SIMPLE_V_DAG_NUM_TASKS_DEFAULT;
    if (conf != null) {
      taskCount = conf.getInt(TEZ_SIMPLE_V_DAG_NUM_TASKS, TEZ_SIMPLE_V_DAG_NUM_TASKS_DEFAULT);
      payload = TezUtils.createUserPayloadFromConf(conf);
    }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
      "tez.simple-test-dag.num-tasks";
  public static int TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT = 2;
 
  public static DAG createDAG(String name,
      Configuration conf) throws Exception {
    UserPayload payload = UserPayload.create(null);
    int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
    if (conf != null) {
      taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
      payload = TezUtils.createUserPayloadFromConf(conf);
    }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
   * @param conf
   * @return
   * @throws Exception
   */
  public static DAG createDAGForVertexOrder(String name, Configuration conf) throws Exception{
    UserPayload payload = UserPayload.create(null);
    int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
    if (conf != null) {
      taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
      payload = TezUtils.createUserPayloadFromConf(conf);
    }
View Full Code Here
318
319
320
321
322
323
324
325
326
327
328


  public static DAG createDAG(String name,
      Configuration conf) throws Exception {
    UserPayload payload = UserPayload.create(null);
    int taskCount = MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS_DEFAULT;
    if (conf != null) {
      taskCount = conf.getInt(MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS, MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS_DEFAULT);
      payload = TezUtils.createUserPayloadFromConf(conf);
    }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
      "tez.simple-test-dag-3-vertices.num-tasks";
  public static int TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT = 2;
 
  public static DAG createDAG(String name,
      Configuration conf) throws Exception {
    UserPayload payload = null;
    int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
    if (conf != null) {
      taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
      payload = TezUtils.createUserPayloadFromConf(conf);
    }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
      "tez.simple-reverse-v-test-dag.num-tasks";
  public static int TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT = 2;
 
  public static DAG createDAG(String name,
      Configuration conf) throws Exception {
    UserPayload payload = UserPayload.create(null);
    int taskCount = TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT;
    if (conf != null) {
      taskCount = conf.getInt(TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS, TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT);
      payload = TezUtils.createUserPayloadFromConf(conf);
    }
View Full Code Here
651
652
653
654
655
656
657
658
659
660
661
662
663
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String
        .valueOf(new Random().nextInt(100000))));
    TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);

    UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf);
    UserPayload stage2Payload = TezUtils.createUserPayloadFromConf(stage2Conf);
    UserPayload stage3Payload = TezUtils.createUserPayloadFromConf(stage3Conf);
   
    DAG dag = DAG.create("testMRRSleepJobDagSubmit-" + random.nextInt(1000));

    Class<? extends InputInitializer> inputInitializerClazz =
        genSplitsInAM ?
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
    }

    @Override
    public void initialize() {
      // Nothing to do. This class isn't currently designed to be used at the DAG API level.
      UserPayload userPayload = getContext().getUserPayload();
      if (userPayload == null || userPayload.getPayload() == null ||
          userPayload.getPayload().limit() == 0) {
        throw new RuntimeException("Could not initialize CustomShuffleEdgeManager"
            + " from provided user payload");
      }
      CustomShuffleEdgeManagerConfig config;
      try {
View Full Code Here
TOP

Related Classes of org.apache.tez.dag.api.UserPayload

  • cascading.flow.tez.Hadoop2TezFlowStep
  • org.apache.hadoop.hive.ql.exec.tez.CustomPartitionVertex
  • org.apache.hadoop.hive.ql.exec.tez.DagUtils
  • org.apache.hadoop.hive.ql.exec.tez.HivePreWarmProcessor
  • org.apache.pig.backend.hadoop.executionengine.tez.runtime.PigProcessor
  • org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder
  • org.apache.pig.tools.pigstats.tez.TezDAGStats
  • org.apache.tez.common.TestTezUtils
  • org.apache.tez.dag.app.dag.impl.Edge
  • org.apache.tez.dag.app.dag.impl.TestVertexImpl
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.