Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

223
224
225
226
227
228
229
230
231
232
233
234
235
236
        taskCount++;
      }
    }

    // Construct the EdgeManager descriptor to be used by all edges which need the routing table.
    EdgeManagerDescriptor hiveEdgeManagerDesc = new EdgeManagerDescriptor(
        CustomPartitionEdge.class.getName());   
    byte[] payload = getBytePayload(bucketToTaskMap);
    hiveEdgeManagerDesc.setUserPayload(payload);

    Map<String, EdgeManagerDescriptor> emMap = Maps.newHashMap();

    // Replace the edge manager for all vertices which have routing type custom.
    for (Entry<String, EdgeProperty> edgeEntry : context.getInputVertexEdgeProperties().entrySet()) {
View Full Code Here
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
      case CUSTOM_EDGE:
       
        dataMovementType = DataMovementType.CUSTOM;
        logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class;
        logicalInputClass = ShuffledUnorderedKVInput.class;
        EdgeManagerDescriptor edgeDesc = new EdgeManagerDescriptor(
            CustomPartitionEdge.class.getName());
        CustomEdgeConfiguration edgeConf =
            new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null);
          DataOutputBuffer dob = new DataOutputBuffer();
          edgeConf.write(dob);
          byte[] userPayload = dob.getData();
        edgeDesc.setUserPayload(userPayload);
        edgeProperty =
          new EdgeProperty(edgeDesc,
              DataSourceType.PERSISTED,
              SchedulingType.SEQUENTIAL,
              new OutputDescriptor(logicalOutputClass.getName()),
View Full Code Here
419
420
421
422
423
424
425
426
427
428
429
430
431
        CustomShuffleEdgeManagerConfig edgeManagerConfig =
            new CustomShuffleEdgeManagerConfig(
                currentParallelism, finalTaskParallelism, basePartitionRange,
                ((remainderRangeForLastShuffler > 0) ?
                    remainderRangeForLastShuffler : basePartitionRange));
        EdgeManagerDescriptor edgeManagerDescriptor =
            new EdgeManagerDescriptor(CustomShuffleEdgeManager.class.getName());
        edgeManagerDescriptor.setUserPayload(edgeManagerConfig.toUserPayload());
        edgeManagers.put(vertex, edgeManagerDescriptor);
      }
     
      context.setVertexParallelism(finalTaskParallelism, null, edgeManagers);
      updatePendingTasks();     
View Full Code Here
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
    TezTaskID firstTask = tasks.keySet().iterator().next();

    startVertex(v3);

    Vertex v1 = vertices.get("vertex1");
    EdgeManagerDescriptor mockEdgeManagerDescriptor =
        new EdgeManagerDescriptor(EdgeManagerForTest.class.getName());

    Map<String, EdgeManagerDescriptor> edgeManagerDescriptors =
        Collections.singletonMap(
       v1.getName(), mockEdgeManagerDescriptor);
    Assert.assertTrue(v3.setParallelism(1, null, edgeManagerDescriptors));
View Full Code Here
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
    EdgeManagerForTest originalEm = (EdgeManagerForTest) em;
    Assert.assertTrue(Arrays.equals(edgePayload, originalEm.getEdgeManagerContext()
        .getUserPayload()));

    byte[] userPayload = new String("foo").getBytes();
    EdgeManagerDescriptor edgeManagerDescriptor =
        new EdgeManagerDescriptor(EdgeManagerForTest.class.getName());
    edgeManagerDescriptor.setUserPayload(userPayload);

    Vertex v1 = vertices.get("vertex1");
    Vertex v3 = vertices.get("vertex3"); // Vertex3 linked to v1 (v1 src, v3
                                         // dest)
View Full Code Here
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
       
        // If CUSTOM without an edge manager, setup a fake edge manager. Avoid
        // referencing the fake edge manager within the API module.
        if (edgeProperty.getDataMovementType() == DataMovementType.CUSTOM
            && edgeProperty.getEdgeManagerDescriptor() == null) {
          EdgeManagerDescriptor edgeDesc = new EdgeManagerDescriptor(
              NullEdgeManager.class.getName());
          EdgeProperty ep = new EdgeProperty(edgeDesc, edgeProperty.getDataSourceType(),
              edgeProperty.getSchedulingType(), edgeProperty.getEdgeSource(),
              edgeProperty.getEdgeDestination());
          edgeProperty = ep;
View Full Code Here
419
420
421
422
423
424
425
426
427
428
429
430
431
        CustomShuffleEdgeManagerConfig edgeManagerConfig =
            new CustomShuffleEdgeManagerConfig(
                currentParallelism, finalTaskParallelism, basePartitionRange,
                ((remainderRangeForLastShuffler > 0) ?
                    remainderRangeForLastShuffler : basePartitionRange));
        EdgeManagerDescriptor edgeManagerDescriptor =
            new EdgeManagerDescriptor(CustomShuffleEdgeManager.class.getName());
        edgeManagerDescriptor.setUserPayload(edgeManagerConfig.toUserPayload());
        edgeManagers.put(vertex, edgeManagerDescriptor);
      }
     
      context.setVertexParallelism(finalTaskParallelism, null, edgeManagers);
      updatePendingTasks();     
View Full Code Here
273
274
275
276
277
278
279
280
281
282
283
284
      logEvents(event, deserializedEvent);
    }
    {
      Map<String,EdgeManagerDescriptor> sourceEdgeManagers
          = new LinkedHashMap<String, EdgeManagerDescriptor>();
      sourceEdgeManagers.put("foo", new EdgeManagerDescriptor("bar"));
      sourceEdgeManagers.put("foo1", new EdgeManagerDescriptor("bar1").setUserPayload(
          new String("payload").getBytes()));
      VertexParallelismUpdatedEvent event =
          new VertexParallelismUpdatedEvent(
              TezVertexID.getInstance(
                  TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 111),
View Full Code Here
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
    TezTaskID firstTask = tasks.keySet().iterator().next();

    startVertex(v3);

    Vertex v1 = vertices.get("vertex1");
    EdgeManagerDescriptor mockEdgeManagerDescriptor =
        new EdgeManagerDescriptor(EdgeManagerForTest.class.getName());

    Map<String, EdgeManagerDescriptor> edgeManagerDescriptors =
        Collections.singletonMap(
       v1.getName(), mockEdgeManagerDescriptor);
    Assert.assertTrue(v3.setParallelism(1, null, edgeManagerDescriptors));
View Full Code Here
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
    EdgeManagerForTest originalEm = (EdgeManagerForTest) em;
    Assert.assertTrue(Arrays.equals(edgePayload, originalEm.getEdgeManagerContext()
        .getUserPayload()));

    byte[] userPayload = new String("foo").getBytes();
    EdgeManagerDescriptor edgeManagerDescriptor =
        new EdgeManagerDescriptor(EdgeManagerForTest.class.getName());
    edgeManagerDescriptor.setUserPayload(userPayload);

    Vertex v1 = vertices.get("vertex1");
    Vertex v3 = vertices.get("vertex3"); // Vertex3 linked to v1 (v1 src, v3
                                         // dest)
View Full Code Here
TOP

Related Classes of org.apache.tez.dag.api.EdgeManagerDescriptor

  • org.apache.hadoop.hive.ql.exec.tez.CustomPartitionVertex
  • org.apache.hadoop.hive.ql.exec.tez.DagUtils
  • org.apache.tez.dag.app.dag.impl.DAGImpl
  • org.apache.tez.dag.app.dag.impl.DAGImpl$InitTransition
  • org.apache.tez.dag.app.dag.impl.TestVertexImpl
  • org.apache.tez.dag.history.events.TestHistoryEventsProtoConversion
  • org.apache.tez.dag.history.events.VertexParallelismUpdatedEvent
  • org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.