Examples of VertexManagerPluginDescriptor

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

228
229
230
231
232
233
234
235
236
237
238
239
240
241
        mergeInputClass = ConcatenatedMergedKeyValueInput.class;
        break;
      case CUSTOM_EDGE:
        mergeInputClass = ConcatenatedMergedKeyValueInput.class;
        int numBuckets = edgeProp.getNumBuckets();
        VertexManagerPluginDescriptor desc = new VertexManagerPluginDescriptor(
            CustomPartitionVertex.class.getName());
        byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array();
        desc.setUserPayload(userPayload);
        w.setVertexManagerPlugin(desc);
        break;

      case CUSTOM_SIMPLE_EDGE:
        mergeInputClass = ConcatenatedMergedKeyValueInput.class;
View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

282
283
284
285
286
287
288
289
290
291
292
293
294
    updateConfigurationForEdge(vConf, v, wConf, w);

    if (edgeProp.getEdgeType() == EdgeType.CUSTOM_EDGE) {
      int numBuckets = edgeProp.getNumBuckets();
      byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array();
      VertexManagerPluginDescriptor desc = new VertexManagerPluginDescriptor(
          CustomPartitionVertex.class.getName());
      desc.setUserPayload(userPayload);
      w.setVertexManagerPlugin(desc);
    }

    return new Edge(v, w, createEdgeProperty(edgeProp));
  }
View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

739
740
741
742
743
744
745
746
747
748
749
750
751
        // TODO: PIG-4049 If standalone Limit we need a new VertexManager or new input
        // instead of ShuffledMergedInput. For limit part of the sort (order by parallel 1) itself
        // need to enhance PartitionerDefinedVertexManager

        if (vmPluginName != null) {
            VertexManagerPluginDescriptor vmPluginDescriptor = VertexManagerPluginDescriptor.create(vmPluginName);
            if (vmPluginConf != null) {
                vmPluginDescriptor.setUserPayload(TezUtils.createUserPayloadFromConf(vmPluginConf));
            }
            vertex.setVertexManagerPlugin(vmPluginDescriptor);
        }
        // Reset udfcontext jobconf. It is not supposed to be set in the front end
        UDFContext.getUDFContext().addJobConf(null);
View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

657
658
659
660
661
662
663
664
665
666
667
668
669
670
          minTaskParallelism);
      return this;
    }

    public VertexManagerPluginDescriptor build() {
      VertexManagerPluginDescriptor desc =
          VertexManagerPluginDescriptor.create(ShuffleVertexManager.class.getName());

      try {
        return desc.setUserPayload(TezUtils.createUserPayloadFromConf(this.conf));
      } catch (IOException e) {
        throw new TezUncheckedException(e);
      }
    }
View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
    when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(4);

    //Check via setters
    ShuffleVertexManager.ShuffleVertexManagerConfigBuilder configurer = ShuffleVertexManager
        .createConfigBuilder(null);
    VertexManagerPluginDescriptor pluginDesc = configurer.setAutoReduceParallelism(true)
        .setDesiredTaskInputSize(1000l)
        .setMinTaskParallelism(10).setSlowStartMaxSrcCompletionFraction(0.5f).build();
    when(mockContext.getUserPayload()).thenReturn(pluginDesc.getUserPayload());


    manager = ReflectionUtils.createClazzInstance(pluginDesc.getClassName(),
        new Class[]{VertexManagerPluginContext.class}, new Object[]{mockContext});
    manager.initialize();

    Assert.assertTrue(manager.enableAutoParallelism == true);
    Assert.assertTrue(manager.desiredTaskInputDataSize == 1000l);
    Assert.assertTrue(manager.minTaskParallelism == 10);
    Assert.assertTrue(manager.slowStartMinSrcCompletionFraction == 0.25f);
    Assert.assertTrue(manager.slowStartMaxSrcCompletionFraction == 0.5f);

    configurer = ShuffleVertexManager.createConfigBuilder(null);
    pluginDesc = configurer.setAutoReduceParallelism(false).build();
    when(mockContext.getUserPayload()).thenReturn(pluginDesc.getUserPayload());

    manager = ReflectionUtils.createClazzInstance(pluginDesc.getClassName(),
        new Class[]{VertexManagerPluginContext.class}, new Object[]{mockContext});
    manager.initialize();

    Assert.assertTrue(manager.enableAutoParallelism == false);
    Assert.assertTrue(manager.desiredTaskInputDataSize ==
View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
      }
     
      boolean hasUserVertexManager = vertex.vertexPlan.hasVertexManagerPlugin();
     
      if (hasUserVertexManager) {
        VertexManagerPluginDescriptor pluginDesc = DagTypeConverters
            .convertVertexManagerPluginDescriptorFromDAGPlan(vertex.vertexPlan
                .getVertexManagerPlugin());
        LOG.info("Setting user vertex manager plugin: "
            + pluginDesc.getClassName() + " on vertex: " + vertex.getName());
        vertex.vertexManager = new VertexManager(pluginDesc, vertex, vertex.appContext);
      } else {
        if (hasBipartite) {
          // setup vertex manager
          // TODO this needs to consider data size and perhaps API.
View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
    }
   
    boolean hasUserVertexManager = vertexPlan.hasVertexManagerPlugin();

    if (hasUserVertexManager) {
      VertexManagerPluginDescriptor pluginDesc = DagTypeConverters
          .convertVertexManagerPluginDescriptorFromDAGPlan(vertexPlan
              .getVertexManagerPlugin());
      LOG.info("Setting user vertex manager plugin: "
          + pluginDesc.getClassName() + " on vertex: " + getName());
      vertexManager = new VertexManager(pluginDesc, this, appContext);
    } else {
      // Intended order of picking a vertex manager
      // If there is an InputInitializer then we use the RootInputVertexManager. May be fixed by TEZ-703
      // If there is a custom edge we fall back to default ImmediateStartVertexManager
View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

433
434
435
436
437
438
439
440
441
442
443
        .setTaskLocalResources(taskLocalResources)
        .setTaskLocationsHint(locations)
        .setJavaOpts(taskJavaOpts);
   
    if (!isMap) {
      vertex.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
          ShuffleVertexManager.class.getName()));
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("Adding vertex to DAG" + ", vertexName="
View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
    Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);

    // Make each vertex manager fail on appropriate attempt
    v1.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
        FailOnAttemptVertexManagerPlugin.class.getName())
        .setUserPayload(new String("1").getBytes()));
    v2.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
        FailOnAttemptVertexManagerPlugin.class.getName())
        .setUserPayload(new String("2").getBytes()));
    v3.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
        FailOnAttemptVertexManagerPlugin.class.getName())
        .setUserPayload(new String("3").getBytes()));
    dag.addVertex(v1).addVertex(v2).addVertex(v3);
    dag.addEdge(new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

229
230
231
232
233
234
235
236
237
238
239
240
241
242
        mergeInputClass = ConcatenatedMergedKeyValueInput.class;
        break;
      case CUSTOM_EDGE:
        mergeInputClass = ConcatenatedMergedKeyValueInput.class;
        int numBuckets = edgeProp.getNumBuckets();
        VertexManagerPluginDescriptor desc = new VertexManagerPluginDescriptor(
            CustomPartitionVertex.class.getName());
        byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array();
        desc.setUserPayload(userPayload);
        w.setVertexManagerPlugin(desc);
        break;

      case CUSTOM_SIMPLE_EDGE:
        mergeInputClass = ConcatenatedMergedKeyValueInput.class;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.