Examples of VertexManagerPluginDescriptor

org.apache.tez.dag.api.VertexManagerPluginDescriptor
Describes the {@link VertexManagerPlugin}

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

    updateConfigurationForEdge(vConf, v, wConf, w);


    if (edgeProp.getEdgeType() == EdgeType.CUSTOM_EDGE) {
      int numBuckets = edgeProp.getNumBuckets();
      byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array();
      VertexManagerPluginDescriptor desc = new VertexManagerPluginDescriptor(
          CustomPartitionVertex.class.getName());
      desc.setUserPayload(userPayload);
      w.setVertexManagerPlugin(desc);
    }


    return new Edge(v, w, createEdgeProperty(edgeProp));
  }

View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

    case CUSTOM_EDGE: {
      int numBuckets = edgeProp.getNumBuckets();
      CustomVertexConfiguration vertexConf = new CustomVertexConfiguration(numBuckets, vertexType);
      DataOutputBuffer dob = new DataOutputBuffer();
      vertexConf.write(dob);
      VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create(
          CustomPartitionVertex.class.getName());
      byte[] userPayloadBytes = dob.getData();
      ByteBuffer userPayload = ByteBuffer.wrap(userPayloadBytes);
      desc.setUserPayload(UserPayload.create(userPayload));
      w.setVertexManagerPlugin(desc);
      break;
    }
    case SIMPLE_EDGE: {
      setupAutoReducerParallelism(edgeProp, w);

View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

        LOG.info("Going through each work and adding MultiMRInput");
        mergeVx.addDataSource(mapWork.getName(),
            MultiMRInput.createConfigBuilder(conf, HiveInputFormat.class).build());
      }


      VertexManagerPluginDescriptor desc =
        VertexManagerPluginDescriptor.create(CustomPartitionVertex.class.getName());
      // the +1 to the size is because of the main work.
      CustomVertexConfiguration vertexConf =
          new CustomVertexConfiguration(mergeJoinWork.getMergeJoinOperator().getConf()
              .getNumBuckets(), vertexType, mergeJoinWork.getBigTableAlias(),
              mapWorkList.size() + 1);
      DataOutputBuffer dob = new DataOutputBuffer();
      vertexConf.write(dob);
      byte[] userPayload = dob.getData();
      desc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload)));
      mergeVx.setVertexManagerPlugin(desc);
      return mergeVx;
    } else {
      Vertex mergeVx =
          createVertex(conf, (ReduceWork) mergeJoinWork.getMainWork(), appJarLr, additionalLr, fs,

View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor


  private void setupAutoReducerParallelism(TezEdgeProperty edgeProp, Vertex v)
    throws IOException {
    if (edgeProp.isAutoReduce()) {
      Configuration pluginConf = new Configuration(false);
      VertexManagerPluginDescriptor desc =
          VertexManagerPluginDescriptor.create(ShuffleVertexManager.class.getName());
      pluginConf.setBoolean(
          ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL, true);
      pluginConf.setInt(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM,
          edgeProp.getMinReducer());
      pluginConf.setLong(
          ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE,
          edgeProp.getInputSizePerReducer());
      UserPayload payload = TezUtils.createUserPayloadFromConf(pluginConf);
      desc.setUserPayload(payload);
      v.setVertexManagerPlugin(desc);
    }
  }

View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

      mergeInputClass = ConcatenatedMergedKeyValueInput.class;
      int numBuckets = edgeProp.getNumBuckets();
      CustomVertexConfiguration vertexConf = new CustomVertexConfiguration(numBuckets, vertexType);
      DataOutputBuffer dob = new DataOutputBuffer();
      vertexConf.write(dob);
      VertexManagerPluginDescriptor desc =
          VertexManagerPluginDescriptor.create(CustomPartitionVertex.class.getName());
      byte[] userPayloadBytes = dob.getData();
      ByteBuffer userPayload = ByteBuffer.wrap(userPayloadBytes);
      desc.setUserPayload(UserPayload.create(userPayload));
      w.setVertexManagerPlugin(desc);
      break;
    }


    case CUSTOM_SIMPLE_EDGE:

View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

    Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);


    // Make each vertex manager fail on appropriate attempt
    v1.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
        FailOnAttemptVertexManagerPlugin.class.getName())
        .setUserPayload(new String("1").getBytes()));
    v2.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
        FailOnAttemptVertexManagerPlugin.class.getName())
        .setUserPayload(new String("2").getBytes()));
    v3.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
        FailOnAttemptVertexManagerPlugin.class.getName())
        .setUserPayload(new String("3").getBytes()));
    dag.addVertex(v1).addVertex(v2).addVertex(v3);
    dag.addEdge(new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,

View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

        mergeInputClass = ConcatenatedMergedKeyValueInput.class;
        break;
      case CUSTOM_EDGE:
        mergeInputClass = ConcatenatedMergedKeyValueInput.class;
        int numBuckets = edgeProp.getNumBuckets();
        VertexManagerPluginDescriptor desc = new VertexManagerPluginDescriptor(
            CustomPartitionVertex.class.getName());
        byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array();
        desc.setUserPayload(userPayload);
        w.setVertexManagerPlugin(desc);
        break;


      case CUSTOM_SIMPLE_EDGE:
        mergeInputClass = ConcatenatedMergedKeyValueInput.class;

View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

    updateConfigurationForEdge(vConf, v, wConf, w);


    if (edgeProp.getEdgeType() == EdgeType.CUSTOM_EDGE) {
      int numBuckets = edgeProp.getNumBuckets();
      byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array();
      VertexManagerPluginDescriptor desc = new VertexManagerPluginDescriptor(
          CustomPartitionVertex.class.getName());
      desc.setUserPayload(userPayload);
      w.setVertexManagerPlugin(desc);
    }


    return new Edge(v, w, createEdgeProperty(edgeProp));
  }

View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

        mergeInputClass = ConcatenatedMergedKeyValueInput.class;
        break;
      case CUSTOM_EDGE:
        mergeInputClass = ConcatenatedMergedKeyValueInput.class;
        int numBuckets = edgeProp.getNumBuckets();
        VertexManagerPluginDescriptor desc = new VertexManagerPluginDescriptor(
            CustomPartitionVertex.class.getName());
        byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array();
        desc.setUserPayload(userPayload);
        w.setVertexManagerPlugin(desc);
        break;


      case CUSTOM_SIMPLE_EDGE:
        mergeInputClass = ConcatenatedMergedKeyValueInput.class;

View Full Code Here

Examples of org.apache.tez.dag.api.VertexManagerPluginDescriptor

    updateConfigurationForEdge(vConf, v, wConf, w);


    if (edgeProp.getEdgeType() == EdgeType.CUSTOM_EDGE) {
      int numBuckets = edgeProp.getNumBuckets();
      byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array();
      VertexManagerPluginDescriptor desc = new VertexManagerPluginDescriptor(
          CustomPartitionVertex.class.getName());
      desc.setUserPayload(userPayload);
      w.setVertexManagerPlugin(desc);
    }


    return new Edge(v, w, createEdgeProperty(edgeProp));
  }

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.