Examples of org.apache.tez.dag.api.PreWarmVertex$PreWarmVertexConfigBuilder

org.apache.tez.dag.api.PreWarmVertex
Setup the prewarm vertex constructor. By default is uses the built-in PreWarmProcessor and sets up the prewarm container number equal to {@link TezConfiguration#TEZ_AM_SESSION_MIN_HELD_CONTAINERS}

    Map<String, LocalResource> lrs = Maps.newHashMap();
    final TezClientForTest client = new TezClientForTest("test", conf, lrs, null);


    configure(client);
    client.start();
    PreWarmVertex vertex = PreWarmVertex.create("PreWarm", 1, Resource.newInstance(1, 1));
    client.preWarm(vertex);
    
    ArgumentCaptor<SubmitDAGRequestProto> captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class);
    verify(client.sessionAmProxy, times(1)).submitDAG((RpcController)any(), (SubmitDAGRequestProto) any());
    SubmitDAGRequestProto proto = captor1.getValue();

View Full Code Here

            doPreWarm = false;
          }
        }
        if (doPreWarm) {
          LOG.info("Pre-warming Session");
          PreWarmVertex preWarmVertex = PreWarmVertex.create("PreWarm", preWarmNumContainers, dag
              .getVertex("initialmap").getTaskResource());
          preWarmVertex.addTaskLocalFiles(dag.getVertex("initialmap").getTaskLocalFiles());
          preWarmVertex.setTaskEnvironment(dag.getVertex("initialmap").getTaskEnvironment());
          preWarmVertex.setTaskLaunchCmdOpts(dag.getVertex("initialmap").getTaskLaunchCmdOpts());
          
          tezSession.preWarm(preWarmVertex);
        }


        if (useTezSession) {

View Full Code Here


    if (HiveConf.getBoolVar(conf, ConfVars.HIVE_PREWARM_ENABLED)) {
      int n = HiveConf.getIntVar(conf, ConfVars.HIVE_PREWARM_NUM_CONTAINERS);
      LOG.info("Prewarming " + n + " containers  (id: " + sessionId
          + ", scratch dir: " + tezScratchDir + ")");
      PreWarmVertex prewarmVertex = utils.createPreWarmVertex(tezConfig, n,
          commonLocalResources);
      try {
        session.preWarm(prewarmVertex);
      } catch (IOException ie) {
        if (ie.getMessage().contains("Interrupted while waiting")) {

View Full Code Here

      IOException, TezException {


    ProcessorDescriptor prewarmProcDescriptor = ProcessorDescriptor.create(HivePreWarmProcessor.class.getName());
    prewarmProcDescriptor.setUserPayload(TezUtils.createUserPayloadFromConf(conf));


    PreWarmVertex prewarmVertex = PreWarmVertex.create("prewarm", prewarmProcDescriptor, numContainers,getContainerResource(conf));


    Map<String, LocalResource> combinedResources = new HashMap<String, LocalResource>();


    if (localResources != null) {
      combinedResources.putAll(localResources);
    }


    prewarmVertex.addTaskLocalFiles(localResources);
    prewarmVertex.setTaskLaunchCmdOpts(getContainerJavaOpts(conf));
    prewarmVertex.setTaskEnvironment(getContainerEnvironment(conf, false));
    return prewarmVertex;
  }

View Full Code Here

TOP

Related Classes of org.apache.tez.dag.api.PreWarmVertex$PreWarmVertexConfigBuilder

org.apache.hadoop.hive.ql.exec.tez.DagUtils

org.apache.hadoop.hive.ql.exec.tez.TezSessionState

org.apache.tez.client.TestTezClient

org.apache.tez.mapreduce.examples.TestOrderedWordCount

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.