Package org.apache.tez.client

Examples of org.apache.tez.client.PreWarmContext

595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
    Configuration conf = sessionConfig.getTezConfiguration();

    ProcessorDescriptor prewarmProcDescriptor = new ProcessorDescriptor(HivePreWarmProcessor.class.getName());
    prewarmProcDescriptor.setUserPayload(MRHelpers.createUserPayloadFromConf(conf));

    PreWarmContext context = new PreWarmContext(prewarmProcDescriptor, getContainerResource(conf),
        numContainers, new VertexLocationHint(null));

    Map<String, LocalResource> combinedResources = new HashMap<String, LocalResource>();

    combinedResources.putAll(sessionConfig.getSessionResources());
    if (localResources != null) {
      combinedResources.putAll(localResources);
    }

    context.setLocalResources(combinedResources);

    /* boiler plate task env */
    Map<String, String> environment = new HashMap<String, String>();
    MRHelpers.updateEnvironmentForMRTasks(conf, environment, true);
    context.setEnvironment(environment);
    context.setJavaOpts(getContainerJavaOpts(conf));
    return context;
  }
View Full Code Here
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
    if (proto.hasLocationHints()) {
      vertexLocationHint =
          DagTypeConverters.convertVertexLocationHintFromProto(
              proto.getLocationHints());
    }
    PreWarmContext context = new PreWarmContext(
      DagTypeConverters.convertProcessorDescriptorFromDAGPlan(
        proto.getProcessorDescriptor()),
        Resource.newInstance(proto.getMemoryMb(), proto.getVirtualCores()),
        proto.getNumTasks(),
        vertexLocationHint);
    if (proto.hasLocalResources()) {
      context.setLocalResources(
        DagTypeConverters.convertFromPlanLocalResources(
          proto.getLocalResources()));
    }
    context.setEnvironment(
      DagTypeConverters.createEnvironmentMapFromDAGPlan(
        proto.getEnvironmentSettingList()));
    if (proto.hasJavaOpts()) {
      context.setJavaOpts(proto.getJavaOpts());
    }
    return context;
  }
View Full Code Here
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
            new ProcessorDescriptor(SleepProcessor.class.getName());
          SleepProcessor.SleepProcessorConfig sleepProcessorConfig =
            new SleepProcessor.SleepProcessorConfig(4000);
          sleepProcDescriptor.setUserPayload(
            sleepProcessorConfig.toUserPayload());
          PreWarmContext context = new PreWarmContext(sleepProcDescriptor,
            dag.getVertex("initialmap").getTaskResource(), preWarmNumContainers,
              vertexLocationHint);

          Map<String, LocalResource> contextLocalRsrcs =
            new TreeMap<String, LocalResource>();
          contextLocalRsrcs.putAll(
            dag.getVertex("initialmap").getTaskLocalResources());
          Map<String, String> contextEnv = new TreeMap<String, String>();
          contextEnv.putAll(dag.getVertex("initialmap").getTaskEnvironment());
          String contextJavaOpts =
            dag.getVertex("initialmap").getJavaOpts();
          context
            .setLocalResources(contextLocalRsrcs)
            .setJavaOpts(contextJavaOpts)
            .setEnvironment(contextEnv);

          tezSession.preWarm(context);
View Full Code Here
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
            new ProcessorDescriptor(SleepProcessor.class.getName());
          SleepProcessor.SleepProcessorConfig sleepProcessorConfig =
            new SleepProcessor.SleepProcessorConfig(4000);
          sleepProcDescriptor.setUserPayload(
            sleepProcessorConfig.toUserPayload());
          PreWarmContext context = new PreWarmContext(sleepProcDescriptor,
            dag.getVertex("initialmap").getTaskResource(), preWarmNumContainers,
              vertexLocationHint);

          Map<String, LocalResource> contextLocalRsrcs =
            new TreeMap<String, LocalResource>();
          contextLocalRsrcs.putAll(
            dag.getVertex("initialmap").getTaskLocalResources());
          Map<String, String> contextEnv = new TreeMap<String, String>();
          contextEnv.putAll(dag.getVertex("initialmap").getTaskEnvironment());
          String contextJavaOpts =
            dag.getVertex("initialmap").getJavaOpts();
          context
            .setLocalResources(contextLocalRsrcs)
            .setJavaOpts(contextJavaOpts)
            .setEnvironment(contextEnv);

          tezSession.preWarm(context);
View Full Code Here
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
    if (proto.hasLocationHints()) {
      vertexLocationHint =
          DagTypeConverters.convertVertexLocationHintFromProto(
              proto.getLocationHints());
    }
    PreWarmContext context = new PreWarmContext(
      DagTypeConverters.convertProcessorDescriptorFromDAGPlan(
        proto.getProcessorDescriptor()),
        Resource.newInstance(proto.getMemoryMb(), proto.getVirtualCores()),
        proto.getNumTasks(),
        vertexLocationHint);
    if (proto.hasLocalResources()) {
      context.setLocalResources(
        DagTypeConverters.convertFromPlanLocalResources(
          proto.getLocalResources()));
    }
    context.setEnvironment(
      DagTypeConverters.createEnvironmentMapFromDAGPlan(
        proto.getEnvironmentSettingList()));
    if (proto.hasJavaOpts()) {
      context.setJavaOpts(proto.getJavaOpts());
    }
    return context;
  }
View Full Code Here
196
197
198
199
200
201
202
203
204
205
206
    if (HiveConf.getBoolVar(conf, ConfVars.HIVE_PREWARM_ENABLED)) {
      int n = HiveConf.getIntVar(conf, ConfVars.HIVE_PREWARM_NUM_CONTAINERS);
      LOG.info("Prewarming " + n + " containers  (id: " + sessionId
          + ", scratch dir: " + tezScratchDir + ")");
      PreWarmContext context = utils.createPreWarmContext(sessionConfig, n, commonLocalResources);
      try {
        session.preWarm(context);
      } catch (InterruptedException ie) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("Hive Prewarm threw an exception ", ie);
View Full Code Here
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
    Configuration conf = sessionConfig.getTezConfiguration();

    ProcessorDescriptor prewarmProcDescriptor = new ProcessorDescriptor(HivePreWarmProcessor.class.getName());
    prewarmProcDescriptor.setUserPayload(MRHelpers.createUserPayloadFromConf(conf));

    PreWarmContext context = new PreWarmContext(prewarmProcDescriptor, getContainerResource(conf),
        numContainers, new VertexLocationHint(null));

    Map<String, LocalResource> combinedResources = new HashMap<String, LocalResource>();

    combinedResources.putAll(sessionConfig.getSessionResources());
    if (localResources != null) {
      combinedResources.putAll(localResources);
    }

    context.setLocalResources(combinedResources);

    /* boiler plate task env */
    Map<String, String> environment = new HashMap<String, String>();
    MRHelpers.updateEnvironmentForMRTasks(conf, environment, true);
    context.setEnvironment(environment);
    context.setJavaOpts(getContainerJavaOpts(conf));
    return context;
  }
View Full Code Here
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
    Configuration conf = sessionConfig.getTezConfiguration();

    ProcessorDescriptor prewarmProcDescriptor = new ProcessorDescriptor(HivePreWarmProcessor.class.getName());
    prewarmProcDescriptor.setUserPayload(MRHelpers.createUserPayloadFromConf(conf));

    PreWarmContext context = new PreWarmContext(prewarmProcDescriptor, getContainerResource(conf),
        numContainers, new VertexLocationHint(null));

    Map<String, LocalResource> combinedResources = new HashMap<String, LocalResource>();

    combinedResources.putAll(sessionConfig.getSessionResources());
    if (localResources != null) {
      combinedResources.putAll(localResources);
    }

    context.setLocalResources(combinedResources);

    /* boiler plate task env */
    Map<String, String> environment = new HashMap<String, String>();
    MRHelpers.updateEnvironmentForMRTasks(conf, environment, true);
    context.setEnvironment(environment);
    context.setJavaOpts(getContainerJavaOpts(conf));
    return context;
  }
View Full Code Here
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
    Configuration conf = sessionConfig.getTezConfiguration();

    ProcessorDescriptor prewarmProcDescriptor = new ProcessorDescriptor(HivePreWarmProcessor.class.getName());
    prewarmProcDescriptor.setUserPayload(MRHelpers.createUserPayloadFromConf(conf));

    PreWarmContext context = new PreWarmContext(prewarmProcDescriptor, getContainerResource(conf),
        numContainers, new VertexLocationHint(null));

    Map<String, LocalResource> combinedResources = new HashMap<String, LocalResource>();

    combinedResources.putAll(sessionConfig.getSessionResources());
    if (localResources != null) {
      combinedResources.putAll(localResources);
    }

    context.setLocalResources(combinedResources);

    /* boiler plate task env */
    Map<String, String> environment = new HashMap<String, String>();
    MRHelpers.updateEnvironmentForMRTasks(conf, environment, true);
    context.setEnvironment(environment);
    context.setJavaOpts(getContainerJavaOpts(conf));
    return context;
  }
View Full Code Here
196
197
198
199
200
201
202
203
204
205
206
    if (HiveConf.getBoolVar(conf, ConfVars.HIVE_PREWARM_ENABLED)) {
      int n = HiveConf.getIntVar(conf, ConfVars.HIVE_PREWARM_NUM_CONTAINERS);
      LOG.info("Prewarming " + n + " containers  (id: " + sessionId
          + ", scratch dir: " + tezScratchDir + ")");
      PreWarmContext context = utils.createPreWarmContext(sessionConfig, n, commonLocalResources);
      try {
        session.preWarm(context);
      } catch (InterruptedException ie) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("Hive Prewarm threw an exception ", ie);
View Full Code Here
TOP

Related Classes of org.apache.tez.client.PreWarmContext

  • org.apache.hadoop.hive.ql.exec.tez.DagUtils
  • org.apache.hadoop.hive.ql.exec.tez.TezSessionState
  • org.apache.tez.dag.api.DagTypeConverters
  • org.apache.tez.mapreduce.examples.OrderedWordCount
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.