Package org.apache.tez.client

Examples of org.apache.tez.client.AMConfiguration

318
319
320
321
322
323
324
325
326
327
328
    // TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir
    // is the same filesystem as the one used for Input/Output.
   
    TezSession tezSession = null;
    AMConfiguration amConfig = new AMConfiguration(null,
        null, tezConf, credentials);
   
    TezSessionConfiguration sessionConfig =
        new TezSessionConfiguration(amConfig, tezConf);
    tezSession = new TezSession("WordCountSession", appId,
View Full Code Here
186
187
188
189
190
191
192
193
194
195
196
        remoteStagingDir.toString());

    Map<String, LocalResource> amLocalResources =
        new HashMap<String, LocalResource>();

    AMConfiguration amConfig = new AMConfiguration(
        commonEnv, amLocalResources,
        tezConf, null);
    TezSessionConfiguration tezSessionConfig =
        new TezSessionConfiguration(amConfig, tezConf);
    TezSession tezSession = new TezSession("testsession", tezSessionConfig);
View Full Code Here
438
439
440
441
442
443
444
445
446
447
448
    TezClient tezClient = new TezClient(tezConf);
    DAGClient dagClient = null;
    TezSession tezSession = null;
    boolean reuseSession = reUseTezSession != null;
    TezSessionConfiguration tezSessionConfig;
    AMConfiguration amConfig = new AMConfiguration(
        commonEnv, amLocalResources,
        tezConf, null);
    if(!dagViaRPC) {
      // TODO Use utility method post TEZ-205 to figure out AM arguments etc.
      dagClient = tezClient.submitDAGApplication(dag, amConfig);
View Full Code Here
834
835
836
837
838
839
840
841
842
843
844
        iReduceSleepTime, iReduceSleepCount, writeSplitsToDfs, generateSplitsInAM);

    conf.set(TezConfiguration.TEZ_AM_JAVA_OPTS,
        MRHelpers.getMRAMJavaOpts(conf));

    AMConfiguration amConfig = new AMConfiguration(null,
        null, conf, this.credentials);

    DAGClient dagClient =
        tezClient.submitDAGApplication(appId, dag, amConfig);
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
        .nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());

    TezClient tezClient = new TezClient(tezConf);
    AMConfiguration amConf = new AMConfiguration(new HashMap<String, String>(),
        new HashMap<String, LocalResource>(), tezConf, null);

    DAGClient dagClient = tezClient.submitDAGApplication(dag, amConf);

    DAGStatus dagStatus = dagClient.getDAGStatus(null);
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
      TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
      tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
          remoteStagingDir.toString());
      tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);

      AMConfiguration amConfig = new AMConfiguration(
          new HashMap<String, String>(), new HashMap<String, LocalResource>(),
          tezConf, null);
      TezSessionConfiguration tezSessionConfig =
          new TezSessionConfiguration(amConfig, tezConf);
      tezSession = new TezSession("TestFaultTolerance", tezSessionConfig);
View Full Code Here
361
362
363
364
365
366
367
368
369
370
371
    // TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir
    // is the same filesystem as the one used for Input/Output.
   
    TezSession tezSession = null;
    AMConfiguration amConfig = new AMConfiguration(null,
        null, tezConf, instance.credentials);
    if (useTezSession) {
      LOG.info("Creating Tez Session");
      TezSessionConfiguration sessionConfig =
          new TezSessionConfiguration(amConfig, tezConf);
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
        remoteJarStatus.getLen(), remoteJarStatus.getModificationTime());
    commonLocalResources.put("dag_job.jar", dagJarLocalRsrc);



    AMConfiguration amConf = new AMConfiguration(null, commonLocalResources, tezConf, null);
    TezSessionConfiguration sessionConf = new TezSessionConfiguration(amConf, tezConf);
    TezSession tezSession = new TezSession("FilterLinesByWordSession", sessionConf);
    tezSession.start(); // Why do I need to start the TezSession.

    Configuration stage1Conf = new JobConf(conf);
View Full Code Here
824
825
826
827
828
829
830
831
832
833
834
        iReduceSleepTime, iReduceSleepCount, writeSplitsToDfs, generateSplitsInAM);

    conf.set(TezConfiguration.TEZ_AM_JAVA_OPTS,
        MRHelpers.getMRAMJavaOpts(conf));

    AMConfiguration amConfig = new AMConfiguration(null,
        null, conf, null);

    DAGClient dagClient =
        tezClient.submitDAGApplication(appId, dag, amConfig);
View Full Code Here
593
594
595
596
597
598
599
600
601
602
603
      int amCores = jobConf.getInt(MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES);
      dagAMConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, amMemMB);
      dagAMConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, amCores);

     
      AMConfiguration amConfig = new AMConfiguration(
          environment,
          jobLocalResources, dagAMConf, ts);
      tezClient.submitDAGApplication(appId, dag, amConfig);
    } catch (TezException e) {
      throw new IOException(e);
View Full Code Here
TOP

Related Classes of org.apache.tez.client.AMConfiguration

  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.hive.ql.exec.tez.TezSessionState
  • org.apache.tez.dag.api.TezConfiguration
  • org.apache.tez.mapreduce.client.YARNRunner
  • org.apache.tez.mapreduce.examples.FilterLinesByWord
  • org.apache.tez.mapreduce.examples.FilterLinesByWordOneToOne
  • org.apache.tez.mapreduce.examples.MRRSleepJob
  • org.apache.tez.mapreduce.examples.OrderedWordCount
  • org.apache.tez.mapreduce.examples.UnionExample
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.