Examples of org.apache.tez.dag.api.TezConfiguration

org.apache.tez.dag.api.TezConfiguration
Defines the configurations for Tez. These configurations are typically specified in tez-site.xml on the client machine where TezClient is used to launch the Tez application. tez-site.xml is expected to be picked up from the classpath of the client process.

  private Credentials credentials = new Credentials();
  
  public boolean run(String inputPath, String outputPath, Configuration conf) throws Exception {
    System.out.println("Running WordCount");
    // conf and UGI
    TezConfiguration tezConf;
    if (conf != null) {
      tezConf = new TezConfiguration(conf);
    } else {
      tezConf = new TezConfiguration();
    }
    UserGroupInformation.setConfiguration(tezConf);
    String user = UserGroupInformation.getCurrentUser().getShortUserName();


    TezClient tezClient = new TezClient(tezConf);
    ApplicationId appId = tezClient.createApplication();
    
    // staging dir
    FileSystem fs = FileSystem.get(tezConf);
    String stagingDirStr = Path.SEPARATOR + "user" + Path.SEPARATOR
        + user + Path.SEPARATOR+ ".staging" + Path.SEPARATOR
        + Path.SEPARATOR + appId.toString();    
    Path stagingDir = new Path(stagingDirStr);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr);
    stagingDir = fs.makeQualified(stagingDir);
    
    // security
    TokenCache.obtainTokensForNamenodes(credentials, new Path[] {stagingDir}, tezConf);
    TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);


    tezConf.set(TezConfiguration.TEZ_AM_JAVA_OPTS,
        MRHelpers.getMRAMJavaOpts(tezConf));


    // No need to add jar containing this class as assumed to be part of
    // the tez jars.

View Full Code Here

    // sleep for *SleepTime duration in Task by recSleepTime per record
    mapSleepCount = (int)Math.ceil(mapSleepTime / ((double)recSleepTime));
    reduceSleepCount = (int)Math.ceil(reduceSleepTime / ((double)recSleepTime));
    iReduceSleepCount = (int)Math.ceil(iReduceSleepTime / ((double)recSleepTime));


    TezConfiguration conf = new TezConfiguration(getConf());
    FileSystem remoteFs = FileSystem.get(conf);


    TezClient tezClient = new TezClient(conf);
    ApplicationId appId =
        tezClient.createApplication();


    conf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        conf.get(
            TezConfiguration.TEZ_AM_STAGING_DIR,
            TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT));


    Path remoteStagingDir =
        remoteFs.makeQualified(new Path(conf.get(
            TezConfiguration.TEZ_AM_STAGING_DIR,
            TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT),
            appId.toString()));
    TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);


    DAG dag = createDAG(remoteFs, conf, appId, remoteStagingDir,
        numMapper, numReducer, iReduceStagesCount, numIReducer,
        mapSleepTime, mapSleepCount, reduceSleepTime, reduceSleepCount,
        iReduceSleepTime, iReduceSleepCount, writeSplitsToDfs, generateSplitsInAM);


    conf.set(TezConfiguration.TEZ_AM_JAVA_OPTS,
        MRHelpers.getMRAMJavaOpts(conf));


    AMConfiguration amConfig = new AMConfiguration(null,
        null, conf, this.credentials);

View Full Code Here

  // FIXME
  // how to handle completed jobs that the RM does not know about?


  public ClientServiceDelegate(Configuration conf, ResourceMgrDelegate rm,
      JobID jobId) {
    this.conf = new TezConfiguration(conf); // Cloning for modifying.
    // For faster redirects from AM to HS.
    this.conf.setInt(
        CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
        this.conf.getInt(MRJobConfig.MR_CLIENT_TO_AM_IPC_MAX_RETRIES,
            MRJobConfig.DEFAULT_MR_CLIENT_TO_AM_IPC_MAX_RETRIES));

View Full Code Here

      Map<String, LocalResource> localResources,
      TezConfiguration conf, Credentials credentials) {
    if (conf != null) {
      this.amConf = conf;
    } else {
      this.amConf = new TezConfiguration();
    }
    this.queueName = this.amConf.get(TezConfiguration.TEZ_QUEUE_NAME);


    if (env != null) {
      this.env = env;

View Full Code Here

    Vertex vertex = new Vertex("SleepVertex", new ProcessorDescriptor(
        SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
        Resource.newInstance(1024, 1));
    dag.addVertex(vertex);


    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(new Random()
        .nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());


    TezClient tezClient = new TezClient(tezConf);
    AMConfiguration amConf = new AMConfiguration(new HashMap<String, String>(),
        new HashMap<String, LocalResource>(), tezConf, null);

View Full Code Here

  InterruptedException, TezException, ClassNotFoundException, YarnException {
    Map<String, String> commonEnv = createCommonEnv();
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String
        .valueOf(new Random().nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    TezConfiguration tezConf = new TezConfiguration(
        mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        remoteStagingDir.toString());


    Map<String, LocalResource> amLocalResources =
        new HashMap<String, LocalResource>();

View Full Code Here


    Map<String, LocalResource> amLocalResources =
        new HashMap<String, LocalResource>();
    amLocalResources.putAll(commonLocalResources);


    TezConfiguration tezConf = new TezConfiguration(
            mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        remoteStagingDir.toString());


    TezClient tezClient = new TezClient(tezConf);
    DAGClient dagClient = null;
    TezSession tezSession = null;

View Full Code Here

      
      Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
          .valueOf(new Random().nextInt(100000))));
      TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
      
      TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
      tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
          remoteStagingDir.toString());
      tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);


      AMConfiguration amConfig = new AMConfiguration(
          new HashMap<String, String>(), new HashMap<String, LocalResource>(),
          tezConf, null);
      TezSessionConfiguration tezSessionConfig =

View Full Code Here

    }


    UserGroupInformation.setConfiguration(conf);
    String user = UserGroupInformation.getCurrentUser().getShortUserName();


    TezConfiguration tezConf = new TezConfiguration(conf);
    TezClient tezClient = new TezClient(tezConf);
    ApplicationId appId = tezClient.createApplication();
    OrderedWordCount instance = new OrderedWordCount();


    FileSystem fs = FileSystem.get(conf);


    String stagingDirStr = Path.SEPARATOR + "user" + Path.SEPARATOR
        + user + Path.SEPARATOR+ ".staging" + Path.SEPARATOR
        + Path.SEPARATOR + appId.toString();
    Path stagingDir = new Path(stagingDirStr);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr);
    stagingDir = fs.makeQualified(stagingDir);
    
    TokenCache.obtainTokensForNamenodes(instance.credentials, new Path[] {stagingDir}, conf);
    TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);


    tezConf.set(TezConfiguration.TEZ_AM_JAVA_OPTS,
        MRHelpers.getMRAMJavaOpts(conf));


    // No need to add jar containing this class as assumed to be part of
    // the tez jars.

View Full Code Here

    if (fs.exists(new Path(outputPath))) {
      System.err.println("Output directory : " + outputPath + " already exists");
      System.exit(2);
    }


    TezConfiguration tezConf = new TezConfiguration(conf);


    fs.getWorkingDirectory();
    Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
    TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);


    tezConf.set(TezConfiguration.TEZ_AM_JAVA_OPTS, MRHelpers.getMRAMJavaOpts(tezConf));


    String jarPath = ClassUtil.findContainingJar(FilterLinesByWord.class);
    if (jarPath == null) {
      throw new TezUncheckedException("Could not find any jar containing"
          + FilterLinesByWord.class.getName() + " in the classpath");

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.tez.dag.api.TezConfiguration

cascading.flow.tez.FlowProcessor

cascading.flow.tez.Hadoop2TezFlow

cascading.flow.tez.Hadoop2TezFlowProcess

cascading.flow.tez.Hadoop2TezFlowStep

cascading.flow.tez.planner.Hadoop2TezFlowStepJob

cascading.flow.tez.planner.Hadoop2TezPlanner

cascading.flow.tez.stream.element.TezSourceStage

cascading.flow.tez.stream.graph.Hadoop2TezStreamGraph

cascading.flow.tez.util.TezUtil

cascading.platform.tez.Hadoop2TezPlatform

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.