Examples of org.apache.tez.dag.api.TezConfiguration

org.apache.tez.dag.api.TezConfiguration
Defines the configurations for Tez. These configurations are typically specified in tez-site.xml on the client machine where TezClient is used to launch the Tez application. tez-site.xml is expected to be picked up from the classpath of the client process.


    job.submit();
    JobID jobId = job.getJobID();
    ApplicationId appId = TypeConverter.toYarn(jobId).getAppId();


    DAGClient dagClient = MRTezClient.getDAGClient(appId, new TezConfiguration(conf), null);
    DAGStatus dagStatus;
    String[] vNames = { "initialmap" , "ireduce1" , "finalreduce" };
    while (true) {
      dagStatus = dagClient.getDAGStatus(null);
      if(dagStatus.getState() == DAGStatus.State.RUNNING ||

View Full Code Here

  Configuration conf = null;
  TezClient tezSession = null;
  Resource defaultResource = Resource.newInstance(100, 0);
  
  void setup() throws Exception {
    TezConfiguration tezConf = null;
    if (conf == null ) {
      tezConf = new TezConfiguration(new YarnConfiguration());
    }else {
       tezConf = new TezConfiguration(new YarnConfiguration(this.conf));
    }
    FileSystem defaultFs = FileSystem.get(tezConf);
    
    Path remoteStagingDir = defaultFs.makeQualified(new Path(TEST_ROOT_DIR, String
        .valueOf(new Random().nextInt(100000))));
    TezClientUtils.ensureStagingDirExists(tezConf, remoteStagingDir);
    
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        remoteStagingDir.toString());


    tezSession = TezClient.create("FaultToleranceTestRunner", tezConf);
    tezSession.start();
  }

View Full Code Here

    LOG.info("Starting session");
    Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
        .valueOf(new Random().nextInt(100000))));
    TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);


    TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
    tezConf.setInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, 0);
    tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "DEBUG");
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        remoteStagingDir.toString());
    tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
    tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
    tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
    tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);


    tezSession = TezClient.create("TestDAGRecovery", tezConf);
    tezSession.start();
  }

View Full Code Here

    // sleep for *SleepTime duration in Task by recSleepTime per record
    mapSleepCount = (int)Math.ceil(mapSleepTime / ((double)recSleepTime));
    reduceSleepCount = (int)Math.ceil(reduceSleepTime / ((double)recSleepTime));
    iReduceSleepCount = (int)Math.ceil(iReduceSleepTime / ((double)recSleepTime));


    TezConfiguration conf = new TezConfiguration(getConf());
    FileSystem remoteFs = FileSystem.get(conf);


    conf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        conf.get(
            TezConfiguration.TEZ_AM_STAGING_DIR,
            TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT));
    
    Path remoteStagingDir =
        remoteFs.makeQualified(new Path(conf.get(
            TezConfiguration.TEZ_AM_STAGING_DIR,
            TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT),
            Long.toString(System.currentTimeMillis())));
    TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);

View Full Code Here

    if (fs.exists(new Path(outputPath))) {
      System.err.println("Output directory : " + outputPath + " already exists");
      return 2;
    }


    TezConfiguration tezConf = new TezConfiguration(conf);


    fs.getWorkingDirectory();
    Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
    TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);


    String jarPath = ClassUtil.findContainingJar(FilterLinesByWordOneToOne.class);
    if (jarPath == null) {
      throw new TezUncheckedException("Could not find any jar containing"

View Full Code Here

  }


  public boolean run(String inputPath, String outputPath, Configuration conf) throws Exception {
    System.out.println("Running UnionExample");
    // conf and UGI
    TezConfiguration tezConf;
    if (conf != null) {
      tezConf = new TezConfiguration(conf);
    } else {
      tezConf = new TezConfiguration();
    }
    UserGroupInformation.setConfiguration(tezConf);
    String user = UserGroupInformation.getCurrentUser().getShortUserName();


    // staging dir
    FileSystem fs = FileSystem.get(tezConf);
    String stagingDirStr = Path.SEPARATOR + "user" + Path.SEPARATOR
        + user + Path.SEPARATOR+ ".staging" + Path.SEPARATOR
        + Path.SEPARATOR + Long.toString(System.currentTimeMillis());    
    Path stagingDir = new Path(stagingDirStr);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr);
    stagingDir = fs.makeQualified(stagingDir);
    


    // No need to add jar containing this class as assumed to be part of
    // the tez jars.

View Full Code Here

    if (result != 0) {
      return result;
    }


    if (tezConf == null) {
      tezConf = new TezConfiguration(getConf());
    }
    UserGroupInformation.setConfiguration(tezConf);
    boolean ownTezClient = false;
    if (tezClient == null) {
      ownTezClient = true;

View Full Code Here

  }
  
  public boolean run(Configuration conf, boolean doLocalityCheck) throws Exception {
    System.out.println("Running BroadcastAndOneToOneExample");
    // conf and UGI
    TezConfiguration tezConf;
    if (conf != null) {
      tezConf = new TezConfiguration(conf);
    } else {
      tezConf = new TezConfiguration();
    }
    tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
    UserGroupInformation.setConfiguration(tezConf);
    String user = UserGroupInformation.getCurrentUser().getShortUserName();


    // staging dir
    FileSystem fs = FileSystem.get(tezConf);
    String stagingDirStr = Path.SEPARATOR + "user" + Path.SEPARATOR
        + user + Path.SEPARATOR+ ".staging" + Path.SEPARATOR
        + Path.SEPARATOR + Long.toString(System.currentTimeMillis());    
    Path stagingDir = new Path(stagingDirStr);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr);
    stagingDir = fs.makeQualified(stagingDir);
    
    // No need to add jar containing this class as assumed to be part of
    // the tez jars.

View Full Code Here

    if (fs.exists(new Path(outputPath))) {
      System.err.println("Output directory : " + outputPath + " already exists");
      return 2;
    }


    TezConfiguration tezConf = new TezConfiguration(conf);


    fs.getWorkingDirectory();
    Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
    TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);


    String jarPath = ClassUtil.findContainingJar(FilterLinesByWord.class);
    if (jarPath == null) {
      throw new TezUncheckedException("Could not find any jar containing"

View Full Code Here

    TezDAGID dagId = createTezDagId();
    TezVertexID vertexId = createTezVertexId(dagId);
    Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<String, ByteBuffer>();
    Multimap<String, String> startedInputsMap = HashMultimap.create();
    TezUmbilical umbilical = mock(TezUmbilical.class);
    TezConfiguration tezConf = new TezConfiguration();


    TezTaskAttemptID taId1 = createTaskAttemptID(vertexId, 1);
    TaskSpec task1 = createTaskSpec(taId1, "vertex1");


    TezTaskAttemptID taId2 = createTaskAttemptID(vertexId, 2);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.tez.dag.api.TezConfiguration

cascading.flow.tez.FlowProcessor

cascading.flow.tez.Hadoop2TezFlow

cascading.flow.tez.Hadoop2TezFlowProcess

cascading.flow.tez.Hadoop2TezFlowStep

cascading.flow.tez.planner.Hadoop2TezFlowStepJob

cascading.flow.tez.planner.Hadoop2TezPlanner

cascading.flow.tez.stream.element.TezSourceStage

cascading.flow.tez.stream.graph.Hadoop2TezStreamGraph

cascading.flow.tez.util.TezUtil

cascading.platform.tez.Hadoop2TezPlatform

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.