Examples of org.apache.tez.dag.api.TezConfiguration

org.apache.tez.dag.api.TezConfiguration
Defines the configurations for Tez. These configurations are typically specified in tez-site.xml on the client machine where TezClient is used to launch the Tez application. tez-site.xml is expected to be picked up from the classpath of the client process.

    when(mockProxy.getVertexStatus(isNull(RpcController.class), argThat(new VertexCounterRequestMatcher())))
      .thenReturn(GetVertexStatusResponseProto.newBuilder().setVertexStatus(vertexStatusProtoWithCounters).build());
   
    
    
    dagClient = new DAGClientRPCImpl(mockAppId, dagIdStr, new TezConfiguration(), null);
    dagClient.appReport = mockAppReport;
    ((DAGClientRPCImpl)dagClient).proxy = mockProxy;
  }

View Full Code Here

      Map<String, LocalResource> localResources,
      TezConfiguration conf, Credentials credentials) {
    if (conf != null) {
      this.amConf = conf;
    } else {
      this.amConf = new TezConfiguration();
    }
    this.queueName = this.amConf.get(TezConfiguration.TEZ_QUEUE_NAME);


    this.env = new HashMap<String, String>();
    TezYARNUtils.setEnvFromInputString(this.env,

View Full Code Here

      
      Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
          .valueOf(new Random().nextInt(100000))));
      TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
      
      TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
      tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
          remoteStagingDir.toString());
      tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);


      AMConfiguration amConfig = new AMConfiguration(
          new HashMap<String, String>(), new HashMap<String, LocalResource>(),
          tezConf, null);
      TezSessionConfiguration tezSessionConfig =

View Full Code Here

  private TezSession createTezSession() throws IOException, TezException {
    Map<String, String> commonEnv = createCommonEnv();
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String
        .valueOf(new Random().nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());


    Map<String, LocalResource> amLocalResources = new HashMap<String, LocalResource>();


    AMConfiguration amConfig = new AMConfiguration(commonEnv, amLocalResources, tezConf, null);
    TezSessionConfiguration tezSessionConfig = new TezSessionConfiguration(amConfig, tezConf);

View Full Code Here

  InterruptedException, TezException, ClassNotFoundException, YarnException {
    Map<String, String> commonEnv = createCommonEnv();
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String
        .valueOf(new Random().nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    TezConfiguration tezConf = new TezConfiguration(
        mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        remoteStagingDir.toString());


    Map<String, LocalResource> amLocalResources =
        new HashMap<String, LocalResource>();

View Full Code Here


    Map<String, LocalResource> amLocalResources =
        new HashMap<String, LocalResource>();
    amLocalResources.putAll(commonLocalResources);


    TezConfiguration tezConf = new TezConfiguration(
            mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        remoteStagingDir.toString());


    TezClient tezClient = new TezClient(tezConf);
    DAGClient dagClient = null;
    TezSession tezSession = null;

View Full Code Here

  TezSession tezSession = null;
  Resource defaultResource = Resource.newInstance(100, 0);
  
  
  void setup() throws Exception {
    TezConfiguration tezConf = new TezConfiguration(new YarnConfiguration());
    FileSystem defaultFs = FileSystem.get(tezConf);
    
    Path remoteStagingDir = defaultFs.makeQualified(new Path(TEST_ROOT_DIR, String
        .valueOf(new Random().nextInt(100000))));
    TezClientUtils.ensureStagingDirExists(tezConf, remoteStagingDir);
    
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        remoteStagingDir.toString());


    AMConfiguration amConfig = new AMConfiguration(
        new HashMap<String, String>(), new HashMap<String, LocalResource>(),
        tezConf, null);

View Full Code Here

    LOG.info("Starting session");
    Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
        .valueOf(new Random().nextInt(100000))));
    TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);


    TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
    tezConf.setInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, 0);
    tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "DEBUG");
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        remoteStagingDir.toString());
    tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
    tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
    tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
    tezConf.set(TezConfiguration.TEZ_AM_JAVA_OPTS, " -Xmx256m");


    AMConfiguration amConfig = new AMConfiguration(
        new HashMap<String, String>(), new HashMap<String, LocalResource>(),
        tezConf, null);
    TezSessionConfiguration tezSessionConfig =

View Full Code Here

   * @param clientCache the client cache object.
   */
  public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate,
      ClientCache clientCache) {
    this.conf = conf;
    this.tezConf = new TezConfiguration(conf);
    try {
      this.resMgrDelegate = resMgrDelegate;
      this.tezClient = new TezClient(tezConf);
      this.clientCache = clientCache;
      this.defaultFileContext = FileContext.getFileContext(this.conf);

View Full Code Here

    }
    return dag;
  }


  private TezConfiguration getDAGAMConfFromMRConf() {
    TezConfiguration finalConf = new TezConfiguration(this.tezConf);
    Map<String, String> mrParamToDAGParamMap = DeprecatedKeys
        .getMRToDAGParamMap();


    for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
      if (finalConf.get(entry.getKey()) != null) {
        finalConf.set(entry.getValue(), finalConf.get(entry.getKey()));
        finalConf.unset(entry.getKey());
        if (LOG.isDebugEnabled()) {
          LOG.debug("MR->DAG Translating MR key: " + entry.getKey()
              + " to Tez key: " + entry.getValue() + " with value "
              + finalConf.get(entry.getValue()));
        }
      }
    }
    return finalConf;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.tez.dag.api.TezConfiguration

cascading.flow.tez.FlowProcessor

cascading.flow.tez.Hadoop2TezFlow

cascading.flow.tez.Hadoop2TezFlowProcess

cascading.flow.tez.Hadoop2TezFlowStep

cascading.flow.tez.planner.Hadoop2TezFlowStepJob

cascading.flow.tez.planner.Hadoop2TezPlanner

cascading.flow.tez.stream.element.TezSourceStage

cascading.flow.tez.stream.graph.Hadoop2TezStreamGraph

cascading.flow.tez.util.TezUtil

cascading.platform.tez.Hadoop2TezPlatform

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.