Examples of org.apache.tez.client.TezClient.submitDAG()

org.apache.tez.client.TezClient.submitDAG()
Submit a DAG.
In non-session mode, it submits a new App Master to the cluster.
In session mode, it submits the DAG to the session App Master. It blocks until either the DAG is submitted to the session or configured timeout period expires. Cleans up session if the submission timed out. @param dag DAG to be submitted to Session @return DAGClient to monitor the DAG @throws TezException @throws IOException @throws DAGSubmissionTimedOut if submission timed out

    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());


    TezClient tezSession = TezClient.create("TezSleepProcessor", tezConf, false);
    tezSession.start();


    DAGClient dagClient = tezSession.submitDAG(dag);


    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
      LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
          + dagStatus.getState());

View Full Code Here

            Resource.newInstance(1024, 1));
        dag.addVertex(vertex);


        DAGClient dagClient = null;
        try {
          dagClient = tezSession.submitDAG(dag);
          if (dagIndex > 1) {
            fail("Should fail due to duplicate dag name for dagIndex: " + dagIndex);
          }
        } catch (TezException tex) {
          if (dagIndex > 1) {

View Full Code Here

    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());


    TezClient tezSession = TezClient.create("TezSleepProcessor", tezConf, false);
    tezSession.start();


    DAGClient dagClient = tezSession.submitDAG(dag);


    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
      LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
          + dagStatus.getState());

View Full Code Here


    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
    TezClient tezSession = TezClient.create("TezSleepProcessorHistoryLogging", tezConf);
    tezSession.start();


    DAGClient dagClient = tezSession.submitDAG(dag);


    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
      LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
          + dagStatus.getState());

View Full Code Here

    } else {
      tezSession = reUseTezSession;
    }
    if(!dagViaRPC) {
      // TODO Use utility method post TEZ-205 to figure out AM arguments etc.
      dagClient = tezSession.submitDAG(dag);
    }


    if (dagViaRPC && closeSessionBeforeSubmit) {
      YarnClient yarnClient = YarnClient.createYarnClient();
      yarnClient.init(mrrTezCluster.getConfig());

View Full Code Here

      LOG.info("Submitting dag to tez session with appId=" + tezSession.getAppMasterApplicationId()
          + " and Dag Name=" + dag.getName());
      if (additionalLocalResources != null) {
        tezSession.addAppMasterLocalFiles(additionalLocalResources);
      }
      dagClient = tezSession.submitDAG(dag);
      Assert.assertEquals(TezAppMasterStatus.RUNNING,
          tezSession.getAppMasterStatus());
    }
    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {

View Full Code Here

        DAG dag = OrderedWordCount.createDAG(tezConf, inputPaths[i], outputPaths[i], numPartitions,
            ("DAG-Iteration-" + i)); // the names of the DAGs must be unique in a session


        tezClient.waitTillReady();
        System.out.println("Running dag number " + i);
        DAGClient dagClient = tezClient.submitDAG(dag);


        // wait to finish
        DAGStatus dagStatus = dagClient.waitForCompletion();
        if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
          System.out.println("Iteration " + i + " failed with diagnostics: "

View Full Code Here

        DAG dag = createDAG(tezConf, inputPath, outputPath, numPartitions);


        // check that the execution environment is ready
        tezClient.waitTillReady();
        // submit the dag and receive a dag client to monitor the progress
        DAGClient dagClient = tezClient.submitDAG(dag);


        // monitor the progress and wait for completion. This method blocks until the dag is done.
        DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
        // check success or failure and print diagnostics
        if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {

View Full Code Here

    TezClient tezClient = TezClient.create("TestVertexOrder", tezConf);
    tezClient.start();


    try {
    DAG dag = SimpleTestDAG.createDAGForVertexOrder("dag1", conf);
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
      LOG.info("Waiting for dag to complete. Sleeping for 500ms."
          + " DAG name: " + dag.getName()
          + " DAG context: " + dagClient.getExecutionContext()

View Full Code Here


    try {
        DAG dag = createDAG(tezConf, inputPath, outputPath, numPartitions, "OrderedWordCount");


        tezClient.waitTillReady();
        DAGClient dagClient = tezClient.submitDAG(dag);


        // monitoring
        DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
        if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
          System.out.println("OrderedWordCount failed with diagnostics: " + dagStatus.getDiagnostics());

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.