Examples of org.apache.hadoop.hive.ql.MapRedStats

org.apache.hadoop.hive.ql.MapRedStats
MapRedStats. A data structure to keep one mapreduce's stats: number of mappers, number of reducers, accumulative CPU time and whether it succeeds.

    EmbeddedAmbroseHiveProgressReporter reporter = getEmbeddedProgressReporter();
    Map<String, String> jobIdToNodeId = reporter.getJobIdToNodeId();
    LOG.info("MapReduce Jobs Launched: ");
    List<MapRedStats> lastMapRedStats = SessionState.get().getLastMapRedStatsList();
    for (int i = 0; i < lastMapRedStats.size(); i++) {
      MapRedStats mrStats = lastMapRedStats.get(i);
      String jobId = mrStats.getJobId();
      String nodeId = jobIdToNodeId.get(jobId);
      StringBuilder sb = new StringBuilder();
      sb.append("Job ")
        .append(i)
        .append(" (")

View Full Code Here


    int expectedReducers = hookContext.getConf().getInt(BUCKET_CONFIG, 0);
    Map<String, MapRedStats> stats = ss.getMapRedStats();
    Assert.assertEquals("Number of MapReduce jobs is incorrect", 1, stats.size());


    MapRedStats stat = stats.values().iterator().next();
    Assert.assertEquals("NumReducers is incorrect", expectedReducers, stat.getNumReduce());
  }

View Full Code Here

          cpuMsec = newCpuMSec;
        }
      }
    }


    MapRedStats mapRedStats = new MapRedStats(numMap, numReduce, cpuMsec, success, rj.getID().toString());
    mapRedStats.setCounters(ctrs);


    // update based on the final value of the counters
    updateCounters(ctrs, rj);


    SessionState ss = SessionState.get();

View Full Code Here


    runningJobs.add(rj);


    ExecDriverTaskHandle th = new ExecDriverTaskHandle(jc, rj, txnMgr);
    jobInfo(rj);
    MapRedStats mapRedStats = progress(th);


    this.task.taskHandle = th;
    // Not always there is a SessionState. Sometimes ExeDriver is directly invoked
    // for special modes. In that case, SessionState.get() is empty.
    if (SessionState.get() != null) {
      SessionState.get().getMapRedStats().put(getId(), mapRedStats);


      // Computes the skew for all the MapReduce irrespective
      // of Success or Failure
      if (this.task.getQueryPlan() != null) {
        computeReducerTimeStatsPerJob(rj);
      }
    }


    boolean success = mapRedStats.isSuccess();


    String statusMesg = getJobEndMsg(rj.getID());
    if (!success) {
      statusMesg += " with errors";
      returnVal = 2;

View Full Code Here

          cpuMsec = newCpuMSec;
        }
      }
    }


    MapRedStats mapRedStats = new MapRedStats(numMap, numReduce, cpuMsec, success, rj.getID().toString());
    mapRedStats.setCounters(ctrs);


    // update based on the final value of the counters
    updateCounters(ctrs, rj);


    SessionState ss = SessionState.get();

View Full Code Here


    runningJobKillURIs.put(rj.getJobID(), rj.getTrackingURL() + "&action=kill");


    ExecDriverTaskHandle th = new ExecDriverTaskHandle(jc, rj);
    jobInfo(rj);
    MapRedStats mapRedStats = progress(th);


    this.task.taskHandle = th;
    // Not always there is a SessionState. Sometimes ExeDriver is directly invoked
    // for special modes. In that case, SessionState.get() is empty.
    if (SessionState.get() != null) {
      SessionState.get().getLastMapRedStatsList().add(mapRedStats);


      // Computes the skew for all the MapReduce irrespective
      // of Success or Failure
      if (this.task.getQueryPlan() != null) {
        computeReducerTimeStatsPerJob(rj);
      }
    }


    boolean success = mapRedStats.isSuccess();


    String statusMesg = getJobEndMsg(rj.getJobID());
    if (!success) {
      statusMesg += " with errors";
      returnVal = 2;

View Full Code Here

          cpuMsec = newCpuMSec;
        }
      }
    }


    MapRedStats mapRedStats = new MapRedStats(numMap, numReduce, cpuMsec, success, rj.getID().toString());
    mapRedStats.setCounters(ctrs);


    // update based on the final value of the counters
    updateCounters(ctrs, rj);


    SessionState ss = SessionState.get();

View Full Code Here


    runningJobs.add(rj);


    ExecDriverTaskHandle th = new ExecDriverTaskHandle(jc, rj, txnMgr);
    jobInfo(rj);
    MapRedStats mapRedStats = progress(th);


    this.task.taskHandle = th;
    // Not always there is a SessionState. Sometimes ExeDriver is directly invoked
    // for special modes. In that case, SessionState.get() is empty.
    if (SessionState.get() != null) {
      SessionState.get().getLastMapRedStatsList().add(mapRedStats);


      // Computes the skew for all the MapReduce irrespective
      // of Success or Failure
      if (this.task.getQueryPlan() != null) {
        computeReducerTimeStatsPerJob(rj);
      }
    }


    boolean success = mapRedStats.isSuccess();


    String statusMesg = getJobEndMsg(rj.getID());
    if (!success) {
      statusMesg += " with errors";
      returnVal = 2;

View Full Code Here

          cpuMsec = newCpuMSec;
        }
      }
    }


    MapRedStats mapRedStats = new MapRedStats(numMap, numReduce, cpuMsec, success, rj.getID().toString());
    mapRedStats.setCounters(ctrs);


    // update based on the final value of the counters
    updateCounters(ctrs, rj);


    SessionState ss = SessionState.get();

View Full Code Here


    runningJobKillURIs.put(rj.getJobID(), rj.getTrackingURL() + "&action=kill");


    ExecDriverTaskHandle th = new ExecDriverTaskHandle(jc, rj);
    jobInfo(rj);
    MapRedStats mapRedStats = progress(th);


    this.task.taskHandle = th;
    // Not always there is a SessionState. Sometimes ExeDriver is directly invoked
    // for special modes. In that case, SessionState.get() is empty.
    if (SessionState.get() != null) {
      SessionState.get().getLastMapRedStatsList().add(mapRedStats);


      // Computes the skew for all the MapReduce irrespective
      // of Success or Failure
      if (this.task.getQueryPlan() != null) {
        computeReducerTimeStatsPerJob(rj);
      }
    }


    boolean success = mapRedStats.isSuccess();


    String statusMesg = getJobEndMsg(rj.getJobID());
    if (!success) {
      statusMesg += " with errors";
      returnVal = 2;

View Full Code Here

0 1

TOP

Related Classes of org.apache.hadoop.hive.ql.MapRedStats

com.twitter.ambrose.hive.AmbroseHiveFinishHook

org.apache.hadoop.hive.ql.exec.HadoopJobExecHelper

org.apache.hadoop.hive.ql.exec.mr.HadoopJobExecHelper

org.apache.hadoop.hive.ql.hooks.VerifyNumReducersHook

org.apache.hadoop.mapred.Counters.Counter

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.