Examples of org.apache.hadoop.hive.ql.MapRedStats

org.apache.hadoop.hive.ql.MapRedStats
MapRedStats. A data structure to keep one mapreduce's stats: number of mappers, number of reducers, accumulative CPU time and whether it succeeds.

          cpuMsec = newCpuMSec;
        }
      }
    }


    MapRedStats mapRedStats = new MapRedStats(numMap, numReduce, cpuMsec, success, rj.getID().toString());
    mapRedStats.setCounters(ctrs);


    // update based on the final value of the counters
    updateCounters(ctrs, rj);


    SessionState ss = SessionState.get();

View Full Code Here


    runningJobKillURIs.put(rj.getJobID(), rj.getTrackingURL() + "&action=kill");


    ExecDriverTaskHandle th = new ExecDriverTaskHandle(jc, rj);
    jobInfo(rj);
    MapRedStats mapRedStats = progress(th);


    this.task.taskHandle = th;
    // Not always there is a SessionState. Sometimes ExeDriver is directly invoked
    // for special modes. In that case, SessionState.get() is empty.
    if (SessionState.get() != null) {
      SessionState.get().getLastMapRedStatsList().add(mapRedStats);


      // Computes the skew for all the MapReduce irrespective
      // of Success or Failure
      if (this.task.getQueryPlan() != null) {
        computeReducerTimeStatsPerJob(rj);
      }
    }


    boolean success = mapRedStats.isSuccess();


    String statusMesg = getJobEndMsg(rj.getJobID());
    if (!success) {
      statusMesg += " with errors";
      returnVal = 2;

View Full Code Here

          cpuMsec = newCpuMSec;
        }
      }
    }


    MapRedStats mapRedStats = new MapRedStats(numMap, numReduce, cpuMsec, success, rj.getID().toString());
    mapRedStats.setCounters(ctrs);


    // update based on the final value of the counters
    updateCounters(ctrs, rj);


    SessionState ss = SessionState.get();

View Full Code Here


    runningJobKillURIs.put(rj.getJobID(), rj.getTrackingURL() + "&action=kill");


    ExecDriverTaskHandle th = new ExecDriverTaskHandle(jc, rj);
    jobInfo(rj);
    MapRedStats mapRedStats = progress(th);


    // Not always there is a SessionState. Sometimes ExeDriver is directly invoked
    // for special modes. In that case, SessionState.get() is empty.
    if (SessionState.get() != null) {
      SessionState.get().getLastMapRedStatsList().add(mapRedStats);


      // Computes the skew for all the MapReduce irrespective
      // of Success or Failure
      if (this.task.getQueryPlan() != null) {
        computeReducerTimeStatsPerJob(rj);
      }
    }


    boolean success = mapRedStats.isSuccess();


    String statusMesg = getJobEndMsg(rj.getJobID());
    if (!success) {
      statusMesg += " with errors";
      returnVal = 2;

View Full Code Here

          cpuMsec = newCpuMSec;
        }
      }
    }


    MapRedStats mapRedStats = new MapRedStats(numMap, numReduce, cpuMsec, success, rj.getID().toString());
    mapRedStats.setCounters(ctrs);


    this.task.setDone();
    // update based on the final value of the counters
    updateCounters(ctrs, rj);

View Full Code Here


    runningJobKillURIs.put(rj.getJobID(), rj.getTrackingURL() + "&action=kill");


    ExecDriverTaskHandle th = new ExecDriverTaskHandle(jc, rj);
    jobInfo(rj);
    MapRedStats mapRedStats = progress(th);


    // Not always there is a SessionState. Sometimes ExeDriver is directly invoked
    // for special modes. In that case, SessionState.get() is empty.
    if (SessionState.get() != null) {
      SessionState.get().getLastMapRedStatsList().add(mapRedStats);
    }


    boolean success = mapRedStats.isSuccess();


    String statusMesg = getJobEndMsg(rj.getJobID());
    if (!success) {
      statusMesg += " with errors";
      returnVal = 2;

View Full Code Here

          cpuMsec = newCpuMSec;
        }
      }
    }


    MapRedStats mapRedStats = new MapRedStats(numMap, numReduce, cpuMsec, success, rj.getID().toString());
    mapRedStats.setCounters(ctrs);


    this.task.setDone();
    // update based on the final value of the counters
    updateCounters(ctrs, rj);

View Full Code Here


    runningJobKillURIs.put(rj.getJobID(), rj.getTrackingURL() + "&action=kill");


    ExecDriverTaskHandle th = new ExecDriverTaskHandle(jc, rj);
    jobInfo(rj);
    MapRedStats mapRedStats = progress(th);


    // Not always there is a SessionState. Sometimes ExeDriver is directly invoked
    // for special modes. In that case, SessionState.get() is empty.
    if (SessionState.get() != null) {
      SessionState.get().getLastMapRedStatsList().add(mapRedStats);
    }


    boolean success = mapRedStats.isSuccess();


    String statusMesg = getJobEndMsg(rj.getJobID());
    if (!success) {
      statusMesg += " with errors";
      returnVal = 2;

View Full Code Here

          cpuMsec = newCpuMSec;
        }
      }
    }


    MapRedStats mapRedStats = new MapRedStats(numMap, numReduce, cpuMsec, success, rj.getID().toString());
    mapRedStats.setCounters(ctrs);


    // update based on the final value of the counters
    updateCounters(ctrs, rj);


    SessionState ss = SessionState.get();

View Full Code Here


    runningJobs.add(rj);


    ExecDriverTaskHandle th = new ExecDriverTaskHandle(jc, rj, txnMgr);
    jobInfo(rj);
    MapRedStats mapRedStats = progress(th);


    this.task.taskHandle = th;
    // Not always there is a SessionState. Sometimes ExeDriver is directly invoked
    // for special modes. In that case, SessionState.get() is empty.
    if (SessionState.get() != null) {
      SessionState.get().getLastMapRedStatsList().add(mapRedStats);


      // Computes the skew for all the MapReduce irrespective
      // of Success or Failure
      if (this.task.getQueryPlan() != null) {
        computeReducerTimeStatsPerJob(rj);
      }
    }


    boolean success = mapRedStats.isSuccess();


    String statusMesg = getJobEndMsg(rj.getID());
    if (!success) {
      statusMesg += " with errors";
      returnVal = 2;

View Full Code Here

0 1

TOP

Related Classes of org.apache.hadoop.hive.ql.MapRedStats

com.twitter.ambrose.hive.AmbroseHiveFinishHook

org.apache.hadoop.hive.ql.exec.HadoopJobExecHelper

org.apache.hadoop.hive.ql.exec.mr.HadoopJobExecHelper

org.apache.hadoop.hive.ql.hooks.VerifyNumReducersHook

org.apache.hadoop.mapred.Counters.Counter

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.