Examples of org.apache.hadoop.mapred.RunningJob

org.apache.hadoop.mapred.RunningJob
Includes details on a running MapReduce job. A client can track a living job using this object. @author Mike Cafarella


            }
          }
        }
      }
      RunningJob running = jc.submitJob(theJobConf);
      this.mapredJobID = running.getJobID();
      this.state = Job.RUNNING;
    } catch (IOException ioe) {
      this.state = Job.FAILED;
      this.message = StringUtils.stringifyException(ioe);
    }

View Full Code Here

   * @throws IOException
   */
  public static boolean runJob(JobConf job) throws IOException {
    JobClient jc = new JobClient(job);
    boolean sucess = true;
    RunningJob running = null;
    try {
      running = jc.submitJob(job);
      String jobId = running.getJobID();
      System.out.println("Job " + jobId + " is submitted");
      while (!running.isComplete()) {
        System.out.println("Job " + jobId + " is still running.");
        try {
          Thread.sleep(60000);
        } catch (InterruptedException e) {
        }
        running = jc.getJob(jobId);
      }
      sucess = running.isSuccessful();
    } finally {
      if (!sucess && (running != null)) {
        running.killJob();
      }
      jc.close();
    }
    return sucess;
  }

View Full Code Here

    }
    
    @SuppressWarnings("deprecation")
    void computeWarningAggregate(Job job, JobClient jobClient, Map<Enum, Long> aggMap) {
        JobID mapRedJobID = job.getAssignedJobID();
        RunningJob runningJob = null;
        try {
            runningJob = jobClient.getJob(mapRedJobID);
            if(runningJob != null) {
                Counters counters = runningJob.getCounters();
                if (counters==null)
                {
                    long nullCounterCount = aggMap.get(PigWarning.NULL_COUNTER_COUNT)==null?0 : aggMap.get(PigWarning.NULL_COUNTER_COUNT);
                    nullCounterCount++;
                    aggMap.put(PigWarning.NULL_COUNTER_COUNT, nullCounterCount);

View Full Code Here

     * @return Returns the percentage progress of this Job
     * @throws IOException
     */
    protected double progressOfRunningJob(Job j, JobClient jobClient) throws IOException{
        JobID mrJobID = j.getAssignedJobID();
        RunningJob rj = jobClient.getJob(mrJobID);
        if(rj==null && j.getState()==Job.SUCCESS)
            return 1;
        else if(rj==null)
            return 0;
        else{
            double mapProg = rj.mapProgress();
            double redProg = rj.reduceProgress();
            return (mapProg + redProg)/2;
        }
    }

View Full Code Here

     * @return Returns the percentage progress of this Job
     * @throws IOException
     */
    protected double progressOfRunningJob(Job j, JobClient jobClient) throws IOException{
        JobID mrJobID = j.getAssignedJobID();
        RunningJob rj = jobClient.getJob(mrJobID);
        if(rj==null && j.getState()==Job.SUCCESS)
            return 1;
        else if(rj==null)
            return 0;
        else{
            double mapProg = rj.mapProgress();
            double redProg = rj.reduceProgress();
            return (mapProg + redProg)/2;
        }
    }

View Full Code Here

        String group = conf.get("group.name");
        JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);


        String launcherId = action.getExternalId();


        final RunningJob launcherJob = jobClient.getJob(JobID.forName(launcherId));


        waitFor(120 * 1000, new Predicate() {
            public boolean evaluate() throws Exception {
                return launcherJob.isComplete();
            }
        });
        assertTrue(launcherJob.isSuccessful());
        assertTrue(LauncherMapper.hasIdSwap(launcherJob));


        new ActionCheckXCommand(action.getId()).call();
        action = jpaService.execute(wfActionGetCmd);
        String mapperId = action.getExternalId();


        assertFalse(launcherId.equals(mapperId));


        final RunningJob mrJob = jobClient.getJob(JobID.forName(mapperId));


        waitFor(120 * 1000, new Predicate() {
            public boolean evaluate() throws Exception {
                return mrJob.isComplete();
            }
        });
        assertTrue(mrJob.isSuccessful());


        new ActionCheckXCommand(action.getId()).call();
        action = jpaService.execute(wfActionGetCmd);


        assertEquals("SUCCEEDED", action.getExternalStatus());

View Full Code Here

        JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker);
        XConfiguration.copy(conf, jobConf);
        String user = jobConf.get("user.name");
        String group = jobConf.get("group.name");
        JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, jobConf);
        final RunningJob runningJob = jobClient.getJob(JobID.forName(jobId));
        assertNotNull(runningJob);
        return runningJob;
    }

View Full Code Here

    }


    private void _testSubmit(String actionXml, boolean checkForSuccess) throws Exception {


        Context context = createContext(actionXml);
        final RunningJob launcherJob = submitAction(context);
        String launcherId = context.getAction().getExternalId();
        evaluateLauncherJob(launcherJob);
        assertTrue(launcherJob.isSuccessful());


        Thread.sleep(2000);
        assertFalse(LauncherMapper.hasIdSwap(launcherJob));
        if (checkForSuccess) {
            assertTrue(LauncherMapper.hasStatsData(launcherJob));

View Full Code Here

     */
    public void testExecutionStats() throws Exception {
        // Set the action xml with the option for retrieving stats to true
        String actionXml = setPigActionXml(PIG_SCRIPT, true);
        Context context = createContext(actionXml);
        final RunningJob launcherJob = submitAction(context);
        evaluateLauncherJob(launcherJob);
        assertTrue(launcherJob.isSuccessful());
        assertTrue(LauncherMapper.hasStatsData(launcherJob));


        PigActionExecutor ae = new PigActionExecutor();
        WorkflowAction wfAction = context.getAction();
        ae.check(context, wfAction);

View Full Code Here

     */
    public void testExternalChildIds() throws Exception {
        // Set the action xml with the option for retrieving stats to false
        String actionXml = setPigActionXml(PIG_SCRIPT, false);
        Context context = createContext(actionXml);
        final RunningJob launcherJob = submitAction(context);
        evaluateLauncherJob(launcherJob);
        assertTrue(launcherJob.isSuccessful());


        PigActionExecutor ae = new PigActionExecutor();
        WorkflowAction wfAction = context.getAction();
        ae.check(context, wfAction);
        ae.end(context, wfAction);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapred.RunningJob

cascading.stats.hadoop.HadoopStepStats

cc.mrlda.ParseCorpus

com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.JobLayer

com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.TaskLayer

com.digitalpebble.behemoth.mahout.util.Mahout2LibSVM

com.ebay.erl.mobius.core.MobiusJobRunner$StatusCheckingThread

com.netflix.lipstick.warnings.JobWarnings

com.nexr.rhive.hadoop.JobManager

com.ngdata.hbaseindexer.master.BatchStateUpdaterTest

com.twitter.ambrose.model.hadoop.MapReduceHelper

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.