Examples of org.apache.hadoop.mapred.jobcontrol.Job

Package org.apache.hadoop.mapred.jobcontrol

Examples of org.apache.hadoop.mapred.jobcontrol.Job

org.apache.hadoop.mapred.jobcontrol.Job
This class encapsulates a MapReduce job and its dependency. It monitors the states of the depending jobs and updates the state of this job. A job starts in the WAITING state. If it does not have any depending jobs, or all of the depending jobs are in SUCCESS state, then the job state will become READY. If any depending jobs fail, the job will fail too. When in READY state, the job can be submitted to Hadoop for execution, with the state changing into RUNNING state. From RUNNING state, the job can get into SUCCESS or FAILED state, depending the status of the job execution.

  public void start(Path inputDir, Path outputDir) throws IOException {
    FileSystem fs = FileSystem.get(this.conf);


    JobConf uniqueListenersConf = getUniqueListenersJobConf(inputDir);
    Path listenersOutputDir = FileOutputFormat.getOutputPath(uniqueListenersConf);
    Job listenersJob = new Job(uniqueListenersConf);
    // delete any output that might exist from a previous run of this job
    if (fs.exists(FileOutputFormat.getOutputPath(uniqueListenersConf))) {
      fs.delete(FileOutputFormat.getOutputPath(uniqueListenersConf), true);
    }


    JobConf sumConf = getSumJobConf(inputDir);
    Path sumOutputDir = FileOutputFormat.getOutputPath(sumConf);
    Job sumJob = new Job(sumConf);
    // delete any output that might exist from a previous run of this job
    if (fs.exists(FileOutputFormat.getOutputPath(sumConf))) {
      fs.delete(FileOutputFormat.getOutputPath(sumConf), true);
    }


    // the merge job depends on the other two jobs
    ArrayList<Job> mergeDependencies = new ArrayList<Job>();
    mergeDependencies.add(listenersJob);
    mergeDependencies.add(sumJob);
    JobConf mergeConf = getMergeConf(outputDir, sumOutputDir, listenersOutputDir);
    Job mergeJob = new Job(mergeConf, mergeDependencies);
    // delete any output that might exist from a previous run of this job
    if (fs.exists(FileOutputFormat.getOutputPath(mergeConf))) {
      fs.delete(FileOutputFormat.getOutputPath(mergeConf), true);
    }

View Full Code Here

                    if (notifyProgress(prog, lastProg)) {
                        List<Job> runnJobs = jc.getRunningJobs();
                        if (runnJobs != null) {
                            StringBuilder msg = new StringBuilder();
                            for (Object object : runnJobs) {
                                Job j = (Job) object;
                                if (j != null) {
                                    msg.append(j.getAssignedJobID()).append(",");
                                }
                            }
                            if (msg.length() > 0) {
                                msg.setCharAt(msg.length() - 1, ']');
                                log.info("Running jobs are [" + msg);

View Full Code Here

        if (stop_on_failure){
            int errCode = 6017;
            StringBuilder msg = new StringBuilder();


            for (int i=0; i<jc.getFailedJobs().size(); i++) {
                Job j = jc.getFailedJobs().get(i);
                msg.append("JobID: " + j.getAssignedJobID() + " Reason: " + j.getMessage());
                if (i!=jc.getFailedJobs().size()-1) {
                    msg.append("\n");
                }
            }

View Full Code Here

        
        try{
            if(pred==null || pred.size()<=0){
                //No dependencies! Create the JobConf
                //Construct the Job object with it and return
                Job ret = null;
                if(seen.containsKey(mro.getOperatorKey()))
                    ret = seen.get(mro.getOperatorKey());
                else{
                    currJC = getJobConf(mro, conf, pigContext);
                    ret = new Job(currJC,null);
                    seen.put(mro.getOperatorKey(), ret);
                }
                return ret;
            }
            
            //Has dependencies. So compile all the inputs
            List<Job> compiledInputs = new ArrayList<Job>(pred.size());
            
            for (MapReduceOper oper : pred) {
                Job ret = null;
                if(seen.containsKey(oper.getOperatorKey()))
                    ret = seen.get(oper.getOperatorKey());
                else{
                    ret = compile(oper, jobCtrl);
                    jobCtrl.addJob(ret);
                    seen.put(oper.getOperatorKey(),ret);
                }
                compiledInputs.add(ret);
            }
            //Get JobConf for the current MapReduceOper
            currJC = getJobConf(mro, conf, pigContext);
            
            //Create a new Job with the obtained JobConf
            //and the compiled inputs as dependent jobs
            return new Job(currJC,(ArrayList<Job>)compiledInputs);
        } catch (JobCreationException jce) {
          throw jce;
        } catch(Exception e) {
            int errCode = 2017;
            String msg = "Internal error creating job configuration.";

View Full Code Here

            lp.set((String)entry.getKey(), (String)entry.getValue());
        }
        FileInputFormat.addInputPath(lp, new Path(inputDir + "/page_views"));
        FileOutputFormat.setOutputPath(lp, new Path(outputDir + "/L4out"));
        lp.setNumReduceTasks(Integer.parseInt(parallel));
        Job group = new Job(lp);


        JobControl jc = new JobControl("L4 join");
        jc.addJob(group);


        new Thread(jc).start();

View Full Code Here

            lp.set((String)entry.getKey(), (String)entry.getValue());
        }
        FileInputFormat.addInputPath(lp, new Path(inputDir + "/page_views"));
        FileOutputFormat.setOutputPath(lp, new Path(outputDir + "/L8out"));
        lp.setNumReduceTasks(1);
        Job group = new Job(lp);


        JobControl jc = new JobControl("L8 join");
        jc.addJob(group);


        new Thread(jc).start();

View Full Code Here

            lp.set((String)entry.getKey(), (String)entry.getValue());
        }
        FileInputFormat.addInputPath(lp, new Path(inputDir + "/page_views"));
        FileOutputFormat.setOutputPath(lp, new Path(outputDir + "/L7out"));
        lp.setNumReduceTasks(Integer.parseInt(parallel));
        Job group = new Job(lp);


        JobControl jc = new JobControl("L7 join");
        jc.addJob(group);


        new Thread(jc).start();

View Full Code Here

            lp.set((String)entry.getKey(), (String)entry.getValue());
        }
        FileInputFormat.addInputPath(lp, new Path(inputDir + "/page_views"));
        FileOutputFormat.setOutputPath(lp, new Path(outputDir + "/L1out"));
        lp.setNumReduceTasks(Integer.parseInt(parallel));
        Job group = new Job(lp);


        JobControl jc = new JobControl("L1 join");
        jc.addJob(group);


        new Thread(jc).start();

View Full Code Here

            lp.set((String)entry.getKey(), (String)entry.getValue());
        }
        FileInputFormat.addInputPath(lp, new Path(inputDir + "/page_views"));
        FileOutputFormat.setOutputPath(lp, new Path(outputDir + "/indexed_pages_3"));
        lp.setNumReduceTasks(0);
        Job loadPages = new Job(lp);


        JobConf lu = new JobConf(L3.class);
        lu.setJobName("L3 Load Users");
        lu.setInputFormat(TextInputFormat.class);
        lu.setOutputKeyClass(Text.class);
        lu.setOutputValueClass(Text.class);
        lu.setMapperClass(ReadUsers.class);
        props = System.getProperties();
        for (Map.Entry<Object,Object> entry : props.entrySet()) {
            lu.set((String)entry.getKey(), (String)entry.getValue());
        }
        FileInputFormat.addInputPath(lu, new Path(inputDir + "/users"));
        FileOutputFormat.setOutputPath(lu, new Path(outputDir + "/indexed_users_3"));
        lu.setNumReduceTasks(0);
        Job loadUsers = new Job(lu);


        JobConf join = new JobConf(L3.class);
        join.setJobName("L3 Join Users and Pages");
        join.setInputFormat(KeyValueTextInputFormat.class);
        join.setOutputKeyClass(Text.class);
        join.setOutputValueClass(Text.class);
        join.setMapperClass(IdentityMapper.class);
        join.setReducerClass(Join.class);
        props = System.getProperties();
        for (Map.Entry<Object,Object> entry : props.entrySet()) {
            join.set((String)entry.getKey(), (String)entry.getValue());
        }
        FileInputFormat.addInputPath(join, new Path(outputDir + "/indexed_pages_3"));
        FileInputFormat.addInputPath(join, new Path(outputDir + "/indexed_users_3"));
        FileOutputFormat.setOutputPath(join, new Path(outputDir + "/L3out"));
        join.setNumReduceTasks(Integer.parseInt(parallel));
        Job joinJob = new Job(join);
        joinJob.addDependingJob(loadPages);
        joinJob.addDependingJob(loadUsers);


        JobControl jc = new JobControl("L3 join");
        jc.addJob(loadPages);
        jc.addJob(loadUsers);
        jc.addJob(joinJob);

View Full Code Here

            lp.set((String)entry.getKey(), (String)entry.getValue());
        }
        FileInputFormat.addInputPath(lp, new Path(inputDir + "/page_views"));
        FileOutputFormat.setOutputPath(lp, new Path(outputDir + "/p"));
        lp.setNumReduceTasks(Integer.parseInt(parallel));
        Job loadPages = new Job(lp);


        JobConf lu = new JobConf(L11.class);
        lu.setJobName("L11 Load Widerow");
        lu.setInputFormat(TextInputFormat.class);
        lu.setOutputKeyClass(Text.class);
        lu.setOutputValueClass(Text.class);
        lu.setMapperClass(ReadWideRow.class);
        lu.setCombinerClass(ReadWideRow.class);
        lu.setReducerClass(ReadWideRow.class);
        props = System.getProperties();
        for (Map.Entry<Object,Object> entry : props.entrySet()) {
            lu.set((String)entry.getKey(), (String)entry.getValue());
        }
        FileInputFormat.addInputPath(lu, new Path(inputDir + "/widerow"));
        FileOutputFormat.setOutputPath(lu, new Path(outputDir + "/wr"));
        lu.setNumReduceTasks(Integer.parseInt(parallel));
        Job loadWideRow = new Job(lu);


        JobConf join = new JobConf(L11.class);
        join.setJobName("L11 Union WideRow and Pages");
        join.setInputFormat(KeyValueTextInputFormat.class);
        join.setOutputKeyClass(Text.class);
        join.setOutputValueClass(Text.class);
        join.setMapperClass(IdentityMapper.class);
        join.setCombinerClass(Union.class);
        join.setReducerClass(Union.class);
        props = System.getProperties();
        for (Map.Entry<Object,Object> entry : props.entrySet()) {
            join.set((String)entry.getKey(), (String)entry.getValue());
        }
        FileInputFormat.addInputPath(join, new Path(outputDir + "/p"));
        FileInputFormat.addInputPath(join, new Path(outputDir + "/wr"));
        FileOutputFormat.setOutputPath(join, new Path(outputDir + "/L11out"));
        join.setNumReduceTasks(Integer.parseInt(parallel));
        Job joinJob = new Job(join);
        joinJob.addDependingJob(loadPages);
        joinJob.addDependingJob(loadWideRow);


        JobControl jc = new JobControl("L11 join");
        jc.addJob(loadPages);
        jc.addJob(loadWideRow);
        jc.addJob(joinJob);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapred.jobcontrol.Job

com.ebay.erl.mobius.core.MobiusJob

fm.last.hadoop.programs.labs.trackstats.TrackStatisticsProgram

org.apache.hadoop.abacus.ValueAggregatorJob

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.Path

org.apache.hadoop.mapred.JobClient

org.apache.hadoop.mapred.JobConf

org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJob

org.apache.hadoop.mapred.RunningJob

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.