Examples of JobGraph

eu.stratosphere.nephele.jobgraph.JobGraph
A job graph represents an entire job in Nephele. A job graph must consists at least of one job vertex and must be acyclic.
org.apache.flink.runtime.jobgraph.JobGraph
A job graph represents an entire Flink runtime job.
org.apache.pig.tools.pigstats.PigStats.JobGraph

Examples of org.apache.pig.tools.pigstats.PigStats.JobGraph


        System.out.println("============================================");
        System.out.println("Test case MapCombineReduce");
        System.out.println("============================================");
        
        JobGraph jp = pigStats.getJobGraph();
        Iterator<JobStats> iter = jp.iterator();
        while (iter.hasNext()) {
            MRJobStats js = (MRJobStats) iter.next();
            System.out.println("Map input records : " + js.getMapInputRecords());
            assertEquals(MAX, js.getMapInputRecords());
            System.out.println("Map output records : " + js.getMapOutputRecords());

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats.JobGraph


        System.out.println("============================================");
        System.out.println("Test case MapCombineReduce");
        System.out.println("============================================");
 
        JobGraph jp = pigStats.getJobGraph();
        Iterator<JobStats> iter = jp.iterator();
        while (iter.hasNext()) {
            MRJobStats js = (MRJobStats) iter.next();
            System.out.println("Map input records : " + js.getMapInputRecords());
            assertEquals(MAX, js.getMapInputRecords());
            System.out.println("Map output records : " + js.getMapOutputRecords());

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats.JobGraph

        
        System.out.println("============================================");
        System.out.println("Test case MultipleMRJobs");
        System.out.println("============================================");
        
        JobGraph jp = pigStats.getJobGraph();
        MRJobStats js = (MRJobStats)jp.getSinks().get(0);
        
        System.out.println("Job id: " + js.getName());
        System.out.println(jp.toString());
        
        System.out.println("Map input records : " + js.getMapInputRecords());
        assertEquals(MAX, js.getMapInputRecords());
        System.out.println("Map output records : " + js.getMapOutputRecords());
        assertEquals(MAX, js.getMapOutputRecords());

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats.JobGraph

            pigServer.registerQuery("b = group a by $0;");
            pigServer.registerQuery("c = foreach b generate group, COUNT(a) as cnt;");
            pigServer.registerQuery("d = group c by cnt;");
            pigServer.registerQuery("e = foreach d generate group;");
            ExecJob job = pigServer.store("e", filePath);
            JobGraph jobGraph = job.getStatistics().getJobGraph();
            assertEquals(2, jobGraph.getJobList().size());


        } catch (IOException e) {
            LOG.error("IOException while creating file ", e);
            fail("Encountered IOException");
        } finally {

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats.JobGraph

     * @param stats
     * @return A list of ExecJob objects
     */
    protected List<ExecJob> getJobs(PigStats stats) {
        LinkedList<ExecJob> jobs = new LinkedList<ExecJob>();
        JobGraph jGraph = stats.getJobGraph();
        Iterator<JobStats> iter = jGraph.iterator();
        while (iter.hasNext()) {
            JobStats js = iter.next();
            for (OutputStats output : js.getOutputs()) {
                if (js.isSuccessful()) {
                    jobs.add(new HJob(HJob.JOB_STATUS.COMPLETED, pigContext, output

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats.JobGraph

     * @param stats
     * @return A list of ExecJob objects
     */
    protected List<ExecJob> getJobs(PigStats stats) {
        LinkedList<ExecJob> jobs = new LinkedList<ExecJob>();
        JobGraph jGraph = stats.getJobGraph();
        Iterator<JobStats> iter = jGraph.iterator();
        while (iter.hasNext()) {
            JobStats js = iter.next();
            for (OutputStats output : js.getOutputs()) {
                if (js.isSuccessful()) {
                    jobs.add(new HJob(HJob.JOB_STATUS.COMPLETED, pigContext, output

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats.JobGraph

            }


            if (!mLoadOnly) {
                mPigServer.executeBatch();
                PigStats stats = PigStats.get();
                JobGraph jg = stats.getJobGraph();
                Iterator<JobStats> iter = jg.iterator();
                while (iter.hasNext()) {
                    JobStats js = iter.next();
                    if (!js.isSuccessful()) {
                        mNumFailedJobs++;
                        Exception exp = (js.getException() != null) ? js.getException()

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats.JobGraph


    private void setJobParents(MapReduceOper mro, Configuration conf) {
        // PigStats maintains a job DAG with the job id being updated
        // upon available. Therefore, before a job is submitted, the ids
        // of its parent jobs are already available.
        JobGraph jg = PigStats.get().getJobGraph();
        JobStats js = null;
        Iterator<JobStats> iter = jg.iterator();
        while (iter.hasNext()) {
            JobStats job = iter.next();
            if (job.getName().equals(mro.getOperatorKey().toString())) {
                js = job;
                break;
            }
        }
        if (js != null) {
            List<Operator> preds = jg.getPredecessors(js);
            if (preds != null) {
                StringBuilder sb = new StringBuilder();
                for (Operator op : preds) {
                    JobStats job = (JobStats)op;
                    if (sb.length() > 0) sb.append(",");

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats.JobGraph

            pigServer.registerQuery("b = group a by $0;");
            pigServer.registerQuery("c = foreach b generate group, COUNT(a) as cnt;");
            pigServer.registerQuery("d = group c by cnt;");
            pigServer.registerQuery("e = foreach d generate group;");
            ExecJob job = pigServer.store("e", filePath);
            JobGraph jobGraph = job.getStatistics().getJobGraph();
            assertEquals(2, jobGraph.getJobList().size());


        } catch (IOException e) {
            LOG.error("IOException while creating file ", e);
            fail("Encountered IOException");
        } finally {

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats.JobGraph


            while(iter.hasNext()) {
                dbfrj.add(iter.next());
            }


            JobGraph jGraph = PigStats.get().getJobGraph();
            assertEquals(3, jGraph.size());
            // find added map-only concatenate job
            MRJobStats js = (MRJobStats)jGraph.getSuccessors(jGraph.getSources().get(0)).get(0);
            assertEquals(1, js.getNumberMaps());
            assertEquals(0, js.getNumberReduces());
        }
        {
            pigServer.getPigContext().getProperties().setProperty(

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.