Examples of MRJobStats

org.apache.pig.tools.pigstats.mapreduce.MRJobStats
This class encapsulates the runtime statistics of a MapReduce job. Job statistics is collected when job is completed.

Examples of org.apache.pig.tools.pigstats.mapreduce.MRJobStats

            }


            JobGraph jGraph = PigStats.get().getJobGraph();
            assertEquals(3, jGraph.size());
            // find added map-only concatenate job
            MRJobStats js = (MRJobStats)jGraph.getSuccessors(jGraph.getSources().get(0)).get(0);
            assertEquals(1, js.getNumberMaps());
            assertEquals(0, js.getNumberReduces());
        }
        {
            pigServer.getPigContext().getProperties().setProperty(
                    "pig.noSplitCombination", "true");
            pigServer.registerQuery("D = join C by $0, B by $0 using 'repl';");

View Full Code Here

Examples of org.apache.pig.tools.pigstats.mapreduce.MRJobStats


    @Override
    public void checkGroupAllWithParallelGraphResult(JobGraph jGraph) {
        assertEquals(1, jGraph.size());
        // find added map-only concatenate job 
        MRJobStats js = (MRJobStats)jGraph.getSources().get(0);
        assertEquals(1, js.getNumberMaps());   
        assertEquals(1, js.getNumberReduces());
    }

View Full Code Here

Examples of org.apache.pig.tools.pigstats.mapreduce.MRJobStats


    MRJobStats createJobStats(String name, JobGraph plan) {
        try {
            Constructor<MRJobStats> con = MRJobStats.class.getDeclaredConstructor(String.class, JobGraph.class);
            con.setAccessible(true);
            MRJobStats jobStats = (MRJobStats) con.newInstance(name, plan);
            return jobStats;
        } catch (Exception e) {
            return null;
        }
    }

View Full Code Here

Examples of org.apache.pig.tools.pigstats.mapreduce.MRJobStats

        // mock methods to return the predefined map and reduce task reports
        Mockito.when(jobClient.getMapTaskReports(jobID)).thenReturn(mapTaskReports);
        Mockito.when(jobClient.getReduceTaskReports(jobID)).thenReturn(reduceTaskReports);


        PigStats.JobGraph jobGraph = new PigStats.JobGraph();
        MRJobStats jobStats = createJobStats("JobStatsTest", jobGraph);
        getJobStatsMethod("setId", JobID.class).invoke(jobStats, jobID);
        jobStats.setSuccessful(true);


        getJobStatsMethod("addMapReduceStatistics", Iterator.class, Iterator.class)
            .invoke(jobStats, Arrays.asList(mapTaskReports).iterator(), Arrays.asList(reduceTaskReports).iterator());
        String msg = (String)getJobStatsMethod("getDisplayString")
            .invoke(jobStats);

View Full Code Here

Examples of org.apache.pig.tools.pigstats.mapreduce.MRJobStats

        reduceTaskReports[0] = Mockito.mock(TaskReport.class);
        Mockito.when(reduceTaskReports[0].getStartTime()).thenReturn(500L * ONE_THOUSAND);
        Mockito.when(reduceTaskReports[0].getFinishTime()).thenReturn(700L * ONE_THOUSAND);


        PigStats.JobGraph jobGraph = new PigStats.JobGraph();
        MRJobStats jobStats = createJobStats("JobStatsTest", jobGraph);
        getJobStatsMethod("setId", JobID.class).invoke(jobStats, jobID);
        jobStats.setSuccessful(true);


        getJobStatsMethod("addMapReduceStatistics", Iterator.class, Iterator.class)
            .invoke(jobStats, Arrays.asList(mapTaskReports).iterator(), Arrays.asList(reduceTaskReports).iterator());
        String msg = (String)getJobStatsMethod("getDisplayString")
            .invoke(jobStats);

View Full Code Here

Examples of org.apache.pig.tools.pigstats.mapreduce.MRJobStats

        // Launch a map-only job
        pigServer.registerQuery("A = load '" + inputFile + "' as (id:int, fruit:chararray);");
        pigServer.registerQuery("store A into 'task_reports';");
        List<ExecJob> jobs = pigServer.executeBatch();
        PigStats pigStats = jobs.get(0).getStatistics();
        MRJobStats jobStats = (MRJobStats) pigStats.getJobGraph().getJobList().get(0);


        // Make sure JobStats includes TaskReports information
        long minMapTime = jobStats.getMinMapTime();
        long maxMapTime = jobStats.getMaxMapTime();
        long avgMapTime = jobStats.getAvgMapTime();
        assertTrue("TaskReports are enabled, so minMapTime shouldn't be -1", minMapTime != -1l);
        assertTrue("TaskReports are enabled, so maxMapTime shouldn't be -1", maxMapTime != -1l);
        assertTrue("TaskReports are enabled, so avgMapTime shouldn't be -1", avgMapTime != -1l);


        // Disable task reports in job statistics
        properties.setProperty(PigConfiguration.PIG_NO_TASK_REPORT, "true");


        // Launch another map-only job
        pigServer.registerQuery("B = load '" + inputFile + "' as (id:int, fruit:chararray);");
        pigServer.registerQuery("store B into 'no_task_reports';");
        jobs = pigServer.executeBatch();
        pigStats = jobs.get(0).getStatistics();
        jobStats = (MRJobStats) pigStats.getJobGraph().getJobList().get(0);


        // Make sure JobStats doesn't include any TaskReports information
        minMapTime = jobStats.getMinMapTime();
        maxMapTime = jobStats.getMaxMapTime();
        avgMapTime = jobStats.getAvgMapTime();
        assertEquals("TaskReports are disabled, so minMapTime should be -1", -1l, minMapTime);
        assertEquals("TaskReports are disabled, so maxMapTime should be -1", -1l, maxMapTime);
        assertEquals("TaskReports are disabled, so avgMapTime should be -1", -1l, avgMapTime);


        cluster.shutDown();

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.