Package org.apache.pig.tools.pigstats.mapreduce

Examples of org.apache.pig.tools.pigstats.mapreduce.MRJobStats

115
116
117
118
119
120
121
122
123
124
125
126
127
        // mock methods to return the predefined map and reduce task reports
        Mockito.when(jobClient.getMapTaskReports(jobID)).thenReturn(mapTaskReports);
        Mockito.when(jobClient.getReduceTaskReports(jobID)).thenReturn(reduceTaskReports);

        PigStats.JobGraph jobGraph = new PigStats.JobGraph();
        MRJobStats jobStats = createJobStats("JobStatsTest", jobGraph);
        getJobStatsMethod("setId", JobID.class).invoke(jobStats, jobID);
        jobStats.setSuccessful(true);

        getJobStatsMethod("addMapReduceStatistics", TaskReport[].class, TaskReport[].class)
            .invoke(jobStats, mapTaskReports, reduceTaskReports);
        String msg = (String)getJobStatsMethod("getDisplayString")
            .invoke(jobStats);
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
156
157
        reduceTaskReports[0] = Mockito.mock(TaskReport.class);
        Mockito.when(reduceTaskReports[0].getStartTime()).thenReturn(500L * ONE_THOUSAND);
        Mockito.when(reduceTaskReports[0].getFinishTime()).thenReturn(700L * ONE_THOUSAND);

        PigStats.JobGraph jobGraph = new PigStats.JobGraph();
        MRJobStats jobStats = createJobStats("JobStatsTest", jobGraph);
        getJobStatsMethod("setId", JobID.class).invoke(jobStats, jobID);
        jobStats.setSuccessful(true);

        getJobStatsMethod("addMapReduceStatistics", TaskReport[].class, TaskReport[].class)
            .invoke(jobStats, mapTaskReports, reduceTaskReports);
        String msg = (String)getJobStatsMethod("getDisplayString")
            .invoke(jobStats);
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, null);
    
            assertTrue(stats.isSuccessful());
            // the sampler job has zero maps
            MRJobStats js = (MRJobStats)stats.getJobGraph().getSources().get(0);
           
            // This assert fails on 205 due to MAPREDUCE-3606
            if (!Util.isHadoop205()&&!Util.isHadoop1_x())
                assertEquals(0, js.getNumberMaps());
           
            FileSystem fs = cluster.getFileSystem();
            FileStatus status = fs.getFileStatus(new Path(OUTPUT_FILE));
            assertTrue(status.isDir());
            assertEquals(0, status.getLen());
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, null);
    
            assertTrue(stats.isSuccessful());   
            // the indexer job has zero maps
            MRJobStats js = (MRJobStats)stats.getJobGraph().getSources().get(0);
           
            // This assert fails on 205 due to MAPREDUCE-3606
            if (!Util.isHadoop205()&&!Util.isHadoop1_x())
                assertEquals(0, js.getNumberMaps());
           
            FileSystem fs = cluster.getFileSystem();
            FileStatus status = fs.getFileStatus(new Path(OUTPUT_FILE));
            assertTrue(status.isDir());
            assertEquals(0, status.getLen());
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, null);
    
            assertTrue(stats.isSuccessful());   
            // the indexer job has zero maps
            MRJobStats js = (MRJobStats)stats.getJobGraph().getSources().get(0);
           
            // This assert fails on 205 due to MAPREDUCE-3606
            if (!Util.isHadoop205()&&!Util.isHadoop1_x())
                assertEquals(0, js.getNumberMaps());
           
            FileSystem fs = cluster.getFileSystem();
            FileStatus status = fs.getFileStatus(new Path(OUTPUT_FILE));
            assertTrue(status.isDir());
            assertEquals(0, status.getLen());
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
133
134
            }
           
            JobGraph jGraph = PigStats.get().getJobGraph();
            assertEquals(3, jGraph.size());
            // find added map-only concatenate job
            MRJobStats js = (MRJobStats)jGraph.getSuccessors(jGraph.getSources().get(0)).get(0);
            assertEquals(1, js.getNumberMaps());  
            assertEquals(0, js.getNumberReduces());
        }
        {
            pigServer.getPigContext().getProperties().setProperty(
                    "pig.noSplitCombination", "true");
           
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
182
183
            }
           
            JobGraph jGraph = PigStats.get().getJobGraph();
            assertEquals(3, jGraph.size());
            // find added map-only concatenate job
            MRJobStats js = (MRJobStats)jGraph.getSuccessors(jGraph.getSources().get(0)).get(0);
            assertEquals(1, js.getNumberMaps());  
            assertEquals(0, js.getNumberReduces());
            Util.checkLogFileMessage(logFile,
                    new String[] {"number of input files: 0", "failed to get number of input files"},
                    false
            );
        }
View Full Code Here
312
313
314
315
316
317
318
319
320
321
322
323
324
            }
           
            JobGraph jGraph = PigStats.get().getJobGraph();
            assertEquals(3, jGraph.size());
            // find added map-only concatenate job
            MRJobStats js = (MRJobStats)jGraph.getSuccessors(jGraph.getSources().get(0)).get(0);
            assertEquals(1, js.getNumberMaps());  
            assertEquals(0, js.getNumberReduces());  
        }
        {
            pigServer.getPigContext().getProperties().setProperty(
                    "pig.noSplitCombination", "true");
            pigServer.registerQuery("D = join C by $0, B by $0 using 'repl';");
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
        System.out.println("============================================");

        JobGraph jg = pigStats.getJobGraph();
        Iterator<JobStats> iter = jg.iterator();
        while (iter.hasNext()) {
            MRJobStats js = (MRJobStats) iter.next();                   

            System.out.println("Map input records : " + js.getMapInputRecords());
            assertEquals(MAX, js.getMapInputRecords());
            System.out.println("Map output records : " + js.getMapOutputRecords());
            assertEquals(count, js.getMapOutputRecords());
            assertEquals(0, js.getReduceInputRecords());
            assertEquals(0, js.getReduceOutputRecords());
            System.out.println("Hdfs bytes written : " + js.getHdfsBytesWritten());
            assertEquals(filesize, js.getHdfsBytesWritten());
        }

    }
View Full Code Here
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
        System.out.println("============================================");

        JobGraph jp = pigStats.getJobGraph();
        Iterator<JobStats> iter = jp.iterator();
        while (iter.hasNext()) {
            MRJobStats js = (MRJobStats) iter.next();
       
            System.out.println("Map input records : " + js.getMapInputRecords());
            assertEquals(MAX, js.getMapInputRecords());
            System.out.println("Map output records : " + js.getMapOutputRecords());
            assertEquals(count, js.getMapOutputRecords());
            assertEquals(0, js.getReduceInputRecords());
            assertEquals(0, js.getReduceOutputRecords());
        }
           
        System.out.println("Hdfs bytes written : " + pigStats.getBytesWritten());
        assertEquals(filesize, pigStats.getBytesWritten());
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.tools.pigstats.mapreduce.MRJobStats

  • org.apache.hadoop.mapred.Counters.Counter
  • org.apache.hadoop.mapred.TaskReport
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore
  • org.apache.pig.impl.io.FileSpec
  • org.apache.pig.test.TestCounters
  • org.apache.pig.test.TestEmptyInputDir
  • org.apache.pig.test.TestFRJoin2
  • org.apache.pig.test.TestGroupConstParallel
  • org.apache.pig.test.TestGroupConstParallelMR
  • org.apache.pig.test.TestMRJobStats
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.