Package org.apache.pig.tools.pigstats

Examples of org.apache.pig.tools.pigstats.JobStats

77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, null);
    
            assertTrue(stats.isSuccessful());
            // the sampler job has zero maps
            JobStats js = (JobStats)stats.getJobGraph().getSources().get(0);
           
            // This assert fails on 205 due to MAPREDUCE-3606
            if (!Util.isHadoop205()&&!Util.isHadoop1_x())
                assertEquals(0, js.getNumberMaps());
           
            FileSystem fs = cluster.getFileSystem();
            FileStatus status = fs.getFileStatus(new Path(OUTPUT_FILE));
            assertTrue(status.isDir());
            assertEquals(0, status.getLen());
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, null);
    
            assertTrue(stats.isSuccessful());   
            // the indexer job has zero maps
            JobStats js = (JobStats)stats.getJobGraph().getSources().get(0);
           
            // This assert fails on 205 due to MAPREDUCE-3606
            if (!Util.isHadoop205()&&!Util.isHadoop1_x())
                assertEquals(0, js.getNumberMaps());
           
            FileSystem fs = cluster.getFileSystem();
            FileStatus status = fs.getFileStatus(new Path(OUTPUT_FILE));
            assertTrue(status.isDir());
            assertEquals(0, status.getLen());
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, null);
    
            assertTrue(stats.isSuccessful());   
            // the indexer job has zero maps
            JobStats js = (JobStats)stats.getJobGraph().getSources().get(0);
           
            // This assert fails on 205 due to MAPREDUCE-3606
            if (!Util.isHadoop205()&&!Util.isHadoop1_x())
                assertEquals(0, js.getNumberMaps());
           
            FileSystem fs = cluster.getFileSystem();
            FileStatus status = fs.getFileStatus(new Path(OUTPUT_FILE));
            assertTrue(status.isDir());
            assertEquals(0, status.getLen());
View Full Code Here
650
651
652
653
654
655
656
657
658
659
660
661
            e.printStackTrace(System.out);
            throw new IOException(e);
        }
        Iterator<JobStats> iter = stats.getJobGraph().iterator();
        while (iter.hasNext()) {
            JobStats js = iter.next();
            failed = !js.isSuccessful();
            if (failed) {
                break;
            }
        }
        return !failed;
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
 
  JobStats createJobStats(String name, JobGraph plan) {
      try {
          Constructor con = JobStats.class.getDeclaredConstructor(String.class, JobGraph.class);
          con.setAccessible(true);
          JobStats jobStats = (JobStats)con.newInstance(name, plan);
          return jobStats;
      } catch (Exception e) {
          return null;
      }
  }
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
    // mock methods to return the predefined map and reduce task reports
    Mockito.when(jobClient.getMapTaskReports(jobID)).thenReturn(mapTaskReports);
    Mockito.when(jobClient.getReduceTaskReports(jobID)).thenReturn(reduceTaskReports);

    PigStats.JobGraph jobGraph = new PigStats.JobGraph();
    JobStats jobStats = createJobStats("JobStatsTest", jobGraph);
    getJobStatsMethod("setId", JobID.class).invoke(jobStats, jobID);
    getJobStatsMethod("setSuccessful", boolean.class).invoke(jobStats, true);

    getJobStatsMethod("addMapReduceStatistics", JobClient.class, Configuration.class)
        .invoke(jobStats, jobClient, jobConf);
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
    Mockito.when(jobClient.getMapTaskReports(jobID)).thenReturn(mapTaskReports);
    Mockito.when(jobClient.getReduceTaskReports(jobID)).thenReturn(reduceTaskReports);
   
    PigStats.JobGraph jobGraph = new PigStats.JobGraph();
    JobStats jobStats = createJobStats("JobStatsTest", jobGraph);
    getJobStatsMethod("setId", JobID.class).invoke(jobStats, jobID);
    getJobStatsMethod("setSuccessful", boolean.class).invoke(jobStats, true);
   
    getJobStatsMethod("addMapReduceStatistics", JobClient.class, Configuration.class)
        .invoke(jobStats, jobClient, jobConf);
View Full Code Here
647
648
649
650
651
652
653
654
655
656
657
658
            e.printStackTrace(System.out);
            throw new IOException(e);
        }
        Iterator<JobStats> iter = stats.getJobGraph().iterator();
        while (iter.hasNext()) {
            JobStats js = iter.next();
            failed = !js.isSuccessful();
            if (failed) {
                break;
            }
        }
        return !failed;
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
            PigStats stats = PigRunner.run(args, null);

            assertTrue(stats.isSuccessful());

            // get the skew-join job stat
            JobStats js = (JobStats) stats.getJobGraph().getSinks().get(0);
            assertEquals(actual_parallel, js.getNumberReduces());

            // estimation should only kick in if parallel and default_parallel are not set
            long estimatedReducers = -1;
            if (parallel < 1 && default_parallel < 1) {
                double fileSize = (double)(new File("test/org/apache/pig/test/data/passwd").length());
                int inputFiles = js.getInputs().size();
                estimatedReducers = Math.min((long)Math.ceil(fileSize/(double)bytes_per_reducer) * inputFiles, 999);
            }

            Util.assertParallelValues(default_parallel, parallel, estimatedReducers,
                    actual_parallel, js.getInputs().get(0).getConf());

        } catch (Exception e) {
            assertNull("Exception thrown during verifySkewJoin", e);
        } finally {
            new File(PIG_FILE).delete();
View Full Code Here
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
            PigStats stats = PigRunner.run(args, new TestNotificationListener());
            assertTrue(stats.isSuccessful());
            assertTrue(stats.getJobGraph().size() == 4);
            assertTrue(stats.getJobGraph().getSinks().size() == 1);
            assertTrue(stats.getJobGraph().getSources().size() == 1);
            JobStats js = (JobStats) stats.getJobGraph().getSinks().get(0);
            assertEquals(OUTPUT_FILE, js.getOutputs().get(0).getName());
            assertEquals(2, js.getOutputs().get(0).getNumberRecords());
            assertEquals(12, js.getOutputs().get(0).getBytes());
            assertEquals(OUTPUT_FILE, stats.getOutputNames().get(0));
            assertEquals(2, stats.getRecordWritten());
            assertEquals(12, stats.getBytesWritten());
           
            assertEquals("A", ((JobStats) stats.getJobGraph().getSources().get(
                    0)).getAlias());
            assertEquals("B", ((JobStats) stats.getJobGraph().getPredecessors(
                    js).get(0)).getAlias());
            assertEquals("B", js.getAlias());
        } finally {
            new File(PIG_FILE).delete();
            Util.deleteFile(cluster, OUTPUT_FILE);
        }
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.tools.pigstats.JobStats

  • com.netflix.lipstick.util.OutputSamplerTest
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.mapred.Counters
  • org.apache.hadoop.mapred.Counters.Counter
  • org.apache.hadoop.mapred.TaskReport
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigStatsOutputSizeReader
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore
  • org.apache.pig.builtin.TestOrcStoragePushdown
  • org.apache.pig.impl.io.FileSpec
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.