Package org.apache.pig.tools.pigstats

Examples of org.apache.pig.tools.pigstats.PigStats

261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group;");
        PigStats pigStats = pigServer.store("c", "output", "BinStorage").getStatistics();

        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output", pigServer.getPigContext()), ExecType.MAPREDUCE, pigServer.getPigContext().getDfs());
        long filesize = 0;
        while(is.read() != -1) filesize++;
       
        is.close();
       
        Map<String, Map<String, String>> stats = pigStats.getPigStats();
        cluster.getFileSystem().delete(new Path(file), true);
        cluster.getFileSystem().delete(new Path("output"), true);

        System.out.println("============================================");
        System.out.println("Test case MapReduce");
        System.out.println("============================================");
        System.out.println("MRPlan : \n" + pigStats.getMRPlan());
        for(Map.Entry<String, Map<String, String>> entry : stats.entrySet()) {
            System.out.println("============================================");
            System.out.println("Job : " + entry.getKey());
            for(Map.Entry<String, String> e1 : entry.getValue().entrySet()) {
                System.out.println(" - " + e1.getKey() + " : \n" + e1.getValue());
            }
            System.out.println("============================================");
        }

        Map.Entry<String, Map<String, String>> e = stats.entrySet().iterator().next();

        Map<String, String> jobStats = e.getValue();

        System.out.println("Map input records : " + jobStats.get("PIG_STATS_MAP_INPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_INPUT_RECORDS")));
        System.out.println("Map output records : " + jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS")));
        System.out.println("Reduce input records : " + jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS")));
        System.out.println("Reduce output records : " + jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS"));
        assertEquals(count, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS")));

        assertNull(jobStats.get("PIG_STATS_COMBINE_PLAN"));
        assertNotNull(jobStats.get("PIG_STATS_MAP_PLAN"));
        assertNotNull(jobStats.get("PIG_STATS_REDUCE_PLAN"));
       
        assertEquals(count, pigStats.getRecordsWritten());
        assertEquals(filesize, pigStats.getBytesWritten());

    }
View Full Code Here
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group, SUM(a.$1);");
        PigStats pigStats = pigServer.store("c", "output").getStatistics();

        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output", pigServer.getPigContext()), ExecType.MAPREDUCE, pigServer.getPigContext().getDfs());
        long filesize = 0;
        while(is.read() != -1) filesize++;
       
        is.close();
        Map<String, Map<String, String>> stats = pigStats.getPigStats();
        cluster.getFileSystem().delete(new Path(file), true);
        cluster.getFileSystem().delete(new Path("output"), true);

        System.out.println("============================================");
        System.out.println("Test case MapCombineReduce");
        System.out.println("============================================");
        System.out.println("MRPlan : \n" + pigStats.getMRPlan());
        for(Map.Entry<String, Map<String, String>> entry : stats.entrySet()) {
            System.out.println("============================================");
            System.out.println("Job : " + entry.getKey());
            for(Map.Entry<String, String> e1 : entry.getValue().entrySet()) {
                System.out.println(" - " + e1.getKey() + " : \n" + e1.getValue());
            }
            System.out.println("============================================");
        }

        Map.Entry<String, Map<String, String>> e = stats.entrySet().iterator().next();

        Map<String, String> jobStats = e.getValue();

        System.out.println("Map input records : " + jobStats.get("PIG_STATS_MAP_INPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_INPUT_RECORDS")));
        System.out.println("Map output records : " + jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS")));
        System.out.println("Reduce input records : " + jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS"));
        assertEquals(count, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS")));
        System.out.println("Reduce output records : " + jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS"));
        assertEquals(count, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS")));

        assertNotNull(jobStats.get("PIG_STATS_COMBINE_PLAN"));
        assertNotNull(jobStats.get("PIG_STATS_MAP_PLAN"));
        assertNotNull(jobStats.get("PIG_STATS_REDUCE_PLAN"));

        assertEquals(count, pigStats.getRecordsWritten());
        assertEquals(filesize, pigStats.getBytesWritten());
    }
View Full Code Here
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group, SUM(a.$1);");
        PigStats pigStats = pigServer.store("c", "output", "BinStorage").getStatistics();

        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output", pigServer.getPigContext()), ExecType.MAPREDUCE, pigServer.getPigContext().getDfs());
        long filesize = 0;
        while(is.read() != -1) filesize++;
       
        is.close();
        Map<String, Map<String, String>> stats = pigStats.getPigStats();
        cluster.getFileSystem().delete(new Path(file), true);
        cluster.getFileSystem().delete(new Path("output"), true);

        System.out.println("============================================");
        System.out.println("Test case MapCombineReduce");
        System.out.println("============================================");
        System.out.println("MRPlan : \n" + pigStats.getMRPlan());
        for(Map.Entry<String, Map<String, String>> entry : stats.entrySet()) {
            System.out.println("============================================");
            System.out.println("Job : " + entry.getKey());
            for(Map.Entry<String, String> e1 : entry.getValue().entrySet()) {
                System.out.println(" - " + e1.getKey() + " : \n" + e1.getValue());
            }
            System.out.println("============================================");
        }

        Map.Entry<String, Map<String, String>> e = stats.entrySet().iterator().next();

        Map<String, String> jobStats = e.getValue();

        System.out.println("Map input records : " + jobStats.get("PIG_STATS_MAP_INPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_INPUT_RECORDS")));
        System.out.println("Map output records : " + jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS")));
        System.out.println("Reduce input records : " + jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS"));
        assertEquals(count, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS")));
        System.out.println("Reduce output records : " + jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS"));
        assertEquals(count, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS")));

        assertNotNull(jobStats.get("PIG_STATS_COMBINE_PLAN"));
        assertNotNull(jobStats.get("PIG_STATS_MAP_PLAN"));
        assertNotNull(jobStats.get("PIG_STATS_REDUCE_PLAN"));

        assertEquals(count, pigStats.getRecordsWritten());
        assertEquals(filesize, pigStats.getBytesWritten());
    }
View Full Code Here
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = order a by $0;");
        pigServer.registerQuery("c = group b by $0;");
        pigServer.registerQuery("d = foreach c generate group, SUM(b.$1);");
        PigStats pigStats = pigServer.store("d", "output").getStatistics();
       
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output", pigServer.getPigContext()), ExecType.MAPREDUCE, pigServer.getPigContext().getDfs());
        long filesize = 0;
        while(is.read() != -1) filesize++;
       
        is.close();
        Map<String, Map<String, String>> stats = pigStats.getPigStats();
        cluster.getFileSystem().delete(new Path(file), true);
        cluster.getFileSystem().delete(new Path("output"), true);
       
        System.out.println("============================================");
        System.out.println("Test case MultipleMRJobs");
        System.out.println("============================================");
        System.out.println("MRPlan : \n" + pigStats.getMRPlan());
        for(Map.Entry<String, Map<String, String>> entry : stats.entrySet()) {
            System.out.println("============================================");
            System.out.println("Job : " + entry.getKey());
            for(Map.Entry<String, String> e1 : entry.getValue().entrySet()) {
                System.out.println(" - " + e1.getKey() + " : \n" + e1.getValue());
            }
            System.out.println("============================================");
        }

        Map<String, String> jobStats = stats.get(pigStats.getRootJobIDs().get(0));

        System.out.println("Map input records : " + jobStats.get("PIG_STATS_MAP_INPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_INPUT_RECORDS")));
        System.out.println("Map output records : " + jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS")));
        System.out.println("Reduce input records : " + jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS"));
        assertEquals(count, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS")));
        System.out.println("Reduce output records : " + jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS"));
        assertEquals(count, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS")));
       
        assertEquals(count, pigStats.getRecordsWritten());
        assertEquals(filesize, pigStats.getBytesWritten());

    }
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
142
143
            String filePath = outputFile.getAbsolutePath();
            outputFile.delete();
            PigServer pig = new PigServer(ExecType.LOCAL);
            pig.registerQuery("A = load 'test/org/apache/pig/test/data/passwd';");
            ExecJob job = pig.store("A", filePath);
            PigStats stats = job.getStatistics();
            File dataFile = new File( outputFile.getAbsoluteFile() + File.separator + "part-00000" );
            assertEquals(dataFile.length(), stats.getBytesWritten());
        } catch (IOException e) {
            e.printStackTrace();
            System.err.println( e.getMessage() );
            fail("IOException happened");
        } finally {
View Full Code Here
377
378
379
380
381
382
383
384
385
386
387
388
389
                if (spec != null)
                    leafMap.put(spec.toString(), physOp);
            }
        }
        try {
            PigStats stats = launcher.launchPig(plan, jobName, pigContext);

            for (OutputStats output : stats.getOutputStats()) {
                POStore store = output.getPOStore();              
                String alias = store.getAlias();
                if (output.isSuccessful()) {
                    jobs.add(new HJob(ExecJob.JOB_STATUS.COMPLETED, pigContext, store, alias, stats));
                } else {
View Full Code Here
513
514
515
516
517
518
519
520
521
522
523
524
525
        w.println("fs -mv nonexist.file dummy.file");
        w.close();
                                       
        try {
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, new TestNotificationListener());
                                                                            
            assertTrue(!stats.isSuccessful());
            assertTrue(stats.getReturnCode() == PigRunner.ReturnCode.IO_EXCEPTION);
        } finally {
            new File(PIG_FILE).delete();
        }
    }
View Full Code Here
534
535
536
537
538
539
540
541
542
543
544
545
546
547
        w.println("store D into '" + OUTPUT_FILE + "';");
        w.close();
       
        try {
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, null);
           
            assertTrue(!stats.isSuccessful());
            assertTrue(stats.getReturnCode() != 0);
           
        } finally {
            new File(PIG_FILE).delete();
            Util.deleteFile(cluster, OUTPUT_FILE);
        }
View Full Code Here
335
336
337
338
339
340
341
342
343
344
345
346
347
     * @return list of jobs being executed
     * @throws FrontendException
     * @throws ExecException
     */
    public List<ExecJob> executeBatch() throws FrontendException, ExecException {
        PigStats stats = executeBatchEx();
        LinkedList<ExecJob> jobs = new LinkedList<ExecJob>();
        JobGraph jGraph = stats.getJobGraph();
        Iterator<JobStats> iter = jGraph.iterator();
        while (iter.hasNext()) {
            JobStats js = iter.next();
            for (OutputStats output : js.getOutputs()) {
                if (js.isSuccessful()) {               
View Full Code Here
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
     * @return {@link ExecJob} containing information about this job
     * @throws IOException
     */
    public ExecJob store(String id, String filename, String func)
            throws IOException {
        PigStats stats = storeEx(id, filename, func);
        if (stats.getOutputStats().size() < 1) {
            throw new IOException("Couldn't retrieve job.");
        }
        OutputStats output = stats.getOutputStats().get(0);

        if(stats.isSuccessful()){
            return  new HJob(JOB_STATUS.COMPLETED, pigContext, output
                    .getPOStore(), output.getAlias(), stats);
        }else{
            HJob job = new HJob(JOB_STATUS.FAILED, pigContext,
                    output.getPOStore(), output.getAlias(), stats);
           
            //check for exception
            Exception ex = null;
            for(JobStats js : stats.getJobGraph()){
                if(js.getException() != null)
                    ex = js.getException();
            }
            job.setException(ex);
            return job;
View Full Code Here
TOP

Related Classes of org.apache.pig.tools.pigstats.PigStats

  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.mapred.Counters
  • org.apache.hadoop.mapred.JobConf
  • org.apache.hadoop.mapred.JobID
  • org.apache.hadoop.mapred.RunningJob
  • org.apache.hive.hcatalog.pig.TestHCatLoader
  • org.apache.oozie.action.hadoop.PigMain
  • org.apache.pig.backend.hadoop.executionengine.HExecutionEngine
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.