Package org.apache.pig.tools.pigstats

Examples of org.apache.pig.tools.pigstats.PigStats$JobComparator

282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
        w.println("store E into '" + OUTPUT_FILE + "';");
        w.close();
       
        try {
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, new TestNotificationListener());
            assertTrue(stats.isSuccessful());
            assertTrue(stats.getJobGraph().size() == 1);
            assertEquals(4, stats.getRecordWritten());          
            assertEquals(18, stats.getBytesWritten());
            assertTrue(stats.getOutputNames().size() == 2);
            for (String fname : stats.getOutputNames()) {              
                assertTrue(fname.equals(OUTPUT_FILE) || fname.equals(OUTPUT_FILE_2));
                if (fname.equals(OUTPUT_FILE)) {
                    assertEquals(2, stats.getNumberRecords(fname));
                } else {
                    assertEquals(2, stats.getNumberRecords(fname));
                }               
            }          
            assertEquals("A,B,C,D,E",
                    ((JobStats)stats.getJobGraph().getSinks().get(0)).getAlias());
        } finally {
            new File(PIG_FILE).delete();
            Util.deleteFile(cluster, OUTPUT_FILE);
            Util.deleteFile(cluster, OUTPUT_FILE_2);
        }
View Full Code Here
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
        w.println("C= order B by $0;");
        w.println("store C into '" + OUTPUT_FILE + "';");
        w.close();
        try {
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, null);
            Iterator<JobStats> iter = stats.getJobGraph().iterator();
            while (iter.hasNext()) {
                 JobStats js=iter.next();
                 if(js.getState().name().equals("FAILED")) {
                     List<Operator> ops=stats.getJobGraph().getSuccessors(js);
                     for(Operator op : ops ) {
                         assertEquals(((JobStats)op).getState().toString(), "UNKNOWN");
                     }
                 }
            }
View Full Code Here
344
345
346
347
348
349
350
351
352
353
354
355
        w.println("B = group A by a;");
        w.println("C = foreach B generate group, COUNT(A);");
        w.println("store C into '" + OUTPUT_FILE + "';");
        w.close();
        String[] args = { "-c", PIG_FILE };
        PigStats stats = PigRunner.run(args, null);
        assertTrue(stats.getReturnCode() == ReturnCode.PIG_EXCEPTION);
        // TODO: error message has changed. Need to catch the new message generated from the
        // new parser.
//        assertTrue(stats.getErrorCode() == 1000);
//        assertEquals("Error during parsing. Invalid alias: a in {a0: int,a1: int,a2: int}",
//                stats.getErrorMessage());
View Full Code Here
356
357
358
359
360
361
362
363
    }
   
    @Test
    public void simpleNegativeTest2() throws Exception {
        String[] args = { "-c", "-e", "this is a test" };
        PigStats stats = PigRunner.run(args, new TestNotificationListener());       
        assertTrue(stats.getReturnCode() == ReturnCode.ILLEGAL_ARGS);
    }
View Full Code Here
363
364
365
366
367
368
369
370
371
372
    }

    @Test
    public void simpleNegativeTest3() throws Exception {
        String[] args = { "-c", "-y" };
        PigStats stats = PigRunner.run(args, new TestNotificationListener());    
        assertTrue(stats.getReturnCode() == ReturnCode.PARSE_EXCEPTION);
        assertEquals("Found unknown option (-y) at position 2",
                stats.getErrorMessage());
    }
View Full Code Here
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
        w.println("store E into 'ee';");
        w.close();
       
        try {
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, null);            
            assertTrue(!stats.isSuccessful());           
            assertTrue(stats.getReturnCode() == ReturnCode.PARTIAL_FAILURE);
            assertTrue(stats.getJobGraph().size() == 2);
            JobStats job = (JobStats)stats.getJobGraph().getSources().get(0);
            assertTrue(job.isSuccessful());
            job = (JobStats)stats.getJobGraph().getSinks().get(0);
            assertTrue(!job.isSuccessful());
            assertTrue(stats.getOutputStats().size() == 3);
            for (OutputStats output : stats.getOutputStats()) {
                if (output.getName().equals("ee")) {
                    assertTrue(!output.isSuccessful());
                } else {
                    assertTrue(output.isSuccessful());
                }
View Full Code Here
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
        w1.println("store C into '" + OUTPUT_FILE + "';");
        w1.close();
       
        try {
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, new TestNotificationListener());
    
            assertTrue(stats.isSuccessful());
           
            assertEquals(1, stats.getNumberJobs());
            List<InputStats> inputs = stats.getInputStats();
            assertEquals(2, inputs.size());
            for (InputStats instats : inputs) {
                assertEquals(5, instats.getNumberRecords());
            }
        } finally {
View Full Code Here
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
        w1.println("store C into '" + OUTPUT_FILE + "';");
        w1.close();
       
        try {
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, new TestNotificationListener());
    
            assertTrue(stats.isSuccessful());
           
            assertEquals(1, stats.getNumberJobs());
            List<InputStats> inputs = stats.getInputStats();
            assertEquals(2, inputs.size());
            for (InputStats instats : inputs) {
                if (instats.getLocation().endsWith("tmp/input")) {
                    assertEquals(2, instats.getNumberRecords());
                } else {
View Full Code Here
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
        w1.println("store B into 'tmp/output';");
        w1.close();
       
        try {
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, new TestNotificationListener());
    
            assertTrue(stats.isSuccessful());
           
            assertEquals(1, stats.getNumberJobs());
            List<OutputStats> outputs = stats.getOutputStats();
            assertEquals(2, outputs.size());
            for (OutputStats outstats : outputs) {
                if (outstats.getLocation().endsWith("tmp/output")) {
                    assertEquals(2, outstats.getNumberRecords());
                } else {
View Full Code Here
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
    @Test
    public void testRegisterExternalJar() throws Exception {
        String[] args = { "-Dpig.additional.jars=pig-withouthadoop.jar",
                "-Dmapred.job.queue.name=default",
                "-e", "A = load '" + INPUT_FILE + "';store A into '" + OUTPUT_FILE + "';\n" };
        PigStats stats = PigRunner.run(args, new TestNotificationListener());       

        Util.deleteFile(cluster, OUTPUT_FILE);
       
        java.lang.reflect.Method getPigContext = stats.getClass()
                .getDeclaredMethod("getPigContext");

        getPigContext.setAccessible(true);

        PigContext ctx = (PigContext) getPigContext.invoke(stats);
View Full Code Here
TOP

Related Classes of org.apache.pig.tools.pigstats.PigStats$JobComparator

  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.mapred.Counters
  • org.apache.hadoop.mapred.JobConf
  • org.apache.hadoop.mapred.JobID
  • org.apache.hadoop.mapred.RunningJob
  • org.apache.hive.hcatalog.pig.TestHCatLoader
  • org.apache.oozie.action.hadoop.PigMain
  • org.apache.pig.backend.hadoop.executionengine.HExecutionEngine
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.