Package org.apache.pig.tools.pigstats

Examples of org.apache.pig.tools.pigstats.OutputStats

775
776
777
778
779
780
781
782
783
784
785
786
                assertEquals(-1, instats.getNumberRecords());
            }
           
            List<OutputStats> outputs = stats.getOutputStats();
            assertEquals(1, outputs.size());
            OutputStats outstats = outputs.get(0);
            assertEquals(9, outstats.getNumberRecords());
        } finally {
            new File(PIG_FILE).delete();
            Util.deleteFile(cluster, OUTPUT_FILE);
        }
    }
View Full Code Here
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
      String[] args = { "-x", "local", PIG_FILE };
      PigStats stats = PigRunner.run(args, null);
      //Pig script was successful
      assertTrue(stats.isSuccessful());
      //Single MapReduce job is launched
      OutputStats outstats = stats.getOutputStats().get(0);
      assertTrue(outstats!= null);
      assertEquals(expectedCols,outstats.getConf()
        .get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR));
      //delete output file on exit
      FileSystem fs = FileSystem.get(outstats.getConf());
      if (fs.exists(new Path(PIGOUTPUT_DIR)))
      {
        fs.delete(new Path(PIGOUTPUT_DIR), true);
      }
    }finally {
View Full Code Here
350
351
352
353
354
355
356
357
358
359
360
361
                    "-Dtfile.io.chunk.size=100", "tfile2.pig" };
            PigStats stats2 = PigRunner.run(args2, null);

            assertTrue(stats2.isSuccessful());

            OutputStats os = stats2.result("B");
            Iterator<Tuple> iter = os.iterator();
            int count = 0;
            String expected = "(1,this is a test for compression of temp files)";
            while (iter.hasNext()) {
                count++;
                assertEquals(expected, iter.next().toString());
View Full Code Here
869
870
871
872
873
874
875
876
877
878
879
880
                }
            }

            List<OutputStats> outputs = stats.getOutputStats();
            assertEquals(1, outputs.size());
            OutputStats outstats = outputs.get(0);
            assertEquals(9, outstats.getNumberRecords());
        } finally {
            new File(PIG_FILE).delete();
            Util.deleteFile(cluster, OUTPUT_FILE);
        }
    }
View Full Code Here
318
319
320
321
322
323
324
325
326
327
328
329
        String[] results = new String[2];
        int i = 0;
        for (PigStats stats : lst) {
            assertTrue("job should succeed", stats.isSuccessful());
            assertEquals(1, stats.getNumberJobs());
            OutputStats os = stats.getOutputStats().get(0);
            Tuple t = os.iterator().next();
            results[i++] = t.get(1).toString();
        }
        assertTrue(results[0] != null);
        assertTrue(results[1] != null);
        assertTrue(!results[0].equals(results[1]));
View Full Code Here
324
325
326
327
328
329
330
331
332
333
334
335
        String[] results = new String[2];
        int i = 0;
        for (PigStats stats : lst) {
            assertTrue("job should succeed", stats.isSuccessful());
            assertEquals(1, stats.getNumberJobs());
            OutputStats os = stats.getOutputStats().get(0);
            Tuple t = os.iterator().next();
            results[i++] = t.get(1).toString();
        }
        assertTrue(results[0] != null);
        assertTrue(results[1] != null);
        assertTrue(!results[0].equals(results[1]));
View Full Code Here
797
798
799
800
801
802
803
804
805
806
807
808
                assertEquals(-1, instats.getNumberRecords());
            }
           
            List<OutputStats> outputs = stats.getOutputStats();
            assertEquals(1, outputs.size());
            OutputStats outstats = outputs.get(0);
            assertEquals(9, outstats.getNumberRecords());
        } finally {
            new File(PIG_FILE).delete();
            Util.deleteFile(cluster, OUTPUT_FILE);
        }
    }
View Full Code Here
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
            throws IOException {
        PigStats stats = storeEx(id, filename, func);
        if (stats.getOutputStats().size() < 1) {
            throw new IOException("Couldn't retrieve job.");
        }
        OutputStats output = stats.getOutputStats().get(0);

        if(stats.isSuccessful()){
            return  new HJob(JOB_STATUS.COMPLETED, pigContext, output
                    .getPOStore(), output.getAlias(), stats);
        }else{
            HJob job = new HJob(JOB_STATUS.FAILED, pigContext,
                    output.getPOStore(), output.getAlias(), stats);

            //check for exception
            Exception ex = null;
            for(JobStats js : stats.getJobGraph()){
                if(js.getException() != null) {
View Full Code Here
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
      String[] args = { "-x", "local", PIG_FILE };
      PigStats stats = PigRunner.run(args, null);
      //Pig script was successful
      assertTrue(stats.isSuccessful());
      //Single MapReduce job is launched
      OutputStats outstats = stats.getOutputStats().get(0);
      assertTrue(outstats!= null);
      assertEquals(expectedCols,outstats.getConf()
        .get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR));
      //delete output file on exit
      FileSystem fs = FileSystem.get(outstats.getConf());
      if (fs.exists(new Path(PIGOUTPUT_DIR)))
      {
        fs.delete(new Path(PIGOUTPUT_DIR), true);
      }
    } finally {
View Full Code Here
773
774
775
776
777
778
779
780
781
782
783
784
                assertEquals(-1, instats.getNumberRecords());
            }
           
            List<OutputStats> outputs = stats.getOutputStats();
            assertEquals(1, outputs.size());
            OutputStats outstats = outputs.get(0);
            assertEquals(9, outstats.getNumberRecords());
        } finally {
            new File(PIG_FILE).delete();
            Util.deleteFile(cluster, OUTPUT_FILE);
        }
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.tools.pigstats.OutputStats

  • org.apache.hadoop.fs.Path
  • org.apache.hive.hcatalog.pig.TestHCatLoader
  • org.apache.pig.data.Tuple
  • org.apache.pig.impl.io.ReadToEndLoader
  • org.apache.pig.impl.PigContext
  • org.apache.pig.LoadFunc
  • org.apache.pig.PigServer
  • org.apache.pig.test.TestPigRunner
  • org.apache.pig.test.TestScriptLanguage
  • org.apache.pig.test.TestTmpFileCompression
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.