Package org.apache.pig.tools.pigstats

Examples of org.apache.pig.tools.pigstats.InputStats

479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
        if (loads.size() == 1) {
            FileSpec fsp = loads.get(0);
            if (!MRPigStatsUtil.isTempFile(fsp.getFileName())) {
                long records = mapInputRecords;
                InputStats is = new InputStats(fsp.getFileName(),
                        hdfsBytesRead, records, (state == JobState.SUCCESS));
                is.setConf(conf);
                if (isSampler()) is.markSampleInput();
                if (isIndexer()) is.markIndexerInput();
                inputs.add(is);
            }
        } else {
            for (int i=0; i<loads.size(); i++) {
                FileSpec fsp = loads.get(i);
View Full Code Here
508
509
510
511
512
513
514
515
516
            if (!disableCounter) records = 0;
            else {
                LOG.warn("unable to get input counter for " + fileName);
            }
        }
        InputStats is = new InputStats(fileName, -1, records, (state == JobState.SUCCESS));
        is.setConf(conf);
        inputs.add(is);
    }
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
                if (counters.get(FS_COUNTER_GROUP) != null &&
                        counters.get(FS_COUNTER_GROUP).get(PigStatsUtil.HDFS_BYTES_READ) != null) {
                    hdfsBytesRead = counters.get(FS_COUNTER_GROUP).get(PigStatsUtil.HDFS_BYTES_READ);
                }
            }
            InputStats is = new InputStats(filename, hdfsBytesRead,
                    records, (state == JobState.SUCCESS));
            is.setConf(conf);
            inputs.add(is);
        }
    }
View Full Code Here
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
        if (loads.size() == 1) {
            FileSpec fsp = loads.get(0);
            if (!MRPigStatsUtil.isTempFile(fsp.getFileName())) {
                long records = mapInputRecords;
                InputStats is = new InputStats(fsp.getFileName(),
                        hdfsBytesRead, records, (state == JobState.SUCCESS));
                is.setConf(conf);
                if (isSampler()) is.markSampleInput();
                if (isIndexer()) is.markIndexerInput();
                inputs.add(is);
            }
        } else {
            for (int i=0; i<loads.size(); i++) {
                FileSpec fsp = loads.get(i);
View Full Code Here
508
509
510
511
512
513
514
515
516
            if (!disableCounter) records = 0;
            else {
                LOG.warn("unable to get input counter for " + fileName);
            }
        }
        InputStats is = new InputStats(fileName, -1, records, (state == JobState.SUCCESS));
        is.setConf(conf);
        inputs.add(is);
    }
View Full Code Here
903
904
905
906
907
908
909
910
911
912
913
914
                assertEquals(-1, outstats.getNumberRecords());
            }
           
            List<InputStats> inputs = stats.getInputStats();
            assertEquals(1, inputs.size());
            InputStats instats = inputs.get(0);
            assertEquals(5, instats.getNumberRecords());
        } finally {
            new File(PIG_FILE).delete();
            Util.deleteFile(cluster, OUTPUT_FILE);
            Util.deleteFile(cluster, "tmp/output");
        }
View Full Code Here
795
796
797
798
799
800
801
802
803
804
805
806
                assertEquals(-1, outstats.getNumberRecords());
            }
           
            List<InputStats> inputs = stats.getInputStats();
            assertEquals(1, inputs.size());
            InputStats instats = inputs.get(0);
            assertEquals(5, instats.getNumberRecords());
        } finally {
            new File(PIG_FILE).delete();
            Util.deleteFile(cluster, OUTPUT_FILE);
            Util.deleteFile(cluster, "tmp/output");
        }
View Full Code Here
852
853
854
855
856
857
858
859
860
861
862
863
                assertEquals(-1, outstats.getNumberRecords());
            }
           
            List<InputStats> inputs = stats.getInputStats();
            assertEquals(1, inputs.size());
            InputStats instats = inputs.get(0);
            assertEquals(5, instats.getNumberRecords());
        } finally {
            new File(PIG_FILE).delete();
            Util.deleteFile(cluster, OUTPUT_FILE);
            Util.deleteFile(cluster, "tmp/output");
        }
View Full Code Here
968
969
970
971
972
973
974
975
976
977
978
979
                }
            }

            List<InputStats> inputs = stats.getInputStats();
            assertEquals(1, inputs.size());
            InputStats instats = inputs.get(0);
            assertEquals(5, instats.getNumberRecords());
        } finally {
            new File(PIG_FILE).delete();
            Util.deleteFile(cluster, OUTPUT_FILE);
            Util.deleteFile(cluster, "tmp/output");
        }
View Full Code Here
874
875
876
877
878
879
880
881
882
883
884
885
                assertEquals(-1, outstats.getNumberRecords());
            }
           
            List<InputStats> inputs = stats.getInputStats();
            assertEquals(1, inputs.size());
            InputStats instats = inputs.get(0);
            assertEquals(5, instats.getNumberRecords());
        } finally {
            new File(PIG_FILE).delete();
            Util.deleteFile(cluster, OUTPUT_FILE);
            Util.deleteFile(cluster, "tmp/output");
        }
View Full Code Here
TOP

Related Classes of org.apache.pig.tools.pigstats.InputStats

  • org.apache.hadoop.fs.Path
  • org.apache.pig.test.TestPigRunner
  • org.apache.pig.tools.pigstats.mapreduce.MRJobStats
  • org.apache.pig.tools.pigstats.tez.TezVertexStats
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.