Package org.apache.hadoop.fs.FileSystem

Examples of org.apache.hadoop.fs.FileSystem.Statistics

475
476
477
478
479
480
481
482
483
484
485
      this.job = job;
      this.inputRecordCounter = reporter.getCounter(MAP_INPUT_RECORDS);
      this.fileInputByteCounter = reporter
          .getCounter(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.Counter.BYTES_READ);

      Statistics matchedStats = null;
      if (split instanceof org.apache.hadoop.mapreduce.lib.input.FileSplit) {
        matchedStats = getFsStatistics(((org.apache.hadoop.mapreduce.lib.input.FileSplit) split)
            .getPath(), job);
      }
      fsStats = matchedStats;
View Full Code Here
610
611
612
613
614
615
616
617
618
619
620
    @SuppressWarnings("unchecked")
    NewDirectOutputCollector(org.apache.hadoop.mapreduce.JobContext jobContext,
        JobConf job, TaskUmbilicalProtocol umbilical, TaskReporter reporter)
    throws IOException, ClassNotFoundException, InterruptedException {
      this.reporter = reporter;
      Statistics matchedStats = null;
      if (outputFormat instanceof org.apache.hadoop.mapreduce.lib.output.FileOutputFormat) {
        matchedStats = getFsStatistics(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
            .getOutputPath(jobContext), job);
      }
      fsStats = matchedStats;
View Full Code Here
804
805
806
807
808
809
810
811
812
813
814
      FileSystem fs = FileSystem.get(job);

     
      OutputFormat<K, V> outputFormat = job.getOutputFormat();
     
      Statistics matchedStats = null;
      if (outputFormat instanceof FileOutputFormat) {
        matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job), job);
      }
      fsStats = matchedStats;
      mapOutputRecordCounter = reporter.getCounter(MAP_OUTPUT_RECORDS);
View Full Code Here
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
   *          the path.
   * @return a Statistics instance, or null if none is found for the scheme.
   */
  protected static Statistics getFsStatistics(Path path, Configuration conf)
      throws IOException {
    Statistics matchedStats = null;
    path = path.getFileSystem(conf).makeQualified(path);
    for (Statistics stats : FileSystem.getAllStatistics()) {
      if (stats.getScheme().equals(path.toUri().getScheme())) {
        matchedStats = stats;
        break;
View Full Code Here
TOP

Related Classes of org.apache.hadoop.fs.FileSystem.Statistics

  • org.apache.hadoop.fs.AbstractFileSystem
  • org.apache.hadoop.fs.FCStatisticsBaseTest
  • org.apache.hadoop.mapred.MapTask$DirectMapOutputCollector
  • org.apache.hadoop.mapred.MapTask$NewDirectOutputCollector
  • org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader
  • org.apache.hadoop.mapred.MapTask$TrackedRecordReader
  • org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter
  • org.apache.hadoop.mapred.ReduceTask$OldTrackingRecordWriter
  • org.apache.hadoop.mapred.Task
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.