Package org.apache.hadoop.fs.FileSystem

Examples of org.apache.hadoop.fs.FileSystem.Statistics

169
170
171
172
173
174
175
176
177
178
179
180
    if (scheme == null) {
      throw new IllegalArgumentException("Scheme not defined in the uri: "
          + uri);
    }
    URI baseUri = getBaseUri(uri);
    Statistics result = STATISTICS_TABLE.get(baseUri);
    if (result == null) {
      result = new Statistics(scheme);
      STATISTICS_TABLE.put(baseUri, result);
    }
    return result;
  }
View Full Code Here
210
211
212
213
214
215
216
217
218
219
220
  protected static synchronized Map<URI, Statistics> getAllStatistics() {
    Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>(
        STATISTICS_TABLE.size());
    for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) {
      URI key = pair.getKey();
      Statistics value = pair.getValue();
      Statistics newStatsObj = new Statistics(value);
      statsMap.put(URI.create(key.toString()), newStatsObj);
    }
    return statsMap;
  }
View Full Code Here
185
186
187
188
189
190
191
192
193
194
195
      inputRecordCounter = reporter.getCounter(MAP_INPUT_RECORDS);
      inputByteCounter = reporter.getCounter(MAP_INPUT_BYTES);
      fileInputByteCounter = reporter
          .getCounter(FileInputFormat.Counter.BYTES_READ);

      Statistics matchedStats = null;
      if (split instanceof FileSplit) {
        matchedStats = getFsStatistics(((FileSplit) split).getPath(), job);
      }
      fsStats = matchedStats;
     
View Full Code Here
476
477
478
479
480
481
482
483
484
485
486
      this.job = job;
      this.inputRecordCounter = reporter.getCounter(MAP_INPUT_RECORDS);
      this.fileInputByteCounter = reporter
          .getCounter(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.Counter.BYTES_READ);

      Statistics matchedStats = null;
      if (split instanceof org.apache.hadoop.mapreduce.lib.input.FileSplit) {
        matchedStats = getFsStatistics(((org.apache.hadoop.mapreduce.lib.input.FileSplit) split)
            .getPath(), job);
      }
      fsStats = matchedStats;
View Full Code Here
611
612
613
614
615
616
617
618
619
620
621
    @SuppressWarnings("unchecked")
    NewDirectOutputCollector(org.apache.hadoop.mapreduce.JobContext jobContext,
        JobConf job, TaskUmbilicalProtocol umbilical, TaskReporter reporter)
    throws IOException, ClassNotFoundException, InterruptedException {
      this.reporter = reporter;
      Statistics matchedStats = null;
      if (outputFormat instanceof org.apache.hadoop.mapreduce.lib.output.FileOutputFormat) {
        matchedStats = getFsStatistics(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
            .getOutputPath(jobContext), job);
      }
      fsStats = matchedStats;
View Full Code Here
805
806
807
808
809
810
811
812
813
814
815
      FileSystem fs = FileSystem.get(job);

     
      OutputFormat<K, V> outputFormat = job.getOutputFormat();
     
      Statistics matchedStats = null;
      if (outputFormat instanceof FileOutputFormat) {
        matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job), job);
      }
      fsStats = matchedStats;
      mapOutputRecordCounter = reporter.getCounter(MAP_OUTPUT_RECORDS);
View Full Code Here
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
   *          the path.
   * @return a Statistics instance, or null if none is found for the scheme.
   */
  protected static Statistics getFsStatistics(Path path, Configuration conf)
      throws IOException {
    Statistics matchedStats = null;
    path = path.getFileSystem(conf).makeQualified(path);
    for (Statistics stats : FileSystem.getAllStatistics()) {
      if (stats.getScheme().equals(path.toUri().getScheme())) {
        matchedStats = stats;
        break;
View Full Code Here
435
436
437
438
439
440
441
442
443
444
445
        JobConf job, TaskReporter reporter, String finalName)
        throws IOException {
      this.outputRecordCounter = outputRecordCounter;
      this.fileOutputByteCounter = reporter
          .getCounter(FileOutputFormat.Counter.BYTES_WRITTEN);
      Statistics matchedStats = null;
      if (job.getOutputFormat() instanceof FileOutputFormat) {
        matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job), job);
      }
      fsStats = matchedStats;
View Full Code Here
554
555
556
557
558
559
560
561
562
563
564
        org.apache.hadoop.mapreduce.TaskAttemptContext taskContext)
        throws InterruptedException, IOException {
      this.outputRecordCounter = recordCounter;
      this.fileOutputByteCounter = reporter
          .getCounter(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.Counter.BYTES_WRITTEN);
      Statistics matchedStats = null;
      // TaskAttemptContext taskContext = new TaskAttemptContext(job,
      // getTaskID());
      if (outputFormat instanceof org.apache.hadoop.mapreduce.lib.output.FileOutputFormat) {
        matchedStats = getFsStatistics(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
            .getOutputPath(taskContext), taskContext.getConfiguration());
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
172
    if (scheme == null) {
      throw new IllegalArgumentException("Scheme not defined in the uri: "
          + uri);
    }
    URI baseUri = getBaseUri(uri);
    Statistics result = STATISTICS_TABLE.get(baseUri);
    if (result == null) {
      result = new Statistics(scheme);
      STATISTICS_TABLE.put(baseUri, result);
    }
    return result;
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.fs.FileSystem.Statistics

  • org.apache.hadoop.fs.AbstractFileSystem
  • org.apache.hadoop.fs.FCStatisticsBaseTest
  • org.apache.hadoop.mapred.MapTask$DirectMapOutputCollector
  • org.apache.hadoop.mapred.MapTask$NewDirectOutputCollector
  • org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader
  • org.apache.hadoop.mapred.MapTask$TrackedRecordReader
  • org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter
  • org.apache.hadoop.mapred.ReduceTask$OldTrackingRecordWriter
  • org.apache.hadoop.mapred.Task
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.