Package org.apache.hadoop.fs.FileSystem

Examples of org.apache.hadoop.fs.FileSystem.Statistics

810
811
812
813
814
815
816
817
818
819
820
      FileSystem fs = FileSystem.get(job);

     
      OutputFormat<K, V> outputFormat = job.getOutputFormat();
     
      Statistics matchedStats = null;
      if (outputFormat instanceof FileOutputFormat) {
        matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job), job);
      }
      fsStats = matchedStats;
      mapOutputRecordCounter = reporter.getCounter(MAP_OUTPUT_RECORDS);
View Full Code Here
152
153
154
155
156
157
158
159
160
161
162
163
   * @param cls the class to lookup
   * @return a statistics object
   */
  protected static synchronized Statistics getStatistics(String scheme,
      Class<? extends AbstractFileSystem> cls) {
    Statistics result = STATISTICS_TABLE.get(cls);
    if (result == null) {
      result = new Statistics(scheme);
      STATISTICS_TABLE.put(cls, result);
    }
    return result;
  }
View Full Code Here
168
169
170
171
172
173
174
175
176
177
178
179
    if (scheme == null) {
      throw new IllegalArgumentException("Scheme not defined in the uri: "
          + uri);
    }
    URI baseUri = getBaseUri(uri);
    Statistics result = STATISTICS_TABLE.get(baseUri);
    if (result == null) {
      result = new Statistics(scheme);
      STATISTICS_TABLE.put(baseUri, result);
    }
    return result;
  }
View Full Code Here
209
210
211
212
213
214
215
216
217
218
219
  protected static synchronized Map<URI, Statistics> getAllStatistics() {
    Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>(
        STATISTICS_TABLE.size());
    for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) {
      URI key = pair.getKey();
      Statistics value = pair.getValue();
      Statistics newStatsObj = new Statistics(value);
      statsMap.put(URI.create(key.toString()), newStatsObj);
    }
    return statsMap;
  }
View Full Code Here
435
436
437
438
439
440
441
442
443
444
445
        JobConf job, TaskReporter reporter, String finalName)
        throws IOException {
      this.outputRecordCounter = outputRecordCounter;
      this.fileOutputByteCounter = reporter
          .getCounter(FileOutputFormat.Counter.BYTES_WRITTEN);
      Statistics matchedStats = null;
      if (job.getOutputFormat() instanceof FileOutputFormat) {
        matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job), job);
      }
      fsStats = matchedStats;
View Full Code Here
554
555
556
557
558
559
560
561
562
563
564
        org.apache.hadoop.mapreduce.TaskAttemptContext taskContext)
        throws InterruptedException, IOException {
      this.outputRecordCounter = recordCounter;
      this.fileOutputByteCounter = reporter
          .getCounter(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.Counter.BYTES_WRITTEN);
      Statistics matchedStats = null;
      // TaskAttemptContext taskContext = new TaskAttemptContext(job,
      // getTaskID());
      if (outputFormat instanceof org.apache.hadoop.mapreduce.lib.output.FileOutputFormat) {
        matchedStats = getFsStatistics(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
            .getOutputPath(taskContext), taskContext.getConfiguration());
View Full Code Here
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
   *          the path.
   * @return a Statistics instance, or null if none is found for the scheme.
   */
  protected static Statistics getFsStatistics(Path path, Configuration conf)
      throws IOException {
    Statistics matchedStats = null;
    path = path.getFileSystem(conf).makeQualified(path);
    for (Statistics stats : FileSystem.getAllStatistics()) {
      if (stats.getScheme().equals(path.toUri().getScheme())) {
        matchedStats = stats;
        break;
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
174
    if (scheme == null) {
      throw new IllegalArgumentException("Scheme not defined in the uri: "
          + uri);
    }
    URI baseUri = getBaseUri(uri);
    Statistics result = STATISTICS_TABLE.get(baseUri);
    if (result == null) {
      result = new Statistics(scheme);
      STATISTICS_TABLE.put(baseUri, result);
    }
    return result;
  }
View Full Code Here
204
205
206
207
208
209
210
211
212
213
214
  protected static synchronized Map<URI, Statistics> getAllStatistics() {
    Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>(
        STATISTICS_TABLE.size());
    for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) {
      URI key = pair.getKey();
      Statistics value = pair.getValue();
      Statistics newStatsObj = new Statistics(value);
      statsMap.put(URI.create(key.toString()), newStatsObj);
    }
    return statsMap;
  }
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
  protected static FileContext fc = null;
 
  @Test
  public void testStatistics() throws IOException, URISyntaxException {
    URI fsUri = getFsUri();
    Statistics stats = FileContext.getStatistics(fsUri);
    Assert.assertEquals(0, stats.getBytesRead());
    Path filePath = getTestRootPath(fc, "file1");
    createFile(fc, filePath, numBlocks, blockSize);

    Assert.assertEquals(0, stats.getBytesRead());
    verifyWrittenBytes(stats);
    FSDataInputStream fstr = fc.open(filePath);
    byte[] buf = new byte[blockSize];
    int bytesRead = fstr.read(buf, 0, blockSize);
    Assert.assertEquals(blockSize, bytesRead);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.fs.FileSystem.Statistics

  • org.apache.hadoop.fs.AbstractFileSystem
  • org.apache.hadoop.fs.FCStatisticsBaseTest
  • org.apache.hadoop.mapred.MapTask$DirectMapOutputCollector
  • org.apache.hadoop.mapred.MapTask$NewDirectOutputCollector
  • org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader
  • org.apache.hadoop.mapred.MapTask$TrackedRecordReader
  • org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter
  • org.apache.hadoop.mapred.ReduceTask$OldTrackingRecordWriter
  • org.apache.hadoop.mapred.Task
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.