Package org.apache.hadoop.raid.Statistics

Examples of org.apache.hadoop.raid.Statistics.Counters

439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
  private void populateMetrics(
      Map<String, Statistics> codeToRaidStatistics) {
    RaidNodeMetrics metrics = RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID);
    for (Codec codec : Codec.getCodecs()) {
      String code = codec.id;
      Counters counters = codeToRaidStatistics.get(code).getParityCounters();
      metrics.parityFiles.get(code).set(counters.getNumFiles());
      metrics.parityBlocks.get(code).set(counters.getNumBlocks());
      metrics.parityBytes.get(code).set(counters.getNumBytes());
      metrics.parityLogical.get(code).set(counters.getNumLogical());
      for (RaidState state : RaidState.values()) {
        counters = codeToRaidStatistics.get(code).getSourceCounters(state);
        metrics.sourceFiles.get(code).get(state).set(counters.getNumFiles());
        metrics.sourceBlocks.get(code).get(state).set(counters.getNumBlocks());
        metrics.sourceBytes.get(code).get(state).set(counters.getNumBytes());
        metrics.sourceLogical.get(code).get(state).set(counters.getNumLogical());
      }
    }
    double repl = getEffectiveReplication();
    populateSaving(metrics);
    if (repl != -1) {
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
188
189
190
    collector.collect(allPolicies);
    Statistics st = collector.getRaidStatistics(codecId);
   
    LOG.info("Dir Statistics collected " + st);
    LOG.info("Dir Statistics html:\n " + st.htmlTable());
    Counters raided = st.getSourceCounters(RaidState.RAIDED);
    Counters tooSmall = st.getSourceCounters(RaidState.NOT_RAIDED_TOO_SMALL);
    Counters tooNew = st.getSourceCounters(RaidState.NOT_RAIDED_TOO_NEW);
    Counters notRaided = st.getSourceCounters(RaidState.NOT_RAIDED_BUT_SHOULD);
   
    assertCounters(raided, 1, 5, 15, 15 * 10240L, 15 * 10240L);
    assertCounters(tooSmall, 1, 2, 2, 3 * 10240L, 2 * 10240L);
    assertCounters(tooNew, 1, 2, 9, 27 * 10240L, 9 * 10240L);
    assertCounters(notRaided, 1, 3, 6, 8 * 10240L, 6 * 10240L);
 
View Full Code Here
219
220
221
222
223
224
225
226
227
228
229
230
231
232
    List<PolicyInfo> allPolicies = Arrays.asList(info, infoTooNew);
    collector.collect(allPolicies);
    Statistics st = collector.getRaidStatistics(codecId);
    LOG.info("Statistics collected " + st);
    LOG.info("Statistics html:\n " + st.htmlTable());
    Counters raided = st.getSourceCounters(RaidState.RAIDED);
    Counters tooSmall = st.getSourceCounters(RaidState.NOT_RAIDED_TOO_SMALL);
    Counters tooNew = st.getSourceCounters(RaidState.NOT_RAIDED_TOO_NEW);
    Counters notRaided = st.getSourceCounters(RaidState.NOT_RAIDED_BUT_SHOULD);
    assertCounters(raided, 2, 7, 7 * 1024L, 7 * 1024L);
    assertCounters(tooSmall, 2, 3, 5 * 1024L, 3 * 1024L);
    assertCounters(tooNew, 2, 9, 27 * 1024L, 9 * 1024L);
    assertCounters(notRaided, 1, 5, 15 * 1024L, 5 * 1024L);
    fs.delete(new Path("/a"), true);
View Full Code Here
283
284
285
286
287
288
289
290
291
292
293
294
295
        new StatisticsCollector(fakeRaidNode, fakeConfigManager, conf);
    List<PolicyInfo> allPolicies = Arrays.asList(info, infoLongPrefix);
    collector.collect(allPolicies);
    Statistics xorSt = collector.getRaidStatistics("xor");
    Statistics rsSt = collector.getRaidStatistics("rs");
    Counters xorShouldRaid = xorSt.getSourceCounters(RaidState.NOT_RAIDED_BUT_SHOULD);
    Counters rsShouldRaid = rsSt.getSourceCounters(RaidState.NOT_RAIDED_BUT_SHOULD);
    Counters rsOther = rsSt.getSourceCounters(RaidState.NOT_RAIDED_OTHER_POLICY);
    assertCounters(xorShouldRaid, 1, 5, 15 * 1024L, 5 * 1024L);
    assertCounters(rsShouldRaid, 1, 4, 12 * 1024L, 4 * 1024L);
    assertCounters(rsOther, 1, 5, 15 * 1024L, 5 * 1024L);
    fs.delete(new Path("/a"), true);
  }
View Full Code Here
313
314
315
316
317
318
319
320
321
322
323
324
325
        new StatisticsCollector(fakeRaidNode, fakeConfigManager, conf);
    List<PolicyInfo> allPolicies = Arrays.asList(info, infoLongPrefix);
    collector.collect(allPolicies);
    Statistics xorSt = collector.getRaidStatistics("xor");
    Statistics rsSt = collector.getRaidStatistics("rs");
    Counters xorShouldRaid = xorSt.getSourceCounters(RaidState.NOT_RAIDED_BUT_SHOULD);
    Counters xorOther = xorSt.getSourceCounters(RaidState.NOT_RAIDED_OTHER_POLICY);
    Counters rsShouldRaid = rsSt.getSourceCounters(RaidState.NOT_RAIDED_BUT_SHOULD);
    assertCounters(xorShouldRaid, 1, 4, 12 * 1024L, 4 * 1024L);
    assertCounters(xorOther, 1, 5, 15 * 1024L, 5 * 1024L);
    assertCounters(rsShouldRaid, 1, 5, 15 * 1024L, 5 * 1024L);
    fs.delete(new Path("/a"), true);
  }
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
                    fs.getConf(), fs, src.getPath());
    if (lfs == null) {
      return false;
    }
    RaidState state = checker.check(info, src, now, true, lfs);
    Counters counters = stateToSourceCounters.get(state);
    counters.inc(lfs);
    if (state == RaidState.RAIDED) {
      long paritySize = computeParitySize(lfs, targetReplication);
      estimatedParitySize += paritySize;
      estimatedDoneParitySize += paritySize;
      estimatedDoneSourceSize += DirectoryStripeReader.getDirPhysicalSize(lfs);
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
                    fs.getConf(), fs, src.getPath());
    if (lfs == null) {
      return false;
    }
    RaidState state = checker.check(info, src, now, false, lfs);
    Counters counters = stateToSourceCounters.get(state);
    counters.inc(lfs);
    if (state == RaidState.RAIDED) {
      long paritySize = computeParitySize(lfs, targetReplication);
      estimatedParitySize += paritySize;
      estimatedDoneParitySize += paritySize;
      estimatedDoneSourceSize += DirectoryStripeReader.getDirPhysicalSize(lfs);
View Full Code Here
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
  private void populateMetrics(
      Map<String, Statistics> codeToRaidStatistics) {
    RaidNodeMetrics metrics = RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID);
    for (Codec codec : Codec.getCodecs()) {
      String code = codec.id;
      Counters counters = codeToRaidStatistics.get(code).getParityCounters();
      metrics.parityFiles.get(code).set(counters.getNumFiles());
      metrics.parityBlocks.get(code).set(counters.getNumBlocks());
      metrics.parityBytes.get(code).set(counters.getNumBytes());
      metrics.parityLogical.get(code).set(counters.getNumLogical());
      for (RaidState state : RaidState.values()) {
        counters = codeToRaidStatistics.get(code).getSourceCounters(state);
        metrics.sourceFiles.get(code).get(state).set(counters.getNumFiles());
        metrics.sourceBlocks.get(code).get(state).set(counters.getNumBlocks());
        metrics.sourceBytes.get(code).get(state).set(counters.getNumBytes());
        metrics.sourceLogical.get(code).get(state).set(counters.getNumLogical());
      }
    }
    double repl = getEffectiveReplication();
    populateSaving(metrics);
    if (repl != -1) {
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
      StatisticsCollector collector =
          new StatisticsCollector(fakeRaidNode, fakeConfigManager, conf);
      collector.collect(allPolicies);
      for (Codec codec : Codec.getCodecs()) {
        Statistics st = collector.getRaidStatistics(codec.id);
        Counters raided = st.getSourceCounters(RaidState.RAIDED);
        if (codec.isDirRaid) {
          assertCounters(raided, 1, 2, 6, 6 * 512L, 6 * 512L);
        } else {
          assertCounters(raided, 2, 6, 6 * 512L, 6 * 512L);
        }
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
    List<PolicyInfo> allPolicies = Arrays.asList(lpInfo, hpInfo);
    collector.collect(allPolicies);
    Statistics lpSt = collector.getRaidStatistics(lpId);
    LOG.info("Statistics collected " + lpSt);
    LOG.info("Statistics html:\n " + lpSt.htmlTable());
    Counters raided = lpSt.getSourceCounters(RaidState.RAIDED);
    Counters tooSmall = lpSt.getSourceCounters(RaidState.NOT_RAIDED_TOO_SMALL);
    Counters tooNew = lpSt.getSourceCounters(RaidState.NOT_RAIDED_TOO_NEW);
    Counters otherPolicy = lpSt.getSourceCounters(RaidState.NOT_RAIDED_OTHER_POLICY);
    Counters notRaided = lpSt.getSourceCounters(RaidState.NOT_RAIDED_BUT_SHOULD);
    assertCounters(raided, lpCodec.isDirRaid? 2 : 0, 2, 7, 7 * 1024L, 7 * 1024L);
    assertCounters(tooSmall, 0, 0, 0, 0L, 0L);
    assertCounters(tooNew, 0, 0, 0, 0L, 0L);
    // the NOT_RAIDED files could be raided by high priority policy
    assertCounters(otherPolicy, lpCodec.isDirRaid? 1 : 0, 1, 5, 15 * 1024L,
 
View Full Code Here
TOP

Related Classes of org.apache.hadoop.raid.Statistics.Counters

  • org.apache.hadoop.raid.DirectoryStatistics
  • org.apache.hadoop.raid.StatisticsCollector
  • org.apache.hadoop.raid.TestStatisticsCollector
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.