Package sizzle.aggregators

Source Code of sizzle.aggregators.TestMrcounterAggregator

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
package sizzle.aggregators;

import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.CounterGroup;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.junit.Assert;
import org.junit.Test;

import sizzle.io.EmitKey;
import sizzle.io.EmitValue;

public class TestMrcounterAggregator {
  @Test
  public void testMrcounterAggregatorCombineUnnamed() {
    final List<EmitValue> values = new ArrayList<EmitValue>();
    values.add(new EmitValue(1));
    values.add(new EmitValue(2));

    final ReduceDriver<EmitKey, EmitValue, EmitKey, EmitValue> reduceDriver = new ReduceDriver<EmitKey, EmitValue, EmitKey, EmitValue>(
        new MrcounterSizzleCombiner());
    reduceDriver.setInput(new EmitKey("test"), values);
    reduceDriver.runTest();
    final Counters counters = reduceDriver.getCounters();
    final CounterGroup group = counters.getGroup("Sizzle Counters");

    Assert.assertEquals("counter value is wrong", 3, group.findCounter("Unnamed counter").getValue());
  }

  @Test
  public void testMrcounterAggregatorCombineNamed() {
    final List<EmitValue> values = new ArrayList<EmitValue>();
    values.add(new EmitValue(1));
    values.add(new EmitValue(2));

    final ReduceDriver<EmitKey, EmitValue, EmitKey, EmitValue> reduceDriver = new ReduceDriver<EmitKey, EmitValue, EmitKey, EmitValue>(
        new MrcounterSizzleCombiner());
    reduceDriver.setInput(new EmitKey("[three]", "test"), values);
    reduceDriver.runTest();
    final Counters counters = reduceDriver.getCounters();
    final CounterGroup group = counters.getGroup("Sizzle Counters");

    Assert.assertEquals("counter value is wrong", 3, group.findCounter("three").getValue());
  }

  @Test
  public void testMrcounterAggregatorCombineGroup() {
    final List<EmitValue> values = new ArrayList<EmitValue>();
    values.add(new EmitValue(1));
    values.add(new EmitValue(2));

    final ReduceDriver<EmitKey, EmitValue, EmitKey, EmitValue> reduceDriver = new ReduceDriver<EmitKey, EmitValue, EmitKey, EmitValue>(
        new MrcounterSizzleCombiner());
    reduceDriver.setInput(new EmitKey("[Job Statistics][Some counter]", "test"), values);
    reduceDriver.runTest();
    final Counters counters = reduceDriver.getCounters();
    final CounterGroup group = counters.getGroup("Job Statistics");

    Assert.assertEquals("counter value is wrong", 3, group.findCounter("Some counter").getValue());
  }

  @Test
  public void testMrcounterAggregatorCombineMultilevel() {
    final List<EmitValue> values = new ArrayList<EmitValue>();
    values.add(new EmitValue(1));
    values.add(new EmitValue(2));

    final ReduceDriver<EmitKey, EmitValue, EmitKey, EmitValue> reduceDriver = new ReduceDriver<EmitKey, EmitValue, EmitKey, EmitValue>(
        new MrcounterSizzleCombiner());
    reduceDriver.setInput(new EmitKey("[Job Statistics][Some][counter]", "test"), values);
    reduceDriver.runTest();
    final Counters counters = reduceDriver.getCounters();
    final CounterGroup group = counters.getGroup("Job Statistics");

    Assert.assertEquals("counter value is wrong", 3, group.findCounter("Somecounter").getValue());
  }

  @Test
  public void testMrcounterAggregatorReduce() {
    final List<EmitValue> values = new ArrayList<EmitValue>();
    values.add(new EmitValue("one", 1));
    values.add(new EmitValue("two", 2));
    values.add(new EmitValue("three", 3));
    values.add(new EmitValue("four", 4));

    final ReduceDriver<EmitKey, EmitValue, Text, NullWritable> reduceDriver = new ReduceDriver<EmitKey, EmitValue, Text, NullWritable>(
        new MrcounterSizzleReducer());
    reduceDriver.setInput(new EmitKey("test"), values);
    reduceDriver.runTest();
    final Counters counters = reduceDriver.getCounters();
    final CounterGroup group = counters.getGroup("test[]");

    Assert.assertEquals("counter value is wrong", 0, group.findCounter("one").getValue());
    Assert.assertEquals("counter value is wrong", 0, group.findCounter("two").getValue());
    Assert.assertEquals("counter value is wrong", 0, group.findCounter("three").getValue());
    Assert.assertEquals("counter value is wrong", 0, group.findCounter("four").getValue());
  }
}

class MrcounterSizzleCombiner extends sizzle.runtime.SizzleCombiner {
  public MrcounterSizzleCombiner() {
    super();

    this.tables.put("test", new Table(new sizzle.aggregators.MrcounterAggregator()));
  }
}

class MrcounterSizzleReducer extends sizzle.runtime.SizzleReducer {
  public MrcounterSizzleReducer() {
    super();

    this.tables.put("test", new Table(new sizzle.aggregators.MrcounterAggregator()));
  }
}
TOP

Related Classes of sizzle.aggregators.TestMrcounterAggregator

  • org.apache.hadoop.mapreduce.CounterGroup
  • org.apache.hadoop.mapreduce.Counters
  • sizzle.io.EmitKey
  • sizzle.io.EmitValue
    • TOP
    Copyright © 2018 www.massapi.com. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.