Examples of CounterSink

Examples of com.cloudera.flume.reporter.aggregator.CounterSink

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
   * Nothing is done until the trigger is toggled "total" times. We trigger in a
   * separate thread.
   */
  @Test
  public void testLatchedDeco() throws IOException, InterruptedException {
    final CounterSink c = new CounterSink("count");
    final FanOutSink<EventSink> s = new FanOutSink<EventSink>();
    s.add(c);
    s.add(new ConsoleEventSink());
    final int total = 10;
    final LatchedDecorator<EventSink> l = new LatchedDecorator<EventSink>(s, 0,
        total);
    l.open();
    Thread t = new Thread() {
      public void run() {
        try {
          long count;
          int i = 0;
          while ((count = c.getCount()) < total) {
            l.trigger();
            i++;
            sleep(10);

            System.out.println("triggered " + i + " times, count is " + count);
          }
        } catch (InterruptedException e) {
          e.printStackTrace();
        }
      }
    };
    t.start();

    for (int i = 0; i < total; i++) {
      Event e = new EventImpl(("message " + i).getBytes());
      l.append(e);
    }
    t.join();
    long count = c.getCount();
    System.out.println("trigger thread joined, count is now: " + count);
    Assert.assertEquals(total, count);
  }
View Full Code Here

Examples of com.cloudera.flume.reporter.aggregator.CounterSink

39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
  @Test
  public void testMultiplier() throws IOException, InterruptedException {
    final int repeat = 9;
    final int msgs = 10;
    CounterSink cnt = new CounterSink("count");
    // send to counter 10x.
    EventSinkDecorator<CounterSink> s = new MultiplierDecorator<CounterSink>(
        cnt, repeat);
    s.open();
    for (int i = 0; i < msgs; i++) {
      Event e = new EventImpl(("" + i).getBytes());
      s.append(e);
    }

    Assert.assertEquals(msgs * repeat, cnt.getCount());
  }
View Full Code Here

Examples of com.cloudera.flume.reporter.aggregator.CounterSink

72
73
74
75
76
77
78
79
    for (int i = 0; i < msgs; i++) {
      Event e = new EventImpl(("" + i).getBytes());
      s.append(e);
    }

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("count");
    Assert.assertEquals(msgs * repeat, cnt.getCount());
  }
View Full Code Here

Examples of com.cloudera.flume.reporter.aggregator.CounterSink

99
100
101
102
103
104
105
106
107
108
      Event e = new EventImpl(("" + i).getBytes());
      s.append(e);
    }
    s.close();

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("count");
    // +3 -> start, first, stop
    Assert.assertEquals(msgs * repeat + 3, cnt.getCount());

  }
View Full Code Here

Examples of com.cloudera.flume.reporter.aggregator.CounterSink

44
45
46
47
48
49
50
    super(name, maxAge, FlumeConfiguration.get().getHistoryMaxLength(), t);
  }

  @Override
  public CounterSink newSink(Tagger format) throws IOException {
    return new CounterSink(format.newTag());
  }
View Full Code Here

Examples of com.cloudera.flume.reporter.aggregator.CounterSink

77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
    FlumeConfiguration conf = FlumeConfiguration.get();
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort() + 1);
    tes.open();

    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (Exception e) {
        }
      }
    };
    t.start(); // drain the sink.

    // mem -> AvroEventSink
    AvroEventSink snk = new AvroEventSink("0.0.0.0",
        conf.getCollectorPort() + 1);
    snk.open();
    EventUtil.dumpAll(mem, snk);
    mem.close();
    snk.close();

    // a little delay to drain events at AvroEventSource queue
    try {
      Thread.sleep(1000);
      t.interrupt();
    } catch (InterruptedException e) {
    }
    tes.close();
    assertEquals(25, cnt.getCount());
    ReportEvent rpt = tes.getMetrics();
    /*
     * The check on BytesIn is different than one on TestThriftSinks tests. This
     * is because currently in the AvroSink version, BytesIn is equal to the
     * number of Bytes of the Event.body shipped.
View Full Code Here

Examples of com.cloudera.flume.reporter.aggregator.CounterSink

140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
    final FlumeConfiguration conf = FlumeConfiguration.get();
    // this is a slight tweak to avoid port conflicts
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort() + 1);
    tes.open();

    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (Exception e) {
        }
      }
    };
    t.start(); // drain the sink.

    // fork off threads threads and have them start all the same time.
    final CountDownLatch sendStarted = new CountDownLatch(threads);
    final CountDownLatch sendDone = new CountDownLatch(threads);
    final AtomicLong sendByteSum = new AtomicLong(0);
    for (int i = 0; i < threads; i++) {
      final int id = i;
      Thread th = new Thread() {
        public void run() {
          try {
            EventSource txt = new NoNlASCIISynthSource(25, 100);
            txt.open();
            MemorySinkSource mem = new MemorySinkSource();
            mem.open();
            EventUtil.dumpAll(txt, mem);
            txt.close();

            // mem -> AvroEventSink
            AvroEventSink snk = new AvroEventSink("0.0.0.0", conf
                .getCollectorPort() + 1);
            snk.open();

            sendStarted.countDown();
            sendStarted.await();
            EventUtil.dumpAll(mem, snk);
            mem.close();
            snk.close();

            sendByteSum.addAndGet(snk.sentBytes.get());
            LOG.info("sink " + id + " sent " + snk.sentBytes + " bytes");
            sendDone.countDown();

          } catch (IOException e) {
            e.printStackTrace();
          } catch (InterruptedException e) {
            e.printStackTrace();
          }
        }
      };
      th.start();
    }

    // wait for senders to send all
    sendDone.await();

    // a little delay get data to the receiving side.
    Thread.sleep(1000);

    tes.close();
    assertEquals(25 * threads, cnt.getCount());
    ReportEvent rpt = tes.getMetrics();
    assertEquals(2500 * threads, sendByteSum.get());
    assertEquals(2500 * threads, rpt.getLongMetric(AvroEventSource.A_BYTES_IN)
        .longValue());
    assertEquals(25 * threads, rpt.getLongMetric(AvroEventSource.A_DEQUEUED)
View Full Code Here

Examples of com.cloudera.flume.reporter.aggregator.CounterSink

58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
  @Test
  public void testCheckCheckerConsole() throws IOException,
      InterruptedException {

    int msgs = 5;
    CounterSink cnt = new CounterSink("count");
    AckChecksumInjector<EventSink> aci = new AckChecksumInjector<EventSink>(
        new FanOutSink<EventSink>(cnt, new ConsoleEventSink()));

    aci.open();
    for (int i = 0; i < msgs; i++) {
      Event e = new EventImpl(("this is a test " + i).getBytes());
      aci.append(e);
    }
    aci.close();
    assertEquals(5 + 2, cnt.getCount()); // should have gotten 5 messages + 1
    // start ack and 1 stop ack.
  }
View Full Code Here

Examples of com.cloudera.flume.reporter.aggregator.CounterSink

149
150
151
152
153
154
155
156
157
158
159
160
    // close and flush
    es.close();
    node.getAckChecker().checkAcks();

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable(rpt);
    assertEquals(count, ctr.getCount());

    // check master state
    mock.ackman.dumpLog();
    Set<String> pending2 = mock.ackman.getPending();
    assertEquals(0, pending2.size());
View Full Code Here

Examples of com.cloudera.flume.reporter.aggregator.CounterSink

69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
    sink.close();
    b.mark("sample dump done");

    e = null;
    mem.open();
    CounterSink sink2 = new CounterSink("counter");

    while ((e = mem.next()) != null) {
      Event e2 = syslog.extract(new String(e.getBody()), 2009);
      sink2.append(e2);
    }
    sink2.close();
    b.mark("count done", sink2.getCount());

    b.done();
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.