Package com.cloudera.flume.reporter.aggregator

Examples of com.cloudera.flume.reporter.aggregator.CounterSink


    ScheduledHistoryReporter<CounterSink> sched = new ScheduledHistoryReporter<CounterSink>(
        "test", longwait, historyLen, new DumbTagger()) {

      @Override
      public CounterSink newSink(Tagger format) throws IOException {
        CounterSink count = new CounterSink("count") {
          public void append(Event e) throws IOException, InterruptedException {
            super.append(e);

            // just add a printf to the counts.
            System.out.println(e.getTimestamp() + " " + e);
View Full Code Here


    ScheduledHistoryReporter<CounterSink> hist = new ScheduledHistoryReporter<CounterSink>(
        "test", longwait, historyLen, new DumbTagger()) {

      @Override
      public CounterSink newSink(Tagger format) throws IOException {
        CounterSink count = new CounterSink("count") {
          public void append(Event e) throws IOException, InterruptedException {
            super.append(e);
            System.out.println(e); // just add a printf to the counts.
          }
        };
View Full Code Here

    mem.open();
    EventUtil.dumpAll(txt, mem);

    b.mark("disk_loaded");

    CounterSink snk = new CounterSink("counter");
    EventUtil.dumpAll(mem, snk);
    b.mark(snk.getName() + " done", snk.getCount());

    b.done();
  }
View Full Code Here

    snk.close(); // this should block until recovery complete.

    // agent checks for ack registrations.
    BenchmarkHarness.node.getAckChecker().checkAcks();

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("count");
    // 1032 in file + 5 from silly driver
    assertEquals(1037, cnt.getCount());

    // check to make sure wal file is gone
    assertFalse(new File(new File(tmp, "import"), acked.getName()).exists());
    assertFalse(new File(new File(tmp, "writing"), acked.getName()).exists());
    assertFalse(new File(new File(tmp, "logged"), acked.getName()).exists());
View Full Code Here

    snk.close(); // this should block until recovery complete.

    // agent checks for ack registrations.
    BenchmarkHarness.node.getAckChecker().checkAcks();

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("count");
    // 1032 in file + 5 from silly driverx
    assertEquals(1037, cnt.getCount());

    // check to make sure wal file is gone
    assertFalse(new File(new File(tmp, "import"), acked.getName()).exists());
    assertFalse(new File(new File(tmp, "writing"), acked.getName()).exists());
    assertFalse(new File(new File(tmp, "logged"), acked.getName()).exists());
View Full Code Here

    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close(); // this should block until recovery complete.

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("count");
    // 461 in file before truncated + 5 from silly driver
    assertEquals(466, cnt.getCount());

    // need to trigger ack checks..

    // BenchmarkHarness.mock.ackman.;
View Full Code Here

  @Test
  public void testFailOverSink() throws IOException, InterruptedException {
    MockClock mock = new MockClock(0);
    Clock.setClock(mock);

    CounterSink primary = new CounterSink("primary");
    CounterSink secondary = new CounterSink("backup");
    ExceptionTwiddleDecorator<CounterSink> twiddle = new ExceptionTwiddleDecorator<CounterSink>(
        primary);
    BackOffFailOverSink failsink = new BackOffFailOverSink(twiddle, secondary,
        100, 10000); // 100 ms
    // initial
    // backoff,
    // 10000ms max
    // backoff
    failsink.open();

    Event e = new EventImpl("event".getBytes());
    // two successful appends to primary.
    failsink.append(e);
    failsink.append(e);
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(2, primary.getCount());
    Assert.assertEquals(0, secondary.getCount());

    mock.forward(100);
    twiddle.setAppendOk(false); // go to fail over.
    failsink.append(e); // primary fails and automatically go to 2ndary
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(1, failsink.getFails()); // one attempt on primary
    // failed.
    Assert.assertEquals(2, primary.getCount()); // same as before,
    Assert.assertEquals(1, secondary.getCount()); // message went to the
    // secondary

    mock.forward(50);
    failsink.append(e); // skip primary and just go to 2ndary
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(1, failsink.getFails()); // still only one attempt on
    // primary
    Assert.assertEquals(2, primary.getCount()); // same as before,
    Assert.assertEquals(2, secondary.getCount()); // message went to the
    // secondary

    mock.forward(50);
    failsink.append(e); // after this fails backoff is now 200
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(2, failsink.getFails()); // try primary
    Assert.assertEquals(0, primary.getCount()); // resets because primary
                                                // restarted (and still fails)
    Assert.assertEquals(3, secondary.getCount()); // but failover to secondary

    mock.forward(200);
    failsink.append(e); // should go to 2ndary, after this fails backoff is now
    // 400
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(3, failsink.getFails());
    Assert.assertEquals(0, primary.getCount());
    Assert.assertEquals(4, secondary.getCount());

    twiddle.setAppendOk(true);
    failsink.append(e); // even through primary is ok, we are backing off
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(3, failsink.getFails());
    Assert.assertEquals(0, primary.getCount());
    Assert.assertEquals(5, secondary.getCount());

    mock.forward(400);
    failsink.append(e); // now that the backoff has expired, we retry the
    // primary and succeed
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(3, failsink.getFails());
    Assert.assertEquals(1, primary.getCount());
    Assert.assertEquals(5, secondary.getCount());

    // this should succeed, with the counts being equal in primary and
    // secondary.
    failsink.close();
  }
View Full Code Here

    Clock.sleep(1000); // need other thread to make progress
    // three blocks should have successfully made it.
    assertEquals(3, master.getAckMan().getPending().size()); // (g0, g1, g3)

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable(rpt);
    LOG.info(" Ack begin dropped, but all true message went through. 20 == "
        + ctr.getCount() + " sent");
    // We dropped a event message, so we should be down one.
    assertEquals(20, ctr.getCount());

    // check master state
    Clock.sleep(1000); // somehow force other thread to go.
    master.getAckMan().dumpLog();
    Set<String> pending = master.getAckMan().getPending();
    assertEquals(3, pending.size());

    // Phase II (5 more messeage, retry succeeds).
    node.getAckChecker().checkAcks();
    node.getAckChecker().forceRetry();
    Clock.sleep(1000);
    master.getAckMan().dumpLog();
    pending = master.getAckMan().getPending();
    LOG.info("Number of pending acks (retry is success)" + pending.size());
    assertEquals(1, pending.size());
    LOG.info("Event count 25 (one message missing/lost) ==" + ctr.getCount());
    assertEquals(25, ctr.getCount());

    node.getAckChecker().checkAcks();
    Clock.sleep(1000); // somehow force other thread to go.

    LOG.info("Nothing new happened " + " 25 == " + ctr.getCount() + " sent");
    // This retry succeeds sending 5 more messages
    assertEquals(25, ctr.getCount());
    pending = master.getAckMan().getPending();
    assertEquals(0, pending.size());

    wal.close();
  }
View Full Code Here

        wal.rotate();
      }
    }

    Clock.sleep(1000); // somehow force other thread to go.
    CounterSink ctr = (CounterSink) ReportManager.get().getReportable(rpt);
    LOG.info(" All should make it through here. " + count + " =="
        + ctr.getCount() + " sent");
    assertEquals(20, ctr.getCount());

    // check master state
    Clock.sleep(1000);
    master.getAckMan().dumpLog();
View Full Code Here

      LOG.info(dfos[i].getMetrics());
    }

    for (int i = 0; i < threads; i++) {

      CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
          "report." + i);
      LOG.info(i + " expected " + (events + i) + " and got " + cnt.getCount());
      success &= ((events + i) == cnt.getCount());
      assertEquals(events + i, cnt.getCount());

    }
    assertTrue("Counts did not line up", success);

    BenchmarkHarness.cleanupLocalWriteDir();
View Full Code Here

TOP

Related Classes of com.cloudera.flume.reporter.aggregator.CounterSink

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.