Examples of com.cloudera.flume.reporter.aggregator.AccumulatorSink

Package com.cloudera.flume.reporter.aggregator

Examples of com.cloudera.flume.reporter.aggregator.AccumulatorSink

com.cloudera.flume.reporter.aggregator.AccumulatorSink
This just counts the number of entries appended. Unlike the CounterSink, this sink does not reset the count when opened/closed

    // Windows semantics for rm different from unix
    Assume.assumeTrue(!OSUtils.isWindowsOS());


    File tmpdir = FileUtil.mktempdir();
    TailDirSource src = new TailDirSource(tmpdir, ".*");
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    DirectDriver drv = new DirectDriver(src, cnt);


    drv.start();


    // This blows up when there are ~2000 files
    genFiles(tmpdir, "foo", 200, 10);
    Clock.sleep(1000);
    assertEquals(2000, cnt.getCount());


    ReportEvent rpt1 = src.getMetrics();
    assertEquals(Long.valueOf(200),
        rpt1.getLongMetric(TailDirSource.A_FILESPRESENT));


    FileUtil.rmr(tmpdir); // This fails in windows because taildir keeps file
    // open
    tmpdir.mkdirs();
    Clock.sleep(1000);
    assertEquals(2000, cnt.getCount());


    ReportEvent rpt = src.getMetrics();
    assertEquals(rpt.getLongMetric(TailDirSource.A_FILESADDED),
        rpt.getLongMetric(TailDirSource.A_FILESDELETED));
    assertEquals(Long.valueOf(0),

View Full Code Here


  void loopUntilCount(long count, LogicalNode coll, LogicalNode coll2)
      throws InterruptedException {
    boolean done = false;
    int loops = 0;
    AccumulatorSink ctr = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    AccumulatorSink ctr2 = (AccumulatorSink) ReportManager.get().getReportable(
        "count2");
    while (!done) {
      Clock.sleep(1000);
      long cnt1 = (ctr == null) ? 0 : ctr.getCount();
      long cnt2 = (ctr2 == null) ? 0 : ctr2.getCount();


      LOG.info("loop " + loops + " collector count = " + cnt1 + " count2 = "
          + cnt2);
      if (coll != null) {
        LOG.info(coll.getMetrics().toText());

View Full Code Here

        + " {diskFailover => {insistentAppend => { lazyOpen "
        + "=> rpcSink(\"localhost\",12345) } } } >";
    LogicalNode agent = setupAgent(count, agentSink);


    // wait until the counts add up properly
    AccumulatorSink ctr = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    AccumulatorSink ctr2 = (AccumulatorSink) ReportManager.get().getReportable(
        "count2");
    loopUntilCount(count, coll, coll2);


    assertTrue("Getting to IDLE state took to long", agent.getDriver()
        .waitForState(DriverState.IDLE, 5000));


    // close off the collector
    Driver c1driver = coll.getDriver();
    Driver c2driver = coll2.getDriver();
    coll.close();
    coll2.close();
    agent.close();


    // check outout
    LOG.info("primary collector count   = " + ctr.getCount());
    LOG.info("secondary collector count = " + ctr2.getCount());
    assertEquals(count, ctr.getCount() + ctr2.getCount());


    // the collector can be in ERROR or IDLE state because of the randomness.
    assertTrue("c1 failed to get to in IDLE state",
        c1driver.waitForAtLeastState(DriverState.IDLE, 1000));
    assertTrue("c2 failed to get to in IDLE state",

View Full Code Here


    // wait for agent done
    // wait until the counts add up properly
    boolean done = false;
    int loops = 0;
    AccumulatorSink ctr = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    AccumulatorSink ctr2 = (AccumulatorSink) ReportManager.get().getReportable(
        "count2");
    long old = 0;
    while (!done) {
      Clock.sleep(1000);


      LOG.info("loop " + loops + " collector count = " + ctr.getCount()
          + " count2 = " + ctr2.getCount());


      LOG.info(coll.getMetrics().toText());
      LOG.info(coll2.getMetrics().toText());
      if (old == ctr.getCount()) {
        break;

View Full Code Here

        + " {diskFailover => { insistentAppend => { stubbornAppend => { insistentOpen "
        + "=> { lazyOpen => {flakeyAppend(.05) => rpcSink(\"localhost\",12346) } } }  } } }> } ";
    LogicalNode agent = setupAgent(count, agentSink);


    // wait until the counts add up properly
    AccumulatorSink ctr = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    AccumulatorSink ctr2 = (AccumulatorSink) ReportManager.get().getReportable(
        "count2");
    loopUntilCount(count, coll, coll2);


    // close off the collector
    Driver c1driver = coll.getDriver();
    Driver c2driver = coll2.getDriver();
    coll.close();
    coll2.close();


    // dump info for debugging
    Map<String, ReportEvent> rpts = new HashMap<String, ReportEvent>();
    agent.getReports(rpts);
    for (Entry<String, ReportEvent> e : rpts.entrySet()) {
      LOG.info(e.getKey() + " : " + e.getValue());
    }


    // check the end states
    assertEquals(count, ctr.getCount() + ctr2.getCount());


    // the collector can be in ERROR or IDLE state because of the randomness.
    assertTrue("c1 failed to get to in IDLE state",
        c1driver.waitForAtLeastState(DriverState.IDLE, 1000));
    assertTrue("c2 failed to get to in IDLE state",

View Full Code Here

    Clock.sleep(2000);
    LogicalNode coll = setupColl(12345, "coll", "count");
    LogicalNode coll2 = setupColl(12346, "coll2", "count2");


    // wait until the counts add up properly
    AccumulatorSink ctr = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    AccumulatorSink ctr2 = (AccumulatorSink) ReportManager.get().getReportable(
        "count2");
    loopUntilCount(count, coll, coll2);


    // close off the collector
    Driver c1driver = coll.getDriver();
    Driver c2driver = coll2.getDriver();
    coll.close();
    coll2.close();


    // dump info for debugging
    Map<String, ReportEvent> rpts = new HashMap<String, ReportEvent>();
    agent.getReports(rpts);
    for (Entry<String, ReportEvent> e : rpts.entrySet()) {
      LOG.info(e.getKey() + " : " + e.getValue());
    }


    // check the end states
    assertEquals(count, ctr.getCount() + ctr2.getCount());
    assertTrue(ctr.getCount() > 0);
    assertTrue(ctr2.getCount() > 0);


    // the collector can be in ERROR or IDLE state because of the randomness.
    assertTrue("c1 failed to get to in IDLE state",
        c1driver.waitForAtLeastState(DriverState.IDLE, 1000));
    assertTrue("c2 failed to get to in IDLE state",

View Full Code Here

    // start collectors
    Clock.sleep(2000);
    LogicalNode coll2 = setupColl(12346, "coll2", "count2");


    // wait until the counts add up properly
    AccumulatorSink ctr2 = (AccumulatorSink) ReportManager.get().getReportable(
        "count2");
    loopUntilCount(count, null, coll2);


    // close off the collector
    Driver c2driver = coll2.getDriver();
    coll2.close();


    // dump info for debugging
    Map<String, ReportEvent> rpts = new HashMap<String, ReportEvent>();
    agent.getReports(rpts);
    for (Entry<String, ReportEvent> e : rpts.entrySet()) {
      LOG.info(e.getKey() + " : " + e.getValue());
    }


    // check the end states
    assertEquals(count, ctr2.getCount());


    // the collector can be in ERROR or IDLE state because of the randomness.f
    assertTrue("c2 failed to get to in IDLE state",
        c2driver.waitForAtLeastState(DriverState.IDLE, 1000));

View Full Code Here

    src.open();
    snk.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close();
    AccumulatorSink cnt = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    assertEquals(LINES, cnt.getCount());
    LOG.info("== multi test stop");
  }

View Full Code Here

    src.open();
    snk.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close();
    AccumulatorSink cnt = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    assertEquals(LINES / 5, cnt.getCount());
    LOG.info("== Decorated stop");
  }

View Full Code Here

    src.open();
    snk.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close();
    AccumulatorSink cnt = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    assertEquals(LINES, cnt.getCount());
    LOG.info("== failover stop");
  }

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of com.cloudera.flume.reporter.aggregator.AccumulatorSink

com.cloudera.flume.agent.diskfailover.TestConcurrentDFOMan

com.cloudera.flume.agent.diskfailover.TestDiskFailoverBehavior

com.cloudera.flume.conf.SinkFactory.SinkBuilder

com.cloudera.flume.conf.TestFlumeBuilderFunctional

com.cloudera.flume.core.EventSink

com.cloudera.flume.handlers.text.TestTailDirSource

com.cloudera.flume.reporter.ReportEvent

com.cloudera.flume.shell.TestFlumeShell

java.util.concurrent.atomic.AtomicLong

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.