Package com.cloudera.flume.reporter.aggregator

Examples of com.cloudera.flume.reporter.aggregator.AccumulatorSink


  void loopUntilCount(long count, LogicalNode coll, LogicalNode coll2)
      throws InterruptedException {
    boolean done = false;
    int loops = 0;
    AccumulatorSink ctr = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    AccumulatorSink ctr2 = (AccumulatorSink) ReportManager.get().getReportable(
        "count2");
    while (!done) {
      Clock.sleep(1000);
      long cnt1 = (ctr == null) ? 0 : ctr.getCount();
      long cnt2 = (ctr2 == null) ? 0 : ctr2.getCount();

      LOG.info("loop " + loops + " collector count = " + cnt1 + " count2 = "
          + cnt2);
      if (coll != null) {
        LOG.info(coll.getReport().toText());
View Full Code Here


        + " {diskFailover => {insistentAppend => { lazyOpen "
        + "=> rpcSink(\"localhost\",12345) } } } >";
    LogicalNode agent = setupAgent(count, agentSink);

    // wait until the counts add up properly
    AccumulatorSink ctr = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    AccumulatorSink ctr2 = (AccumulatorSink) ReportManager.get().getReportable(
        "count2");
    loopUntilCount(count, coll, coll2);

    assertEquals(NodeState.IDLE, agent.getStatus().state);

    // close off the collector
    coll.close();
    coll2.close();
    agent.close();

    // check outout
    LOG.info("primary collector count   = " + ctr.getCount());
    LOG.info("secondary collector count = " + ctr2.getCount());
    assertEquals(count, ctr.getCount() + ctr2.getCount());

    // the collector can be in ERROR or IDLE state because of the randomness.
    NodeState stateColl = coll.getStatus().state;
    LOG.info("coll exited in state: " + stateColl);
    assertTrue(stateColl.equals(NodeState.IDLE)
View Full Code Here

    // wait for agent done
    // wait until the counts add up properly
    boolean done = false;
    int loops = 0;
    AccumulatorSink ctr = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    AccumulatorSink ctr2 = (AccumulatorSink) ReportManager.get().getReportable(
        "count2");
    long old = 0;
    while (!done) {
      Clock.sleep(1000);

      LOG.info("loop " + loops + " collector count = " + ctr.getCount()
          + " count2 = " + ctr2.getCount());

      LOG.info(coll.getReport().toText());
      LOG.info(coll2.getReport().toText());
      if (old == ctr.getCount()) {
        break;
View Full Code Here

        + " {diskFailover => { insistentAppend => { stubbornAppend => { insistentOpen "
        + "=> { lazyOpen => {flakeyAppend(.05) => rpcSink(\"localhost\",12346) } } }  } } }> } ";
    LogicalNode agent = setupAgent(count, agentSink);

    // wait until the counts add up properly
    AccumulatorSink ctr = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    AccumulatorSink ctr2 = (AccumulatorSink) ReportManager.get().getReportable(
        "count2");
    loopUntilCount(count, coll, coll2);

    // close off the collector
    coll.close();
    coll2.close();

    // dump info for debugging
    Map<String, ReportEvent> rpts = new HashMap<String, ReportEvent>();
    agent.getReports(rpts);
    for (Entry<String, ReportEvent> e : rpts.entrySet()) {
      LOG.info(e.getKey() + " : " + e.getValue());
    }

    // check the end states
    assertEquals(count, ctr.getCount() + ctr2.getCount());

    // the collector can be in ERROR or IDLE state because of the randomness.
    NodeState stateColl = coll.getStatus().state;
    LOG.info("coll exited in state: " + stateColl);
    assertTrue(stateColl.equals(NodeState.IDLE)
View Full Code Here

    Clock.sleep(2000);
    LogicalNode coll = setupColl(12345, "coll", "count");
    LogicalNode coll2 = setupColl(12346, "coll2", "count2");

    // wait until the counts add up properly
    AccumulatorSink ctr = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    AccumulatorSink ctr2 = (AccumulatorSink) ReportManager.get().getReportable(
        "count2");
    loopUntilCount(count, coll, coll2);

    // close off the collector
    coll.close();
    coll2.close();

    // dump info for debugging
    Map<String, ReportEvent> rpts = new HashMap<String, ReportEvent>();
    agent.getReports(rpts);
    for (Entry<String, ReportEvent> e : rpts.entrySet()) {
      LOG.info(e.getKey() + " : " + e.getValue());
    }

    // check the end states
    assertEquals(count, ctr.getCount() + ctr2.getCount());
    assertTrue(ctr.getCount() > 0);
    assertTrue(ctr2.getCount() > 0);

    // the collector can be in ERROR or IDLE state because of the randomness.
    NodeState stateColl = coll.getStatus().state;
    LOG.info("coll exited in state: " + stateColl);
    assertTrue(stateColl.equals(NodeState.IDLE)
View Full Code Here

    // start collectors
    Clock.sleep(2000);
    LogicalNode coll2 = setupColl(12346, "coll2", "count2");

    // wait until the counts add up properly
    AccumulatorSink ctr2 = (AccumulatorSink) ReportManager.get().getReportable(
        "count2");
    loopUntilCount(count, null, coll2);

    // close off the collector
    coll2.close();

    // dump info for debugging
    Map<String, ReportEvent> rpts = new HashMap<String, ReportEvent>();
    agent.getReports(rpts);
    for (Entry<String, ReportEvent> e : rpts.entrySet()) {
      LOG.info(e.getKey() + " : " + e.getValue());
    }

    // check the end states
    assertEquals(count, ctr2.getCount());

    // the collector can be in ERROR or IDLE state because of the randomness.f
    NodeState stateColl2 = coll2.getStatus().state;
    LOG.info("coll2 exited in state: " + stateColl2);
    assertTrue(stateColl2.equals(NodeState.IDLE)
View Full Code Here

    sh.executeLine("waitForNodesDone 0 foo bar baz");
    n.getLivenessManager().heartbeatChecks();
    NodeState status = flumeMaster.getStatMan().getNodeStatuses().get(nodename).state;
    NodeState idle = NodeState.IDLE;
    assertEquals(status, idle);
    AccumulatorSink cnt = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    assertEquals(100, cnt.getCount());
    AccumulatorSink cnt2 = (AccumulatorSink) ReportManager.get().getReportable(
        "count2");
    assertEquals(50, cnt2.getCount());
    AccumulatorSink cnt3 = (AccumulatorSink) ReportManager.get().getReportable(
        "count3");
    assertEquals(75, cnt3.getCount());
    n.stop();
    n2.stop();
    n3.stop();
  }
View Full Code Here

    sh.executeLine("waitForNodesDone 0 foo");
    n.getLivenessManager().heartbeatChecks();
    status = flumeMaster.getStatMan().getNodeStatuses().get(nodename);
    NodeState idle = NodeState.IDLE;
    assertEquals(status.state, idle);
    AccumulatorSink cnt = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    assertEquals(100, cnt.getCount());
    n.stop();
  }
View Full Code Here

      new Thread("Concurrent-" + i) {
        @Override
        public void run() {
          try {
            File f1 = FileUtil.mktempdir();
            AccumulatorSink cnt1 = new AccumulatorSink("count." + idx);
            DiskFailoverManager dfoMan = new NaiveFileFailoverManager(f1);
            dfos[idx] = dfoMan; // save for checking.

            // short trigger causes lots of rolls
            EventSink snk = new DiskFailoverDeco<EventSink>(cnt1, dfoMan,
                new TimeTrigger(100), 50);

            ReportManager.get().add(cnt1);
            // make each parallel instance send a slightly different number of
            // messages.
            EventSource src = new NoNlASCIISynthSource(events + idx, 100);

            src.open();
            snk.open();

            started.countDown();

            EventUtil.dumpAll(src, snk);
            src.close();
            snk.close(); // this triggers a flush of current file!?
            FileUtil.rmr(f1);
          } catch (Exception e) {
            LOG.error(e, e);
          } finally {
            done.countDown();
          }

        }
      }.start();

    }

    started.await();
    boolean ok = done.await(timeout, TimeUnit.MILLISECONDS);
    assertTrue("Test timed out", ok);

    for (int i = 0; i < threads; i++) {
      AccumulatorSink cnt = (AccumulatorSink) ReportManager.get()
          .getReportable("count." + i);
      // check for the slightly different counts based on thread.
      int exp = events + i;
      LOG
          .info("count." + i + " expected " + exp + " and got "
              + cnt.getCount());
      assertEquals(exp, (int) cnt.getCount());

      // check dfo reports to see if they are sane.
      ReportEvent rpt = dfos[i].getReport();
      LOG.info(rpt);
      long failovered = rpt.getLongMetric(DiskFailoverManager.A_MSG_WRITING);
View Full Code Here

TOP

Related Classes of com.cloudera.flume.reporter.aggregator.AccumulatorSink

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.