Examples of com.cloudera.flume.reporter.aggregator.CounterSink

Package com.cloudera.flume.reporter.aggregator

Examples of com.cloudera.flume.reporter.aggregator.CounterSink

com.cloudera.flume.reporter.aggregator.CounterSink
This just counts the number of entries appended.

        public void run() {


          try {
            File f1 = FileUtil.mktempdir();


            CounterSink cnt1 = new CounterSink("count." + idx);
            AckChecksumChecker<EventSink> chk = new AckChecksumChecker<EventSink>(
                cnt1);
            NaiveFileWALManager wman1 = new NaiveFileWALManager(f1);
            EventSink snk = new NaiveFileWALDeco(new Context(), chk, wman1,
                new TimeTrigger(1000000), new AckListener.Empty(), 100);


            ReportManager.get().add(cnt1);
            // make each parallel instance send a slightly different number of
            // messages.
            EventSource src = new NoNlASCIISynthSource(events + idx, 100);


            src.open();
            snk.open();


            started.countDown();


            EventUtil.dumpAll(src, snk);
            src.close();
            snk.close();
            FileUtil.rmr(f1);
          } catch (Exception e) {
            LOG.error(e, e);
          } finally {
            done.countDown();
          }


        }
      }.start();


    }


    started.await();
    boolean ok = done.await(timeout, TimeUnit.MILLISECONDS);
    assertTrue("Test timed out", ok);


    for (int i = 0; i < threads; i++) {
      CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
          "count." + i);
      // check for the slightly different counts based on thread.
      int exp = events + i;
      LOG.info("expected " + exp + " but got " + cnt.getCount());
      assertEquals(exp, (int) cnt.getCount());
    }
  }

View Full Code Here

    waitForEmptyWALs(master, node, timeout);


    // check to make sure everyone got the right number of events
    boolean success = true;
    for (int i = 0; i < threads; i++) {
      CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
          "report." + i);
      LOG.info("expected " + (events + i) + " but got " + cnt.getCount());
      success &= ((events + i) == cnt.getCount());
      assertEquals(events + i, cnt.getCount());
    }
    assertTrue("Counts did not line up", success);
    BenchmarkHarness.cleanupLocalWriteDir();
  }

View Full Code Here

    }


    for (int i = 0; i < res_size.length; i++) {
      mem.open();
      int sz = res_size[i];
      CounterSink cnt = new CounterSink("null");
      EventSink res = new ReservoirSamplerDeco<CounterSink>(cnt, sz);
      EventUtil.dumpAll(mem, res);
      b.mark("reservoir", sz);


      res.close();
      b.mark("count", cnt.getCount());
    }
    b.done();
  }

View Full Code Here

    }


    for (int i = 0; i < interval_size.length; i++) {
      mem.open();
      int sz = interval_size[i];
      CounterSink cnt = new CounterSink("null");
      EventSink res = new IntervalSampler<CounterSink>(cnt, sz);
      EventUtil.dumpAll(mem, res);
      b.mark("interval", sz);


      res.close();
      b.mark("count", cnt.getCount());
    }
    b.done();
  }

View Full Code Here

    }


    for (int i = 0; i < probs.length; i++) {
      mem.open();
      double prob = probs[i];
      CounterSink cnt = new CounterSink("null");
      EventSink res = new ProbabilitySampler<CounterSink>(cnt, prob);
      EventUtil.dumpAll(mem, res);
      b.mark("probability", prob);


      res.close();
      b.mark("count", cnt.getCount());
    }
    b.done();
  }

View Full Code Here

      if (i % 5 == 4) {
        wal.rotate();
      }
    }


    CounterSink ctr = (CounterSink) ReportManager.get().getReportable(rpt);
    LOG.info("Last batch should be missing. " + count + " > " + ctr.getCount()
        + " sent");


    wal.close();
    LOG.info("All should make it through here. " + count + " =="
        + ctr.getCount() + " sent");
    assertEquals(20, ctr.getCount());


    // check master state -- there should be 5 pending acks.
    assertEquals(5, master.getAckMan().getPending().size());


    // after checking, there shuld be 0 acks left.

View Full Code Here

    // arbitrary sleep for now.
    Clock.sleep(1000);


    // three blocks should have successfully made it.
    assertEquals(3, master.getAckMan().getPending().size()); // (g0, g1, g3)
    CounterSink ctr = (CounterSink) ReportManager.get().getReportable(rpt);
    LOG.info(" Ack begin dropped, but all true messages went through. 20 == "
        + ctr.getCount() + " sent");
    // We dropped an ack begin message. Messages still make it through and there
    // should be some sort of ack warning on the master.
    assertEquals(20, ctr.getCount());


    // check master state
    master.getAckMan().dumpLog();
    Set<String> pending = master.getAckMan().getPending();
    assertEquals(3, pending.size());


    // check to make sure wal files are gone.
    node.getAckChecker().checkAcks();
    node.getAckChecker().checkRetry();
    master.getAckMan().dumpLog();
    pending = master.getAckMan().getPending();
    LOG.info("Number of pending acks (retry had failure)" + pending.size());
    assertEquals(0, pending.size());
    LOG.info("Event count 20  (one message missing/lost) ==" + ctr.getCount());
    assertEquals(20, ctr.getCount());


    Clock.sleep(1000); // somehow force other thread to go.


    // // Phase II - force message resent retry but one is dropped. (only 4 make
    // it)
    node.getAckChecker().checkAcks();
    node.getAckChecker().forceRetry();


    Clock.sleep(500); // somehow force other thread to go.


    LOG.info("Was stuff resent? " + " 24 == " + ctr.getCount() + " sent");
    // This retry succeeds sending 4 more messages
    assertEquals(24, ctr.getCount());
    assertEquals(0, master.getAckMan().getPending().size());


    // // Phase III - force another retry
    node.getAckChecker().checkAcks();
    node.getAckChecker().forceRetry();
    LOG
        .info("After another check. " + count + " < " + ctr.getCount()
            + " sent");
    Clock.sleep(500);


    // suceeded sending 5 more messages.
    assertEquals(29, ctr.getCount());
    assertEquals(1, master.getAckMan().getPending().size());


    // check the ack.
    node.getAckChecker().checkAcks();


    // we are clean.
    assertEquals(29, ctr.getCount());
    assertEquals(0, master.getAckMan().getPending().size());


    wal.close();
  }

View Full Code Here


    Clock.sleep(1000); // need other thread to make progress
    // 3 blocks should have successfully made it.
    assertEquals(3, master.getAckMan().getPending().size()); // (g0, g1, g3)


    CounterSink ctr = (CounterSink) ReportManager.get().getReportable(rpt);
    LOG.info(" Ack begin dropped, but all true message went through. 20 == "
        + ctr.getCount() + " sent");
    // We dropped a event message, so we should be down one, 19 of 20 events.
    assertEquals(19, ctr.getCount());


    // check master state
    master.getAckMan().dumpLog();
    Set<String> pending = master.getAckMan().getPending();
    assertEquals(3, pending.size());


    // // Phase II (4 more messages, 23 total, still only 3 groups)
    // check to make sure wal files are gone.
    node.getAckChecker().checkAcks();
    node.getAckChecker().forceRetry();
    Clock.sleep(1000);
    master.getAckMan().dumpLog();
    pending = master.getAckMan().getPending();
    LOG.info("Number of pending acks (retry had failure)" + pending.size());
    assertEquals(0, pending.size());
    LOG.info("Event count 23  (one message missing/lost) ==" + ctr.getCount());
    assertEquals(23, ctr.getCount());


    // //Phase III (5 more messages, 28 total, 4 groups good)
    node.getAckChecker().checkAcks();
    node.getAckChecker().forceRetry();
    Clock.sleep(1000); // somehow force other thread to go.


    LOG.info("Was stuff resent? " + " 28 == " + ctr.getCount() + " sent");
    // This retry succeeds sending 5 more messages
    assertEquals(28, ctr.getCount());
    Clock.sleep(1000); // somehow force other thread to go.
    node.getAckChecker().checkAcks();
    node.getAckChecker().forceRetry();
    LOG.info("After another check. " + " 28 == " + ctr.getCount() + " sent");


    // Nothing happened
    assertEquals(28, ctr.getCount());
    assertEquals(0, master.getAckMan().getPending().size());
    wal.close();


  }

View Full Code Here

    ScheduledHistoryReporter<CounterSink> hist = new ScheduledHistoryReporter<CounterSink>(
        "test", longwait, historyLen, new DumbTagger()) {


      @Override
      public CounterSink newSink(Tagger format) throws IOException {
        CounterSink count = new CounterSink("count") {
          public void append(Event e) throws IOException, InterruptedException {
            super.append(e);
            System.out.println(e); // just add a printf to the counts.
          }
        };

View Full Code Here


  @Test
  public void testSimpleIntervalSamplerSink() throws IOException,
      InterruptedException {
    System.out.println("Simple interval sampler sink");
    CounterSink count = new CounterSink("count");
    IntervalSampler<CounterSink> sink = new IntervalSampler<CounterSink>(count,
        10);
    sink.open();
    for (int i = 0; i < 30; i++) {
      sink.append(new EventImpl(("test " + i).getBytes()));
    }
    Assert.assertEquals(3, count.getCount());
    sink.close();


    // do the boundary condition.
    CounterSink count2 = new CounterSink("count");
    IntervalSampler<CounterSink> sink2 = new IntervalSampler<CounterSink>(
        count2, 10);
    sink2.open();
    for (int i = 0; i < 31; i++) {
      sink2.append(new EventImpl(("test " + i).getBytes()));
    }
    Assert.assertEquals(4, count2.getCount());
    sink2.close();
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.reporter.aggregator.CounterSink

com.cloudera.flume.agent.diskfailover.TestConcurrentDFOMan

com.cloudera.flume.agent.diskfailover.TestDiskFailoverBenchmarking

com.cloudera.flume.agent.durability.TestConcurrentWALMan

com.cloudera.flume.agent.durability.TestNaiveFileWALDeco

com.cloudera.flume.agent.durability.TestNaiveFileWALManagerConcurrently

com.cloudera.flume.agent.TestAckedWALDecorator

com.cloudera.flume.agent.TestEndToEndAckFailureRecovery

com.cloudera.flume.agent.TestWriteAheadLogDecorator

com.cloudera.flume.conf.SinkFactory.SinkBuilder

com.cloudera.flume.core.EventSink

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.