Package com.cloudera.flume.reporter.aggregator

Examples of com.cloudera.flume.reporter.aggregator.CounterSink


    Thread.sleep(1000);

    long sum = 0;
    for (int i = 0; i < threads; i++) {

      CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
          "total." + i);
      long ci = cnt.getCount();
      LOG.info("count " + i + " : " + ci);
      sum += ci;

    }
View Full Code Here


    assertTrue("Test timed out!", ok);
    Thread.sleep(1000);

    long sum = 0;
    for (int i = 0; i < threads; i++) {
      CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
          "total." + i);
      long ci = cnt.getCount();
      LOG.info("count " + i + " : " + ci);
      sum += ci;

    }
View Full Code Here

        public void run() {

          try {
            File f1 = FileUtil.mktempdir();

            CounterSink cnt1 = new CounterSink("count." + idx);
            AckChecksumChecker<EventSink> chk = new AckChecksumChecker<EventSink>(
                cnt1);
            NaiveFileWALManager wman1 = new NaiveFileWALManager(f1);
            EventSink snk = new NaiveFileWALDeco<EventSink>(new Context(), chk,
                wman1, new TimeTrigger(1000000), new AckListener.Empty(), 100);

            ReportManager.get().add(cnt1);
            // make each parallel instance send a slightly different number of
            // messages.
            EventSource src = new NoNlASCIISynthSource(events + idx, 100);

            src.open();
            snk.open();

            started.countDown();

            EventUtil.dumpAll(src, snk);
            src.close();
            snk.close();
            FileUtil.rmr(f1);
          } catch (Exception e) {
            LOG.error(e, e);
          } finally {
            done.countDown();
          }

        }
      }.start();

    }

    started.await();
    boolean ok = done.await(timeout, TimeUnit.MILLISECONDS);
    assertTrue("Test timed out", ok);

    for (int i = 0; i < threads; i++) {
      CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
          "count." + i);
      // check for the slightly different counts based on thread.
      int exp = events + i;
      LOG.info("expected " + exp + " but got " + cnt.getCount());
      assertEquals(exp, (int) cnt.getCount());
    }
  }
View Full Code Here

    waitForEmptyWALs(master, node, timeout);

    // check to make sure everyone got the right number of events
    boolean success = true;
    for (int i = 0; i < threads; i++) {
      CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
          "report." + i);
      LOG.info("expected " + (events + i) + " but got " + cnt.getCount());
      success &= ((events + i) == cnt.getCount());
      assertEquals(events + i, cnt.getCount());
    }
    assertTrue("Counts did not line up", success);
    BenchmarkHarness.cleanupLocalWriteDir();
  }
View Full Code Here

    final ThriftEventSource tes = new ThriftEventSource(
        conf.getCollectorPort() + 1); // this is a slight
    // tweak to avoid port conflicts
    tes.open();

    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (IOException e) {
        }
      }
    };
    t.start(); // drain the sink.

    // mem -> ThriftEventSink
    ThriftEventSink snk = new ThriftEventSink("0.0.0.0", conf
        .getCollectorPort() + 1);
    snk.open();
    EventUtil.dumpAll(mem, snk);
    mem.close();
    snk.close();

    // a little delay to drain events at ThriftEventSource queue
    try {
      Thread.sleep(1000);
      t.interrupt();
    } catch (InterruptedException e) {
    }
    tes.close();
    assertEquals(25, cnt.getCount());
    ReportEvent rpt = tes.getReport();
    assertEquals(4475, rpt.getLongMetric(ThriftEventSource.A_BYTES_IN)
        .longValue());
    assertEquals(25, rpt.getLongMetric(ThriftEventSource.A_DEQUEUED)
        .longValue());
View Full Code Here

    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();

    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (IOException e) {
        }
      }
    };
    t.start(); // drain the sink.

    // mem -> thriftRawEventSink
    ThriftRawEventSink snk = new ThriftRawEventSink("0.0.0.0", conf
        .getCollectorPort());
    snk.open();
    EventUtil.dumpAll(mem, snk);
    mem.close();
    snk.close();

    // a little delay to drain events at ThriftEventSource queue
    try {
      Thread.sleep(5000);
      t.interrupt();
    } catch (InterruptedException e) {
    }
    tes.close();
    assertEquals(25, cnt.getCount());

  }
View Full Code Here

    // this is a slight tweak to avoid port conflicts
    final ThriftEventSource tes = new ThriftEventSource(
        conf.getCollectorPort() + 1);
    tes.open();

    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (IOException e) {
        }
      }
    };
    t.start(); // drain the sink.

    // fork off threads threads and have them start all the same time.
    final CountDownLatch sendStarted = new CountDownLatch(threads);
    final CountDownLatch sendDone = new CountDownLatch(threads);
    final AtomicLong sendByteSum = new AtomicLong(0);
    for (int i = 0; i < threads; i++) {
      final int id = i;
      Thread th = new Thread() {
        public void run() {
          try {
            // TODO (jon) this may have different sizes due to the host it is
            // running on . Needs to be fixed.
            EventSource txt = new NoNlASCIISynthSource(25, 100);

            txt.open();
            MemorySinkSource mem = new MemorySinkSource();
            mem.open();
            EventUtil.dumpAll(txt, mem);
            txt.close();

            // mem -> ThriftEventSink
            ThriftEventSink snk = new ThriftEventSink("0.0.0.0", conf
                .getCollectorPort() + 1);
            snk.open();

            sendStarted.countDown();
            sendStarted.await();
            EventUtil.dumpAll(mem, snk);
            mem.close();
            snk.close();

            sendByteSum.addAndGet(snk.sentBytes.get());
            LOG.info("sink " + id + " sent " + snk.sentBytes + " bytes");
            sendDone.countDown();

          } catch (IOException e) {
            e.printStackTrace();
          } catch (InterruptedException e) {
            e.printStackTrace();
          }

        }
      };
      th.start();
    }

    // wait for senders to send all
    sendDone.await();

    // a little delay get data to the receiving side.
    Thread.sleep(1000);

    tes.close();
    assertEquals(25 * threads, cnt.getCount());
    ReportEvent rpt = tes.getReport();
    assertEquals(4475 * threads, sendByteSum.get());
    assertEquals(4475 * threads, rpt
        .getLongMetric(ThriftEventSource.A_BYTES_IN).longValue());
    assertEquals(25 * threads, rpt.getLongMetric(ThriftEventSource.A_DEQUEUED)
View Full Code Here

    FlumeConfiguration conf = FlumeConfiguration.get();
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort() + 1);
    tes.open();

    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (IOException e) {
        }
      }
    };
    t.start(); // drain the sink.

    // mem -> AvroEventSink
    AvroEventSink snk = new AvroEventSink("0.0.0.0",
        conf.getCollectorPort() + 1);
    snk.open();
    EventUtil.dumpAll(mem, snk);
    mem.close();
    snk.close();

    // a little delay to drain events at AvroEventSource queue
    try {
      Thread.sleep(1000);
      t.interrupt();
    } catch (InterruptedException e) {
    }
    tes.close();
    assertEquals(25, cnt.getCount());
    ReportEvent rpt = tes.getReport();
    /*
     * The check on BytesIn is different than one on TestThriftSinks tests. This
     * is because currently in the AvroSink version, BytesIn is equal to the
     * number of Bytes of the Event.body shipped.
View Full Code Here

    final FlumeConfiguration conf = FlumeConfiguration.get();
    // this is a slight tweak to avoid port conflicts
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort() + 1);
    tes.open();

    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (IOException e) {
        }
      }
    };
    t.start(); // drain the sink.

    // fork off threads threads and have them start all the same time.
    final CountDownLatch sendStarted = new CountDownLatch(threads);
    final CountDownLatch sendDone = new CountDownLatch(threads);
    final AtomicLong sendByteSum = new AtomicLong(0);
    for (int i = 0; i < threads; i++) {
      final int id = i;
      Thread th = new Thread() {
        public void run() {
          try {
            EventSource txt = new NoNlASCIISynthSource(25, 100);
            txt.open();
            MemorySinkSource mem = new MemorySinkSource();
            mem.open();
            EventUtil.dumpAll(txt, mem);
            txt.close();

            // mem -> AvroEventSink
            AvroEventSink snk = new AvroEventSink("0.0.0.0", conf
                .getCollectorPort() + 1);
            snk.open();

            sendStarted.countDown();
            sendStarted.await();
            EventUtil.dumpAll(mem, snk);
            mem.close();
            snk.close();

            sendByteSum.addAndGet(snk.sentBytes.get());
            LOG.info("sink " + id + " sent " + snk.sentBytes + " bytes");
            sendDone.countDown();

          } catch (IOException e) {
            e.printStackTrace();
          } catch (InterruptedException e) {
            e.printStackTrace();
          }
        }
      };
      th.start();
    }

    // wait for senders to send all
    sendDone.await();

    // a little delay get data to the receiving side.
    Thread.sleep(1000);

    tes.close();
    assertEquals(25 * threads, cnt.getCount());
    ReportEvent rpt = tes.getReport();
    assertEquals(2500 * threads, sendByteSum.get());
    assertEquals(2500 * threads, rpt.getLongMetric(AvroEventSource.A_BYTES_IN)
        .longValue());
    assertEquals(25 * threads, rpt.getLongMetric(AvroEventSource.A_DEQUEUED)
View Full Code Here

    snk.open();
    snk.append(new EventImpl(new byte[0]));
    snk.append(new EventImpl(new byte[0]));
    snk.close();

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("test");
    Assert.assertEquals(1, ctr.getCount());

  }
View Full Code Here

TOP

Related Classes of com.cloudera.flume.reporter.aggregator.CounterSink

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.