Package com.cloudera.flume.reporter.aggregator

Examples of com.cloudera.flume.reporter.aggregator.CounterSink


    waitForEmptyWALs(master, node, timeout);

    // check to make sure everyone got the right number of events
    boolean success = true;
    for (int i = 0; i < threads; i++) {
      CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
          "report." + i);
      LOG.info("expected " + (events + i) + " but got " + cnt.getCount());
      success &= ((events + i) == cnt.getCount());
      assertEquals(events + i, cnt.getCount());
    }
    assertTrue("Counts did not line up", success);
    FlumeTestHarness.cleanupLocalWriteDir();
  }
View Full Code Here


    // this is a slight tweak to avoid port conflicts
    final int port = conf.getCollectorPort() + 1;
    final ThriftEventSource tes = new ThriftEventSource(port);
    tes.open();

    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (IOException e) {
        } catch (InterruptedException e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    t.start(); // drain the sink.

    // mem -> ThriftEventSink
    ThriftEventSink snk = new ThriftEventSink("0.0.0.0", port);
    snk.open();
    EventUtil.dumpAll(mem, snk);
    mem.close();
    snk.close();

    // a little delay to drain events at ThriftEventSource queue
    try {
      Thread.sleep(1000);
      t.interrupt();
    } catch (InterruptedException e) {
    }
    tes.close();
    assertEquals(25, cnt.getCount());
    ReportEvent rpt = tes.getMetrics();
    assertEquals(4475, rpt.getLongMetric(ThriftEventSource.A_BYTES_IN)
        .longValue());
    assertEquals(25, rpt.getLongMetric(ThriftEventSource.A_DEQUEUED)
        .longValue());
View Full Code Here

    // this is a slight tweak to avoid port conflicts
    final int port = conf.getCollectorPort() + 1;
    final ThriftEventSource tes = new ThriftEventSource(port);
    tes.open();

    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (Exception e) {
        }
      }
    };
    t.start(); // drain the sink.

    // fork off threads threads and have them start all the same time.
    final CountDownLatch sendStarted = new CountDownLatch(threads);
    final CountDownLatch sendDone = new CountDownLatch(threads);
    final AtomicLong sendByteSum = new AtomicLong(0);
    for (int i = 0; i < threads; i++) {
      final int id = i;
      Thread th = new Thread() {
        public void run() {
          try {
            // TODO (jon) this may have different sizes due to the host it is
            // running on . Needs to be fixed.
            EventSource txt = new NoNlASCIISynthSource(25, 100);

            txt.open();
            MemorySinkSource mem = new MemorySinkSource();
            mem.open();
            EventUtil.dumpAll(txt, mem);
            txt.close();

            // mem -> ThriftEventSink
            ThriftEventSink snk = new ThriftEventSink("0.0.0.0", port);
            snk.open();

            sendStarted.countDown();
            sendStarted.await();
            EventUtil.dumpAll(mem, snk);
            mem.close();
            snk.close();

            sendByteSum.addAndGet(snk.getSentBytes());
            LOG.info("sink " + id + " sent " + snk.getSentBytes() + " bytes");
            sendDone.countDown();

          } catch (IOException e) {
            e.printStackTrace();
          } catch (InterruptedException e) {
            e.printStackTrace();
          }

        }
      };
      th.start();
    }

    // wait for senders to send all
    sendDone.await();

    // a little delay get data to the receiving side.
    Thread.sleep(1000);

    tes.close();
    assertEquals(25 * threads, cnt.getCount());
    ReportEvent rpt = tes.getMetrics();
    assertEquals(4475 * threads, sendByteSum.get());
    assertEquals(4475 * threads, rpt
        .getLongMetric(ThriftEventSource.A_BYTES_IN).longValue());
    assertEquals(25 * threads, rpt.getLongMetric(ThriftEventSource.A_DEQUEUED)
View Full Code Here

    snk.open();
    snk.append(new EventImpl(new byte[0]));
    snk.append(new EventImpl(new byte[0]));
    snk.close();

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("test");
    Assert.assertEquals(1, ctr.getCount());

  }
View Full Code Here

      LOG.info(dfos[i].getMetrics());
    }

    for (int i = 0; i < threads; i++) {

      CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
          "report." + i);
      LOG.info(i + " expected " + (events + i) + " and got " + cnt.getCount());
      success &= ((events + i) == cnt.getCount());
      assertEquals(events + i, cnt.getCount());

    }
    assertTrue("Counts did not line up", success);

    FlumeTestHarness.cleanupLocalWriteDir();
View Full Code Here

    // second sleeper, but check period is really long

    Event e = new EventImpl("this is a test message".getBytes());
    snk.open();
    snk.append(e);
    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("foo");
    Clock.sleep(3000); // sleep 3s

    // the roller automatically flushed!
    assertEquals(1, cnt.getCount());
    snk.close();
  }
View Full Code Here

    // this is a slight tweak to avoid port conflicts
    final int port = conf.getCollectorPort() + 1;
    final AvroEventSource tes = new AvroEventSource(port);
    tes.open();

    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (Exception e) {
        }
      }
    };
    t.start(); // drain the sink.

    // mem -> AvroEventSink
    AvroEventSink snk = new AvroEventSink("0.0.0.0", port);
    snk.open();
    EventUtil.dumpAll(mem, snk);
    mem.close();
    snk.close();

    // a little delay to drain events at AvroEventSource queue
    try {
      Thread.sleep(1000);
      t.interrupt();
    } catch (InterruptedException e) {
    }
    tes.close();
    assertEquals(25, cnt.getCount());
    ReportEvent rpt = tes.getMetrics();
    /*
     * The check on BytesIn is different than one on TestThriftSinks tests. This
     * is because currently in the AvroSink version, BytesIn is equal to the
     * number of Bytes of the Event.body shipped.
View Full Code Here

    // this is a slight tweak to avoid port conflicts
    final int port = conf.getCollectorPort() + 1;
    final AvroEventSource tes = new AvroEventSource(port);
    tes.open();

    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (Exception e) {
        }
      }
    };
    t.start(); // drain the sink.

    // fork off threads threads and have them start all the same time.
    final CountDownLatch sendStarted = new CountDownLatch(threads);
    final CountDownLatch sendDone = new CountDownLatch(threads);
    final AtomicLong sendByteSum = new AtomicLong(0);
    for (int i = 0; i < threads; i++) {
      final int id = i;
      Thread th = new Thread() {
        public void run() {
          try {
            EventSource txt = new NoNlASCIISynthSource(25, 100);
            txt.open();
            MemorySinkSource mem = new MemorySinkSource();
            mem.open();
            EventUtil.dumpAll(txt, mem);
            txt.close();

            // mem -> AvroEventSink
            AvroEventSink snk = new AvroEventSink("0.0.0.0", port);
            snk.open();

            sendStarted.countDown();
            sendStarted.await();
            EventUtil.dumpAll(mem, snk);
            mem.close();
            snk.close();

            sendByteSum.addAndGet(snk.getSentBytes());
            LOG.info("sink " + id + " sent " + snk.getSentBytes() + " bytes");
            sendDone.countDown();

          } catch (IOException e) {
            e.printStackTrace();
          } catch (InterruptedException e) {
            e.printStackTrace();
          }
        }
      };
      th.start();
    }

    // wait for senders to send all
    sendDone.await();

    // a little delay get data to the receiving side.
    Thread.sleep(1000);

    tes.close();
    assertEquals(25 * threads, cnt.getCount());
    ReportEvent rpt = tes.getMetrics();
    assertEquals(2500 * threads, sendByteSum.get());
    assertEquals(2500 * threads, rpt.getLongMetric(AvroEventSource.A_BYTES_IN)
        .longValue());
    assertEquals(25 * threads, rpt.getLongMetric(AvroEventSource.A_DEQUEUED)
View Full Code Here

    super(name, maxAge, FlumeConfiguration.get().getHistoryMaxLength(), t);
  }

  @Override
  public CounterSink newSink(Tagger format) throws IOException {
    return new CounterSink(format.newTag());
  }
View Full Code Here

    boolean ok = done.await(30, TimeUnit.SECONDS);
    assertTrue("Test timed out!", ok);
    Thread.sleep(1000);
    snk.close();

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("total");
    long ci = cnt.getCount();
    LOG.info("count : " + ci);
    assertEquals((count * threads) + 2, ci);

    FileUtil.rmr(dir);
  }
View Full Code Here

TOP

Related Classes of com.cloudera.flume.reporter.aggregator.CounterSink

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.