Package com.cloudera.flume.reporter.aggregator

Examples of com.cloudera.flume.reporter.aggregator.CounterSink


    final ThriftEventSource tes = new ThriftEventSource(
        conf.getCollectorPort() + 1); // this is a slight
    // tweak to avoid port conflicts
    tes.open();

    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (IOException e) {
        } catch (InterruptedException e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    t.start(); // drain the sink.

    // mem -> ThriftEventSink
    ThriftEventSink snk = new ThriftEventSink("0.0.0.0", conf
        .getCollectorPort() + 1);
    snk.open();
    EventUtil.dumpAll(mem, snk);
    mem.close();
    snk.close();

    // a little delay to drain events at ThriftEventSource queue
    try {
      Thread.sleep(1000);
      t.interrupt();
    } catch (InterruptedException e) {
    }
    tes.close();
    assertEquals(25, cnt.getCount());
    ReportEvent rpt = tes.getMetrics();
    assertEquals(4475, rpt.getLongMetric(ThriftEventSource.A_BYTES_IN)
        .longValue());
    assertEquals(25, rpt.getLongMetric(ThriftEventSource.A_DEQUEUED)
        .longValue());
View Full Code Here


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();

    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (Exception e) {
        }
      }
    };
    t.start(); // drain the sink.

    // mem -> thriftRawEventSink
    ThriftRawEventSink snk = new ThriftRawEventSink("0.0.0.0", conf
        .getCollectorPort());
    snk.open();
    EventUtil.dumpAll(mem, snk);
    mem.close();
    snk.close();

    // a little delay to drain events at ThriftEventSource queue
    try {
      Thread.sleep(5000);
      t.interrupt();
    } catch (InterruptedException e) {
    }
    tes.close();
    assertEquals(25, cnt.getCount());

  }
View Full Code Here

    // this is a slight tweak to avoid port conflicts
    final ThriftEventSource tes = new ThriftEventSource(
        conf.getCollectorPort() + 1);
    tes.open();

    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (Exception e) {
        }
      }
    };
    t.start(); // drain the sink.

    // fork off threads threads and have them start all the same time.
    final CountDownLatch sendStarted = new CountDownLatch(threads);
    final CountDownLatch sendDone = new CountDownLatch(threads);
    final AtomicLong sendByteSum = new AtomicLong(0);
    for (int i = 0; i < threads; i++) {
      final int id = i;
      Thread th = new Thread() {
        public void run() {
          try {
            // TODO (jon) this may have different sizes due to the host it is
            // running on . Needs to be fixed.
            EventSource txt = new NoNlASCIISynthSource(25, 100);

            txt.open();
            MemorySinkSource mem = new MemorySinkSource();
            mem.open();
            EventUtil.dumpAll(txt, mem);
            txt.close();

            // mem -> ThriftEventSink
            ThriftEventSink snk = new ThriftEventSink("0.0.0.0", conf
                .getCollectorPort() + 1);
            snk.open();

            sendStarted.countDown();
            sendStarted.await();
            EventUtil.dumpAll(mem, snk);
            mem.close();
            snk.close();

            sendByteSum.addAndGet(snk.sentBytes.get());
            LOG.info("sink " + id + " sent " + snk.sentBytes + " bytes");
            sendDone.countDown();

          } catch (IOException e) {
            e.printStackTrace();
          } catch (InterruptedException e) {
            e.printStackTrace();
          }

        }
      };
      th.start();
    }

    // wait for senders to send all
    sendDone.await();

    // a little delay get data to the receiving side.
    Thread.sleep(1000);

    tes.close();
    assertEquals(25 * threads, cnt.getCount());
    ReportEvent rpt = tes.getMetrics();
    assertEquals(4475 * threads, sendByteSum.get());
    assertEquals(4475 * threads, rpt
        .getLongMetric(ThriftEventSource.A_BYTES_IN).longValue());
    assertEquals(25 * threads, rpt.getLongMetric(ThriftEventSource.A_DEQUEUED)
View Full Code Here

  @Test
  public void testLazyOpenBuild() throws IOException, FlumeSpecException,
      InterruptedException {
    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(),
        "{ lazyOpen => counter(\"count\") } ");
    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("count");

    boolean ok = false;
    Event e = new EventImpl("event".getBytes());
    snk.open();
    try {
      cnt.append(e);
    } catch (Exception ex) {
      ok = true;
      // should be thrown because not actually open
    }
    assertTrue(ok);

    snk.append(e);

    assertEquals(1, cnt.getCount());
    snk.close();
  }
View Full Code Here

    // second sleeper, but check period is really long

    Event e = new EventImpl("this is a test message".getBytes());
    snk.open();
    snk.append(e);
    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("foo");
    Clock.sleep(3000); // sleep 3s

    // the roller automatically flushed!
    assertEquals(1, cnt.getCount());
    snk.close();
  }
View Full Code Here

    }
    fw.close();

    assertTrue(done.await(10, TimeUnit.SECONDS));

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("count");
    assertEquals(count, ctr.getCount());
  }
View Full Code Here

    // This sleep is necessary to make sure the file is not moved before tail
    // sees it.
    Clock.sleep(2000);
    f.renameTo(f2);

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("count");
    LOG.info("counted " + ctr.getCount());

    FileWriter fw2 = new FileWriter(f);
    for (int i = count / 2; i < count; i++) {
      fw2.append("Line " + i + "\n");
      fw2.flush();
    }
    fw2.close();

    done.await();

    ctr = (CounterSink) ReportManager.get().getReportable("count");
    assertEquals(count, ctr.getCount());
  }
View Full Code Here

    fw.close();
    fw2.close();

    assertTrue("Test timed out", done.await(30, TimeUnit.SECONDS));

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("count");
    LOG.info("events in file1: " + log1 + " events in file2: " + log2);
    assertEquals(count, ctr.getCount());

  }
View Full Code Here

    }
    fw2.close();

    assertTrue(done.await(10, TimeUnit.SECONDS));

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("count");
    assertEquals(count, ctr.getCount());

    Cursor cursor = src.cursors.get(0);
    assertEquals(cursor.lastChannelSize, cursor.lastChannelPos);
  }
View Full Code Here

    }

    for (int i = 0; i < res_size.length; i++) {
      mem.open();
      int sz = res_size[i];
      CounterSink cnt = new CounterSink("null");
      EventSink res = new ReservoirSamplerDeco<CounterSink>(cnt, sz);
      EventUtil.dumpAll(mem, res);
      b.mark("reservoir", sz);

      res.close();
      b.mark("count", cnt.getCount());
    }
    b.done();
  }
View Full Code Here

TOP

Related Classes of com.cloudera.flume.reporter.aggregator.CounterSink

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.