Examples of com.cloudera.flume.handlers.debug.MemorySinkSource

Package com.cloudera.flume.handlers.debug

Examples of com.cloudera.flume.handlers.debug.MemorySinkSource

com.cloudera.flume.handlers.debug.MemorySinkSource
This is for throughput testing. Here we put a set of events into memory and then stream them through the system at max rate. It is both a sink and a source. The idea is to use it as a sink where everything is streamed in and then use it as a source for some downstream sink. This will let us determine what network and disk bandwidth limiters are.

    assertEquals("", Attributes.readString(e1, "outofrange"));
  }


  @Test
  public void testSplitExtractor() throws IOException, InterruptedException {
    MemorySinkSource mem = new MemorySinkSource();
    SplitExtractor re1 = new SplitExtractor(mem, "\\.", 1, "dot");
    SplitExtractor re2 = new SplitExtractor(re1, ":", 1, "colon");
    SplitExtractor re3 = new SplitExtractor(re2, "foobar", 1, "foobar");
    SplitExtractor re4 = new SplitExtractor(re3, "#", 1, "empty");
    SplitExtractor re5 = new SplitExtractor(re4, "foobar", 2, "outofrange");
    SplitExtractor re = new SplitExtractor(re5, "\\.|:|foobar", 3, "disj");


    re.open();
    re.append(new EventImpl("1:2:3.4foobar5".getBytes()));
    re.close();


    mem.close();
    mem.open();
    Event e1 = mem.next();
    assertEquals("4foobar5", Attributes.readString(e1, "dot"));
    assertEquals("2", Attributes.readString(e1, "colon"));
    assertEquals("5", Attributes.readString(e1, "foobar"));
    assertEquals("4", Attributes.readString(e1, "disj"));
    assertEquals("", Attributes.readString(e1, "empty"));

View Full Code Here

   */
  @Test
  public void testAvroSend() throws IOException, InterruptedException {
    EventSource txt = new NoNlASCIISynthSource(25, 100);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();


    FlumeConfiguration conf = FlumeConfiguration.get();
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort() + 1);
    tes.open();


    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (Exception e) {
        }
      }
    };
    t.start(); // drain the sink.


    // mem -> AvroEventSink
    AvroEventSink snk = new AvroEventSink("0.0.0.0",
        conf.getCollectorPort() + 1);
    snk.open();
    EventUtil.dumpAll(mem, snk);
    mem.close();
    snk.close();


    // a little delay to drain events at AvroEventSource queue
    try {
      Thread.sleep(1000);

View Full Code Here

      Thread th = new Thread() {
        public void run() {
          try {
            EventSource txt = new NoNlASCIISynthSource(25, 100);
            txt.open();
            MemorySinkSource mem = new MemorySinkSource();
            mem.open();
            EventUtil.dumpAll(txt, mem);
            txt.close();


            // mem -> AvroEventSink
            AvroEventSink snk = new AvroEventSink("0.0.0.0", conf
                .getCollectorPort() + 1);
            snk.open();


            sendStarted.countDown();
            sendStarted.await();
            EventUtil.dumpAll(mem, snk);
            mem.close();
            snk.close();


            sendByteSum.addAndGet(snk.sentBytes.get());
            LOG.info("sink " + id + " sent " + snk.sentBytes + " bytes");
            sendDone.countDown();

View Full Code Here

      FlumeSpecException, InterruptedException {
    File f = File.createTempFile("multitemp1", ".tmp");
    f.deleteOnExit();
    File f2 = File.createTempFile("multitemp2", ".tmp");
    f2.deleteOnExit();
    final MemorySinkSource snk = new MemorySinkSource();
    final EventSource src = TailSource.multiTailBuilder().build(
        f.getAbsolutePath(), f2.getAbsolutePath());
    final CountDownLatch done = new CountDownLatch(1);
    final int count = 60;
    Thread t = new Thread() {
      @Override
      public void run() {
        try {
          src.open();
          snk.open();
          EventUtil.dumpN(count, src, snk);
          src.close();
          snk.close();
          done.countDown();
        } catch (IOException e) {
          e.printStackTrace();
        } catch (InterruptedException e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    t.start();


    int log1 = 0, log2 = 0;
    FileWriter fw = new FileWriter(f);
    FileWriter fw2 = new FileWriter(f2);
    for (int i = 0; i < count; i++) {
      if (Math.random() > 0.5) {
        fw.append("Line " + i + "\n");
        fw.flush();
        log1++;
      } else {
        fw2.append("Line " + i + "\n");
        fw2.flush();
        log2++;
      }
    }
    fw.close();
    fw2.close();


    assertTrue(done.await(15, TimeUnit.SECONDS));


    Event e = null;
    while ((e = snk.next()) != null) {
      byte[] fn = e.get(TailSource.A_TAILSRCFILE);
      String sfn = new String(fn);
      if (!sfn.equals(f.getName()) && !sfn.equals(f2.getName())) {
        Assert.fail("didn't have tail src file metadata! " + sfn + " != "
            + f.getName() + " or " + f2.getName());

View Full Code Here


  @Test
  public void testWrite() throws IOException, InterruptedException {
    Benchmark b = new Benchmark("seqfile write");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("receiver_started");


    EventUtil.dumpAll(mem, sink);


    b.mark("seqfile_disk_write");


    sink.close();
    b.mark("seqfile size", tmp.length());
    b.done();
    mem = null; // allow mem to be freed.


    // //////// second phase using the file written in previous phase.
    Benchmark b2 = new Benchmark("seqfile_disk_read");
    b2.mark("begin");


    SeqfileEventSource seq = new SeqfileEventSource(tmp.getAbsolutePath());
    seq.open();
    MemorySinkSource mem2 = new MemorySinkSource();
    EventUtil.dumpAll(seq, mem2);
    seq.close();
    b2.mark("seqfile_loaded");


    b2.done();

View Full Code Here

  @Test
  public void testAvroSend() throws IOException, InterruptedException {


    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort());
    tes.open();

View Full Code Here

  @Test
  public void testAvroBatchSend10() throws IOException, InterruptedException {


    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort());
    tes.open();

View Full Code Here

  @Test
  public void testAvroBatchSend100() throws IOException, InterruptedException {


    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort());
    tes.open();

View Full Code Here

  public void testAvroSendMulti() throws IOException, InterruptedException {


    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");


    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort());
    tes.open();

View Full Code Here

  public void testAvroRawSend() throws IOException, InterruptedException {


    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");


    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort());
    tes.open();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.handlers.debug.MemorySinkSource

com.cloudera.flume.agent.durability.NaiveFileWALManager

com.cloudera.flume.agent.TestAckedWALDecorator

com.cloudera.flume.collector.TestCollectorSink

com.cloudera.flume.core.Event

com.cloudera.flume.core.EventImpl

com.cloudera.flume.core.extractors.TestExtractors

com.cloudera.flume.core.TestDigestDecorator

com.cloudera.flume.core.TestFormatterDecorator

com.cloudera.flume.core.TestSelectMaskDecos

com.cloudera.flume.FlumeBenchmarkHarness

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.