Examples of com.cloudera.util.Benchmark

com.cloudera.util.Benchmark
Simple class to consolidate benchmarking and timing code. This will help me find the bottlenecks. Usage: create new benchmark. call mark with comments whenever wanted. call done to flush print writer. std out is used for human readable results, stderr is used for csv formatted data.

   * mem -> batch(10) AvroEventSink -> AvroEventSource -> NullSink
   */
  @Test
  public void testAvroBatchSend100() throws IOException, InterruptedException {


    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final AvroEventSink tsnk = new AvroEventSink("0.0.0.0",
        conf.getCollectorPort());
    // make size happen first all the time.
    final BatchingDecorator snk = new BatchingDecorator(tsnk, 100, 10000000);
    snk.open();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);
    b.mark("Avro sink to Avro source done");
    // MB/s = B/us
    b.mark("MB/s", (double) tsnk.getSentBytes()
        / (double) (b.getLastDelta() / 1000));


    tes.close();
    snk.close();
    drain.interrupt();
    b.done();
  }

View Full Code Here

   * mem -> AvroEventSink -> AvroEventSource -> NullSink
   **/
  @Test
  public void testAvroSendMulti() throws IOException, InterruptedException {


    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");


    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final AvroEventSink snk = new AvroEventSink("0.0.0.0",
        conf.getCollectorPort());


    Thread t = new Thread() {
      public void run() {
        try {
          snk.open();
        } catch (IOException e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    t.start();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);
    b.mark("Avro sink to Avro source done");
    // MB/s = B/us
    b.mark("MB/s", (double) snk.getSentBytes()
        / (double) (b.getLastDelta() / 1000));


    Thread.sleep(1000);
    tes.close();
    snk.close();
    t.interrupt();
    drain.interrupt();
    b.done();
  }

View Full Code Here

   * mem -> ThriftEventSink -> ThriftEventSource -> NullSink
   */
  @Test
  public void testThriftSend() throws IOException, InterruptedException {


    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink snk = new ThriftEventSink("0.0.0.0",
        conf.getCollectorPort());
    snk.open();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);
    b.mark("thrift sink to thrift source done");
    // MB/s = B/us
    b.mark("MB/s", (double) snk.getSentBytes()
        / (double) (b.getLastDelta() / 1000));
    tes.close();
    snk.close();
    drain.interrupt();
    b.done();
  }

View Full Code Here

   * mem -> batch(10) ThriftEventSink -> ThriftEventSource -> NullSink
   */
  @Test
  public void testThriftBatchSend10() throws IOException, InterruptedException {


    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink tsnk = new ThriftEventSink("0.0.0.0",
        conf.getCollectorPort());
    // make size happen first all the time.
    final BatchingDecorator snk = new BatchingDecorator(tsnk, 10, 10000000);
    snk.open();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);
    b.mark("thrift sink to thrift source done");
    // MB/s = B/us
    b.mark("MB/s", (double) tsnk.getSentBytes()
        / (double) (b.getLastDelta() / 1000));


    tes.close();
    snk.close();
    drain.interrupt();
    b.done();
  }

View Full Code Here

   * mem -> batch(10) ThriftEventSink -> ThriftEventSource -> NullSink
   */
  @Test
  public void testThriftBatchSend100() throws IOException, InterruptedException {


    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink tsnk = new ThriftEventSink("0.0.0.0",
        conf.getCollectorPort());
    // make size happen first all the time.
    final BatchingDecorator snk = new BatchingDecorator(tsnk, 100, 10000000);
    snk.open();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);
    b.mark("thrift sink to thrift source done");
    // MB/s = B/us
    b.mark("MB/s", (double) tsnk.getSentBytes()
        / (double) (b.getLastDelta() / 1000));


    tes.close();
    snk.close();
    drain.interrupt();
    b.done();
  }

View Full Code Here

   * mem -> ThriftEventSink -> ThriftEventSource -> NullSink
   **/
  @Test
  public void testThriftSendMulti() throws IOException, InterruptedException {


    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");


    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink snk = new ThriftEventSink("0.0.0.0",
        conf.getCollectorPort());


    Thread t = new Thread() {
      public void run() {
        try {
          snk.open();
        } catch (IOException e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    t.start();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);
    b.mark("thrift sink to thrift source done");
    // MB/s = B/us
    b.mark("MB/s", (double) snk.getSentBytes()
        / (double) (b.getLastDelta() / 1000));


    Thread.sleep(1000);
    tes.close();
    snk.close();
    t.interrupt();
    drain.interrupt();
    b.done();
  }

View Full Code Here

   * @throws InterruptedException
   */
  @Test
  public void testThriftRawSend() throws IOException, InterruptedException {


    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");


    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink snk = new ThriftEventSink("0.0.0.0",
        conf.getCollectorPort());
    snk.open();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);
    b.mark("thrift sink to thrift source done");
    // MB/s = B/us
    b.mark("MB/s", (double) snk.getSentBytes()
        / (double) (b.getLastDelta() / 1000));


    tes.close();
    snk.close();
    drain.interrupt();
    b.done();
  }

View Full Code Here

 */
public class PerfSamplers {


  @Test
  public void testReservoirSampler() throws IOException, InterruptedException {
    Benchmark b = new Benchmark("Reservoir sampler + nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    NullSink nullsnk = new NullSink();
    EventUtil.dumpAll(mem, nullsnk);
    b.mark("warmup");


    int res_size[] = { 100, 1000, 10000, 100000 };
    for (int i = 0; i < res_size.length; i++) {
      mem.open();
      int sz = res_size[i];
      EventSink res = new ReservoirSamplerDeco<NullSink>(new NullSink(), sz);
      EventUtil.dumpAll(mem, res);
      b.mark("reservoir " + sz + " sampling done", sz);


      res.close();
      b.mark("sample dump done");
    }


    for (int i = 0; i < res_size.length; i++) {
      mem.open();
      int sz = res_size[i];
      CounterSink cnt = new CounterSink("null");
      EventSink res = new ReservoirSamplerDeco<CounterSink>(cnt, sz);
      EventUtil.dumpAll(mem, res);
      b.mark("reservoir", sz);


      res.close();
      b.mark("count", cnt.getCount());
    }
    b.done();
  }

View Full Code Here

    b.done();
  }


  @Test
  public void testIntervalSampler() throws IOException, InterruptedException {
    Benchmark b = new Benchmark("Interval sampler + nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    NullSink nullsnk = new NullSink();
    EventUtil.dumpAll(mem, nullsnk);
    b.mark("warmup");


    int interval_size[] = { 100000, 10000, 1000, 100 };
    for (int i = 0; i < interval_size.length; i++) {
      mem.open();
      int sz = interval_size[i];
      EventSink res = new IntervalSampler<NullSink>(new NullSink(), sz);
      EventUtil.dumpAll(mem, res);
      b.mark("interval " + sz + " sampling done", sz);


      res.close();
      b.mark("sample dump done");
    }


    for (int i = 0; i < interval_size.length; i++) {
      mem.open();
      int sz = interval_size[i];
      CounterSink cnt = new CounterSink("null");
      EventSink res = new IntervalSampler<CounterSink>(cnt, sz);
      EventUtil.dumpAll(mem, res);
      b.mark("interval", sz);


      res.close();
      b.mark("count", cnt.getCount());
    }
    b.done();
  }

View Full Code Here

    b.done();
  }


  @Test
  public void testProbabilitySampler() throws IOException, InterruptedException {
    Benchmark b = new Benchmark("Reservoir sampler + nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    NullSink nullsnk = new NullSink();
    EventUtil.dumpAll(mem, nullsnk);
    b.mark("warmup");


    double probs[] = { .00001, .0001, .001, .01 };
    for (int i = 0; i < probs.length; i++) {
      mem.open();
      double prob = probs[i];
      EventSink res = new ProbabilitySampler<NullSink>(new NullSink(), prob);
      EventUtil.dumpAll(mem, res);
      b.mark("probability" + prob + " sampling done", prob);


      res.close();
      b.mark("sample dump done");
    }


    for (int i = 0; i < probs.length; i++) {
      mem.open();
      double prob = probs[i];
      CounterSink cnt = new CounterSink("null");
      EventSink res = new ProbabilitySampler<CounterSink>(cnt, prob);
      EventUtil.dumpAll(mem, res);
      b.mark("probability", prob);


      res.close();
      b.mark("count", cnt.getCount());
    }
    b.done();
  }

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of com.cloudera.util.Benchmark

com.cloudera.flume.handlers.debug.BenchmarkReportDecorator

com.cloudera.flume.handlers.debug.TextToCollector

com.cloudera.flume.handlers.syslog.PerfSyslogWireExtract

com.cloudera.flume.master.EvilExhaustConnections

com.cloudera.flume.PerfAvroSinks

com.cloudera.flume.PerfDiskIO

com.cloudera.flume.PerfGrepReportSinks

com.cloudera.flume.PerfHdfsIO

com.cloudera.flume.PerfReportSinks

com.cloudera.flume.PerfSamplers

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.