Package com.cloudera.util

Examples of com.cloudera.util.Benchmark


   * @throws FlumeSpecException
   * @throws InterruptedException
   */
  @Test
  public void testTooManyOpens() throws IOException, FlumeSpecException, InterruptedException {
    Benchmark b = new Benchmark("connection exhaust");

    EventSource src = FlumeBuilder.buildSource(LogicalNodeContext.testingContext(),"thrift(31337)");

    EventSink snk = FlumeBuilder.buildSink("thrift(\"0.0.0.0\",31337)");

    src.open();

    // iterate until an exception is thrown
    int i = 0;
    try {
      for (i = 0; true; i++) {

        snk.open();
        System.out.println(i + " connections...");
      }
    } catch (IOException io) {
      System.out.println(io);
      b.mark("conns", i);
    }
    src.close();
    b.done();
  }
View Full Code Here


  }

  @Test
  public void testSurviveManyOpens() throws IOException, FlumeSpecException,
      InterruptedException {
    Benchmark b = new Benchmark("connection exhaust");

    EventSource src = FlumeBuilder.buildSource(LogicalNodeContext.testingContext(), "thrift(31337)");
    src.open();

    // iterate until an exception is thrown
    for (int i = 0; i < 10000; i++) { // previous fails at 1000, make sure ok at
      // an order of magnitude bigger.

      EventSink snk = FlumeBuilder.buildSink(LogicalNodeContext.testingContext(),
          "thrift(\"0.0.0.0\",31337)");
      snk.open();
      System.out.println(i + " connections...");
      snk.close();
    }

    src.close();
    b.done();
  }
View Full Code Here

    String tag = new String(tagbytes, CharEncUtils.RAW);

    if (Arrays.equals(bench, BenchmarkInjectDecorator.BENCH_START)) {
      StringWriter out = new StringWriter();
      PrintWriter pw = new PrintWriter(out);
      Benchmark b = new Benchmark(tag, pw, pw);
      b.mark("benchmarkStart");
      benchmarks.put(tag, new Pair<Benchmark, StringWriter>(b, out));
    } else if (Arrays.equals(bench, BenchmarkInjectDecorator.BENCH_FIRST)) {
      Benchmark b = benchmarks.get(tag).getLeft();
      b.mark("benchmarkFirst");
    } else if (Arrays.equals(bench, BenchmarkInjectDecorator.BENCH_STOP)) {
      Benchmark b = benchmarks.get(tag).getLeft();
      b.mark("benchmarkDone");
      b.done();

      ReportEvent rpt = getMetrics();
      LOG.info(rpt.toText());
      reportSink.append(rpt);

    } else if (Arrays.equals(bench, BenchmarkInjectDecorator.BENCH_ERROR)) {
      Benchmark b = benchmarks.get(tag).getLeft();
      b.mark("benchmarkError");
      b.done();
      LOG.info(getMetrics().toText());

      ReportEvent rpt = getMetrics();
      LOG.info(rpt.toText());
      reportSink.append(rpt);
View Full Code Here

*/
public class PerfDiskIO implements ExamplePerfData {

  @Test
  public void testWrite() throws IOException, InterruptedException {
    Benchmark b = new Benchmark("seqfile write");
    b.mark("begin");

    TextFileSource txt = new TextFileSource(HADOOP_DATA[0]);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("receiver_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("seqfile_disk_write");

    sink.close();
    b.mark("seqfile size", tmp.length());
    b.done();

    // //////// second phase using the file written in previous phase.
    Benchmark b2 = new Benchmark("seqfile_disk_read");
    b2.mark("begin");

    SeqfileEventSource seq = new SeqfileEventSource(tmp.getAbsolutePath());
    seq.open();
    MemorySinkSource mem2 = new MemorySinkSource();
    EventUtil.dumpAll(seq, mem2);
    seq.close();
    b2.mark("seqfile_loaded");

    b2.done();
  }
View Full Code Here

    b2.done();
  }

  @Test
  public void testReadFormat() throws IOException, InterruptedException {
    Benchmark b = new Benchmark("log4j format read");
    b.mark("begin");

    Log4jTextFileSource txt = new Log4jTextFileSource(HADOOP_DATA[0]);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();
    b.mark("log4j_disk_loaded");
    b.done();
  }
View Full Code Here

public class PerfHdfsIO implements ExamplePerfData {

  @Test
  public void testCopy() throws IOException, InterruptedException {

    Benchmark b = new Benchmark("hdfs seqfile copy");
    b.mark("begin");

    TextFileSource txt = new TextFileSource(HADOOP_DATA[0]);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("localdisk_write_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("local_disk_write done");

    sink.close();

    FlumeConfiguration conf = FlumeConfiguration.get();
    Path src = new Path(tmp.getAbsolutePath());
    Path dst = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = dst.getFileSystem(conf);
    hdfs.deleteOnExit(dst);

    b.mark("hdfs_copy_started");
    hdfs.copyFromLocalFile(src, dst);
    b.mark("hdfs_copy_done");
    hdfs.close();
    b.done();
  }
View Full Code Here

  }

  @Test
  public void testDirectWrite() throws IOException, InterruptedException {

    Benchmark b = new Benchmark("hdfs seqfile write");
    b.mark("begin");

    TextFileSource txt = new TextFileSource(HADOOP_DATA[0]);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();
    b.mark("disk_loaded");

    FlumeConfiguration conf = FlumeConfiguration.get();
    Path path = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = path.getFileSystem(conf);
    hdfs.deleteOnExit(path);

    Writer w = SequenceFile.createWriter(hdfs, conf, path,
        WriteableEventKey.class, WriteableEvent.class);
    b.mark("hdfs_fileopen_started");

    Event e = null;
    while ((e = mem.next()) != null) {
      // writing
      w.append(new WriteableEventKey(e), new WriteableEvent(e));
    }
    w.close();
    b.mark("seqfile_hdfs_write");

    hdfs.close();
    b.done();
  }
View Full Code Here

public class PerfSyslogFormats implements ExamplePerfData {

  @Test
  public void testSyslogFormat() throws IOException, EventExtractException,
      InterruptedException {
    Benchmark b = new Benchmark("Syslog format + nullsink");
    b.mark("begin");
    TextFileSource txt = new TextFileSource(SYSLOG_LOG); // 23244 entires
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();

    b.mark("disk_loaded");
    Event e = null;
    NullSink sink = new NullSink();
    SyslogEntryFormat syslog = new SyslogEntryFormat();
    while ((e = mem.next()) != null) {
      Event e2 = syslog.extract(new String(e.getBody()), 2009);
      sink.append(e2);
    }
    sink.close();
    b.mark("warmup done");

    e = null;
    mem.open();
    while ((e = mem.next()) != null) {
      Event e2 = syslog.extract(new String(e.getBody()), 2009);
      sink.append(e2);
    }
    sink.close();
    b.mark("sample dump done");

    e = null;
    mem.open();
    CounterSink sink2 = new CounterSink("counter");

    while ((e = mem.next()) != null) {
      Event e2 = syslog.extract(new String(e.getBody()), 2009);
      sink2.append(e2);
    }
    sink2.close();
    b.mark("count done", sink2.getCount());

    b.done();
  }
View Full Code Here

   *
   * 1M x 100 bytes, 5 times
   */
  @Test
  public void testNewExtractScan100() throws IOException, EventExtractException {
    Benchmark b = new Benchmark("new extract - scan 100 blocks");

    b.mark("build dataset");
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    // 1M x 100 byte messages, 0 is the rand seed
    NoNlSynthSource src = new NoNlSynthSource(1000000, 100, 0);

    src.open();
    Event e = null;
    while ((e = src.next()) != null) {
      out.write("<33>".getBytes());
      out.write(e.getBody());
      out.write('\n');
    }

    b.mark("start parsing dataset");
    int good = 0;
    int bad = 0;
    int lines = 0;

    // We do this test 100 times!
    for (int i = 0; i < 5; i++) {
      DataInputStream in = new DataInputStream(new ByteArrayInputStream(out
          .toByteArray()));
      lines++;
      try {
        byte[] data = new byte[100];
        while (true)
          in.readFully(data);
      } catch (EOFException eof) {
        // expected.
      }

    }
    b.mark("complete-good-bad", good, bad, lines);
    b.done();
  }
View Full Code Here

  String base = "this is a test";

  @Test
  public void testStringToByteArray() {
    Benchmark b = new Benchmark("String to ByteArray");

    b.mark("begin");
    for (int i = 0; i < times; i++) {
      base.getBytes();
    }
    b.mark("after " + times);
    b.done();
  }
View Full Code Here

TOP

Related Classes of com.cloudera.util.Benchmark

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.