Package com.cloudera.util

Examples of com.cloudera.util.Benchmark

34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
*/
public class PerfReportSinks {

  @Test
  public void testNullSink() throws IOException, InterruptedException {
    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    EventSink nullsnk = new NullSink();
    EventUtil.dumpAll(mem, nullsnk);
    b.mark("nullsink done");

    b.done();
  }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
    b.done();
  }

  @Test
  public void testCountSink() throws IOException, InterruptedException {
    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    CounterSink snk = new CounterSink("counter");
    EventUtil.dumpAll(mem, snk);
    b.mark(snk.getName() + " done", snk.getCount());

    b.done();
  }
View Full Code Here
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
   * @throws InterruptedException
   */
  @Test
  public void testAvroRawSend() throws IOException, InterruptedException {

    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");

    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    FlumeConfiguration conf = FlumeConfiguration.get();
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");

    final AvroEventSink snk = new AvroEventSink("0.0.0.0",
        conf.getCollectorPort());
    snk.open();
    b.mark("sink_started");

    EventUtil.dumpAll(mem, snk);
    b.mark("Avro sink to Avro source done");
    // MB/s = B/us
    b.mark("MB/s", (double) snk.getSentBytes()
        / (double) (b.getLastDelta() / 1000));

    tes.close();
    snk.close();
    drain.interrupt();
    b.done();
  }
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
public class PerfHdfsIO {

  @Test
  public void testCopy() throws IOException, InterruptedException {

    Benchmark b = new Benchmark("hdfs seqfile copy");
    b.mark("begin");

    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("localdisk_write_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("local_disk_write done");

    sink.close();

    FlumeConfiguration conf = FlumeConfiguration.get();
    Path src = new Path(tmp.getAbsolutePath());
    Path dst = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = dst.getFileSystem(conf);
    hdfs.deleteOnExit(dst);

    b.mark("hdfs_copy_started");
    hdfs.copyFromLocalFile(src, dst);
    b.mark("hdfs_copy_done");
    hdfs.close();
    b.done();
  }
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
  }

  @Test
  public void testDirectWrite() throws IOException, InterruptedException {

    Benchmark b = new Benchmark("hdfs seqfile write");
    b.mark("begin");

    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    FlumeConfiguration conf = FlumeConfiguration.get();
    Path path = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = path.getFileSystem(conf);
    hdfs.deleteOnExit(path);

    Writer w = SequenceFile.createWriter(hdfs, conf, path,
        WriteableEventKey.class, WriteableEvent.class);
    b.mark("hdfs_fileopen_started");

    Event e = null;
    while ((e = mem.next()) != null) {
      // writing
      w.append(new WriteableEventKey(e), new WriteableEvent(e));
    }
    w.close();
    b.mark("seqfile_hdfs_write");

    hdfs.close();
    b.done();
  }
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
   *
   * @throws FlumeSpecException
   */
  @Test
  public void testTooManyOpens() throws IOException, FlumeSpecException {
    Benchmark b = new Benchmark("connection exhaust");

    EventSource src = FlumeBuilder.buildSource("thrift(31337)");

    EventSink snk = FlumeBuilder.buildSink("thrift(\"0.0.0.0\",31337)");

    src.open();

    // iterate until an exception is thrown
    int i = 0;
    try {
      for (i = 0; true; i++) {

        snk.open();
        System.out.println(i + " connections...");
      }
    } catch (IOException io) {
      System.out.println(io);
      b.mark("conns", i);
    }
    src.close();
    b.done();
  }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
    b.done();
  }

  @Test
  public void testSurviveManyOpens() throws IOException, FlumeSpecException {
    Benchmark b = new Benchmark("connection exhaust");

    EventSource src = FlumeBuilder.buildSource("thrift(31337)");
    src.open();

    // iterate until an exception is thrown
    for (int i = 0; i < 10000; i++) { // previous fails at 1000, make sure ok at
      // an order of magnitude bigger.

      EventSink snk = FlumeBuilder.buildSink("thrift(\"0.0.0.0\",31337)");
      snk.open();
      System.out.println(i + " connections...");
      snk.close();
    }

    src.close();
    b.done();
  }
View Full Code Here
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
*/
public class PerfSamplers implements ExamplePerfData {

  @Test
  public void testReservoirSampler() throws IOException {
    Benchmark b = new Benchmark("Reservoir sampler + nullsink");
    b.mark("begin");
    TextFileSource txt = new TextFileSource(HADOOP_DATA[0]);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    b.mark("disk_loaded");

    NullSink nullsnk = new NullSink();
    EventUtil.dumpAll(mem, nullsnk);
    b.mark("warmup");

    int res_size[] = { 100, 1000, 10000, 100000 };
    for (int i = 0; i < res_size.length; i++) {
      mem.open();
      int sz = res_size[i];
      EventSink res = new ReservoirSamplerDeco<NullSink>(new NullSink(), sz);
      EventUtil.dumpAll(mem, res);
      b.mark("reservoir " + sz + " sampling done", sz);

      res.close();
      b.mark("sample dump done");
    }

    for (int i = 0; i < res_size.length; i++) {
      mem.open();
      int sz = res_size[i];
      CounterSink cnt = new CounterSink("null");
      EventSink res = new ReservoirSamplerDeco<CounterSink>(cnt, sz);
      EventUtil.dumpAll(mem, res);
      b.mark("reservoir", sz);

      res.close();
      b.mark("count", cnt.getCount());
    }
    b.done();
  }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
    b.done();
  }

  @Test
  public void testIntervalSampler() throws IOException {
    Benchmark b = new Benchmark("Interval sampler + nullsink");
    b.mark("begin");
    TextFileSource txt = new TextFileSource(HADOOP_DATA[0]);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    b.mark("disk_loaded");

    NullSink nullsnk = new NullSink();
    EventUtil.dumpAll(mem, nullsnk);
    b.mark("warmup");

    int interval_size[] = { 100000, 10000, 1000, 100 };
    for (int i = 0; i < interval_size.length; i++) {
      mem.open();
      int sz = interval_size[i];
      EventSink res = new IntervalSampler<NullSink>(new NullSink(), sz);
      EventUtil.dumpAll(mem, res);
      b.mark("interval " + sz + " sampling done", sz);

      res.close();
      b.mark("sample dump done");
    }

    for (int i = 0; i < interval_size.length; i++) {
      mem.open();
      int sz = interval_size[i];
      CounterSink cnt = new CounterSink("null");
      EventSink res = new IntervalSampler<CounterSink>(cnt, sz);
      EventUtil.dumpAll(mem, res);
      b.mark("interval", sz);

      res.close();
      b.mark("count", cnt.getCount());
    }
    b.done();
  }
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
    b.done();
  }

  @Test
  public void testProbabilitySampler() throws IOException {
    Benchmark b = new Benchmark("Reservoir sampler + nullsink");
    b.mark("begin");
    TextFileSource txt = new TextFileSource(HADOOP_DATA[0]);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);

    b.mark("disk_loaded");

    NullSink nullsnk = new NullSink();
    EventUtil.dumpAll(mem, nullsnk);
    b.mark("warmup");

    double probs[] = { .00001, .0001, .001, .01 };
    for (int i = 0; i < probs.length; i++) {
      mem.open();
      double prob = probs[i];
      EventSink res = new ProbabilitySampler<NullSink>(new NullSink(), prob);
      EventUtil.dumpAll(mem, res);
      b.mark("probability" + prob + " sampling done", prob);

      res.close();
      b.mark("sample dump done");
    }

    for (int i = 0; i < probs.length; i++) {
      mem.open();
      double prob = probs[i];
      CounterSink cnt = new CounterSink("null");
      EventSink res = new ProbabilitySampler<CounterSink>(cnt, prob);
      EventUtil.dumpAll(mem, res);
      b.mark("probability", prob);

      res.close();
      b.mark("count", cnt.getCount());
    }
    b.done();
  }
View Full Code Here
TOP

Related Classes of com.cloudera.util.Benchmark

  • com.cloudera.flume.handlers.debug.BenchmarkReportDecorator
  • com.cloudera.flume.handlers.debug.TextToCollector
  • com.cloudera.flume.handlers.syslog.PerfSyslogWireExtract
  • com.cloudera.flume.master.EvilExhaustConnections
  • com.cloudera.flume.PerfAvroSinks
  • com.cloudera.flume.PerfDiskIO
  • com.cloudera.flume.PerfGrepReportSinks
  • com.cloudera.flume.PerfHdfsIO
  • com.cloudera.flume.PerfReportSinks
  • com.cloudera.flume.PerfSamplers
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.