Package com.cloudera.flume.core

Examples of com.cloudera.flume.core.EventSource$Base

196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
    assertEquals(11, (long) rpt.getLongMetric(BatchingDecorator.R_TRIGGERS));
  }

  @Test
  public void testBatchingMetrics() throws IOException, InterruptedException {
    EventSource src = MemorySinkSource.cannedData("this is a data", 200);
    MemorySinkSource mem = new MemorySinkSource();
    UnbatchingDecorator<EventSink> unbatch = new UnbatchingDecorator<EventSink>(
        mem);
    GunzipDecorator<EventSink> gunzip = new GunzipDecorator<EventSink>(unbatch);

    int evts = 5;
    int latency = 0; // 0 never triggers on time
    GzipDecorator<EventSink> gzip = new GzipDecorator<EventSink>(gunzip);
    BatchingDecorator<EventSink> batch = new BatchingDecorator<EventSink>(gzip,
        evts, latency);

    src.open();
    batch.open();
    EventUtil.dumpAll(src, batch);
    src.close();
    batch.close();

    // check metrics.
    ReportEvent ubRpt = unbatch.getMetrics();
    assertEquals(40, (long) ubRpt
View Full Code Here
555
556
557
558
559
560
561
562
    LOG.info("opening log file  " + sendingTag);
    changeState(sendingTag, State.LOGGED, State.SENDING);
    sendingCount.incrementAndGet();
    File curFile = getFile(sendingTag);
    EventSource curSource = new SeqfileEventSource(curFile.getAbsolutePath());
    return new StateChangeDeco(curSource, sendingTag);
  }
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
*/
public class TestFileSource implements ExampleData {

  @Test
  public void testFileSource() throws IOException, InterruptedException {
    EventSource src = new TextFileSource(TEST);
    src.open();
    int count = 0;
    while (true) {
      Event e = src.next();
      if (e == null)
        break;
      count++;
      System.out.println(e);
    }
    System.out.println("read " + count + " lines");
    Assert.assertEquals(56, count);
    src.close();
  }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
    // "{ benchinject => { benchreport(\"pre\") =>  { diskFailover => [console, counter(\"beforecount\")] } } }";
    String spec = "{ benchinject => { benchreport(\"pre\") =>  { diskFailover => counter(\"beforecount\") } } }";
    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(
        LogicalNodeContext.testingContext()), spec);

    EventSource src = MemorySinkSource.cannedData("test ", 5);
    snk.open();
    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close();

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
        "beforecount");
    Assert.assertEquals(5, cnt.getCount());
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
    BenchmarkHarness.setupLocalWriteDir();
    String spec = "{ benchinject => { diskFailover => { benchreport(\"post\") =>  counter(\"beforecount\") } } }";
    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(
        LogicalNodeContext.testingContext()), spec);

    EventSource src = MemorySinkSource.cannedData("test ", 5);
    snk.open();
    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close();

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
        "beforecount");
    Assert.assertEquals(5, cnt.getCount());
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
    BenchmarkHarness.setupLocalWriteDir();
    String spec = "{ benchinject => { benchreport(\"pre\") =>  { diskFailover => counter(\"beforecount\") } } }";
    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(
        LogicalNodeContext.testingContext()), spec);

    EventSource src = MemorySinkSource.cannedData("test ", 5);
    snk.open();
    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close();

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
        "beforecount");
    Assert.assertEquals(5, cnt.getCount());
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
    String spec = "{ benchinject => { ackedWriteAhead => { benchreport(\"post\") =>  counter(\"beforecount\") } } }";
    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(
        LogicalNodeContext.testingContext()), spec);

    EventSource src = MemorySinkSource.cannedData("test ", 5);
    snk.open();
    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close();

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable(
        "beforecount");
View Full Code Here
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
  @Test
  public void testBloomDecos() throws FlumeSpecException, IOException,
      InterruptedException {
    String spec = "{ bloomGen(10000,2) => { bloomCheck(10000,2) => counter(\"test\")} } ";
    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(), spec);
    EventSource src = FlumeBuilder.buildSource(LogicalNodeContext
        .testingContext(), "asciisynth(10000)");
    snk.open();
    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close();

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("test");
    assertEquals(ctr.getCount(), 10000);
View Full Code Here
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
    tooLarge.append("\n");

    FileOutputStream f = new FileOutputStream(tmp);
    f.write(tooLarge.toString().getBytes());
    f.close();
    EventSource source = new ExecNioSource.Builder().build("cat "
        + tmp.getCanonicalPath());
    source.open();
    Event e = source.next();
    assertNotNull(e);
    assertEquals(max, e.getBody().length); // check that event was truncated.
    source.close();
    FileUtil.rmr(tmpdir);
    // Check that the stdout reader closed correctly
    assertTrue(((ExecNioSource) source).readOut.signalDone.get());
  }
View Full Code Here
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
    tooLarge.append("\n");

    FileOutputStream f = new FileOutputStream(tmp);
    f.write(tooLarge.toString().getBytes());
    f.close();
    EventSource source = new ExecNioSource.Builder().build(
        "cat " + tmp.getCanonicalPath(), "true");
    source.open();
    Event e = source.next();
    assertNotNull(e);
    assertEquals(max, e.getBody().length); // check that event was truncated.
    source.close();
    FileUtil.rmr(tmpdir);
    // Check that the stdout reader closed correctly
    assertTrue(((ExecNioSource) source).readOut.signalDone.get());
  }
View Full Code Here
TOP

Related Classes of com.cloudera.flume.core.EventSource$Base

  • com.cloudera.flume.agent.diskfailover.NaiveFileFailoverManager
  • com.cloudera.flume.agent.diskfailover.TestConcurrentDFOMan
  • com.cloudera.flume.agent.diskfailover.TestDiskFailoverBenchmarking
  • com.cloudera.flume.agent.diskfailover.TestDiskFailoverDeco
  • com.cloudera.flume.agent.diskfailover.TestDiskFailoverManager
  • com.cloudera.flume.agent.diskfailover.TestDiskFailoverSource
  • com.cloudera.flume.agent.durability.NaiveFileWALManager
  • com.cloudera.flume.agent.durability.TestConcurrentWALMan
  • com.cloudera.flume.agent.durability.TestNaiveFileWALDeco
  • com.cloudera.flume.agent.durability.TestNaiveFileWALManager
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.