Package org.apache.flume.instrumentation

Examples of org.apache.flume.instrumentation.SinkCounter

438
439
440
441
442
443
444
445
446
447
448
      numberOfClosesRequired);
    BucketWriter bucketWriter = new BucketWriter(0, 0, 1, 1, ctx,
      hdfsPath, hdfsPath, "singleBucket", ".tmp", null, null,
      null, new MockDataStream(mockFs),
      timedRollerPool, null,
      new SinkCounter(
        "test-bucket-writer-" + System.currentTimeMillis()),
      0, null, null, 30000, Executors.newSingleThreadExecutor(), 1,
      numberOfClosesRequired);

    bucketWriter.setFileSystem(mockFs);
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
    int maxEvents = 100;
    MockHDFSWriter hdfsWriter = new MockHDFSWriter();
    BucketWriter bucketWriter = new BucketWriter(0, 0, maxEvents, 0, ctx,
        "/tmp", "file", "", ".tmp", null, null, SequenceFile.CompressionType.NONE,
        hdfsWriter, timedRollerPool, null,
        new SinkCounter("test-bucket-writer-" + System.currentTimeMillis()), 0,
        null, null, 30000, Executors.newSingleThreadExecutor(), 0, 0);

    Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
    for (int i = 0; i < 1000; i++) {
      bucketWriter.append(e);
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
    int maxBytes = 300;
    MockHDFSWriter hdfsWriter = new MockHDFSWriter();
    BucketWriter bucketWriter = new BucketWriter(0, maxBytes, 0, 0,
      ctx, "/tmp", "file", "", ".tmp", null, null,
      SequenceFile.CompressionType.NONE, hdfsWriter,timedRollerPool,
      null, new SinkCounter("test-bucket-writer-" +
      System.currentTimeMillis()),0, null, null, 30000,
      Executors.newSingleThreadExecutor(), 0, 0);

    Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
    for (int i = 0; i < 1000; i++) {
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
    MockHDFSWriter hdfsWriter = new MockHDFSWriter();
    BucketWriter bucketWriter = new BucketWriter(ROLL_INTERVAL, 0, 0, 0, ctx,
      "/tmp", "file", "", ".tmp", null, null, SequenceFile.CompressionType.NONE,
      hdfsWriter, timedRollerPool, null,
      new SinkCounter("test-bucket-writer-" + System.currentTimeMillis()),
      0, new HDFSEventSink.WriterCallback() {
      @Override
      public void run(String filePath) {
        calledBack.set(true);
      }
    }, null, 30000, Executors.newSingleThreadExecutor(), 0, 0);

    Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
    long startNanos = System.nanoTime();
    for (int i = 0; i < NUM_EVENTS - 1; i++) {
      bucketWriter.append(e);
    }

    // sleep to force a roll... wait 2x interval just to be sure
    Thread.sleep(2 * ROLL_INTERVAL * 1000L);

    Assert.assertTrue(bucketWriter.closed);
    Assert.assertTrue(calledBack.get());

    bucketWriter = new BucketWriter(ROLL_INTERVAL, 0, 0, 0, ctx,
      "/tmp", "file", "", ".tmp", null, null, SequenceFile.CompressionType.NONE,
      hdfsWriter, timedRollerPool, null,
      new SinkCounter("test-bucket-writer-"
        + System.currentTimeMillis()), 0, null, null, 30000,
      Executors.newSingleThreadExecutor(), 0, 0);
    // write one more event (to reopen a new file so we will roll again later)
    bucketWriter.append(e);

View Full Code Here
228
229
230
231
232
233
234
235
236
237
238
    String name = tmpFile.getName();

    BucketWriter bucketWriter = new BucketWriter(ROLL_INTERVAL, 0, 0,
      0, ctx, path, name, "", ".tmp", null, null,
      SequenceFile.CompressionType.NONE, hdfsWriter,
      timedRollerPool, null, new SinkCounter("test-bucket-writer-"
      + System.currentTimeMillis()),
      0, null, null, 30000, Executors.newSingleThreadExecutor(),
      0, 0);

    Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
View Full Code Here
253
254
255
256
257
258
259
260
261
262
263
      MockHDFSWriter hdfsWriter = new MockHDFSWriter();
      BucketWriter bucketWriter = new BucketWriter(ROLL_INTERVAL, 0,
        0, 0, ctx, "/tmp", "file", "", ".tmp", suffix, null,
        SequenceFile.CompressionType.NONE, hdfsWriter,
        timedRollerPool, null, new SinkCounter("test-bucket-writer-"
        + System.currentTimeMillis()), 0, null, null, 30000,
        Executors.newSingleThreadExecutor(), 0, 0);

      // Need to override system time use for test so we know what to expect
      final long testTime = System.currentTimeMillis();
View Full Code Here
281
282
283
284
285
286
287
288
289
290
291
      MockHDFSWriter hdfsWriter = new MockHDFSWriter();
      BucketWriter bucketWriter = new BucketWriter(ROLL_INTERVAL, 0,
        0, 0, ctx, "/tmp", "file", "", ".tmp", suffix, null,
        SequenceFile.CompressionType.NONE, hdfsWriter,
        timedRollerPool, null, new SinkCounter(
        "test-bucket-writer-" + System.currentTimeMillis()), 0,
        null, null, 30000, Executors.newSingleThreadExecutor(), 0, 0);

        // Need to override system time use for test so we know what to expect
View Full Code Here
314
315
316
317
318
319
320
321
322
323
324
    MockHDFSWriter hdfsWriter = new MockHDFSWriter();
    BucketWriter bucketWriter = new BucketWriter(ROLL_INTERVAL, 0, 0,
      0, ctx, "/tmp", "file", "", ".tmp", suffix,
      HDFSEventSink.getCodec("gzip"),
      SequenceFile.CompressionType.BLOCK, hdfsWriter,
      timedRollerPool, null, new SinkCounter("test-bucket-writer-"
      + System.currentTimeMillis()), 0, null, null, 30000,
      Executors.newSingleThreadExecutor(), 0, 0
    );

    // Need to override system time use for test so we know what to expect
View Full Code Here
64
65
66
67
68
69
70
71
    this.context = context;
    maxBatchSize = context.getInteger(BATCH_SIZE, maxBatchSize);
    maxBatchDurationMillis = context.getLong(BATCH_DURATION_MILLIS, maxBatchDurationMillis);
    handlerClass = context.getString(HANDLER_CLASS, MorphlineHandlerImpl.class.getName());   
    if (sinkCounter == null) {
      sinkCounter = new SinkCounter(getName());
    }
  }
View Full Code Here
303
304
305
306
307
308
309
310
    if(useLocalTime) {
      clock = new SystemClock();
    }

    if (sinkCounter == null) {
      sinkCounter = new SinkCounter(getName());
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.flume.instrumentation.SinkCounter

  • com.github.kristofa.flume.ZipkinMetricsSink
  • com.github.kristofa.flume.ZipkinSpanCollectorSink
  • com.salesforce.phoenix.flume.sink.PhoenixSink
  • org.apache.flume.sink.AbstractRpcSink
  • org.apache.flume.sink.AvroSink
  • org.apache.flume.sink.elasticsearch.ElasticSearchSink
  • org.apache.flume.sink.hbase.AsyncHBaseSink
  • org.apache.flume.sink.hbase.HBaseSink
  • org.apache.flume.sink.hdfs.HDFSEventSink
  • org.apache.flume.sink.hdfs.TestBucketWriter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.