Examples of SeqfileEventSink

Examples of com.cloudera.flume.handlers.hdfs.SeqfileEventSink

238
239
240
241
242
243
244
245
246
247
248
   */
  synchronized public EventSink newWritingSink(final Tagger tagger)
      throws IOException {
    File dir = getDir(State.WRITING);
    final String tag = tagger.newTag();
    EventSink curSink = new SeqfileEventSink(new File(dir, tag)
        .getAbsoluteFile());
    writingQ.add(tag);
    DFOData data = new DFOData(tag);
    table.put(tag, data);
    writingCount.incrementAndGet();
View Full Code Here

Examples of com.cloudera.flume.handlers.hdfs.SeqfileEventSink

255
256
257
258
259
260
261
262
263
264
265
  synchronized public EventSink newAckWritingSink(final Tagger tagger,
      AckListener al) throws IOException {
    File dir = getDir(State.WRITING);
    final String tag = tagger.newTag();

    EventSink bareSink = new SeqfileEventSink(new File(dir, tag)
        .getAbsoluteFile());
    EventSink curSink = new AckChecksumInjector<EventSink>(bareSink, tag
        .getBytes(), al);

    writingQ.add(tag);
View Full Code Here

Examples of com.cloudera.flume.handlers.hdfs.SeqfileEventSink

295
296
297
298
299
300
301
302
303
304
305
   */
  synchronized public EventSink newWritingSink(final Tagger tagger)
      throws IOException {
    File dir = getDir(State.WRITING);
    final String tag = tagger.newTag();
    EventSink curSink = new SeqfileEventSink(new File(dir, tag)
        .getAbsoluteFile());
    writingQ.add(tag);
    WALData data = new WALData(tag);
    table.put(tag, data);

View Full Code Here

Examples of com.cloudera.flume.handlers.hdfs.SeqfileEventSink

49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
    txt.close();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("receiver_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("seqfile_disk_write");

    sink.close();
    b.mark("seqfile size", tmp.length());
    b.done();

    // //////// second phase using the file written in previous phase.
    Benchmark b2 = new Benchmark("seqfile_disk_read");
View Full Code Here

Examples of com.cloudera.flume.handlers.hdfs.SeqfileEventSink

52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
    txt.close();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("localdisk_write_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("local_disk_write done");

    sink.close();

    FlumeConfiguration conf = FlumeConfiguration.get();
    Path src = new Path(tmp.getAbsolutePath());
    Path dst = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = dst.getFileSystem(conf);
View Full Code Here

Examples of com.cloudera.flume.handlers.hdfs.SeqfileEventSink

48
49
50
51
52
53
54
55
56
57
58
59
60
    BenchmarkHarness.setupLocalWriteDir();
    File tmp = BenchmarkHarness.tmpdir;

    for (int i = 0; i < 3000; i++) {
      File path = new File(tmp, "" + i);
      EventSink snk = new SeqfileEventSink(path);
      snk.open();
      Event e = new EventImpl(("foo " + i).getBytes());
      snk.append(e);
      snk.close();
    }

  }
View Full Code Here

Examples of com.cloudera.flume.handlers.hdfs.SeqfileEventSink

39
40
41
42
43
44
45
46
    super();
    path = null;
  }

  public SeqfileAppender(File path) throws IOException {
    super(new SeqfileEventSink(path));
    this.path = path.getCanonicalPath();
  }
View Full Code Here

Examples of com.cloudera.flume.handlers.hdfs.SeqfileEventSink

45
46
47
48
49
50
51
52
53
54
55
56
    this.path = path.getCanonicalPath();
  }

  public void activateOptions() {
    try {
      SeqfileEventSink s = new SeqfileEventSink(new File(path));
      s.setBufferIO(bufferedIO);
      setEventSink(s);
    } catch (IOException e) {
      e.printStackTrace();
      // this will fail, but allow the program to continue.
    }
View Full Code Here

Examples of com.cloudera.flume.handlers.hdfs.SeqfileEventSink

41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("receiver_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("seqfile_disk_write");

    sink.close();
    b.mark("seqfile size", tmp.length());
    b.done();
    mem = null; // allow mem to be freed.

    // //////// second phase using the file written in previous phase.
View Full Code Here

Examples of com.cloudera.flume.handlers.hdfs.SeqfileEventSink

46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("localdisk_write_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("local_disk_write done");

    sink.close();

    FlumeConfiguration conf = FlumeConfiguration.get();
    Path src = new Path(tmp.getAbsolutePath());
    Path dst = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = dst.getFileSystem(conf);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.