Package com.cloudera.flume.handlers.hdfs

Examples of com.cloudera.flume.handlers.hdfs.SeqfileEventSink

238
239
240
241
242
243
244
245
246
247
248
   */
  synchronized public EventSink newWritingSink(final Tagger tagger)
      throws IOException {
    File dir = getDir(State.WRITING);
    final String tag = tagger.newTag();
    EventSink curSink = new SeqfileEventSink(new File(dir, tag)
        .getAbsoluteFile());
    writingQ.add(tag);
    DFOData data = new DFOData(tag);
    table.put(tag, data);
    writingCount.incrementAndGet();
View Full Code Here
255
256
257
258
259
260
261
262
263
264
265
  synchronized public EventSink newAckWritingSink(final Tagger tagger,
      AckListener al) throws IOException {
    File dir = getDir(State.WRITING);
    final String tag = tagger.newTag();

    EventSink bareSink = new SeqfileEventSink(new File(dir, tag)
        .getAbsoluteFile());
    EventSink curSink = new AckChecksumInjector<EventSink>(bareSink, tag
        .getBytes(), al);

    writingQ.add(tag);
View Full Code Here
295
296
297
298
299
300
301
302
303
304
305
   */
  synchronized public EventSink newWritingSink(final Tagger tagger)
      throws IOException {
    File dir = getDir(State.WRITING);
    final String tag = tagger.newTag();
    EventSink curSink = new SeqfileEventSink(new File(dir, tag)
        .getAbsoluteFile());
    writingQ.add(tag);
    WALData data = new WALData(tag);
    table.put(tag, data);

View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
    txt.close();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("receiver_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("seqfile_disk_write");

    sink.close();
    b.mark("seqfile size", tmp.length());
    b.done();

    // //////// second phase using the file written in previous phase.
    Benchmark b2 = new Benchmark("seqfile_disk_read");
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
    txt.close();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("localdisk_write_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("local_disk_write done");

    sink.close();

    FlumeConfiguration conf = FlumeConfiguration.get();
    Path src = new Path(tmp.getAbsolutePath());
    Path dst = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = dst.getFileSystem(conf);
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
60
    BenchmarkHarness.setupLocalWriteDir();
    File tmp = BenchmarkHarness.tmpdir;

    for (int i = 0; i < 3000; i++) {
      File path = new File(tmp, "" + i);
      EventSink snk = new SeqfileEventSink(path);
      snk.open();
      Event e = new EventImpl(("foo " + i).getBytes());
      snk.append(e);
      snk.close();
    }

  }
View Full Code Here
39
40
41
42
43
44
45
46
    super();
    path = null;
  }

  public SeqfileAppender(File path) throws IOException {
    super(new SeqfileEventSink(path));
    this.path = path.getCanonicalPath();
  }
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
56
    this.path = path.getCanonicalPath();
  }

  public void activateOptions() {
    try {
      SeqfileEventSink s = new SeqfileEventSink(new File(path));
      s.setBufferIO(bufferedIO);
      setEventSink(s);
    } catch (IOException e) {
      e.printStackTrace();
      // this will fail, but allow the program to continue.
    }
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("receiver_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("seqfile_disk_write");

    sink.close();
    b.mark("seqfile size", tmp.length());
    b.done();
    mem = null; // allow mem to be freed.

    // //////// second phase using the file written in previous phase.
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("localdisk_write_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("local_disk_write done");

    sink.close();

    FlumeConfiguration conf = FlumeConfiguration.get();
    Path src = new Path(tmp.getAbsolutePath());
    Path dst = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = dst.getFileSystem(conf);
View Full Code Here
TOP

Related Classes of com.cloudera.flume.handlers.hdfs.SeqfileEventSink

  • com.cloudera.flume.agent.diskfailover.NaiveFileFailoverManager
  • com.cloudera.flume.agent.durability.NaiveFileWALManager
  • com.cloudera.flume.agent.durability.TestNaiveFileWALHandles
  • com.cloudera.flume.agent.durability.TestNaiveFileWALManager
  • com.cloudera.flume.conf.FlumeConfiguration
  • com.cloudera.flume.conf.SinkFactory.SinkBuilder
  • com.cloudera.flume.handlers.log4j.SeqfileAppender
  • com.cloudera.flume.PerfDiskIO
  • com.cloudera.flume.PerfHdfsIO
  • org.apache.hadoop.conf.Configuration
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.