Package com.cloudera.flume.handlers.hdfs

Examples of com.cloudera.flume.handlers.hdfs.SeqfileEventSink


   */
  synchronized public EventSink newWritingSink(final Tagger tagger)
      throws IOException {
    File dir = getDir(State.WRITING);
    final String tag = tagger.newTag();
    EventSink curSink = new SeqfileEventSink(new File(dir, tag)
        .getAbsoluteFile());
    writingQ.add(tag);
    DFOData data = new DFOData(tag);
    table.put(tag, data);
    writingCount.incrementAndGet();
View Full Code Here


  synchronized public EventSink newAckWritingSink(final Tagger tagger,
      AckListener al) throws IOException {
    File dir = getDir(State.WRITING);
    final String tag = tagger.newTag();

    EventSink bareSink = new SeqfileEventSink(new File(dir, tag)
        .getAbsoluteFile());
    EventSink curSink = new AckChecksumInjector<EventSink>(bareSink, tag
        .getBytes(), al);

    writingQ.add(tag);
View Full Code Here

   */
  synchronized public EventSink newWritingSink(final Tagger tagger)
      throws IOException {
    File dir = getDir(State.WRITING);
    final String tag = tagger.newTag();
    EventSink curSink = new SeqfileEventSink(new File(dir, tag)
        .getAbsoluteFile());
    writingQ.add(tag);
    WALData data = new WALData(tag);
    table.put(tag, data);

View Full Code Here

    txt.close();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("receiver_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("seqfile_disk_write");

    sink.close();
    b.mark("seqfile size", tmp.length());
    b.done();

    // //////// second phase using the file written in previous phase.
    Benchmark b2 = new Benchmark("seqfile_disk_read");
View Full Code Here

    txt.close();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("localdisk_write_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("local_disk_write done");

    sink.close();

    FlumeConfiguration conf = FlumeConfiguration.get();
    Path src = new Path(tmp.getAbsolutePath());
    Path dst = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = dst.getFileSystem(conf);
View Full Code Here

    BenchmarkHarness.setupLocalWriteDir();
    File tmp = BenchmarkHarness.tmpdir;

    for (int i = 0; i < 3000; i++) {
      File path = new File(tmp, "" + i);
      EventSink snk = new SeqfileEventSink(path);
      snk.open();
      Event e = new EventImpl(("foo " + i).getBytes());
      snk.append(e);
      snk.close();
    }

  }
View Full Code Here

    super();
    path = null;
  }

  public SeqfileAppender(File path) throws IOException {
    super(new SeqfileEventSink(path));
    this.path = path.getCanonicalPath();
  }
View Full Code Here

    this.path = path.getCanonicalPath();
  }

  public void activateOptions() {
    try {
      SeqfileEventSink s = new SeqfileEventSink(new File(path));
      s.setBufferIO(bufferedIO);
      setEventSink(s);
    } catch (IOException e) {
      e.printStackTrace();
      // this will fail, but allow the program to continue.
    }
View Full Code Here

    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("receiver_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("seqfile_disk_write");

    sink.close();
    b.mark("seqfile size", tmp.length());
    b.done();
    mem = null; // allow mem to be freed.

    // //////// second phase using the file written in previous phase.
View Full Code Here

    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("localdisk_write_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("local_disk_write done");

    sink.close();

    FlumeConfiguration conf = FlumeConfiguration.get();
    Path src = new Path(tmp.getAbsolutePath());
    Path dst = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = dst.getFileSystem(conf);
View Full Code Here

TOP

Related Classes of com.cloudera.flume.handlers.hdfs.SeqfileEventSink

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.