Package com.cloudera.flume.handlers.hdfs

Examples of com.cloudera.flume.handlers.hdfs.SeqfileEventSource

524
525
526
527
528
529
530
531
    LOG.info("opening new file for " + sendingTag);
    changeState(sendingTag, State.LOGGED, State.SENDING);
    sendingCount.incrementAndGet();
    File curFile = getFile(sendingTag);
    EventSource curSource = new SeqfileEventSource(curFile.getAbsolutePath());
    return new StateChangeDeco(curSource, sendingTag);
  }
View Full Code Here
555
556
557
558
559
560
561
562
    LOG.info("opening log file  " + sendingTag);
    changeState(sendingTag, State.LOGGED, State.SENDING);
    sendingCount.incrementAndGet();
    File curFile = getFile(sendingTag);
    EventSource curSource = new SeqfileEventSource(curFile.getAbsolutePath());
    return new StateChangeDeco(curSource, sendingTag);
  }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
76
77
78
    // //////// second phase using the file written in previous phase.
    Benchmark b2 = new Benchmark("seqfile_disk_read");
    b2.mark("begin");

    SeqfileEventSource seq = new SeqfileEventSource(tmp.getAbsolutePath());
    seq.open();
    MemorySinkSource mem2 = new MemorySinkSource();
    EventUtil.dumpAll(seq, mem2);
    seq.close();
    b2.mark("seqfile_loaded");

    b2.done();
  }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
80
81
82
    corrupt.deleteOnExit();

    // check now, and any age is too old.
    File commit = new File(tmpdir, "committed");
    commit.deleteOnExit();
    EventSource src = new SeqfileEventSource(corrupt.getAbsolutePath());

    try {
      src.open();
    } catch (IOException e) {
      return;
    }

    Assert.fail("should have failed with io exception");
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
87
    corrupt.deleteOnExit();

    // check now, and any age is too old.
    File commit = new File(tmpdir, "committed");
    commit.deleteOnExit();
    EventSource src = new SeqfileEventSource(corrupt.getAbsolutePath());

    try {
      src.open(); // expect IOException
    } catch (IOException e) {
      throw e;
    } finally {
      FileUtil.rmr(tmpdir);
    }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
    // //////// second phase using the file written in previous phase.
    Benchmark b2 = new Benchmark("seqfile_disk_read");
    b2.mark("begin");

    SeqfileEventSource seq = new SeqfileEventSource(tmp.getAbsolutePath());
    seq.open();
    MemorySinkSource mem2 = new MemorySinkSource();
    EventUtil.dumpAll(seq, mem2);
    seq.close();
    b2.mark("seqfile_loaded");

    b2.done();
  }
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
90
    corrupt.deleteOnExit();

    // check now, and any age is too old.
    File commit = new File(tmpdir, "committed");
    commit.deleteOnExit();
    EventSource src = new SeqfileEventSource(corrupt.getAbsolutePath());

    try {
      src.open(); // expect IOException
    } catch (IOException e) {
      throw e;
    } finally {
      FileUtil.rmr(tmpdir);
    }
View Full Code Here
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
   * buffer.
   */
  private AckFramingState checkAndStripAckFraming(final File dir,
      final String f, MemorySinkSource mem) throws InterruptedException,
      IOException {
    EventSource src = new SeqfileEventSource(new File(dir, f).getAbsolutePath());
    AckFramingState state = new AckFramingState();
    // strip previous ack tagged attributes out of events before putting raw
    // events.
    EventSink mask = new MaskDecorator<EventSink>(mem,
        AckChecksumInjector.ATTR_ACK_TYPE, AckChecksumInjector.ATTR_ACK_TAG,
        AckChecksumInjector.ATTR_ACK_HASH);
    // check for and extract the ack events.
    AckChecksumChecker<EventSink> check = new AckChecksumChecker<EventSink>(
        mask, state);

    try {
      // copy all raw events into mem buffer
      src.open();
      check.open();
      EventUtil.dumpAll(src, check);
      src.close();
      check.close();
    } catch (IOException e) {
      LOG.warn("Recovered log file {}  was corrupt", f);
      throw e;
    }
View Full Code Here
759
760
761
762
763
764
765
766
    }
    LOG.info("opening log file {}", sendingTag);
    changeState(sendingTag, State.LOGGED, State.SENDING);
    sendingCount.incrementAndGet();
    File curFile = getFile(sendingTag);
    EventSource curSource = new SeqfileEventSource(curFile.getAbsolutePath());
    return new StateChangeDeco(curSource, sendingTag);
  }
View Full Code Here
534
535
536
537
538
539
540
541
    LOG.info("opening new file for " + sendingTag);
    changeState(sendingTag, State.LOGGED, State.SENDING);
    sendingCount.incrementAndGet();
    File curFile = getFile(sendingTag);
    EventSource curSource = new SeqfileEventSource(curFile.getAbsolutePath());
    return new StateChangeDeco(curSource, sendingTag);
  }
View Full Code Here
TOP

Related Classes of com.cloudera.flume.handlers.hdfs.SeqfileEventSource

  • com.cloudera.flume.agent.diskfailover.NaiveFileFailoverManager
  • com.cloudera.flume.agent.diskfailover.TestDiskFailoverSource
  • com.cloudera.flume.agent.durability.NaiveFileWALManager
  • com.cloudera.flume.agent.durability.TestNaiveFileWALSource
  • com.cloudera.flume.conf.SourceFactory.SourceBuilder
  • com.cloudera.flume.core.Event
  • com.cloudera.flume.PerfDiskIO
  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.Path
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.