Examples of com.cloudera.flume.handlers.rolling.RollSink

Package com.cloudera.flume.handlers.rolling

Examples of com.cloudera.flume.handlers.rolling.RollSink

com.cloudera.flume.handlers.rolling.RollSink
This rolling configurations based on a trigger such as a period of time or a certain size of file. When a roll happens, the current instance of the subordinate configuration is closed and a new instance of the sink is opened.

  @Override
  public RollSink getAckingSink(Context ctx, final RollTrigger t,
      final AckListener ackQueue, long checkMs) throws IOException {
    // TODO (jon) make this expressible using the DSL instead of only in
    // javacode
    return new RollSink(ctx, "ackingWal", t, checkMs) {
      @Override
      public EventSink newSink(Context ctx) throws IOException {
        return newAckWritingSink(t.getTagger(), ackQueue);
      }
    };

View Full Code Here

   */
  @Override
  public RollSink getEventSink(Context ctx, final RollTrigger t)
      throws IOException {
    // NaiveFileFailover is just a place holder
    return new RollSink(ctx, "NaiveFileFailover", t, 250) {


      @Override
      public EventSink newSink(Context ctx) throws IOException {
        // TODO (jon) clean this up -- want to deprecate Tagger
        return newWritingSink(t.getTagger());

View Full Code Here

    deco = (EventSinkDecorator<EventSink>) deco.getSink();


    // insistent append -> mask
    deco = (EventSinkDecorator<EventSink>) deco.getSink();


    RollSink roll = (RollSink) deco.getSink();


    // normally inside wal
    NaiveFileWALDeco.AckChecksumRegisterer<EventSink> snk = new NaiveFileWALDeco.AckChecksumRegisterer(
        coll, node.getAckChecker().getAgentAckQueuer());
    return new Pair<RollSink, EventSink>(roll, snk);

View Full Code Here

    File tmpdir = FileUtil.mktempdir();


    EventSource ackedmem = setupAckRoll();
    Pair<RollSink, EventSink> p = setupSink(node, tmpdir);
    EventSink snk = p.getRight();
    RollSink roll = p.getLeft();
    snk.open();


    String tag1 = roll.getCurrentTag();
    LOG.info(tag1);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    // don't rotate the first one.
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // still one ack pending.
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());


    String tag2 = roll.getCurrentTag();
    LOG.info(tag2);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // two acks pending.
    assertEquals(2, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no more acks pending.
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());


    String tag3 = roll.getCurrentTag();
    LOG.info(tag3);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // one ack pending
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no more acks pending.
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());

View Full Code Here

    File tmpdir = FileUtil.mktempdir();


    EventSource ackedmem = setupAckRoll();
    Pair<RollSink, EventSink> p = setupSink(node, tmpdir);
    EventSink snk = p.getRight();
    RollSink roll = p.getLeft();
    snk.open();


    String tag1 = roll.getCurrentTag();
    LOG.info(tag1);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate(); // we should have the first batch and part of the second
    // one ack pending
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no acks pending
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());


    // note, we still are checking state for the 2nd batch of messages


    String tag2 = roll.getCurrentTag();
    LOG.info(tag2);
    // This is the end msg closes the 2nd batch
    snk.append(ackedmem.next()); // ack end
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // now 2nd batch and 3rd batch are pending.
    assertEquals(2, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();


    // no more acks out standing

View Full Code Here

    File tmpdir = FileUtil.mktempdir();


    EventSource ackedmem = setupAckRoll();
    Pair<RollSink, EventSink> p = setupSink(node, tmpdir);
    EventSink snk = p.getRight();
    RollSink roll = p.getLeft();
    snk.open();


    String tag1 = roll.getCurrentTag();
    LOG.info(tag1);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    snk.append(ackedmem.next()); // ack beg
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate(); // we should have the first batch and part of the second
    // one ack pending
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no acks pending
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());


    // we are partially through the second batch, at a different split point


    String tag2 = roll.getCurrentTag();
    LOG.info(tag2);
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // now we have closed off group2 and group3
    assertEquals(2, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    Clock.sleep(10); // have to make sure it is not in the same millisecond

View Full Code Here

    // max 5s, backoff initially at 10ms
    BackoffPolicy bop = new CappedExponentialBackoff(10, 5000);
    final EventSink insistent = new InsistentAppendDecorator<EventSink>(
        fail4eva, bop);
    final EventSink sink = new LazyOpenDecorator<EventSink>(insistent);
    final EventSink roll = new RollSink(new ReportTestingContext(), "mock",
        10000, 100) {
      @Override
      public EventSink newSink(Context ctx) throws IOException {
        return sink;
      }
    };
    roll.open();


    // create an endless stream of data
    final EventSource source = new EventSource.Base() {
      @Override
      public Event next() {

View Full Code Here

   * This instantiates a roller where all input is sent to.
   */
  @Override
  public RollSink getEventSink(final RollTrigger t) throws IOException {
    // NaiveFileFailover is just a place holder
    return new RollSink(new Context(), "NaiveFileFailover", t, 250) {


      @Override
      public EventSink newSink(Context ctx) throws IOException {
        // TODO (jon) clean this up -- want to deprecate Tagger
        return newWritingSink(t.getTagger());

View Full Code Here

    this(path, filename, hiveTableName, millis, new ProcessTagger(), 250, elasticSearchUrl, elasticIndex, elasticType, runMarkerQueries);
  }


  CollectorSink(final String logdir, final String filename, final long millis,
      final Tagger tagger, long checkmillis) {
    EventSink s = new RollSink(new Context(), "collectorSink", new TimeTrigger(
        tagger, millis), checkmillis) {
      @Override
      public EventSink newSink(Context ctx) throws IOException {
        String tag = tagger.newTag();
        String path = logdir + Path.SEPARATOR_CHAR;

View Full Code Here

    snk = new AckChecksumChecker<EventSink>(tmp, accum);
  }
  
  CollectorSink(final String logdir, final String filename, final String hiveTableName, final long millis,
      final Tagger tagger, long checkmillis) {
    EventSink s = new RollSink(new Context(), null, new TimeTrigger(tagger,
        millis), checkmillis) {
      @Override
      public EventSink newSink(Context ctx) throws IOException {
        String tag = tagger.newTag();
        String path = logdir + Path.SEPARATOR_CHAR;

View Full Code Here

0 1 2 3

TOP

Related Classes of com.cloudera.flume.handlers.rolling.RollSink

com.cloudera.flume.agent.diskfailover.NaiveFileFailoverManager

com.cloudera.flume.agent.durability.NaiveFileWALManager

com.cloudera.flume.agent.TestAgentFailChainSink

com.cloudera.flume.collector.CollectorSink

com.cloudera.flume.collector.TestCollectorSink

com.cloudera.flume.conf.FlumeBuilder

com.cloudera.flume.conf.FlumeBuilder.FunctionSpec

com.cloudera.flume.conf.SinkFactory.SinkBuilder

com.cloudera.flume.core.CompositeSink

com.cloudera.flume.core.connector.TestDirectDriver

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.