Examples of com.cloudera.flume.handlers.rolling.RollSink

Package com.cloudera.flume.handlers.rolling

Examples of com.cloudera.flume.handlers.rolling.RollSink

com.cloudera.flume.handlers.rolling.RollSink
This rolling configurations based on a trigger such as a period of time or a certain size of file. When a roll happens, the current instance of the subordinate configuration is closed and a new instance of the sink is opened.

            + t.toStringTree() + "roll only takes two arguments");
        CommonTree ctbody = rollArgs.get(0);
        Long period = Long.parseLong(buildSimpleArg(rollArgs.get(1)));
        String body = FlumeSpecGen.genEventSink(ctbody);
        // TODO (jon) replace the hard coded 250 with a parameterizable value
        RollSink roller = new RollSink(context, body, period, 250);
        return roller;
      } catch (IllegalArgumentException iae) {
        throw new FlumeSpecException(iae.getMessage());
      }

View Full Code Here

  public FlumeSeqfileAppender() {
    MemoryMonitor.setupHardExitMemMonitor(.95);
  }


  public void activateOptions() {
    appender = new RollSink(new Context(), "dfs(\"file://" + dir
        + File.pathSeparator + "%{rolltag}\")", maxage, checkmillis);
  }

View Full Code Here


  CollectorSink(Context ctx, final String snkSpec, final long millis,
      final Tagger tagger, long checkmillis, AckListener ackDest) {
    this.ackDest = ackDest;
    this.snkSpec = snkSpec;
    roller = new RollSink(ctx, snkSpec, new TimeTrigger(tagger, millis),
        checkmillis) {
      // this is wraps the normal roll sink with an extra roll detection
      // decorator that triggers ack delivery on close.
      @Override
      public EventSink newSink(Context ctx) throws IOException {

View Full Code Here

    // thus closes hdfs handles).
    String snkspec = "collectorSink(\"file:///" + tmpdir.getAbsolutePath()
        + "\",\"\")";
    CollectorSink coll = (CollectorSink) FlumeBuilder.buildSink(new Context(),
        snkspec);
    RollSink roll = coll.roller; // shortcut to roller exposed for testing


    // normally inside wal
    NaiveFileWALDeco.AckChecksumRegisterer<EventSink> snk = new NaiveFileWALDeco.AckChecksumRegisterer<EventSink>(
        coll, node.getAckChecker().getAgentAckQueuer());
    return new Pair<RollSink, EventSink>(roll, snk);

View Full Code Here

    File tmpdir = FileUtil.mktempdir();


    EventSource ackedmem = setupAckRoll();
    Pair<RollSink, EventSink> p = setupSink(node, tmpdir);
    EventSink snk = p.getRight();
    RollSink roll = p.getLeft();
    snk.open();


    String tag1 = roll.getCurrentTag();
    LOG.info(tag1);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    // don't rotate the first one.
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // still one ack pending.
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());


    String tag2 = roll.getCurrentTag();
    LOG.info(tag2);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // two acks pending.
    assertEquals(2, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no more acks pending.
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());


    String tag3 = roll.getCurrentTag();
    LOG.info(tag3);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // one ack pending
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no more acks pending.
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());

View Full Code Here

    File tmpdir = FileUtil.mktempdir();


    EventSource ackedmem = setupAckRoll();
    Pair<RollSink, EventSink> p = setupSink(node, tmpdir);
    EventSink snk = p.getRight();
    RollSink roll = p.getLeft();
    snk.open();


    String tag1 = roll.getCurrentTag();
    LOG.info(tag1);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate(); // we should have the first batch and part of the second
    // one ack pending
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no acks pending
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());


    // note, we still are checking state for the 2nd batch of messages


    String tag2 = roll.getCurrentTag();
    LOG.info(tag2);
    // This is the end msg closes the 2nd batch
    snk.append(ackedmem.next()); // ack end
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // now 2nd batch and 3rd batch are pending.
    assertEquals(2, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();


    // no more acks out standing

View Full Code Here

    File tmpdir = FileUtil.mktempdir();


    EventSource ackedmem = setupAckRoll();
    Pair<RollSink, EventSink> p = setupSink(node, tmpdir);
    EventSink snk = p.getRight();
    RollSink roll = p.getLeft();
    snk.open();


    String tag1 = roll.getCurrentTag();
    LOG.info(tag1);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    snk.append(ackedmem.next()); // ack beg
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate(); // we should have the first batch and part of the second
    // one ack pending
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no acks pending
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());


    // we are partially through the second batch, at a different split point


    String tag2 = roll.getCurrentTag();
    LOG.info(tag2);
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // now we have closed off group2 and group3
    assertEquals(2, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    Clock.sleep(10); // have to make sure it is not in the same millisecond

View Full Code Here

              + e.getAttrs().get(AckChecksumInjector.ATTR_ACK_HASH));
        }
      };


      // setup a roller that will roll like crazy from a separate thread
      final RollSink roll = cs.roller;
      Thread t = new Thread("roller") {
        @Override
        public void run() {
          try {
            for (int i = 0; i < ROLLS; i++) {
              roll.rotate();
            }
          } catch (InterruptedException e) {
            e.printStackTrace();
          }
        }
      };
      t.start();


      // pump it through and wait for exception.
      cs.open();


      EventUtil.dumpAll(mem, cs);
      t.join();


      cs.close();
      long rolls = roll.getMetrics().getLongMetric(RollSink.A_ROLLS);
      LOG.info("rolls {} ", rolls);
      LOG.info("start={} end={}", fakeMasterRpc.startCount,
          fakeMasterRpc.endCount);
      LOG.info("endset size={}", fakeMasterRpc.ends.size());
      LOG.info("expire={} err={}", fakeMasterRpc.expireCount,

View Full Code Here

    // max 5s, backoff initially at 10ms
    BackoffPolicy bop = new CappedExponentialBackoff(10, 5000);
    final EventSink insistent = new InsistentAppendDecorator<EventSink>(
        fail4eva, bop);
    final EventSink sink = new LazyOpenDecorator<EventSink>(insistent);
    final EventSink roll = new RollSink(new ReportTestingContext(), "mock",
        10000, 100) {
      @Override
      public EventSink newSink(Context ctx) throws IOException {
        return sink;
      }

View Full Code Here

    // get handle to roller in dfo log roller to provide data
    AgentFailChainSink afcs = (AgentFailChainSink) snk.getSink();
    BackOffFailOverSink bofos = (BackOffFailOverSink) ((CompositeSink) afcs.snk)
        .getSink();
    DiskFailoverDeco dfo = (DiskFailoverDeco) bofos.getBackup();
    RollSink dfoWriter = dfo.getDFOWriter();
    dfoWriter.rotate(); // allow dfo retry thread to go.


    // give data some time to show up.
    Clock.sleep(1000);
    c1Src.next();
    c1Src.next();

View Full Code Here

0 1 2 3

TOP

Related Classes of com.cloudera.flume.handlers.rolling.RollSink

com.cloudera.flume.agent.diskfailover.NaiveFileFailoverManager

com.cloudera.flume.agent.durability.NaiveFileWALManager

com.cloudera.flume.agent.TestAgentFailChainSink

com.cloudera.flume.collector.CollectorSink

com.cloudera.flume.collector.TestCollectorSink

com.cloudera.flume.conf.FlumeBuilder

com.cloudera.flume.conf.FlumeBuilder.FunctionSpec

com.cloudera.flume.conf.SinkFactory.SinkBuilder

com.cloudera.flume.core.CompositeSink

com.cloudera.flume.core.connector.TestDirectDriver

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.