Examples of com.cloudera.flume.handlers.rolling.ProcessTagger

Package com.cloudera.flume.handlers.rolling

Examples of com.cloudera.flume.handlers.rolling.ProcessTagger

com.cloudera.flume.handlers.rolling.ProcessTagger
This tagger uses process name and pid as its prefix to manage log write-ahead files. It uses a file name convention and to tag on batches of events.

  // References package exposed for testing
  final RollSink roller;


  CollectorSink(Context ctx, String snkSpec, long millis, AckListener ackDest)
      throws FlumeSpecException {
    this(ctx, snkSpec, millis, new ProcessTagger(), 250, ackDest);
  }

View Full Code Here

        if (argv.length >= 3) {
          millis = Long.parseLong(argv[2]);
        }
        try {
          EventSink snk = new CollectorSink(context, logdir, prefix, millis,
              new ProcessTagger(), 250, FlumeNode.getInstance()
                  .getCollectorAckListener());
          return snk;
        } catch (FlumeSpecException e) {
          LOG.error("CollectorSink spec error " + e, e);
          throw new IllegalArgumentException(

View Full Code Here

      TestAckListener fakeMasterRpc = new TestAckListener();


      // massive roll millis because the test will force fast and frequent rolls
      CollectorSink cs = new CollectorSink(LogicalNodeContext.testingContext(),
          "file:///" + dir.getAbsolutePath(), "test", 1000000,
          new ProcessTagger(), 250, fakeMasterRpc) {
        @Override
        public void append(Event e) throws IOException, InterruptedException {
          LOG.info("Pre  append: "
              + e.getAttrs().get(AckChecksumInjector.ATTR_ACK_HASH));
          super.append(e);

View Full Code Here


    Context ctx = new ReportTestingContext();
    EventSink cntsnk = new CompositeSink(ctx, "counter(\"total\")");
    // use the same wal, but different counter.
    final EventSink snk = new NaiveFileWALDeco(ctx, cntsnk, wal,
        new TimeTrigger(new ProcessTagger(), 1000000), new AckListener.Empty(),
        1000000);
    snk.open();


    for (int i = 0; i < threads; i++) {
      new Thread() {

View Full Code Here

            Context ctx = new ReportTestingContext();
            EventSink snk = new CompositeSink(ctx, "counter(\"total." + idx
                + "\")");
            // use the same wal, but different counter.
            snk = new NaiveFileWALDeco(ctx, snk, wal, new TimeTrigger(
                new ProcessTagger(), 1000000), new AckListener.Empty(), 1000000);
            src.open();
            snk.open();


            start.await();

View Full Code Here

            EventSink snk = new CompositeSink(ctx, "counter(\"total." + idx
                + "\")");
            // use the same wal, but different counter.


            snk = new NaiveFileWALDeco(ctx, snk, wal, new TimeTrigger(
                new ProcessTagger(), 1000000), new AckListener.Empty(), 1000000);


            start.await();


            // allow for contention on the open call.
            src.open();

View Full Code Here

    String snk = "{ ackChecker => [ console, counter(\"" + rpt + "\") ] }";
    Context ctx = new ReportTestingContext();
    EventSink es = FlumeBuilder.buildSink(ctx, snk);
    // Excessively large timeout because I am manually triggering it.
    NaiveFileWALDeco wal = new NaiveFileWALDeco(ctx, es, node.getWalManager(),
        new TimeTrigger(new ProcessTagger(), 100000), node.getAckChecker()
            .getAgentAckQueuer(), 1000);


    // open and send data.
    wal.open();
    for (int i = 0; i < count; i++) {

View Full Code Here

        + rpt + "\") ] } }  ";
    Context ctx = new ReportTestingContext();
    EventSink es = FlumeBuilder.buildSink(ctx, snk);


    NaiveFileWALDeco wal = new NaiveFileWALDeco(ctx, es, node.getWalManager(),
        new TimeTrigger(new ProcessTagger(), 100000), node.getAckChecker()
            .getAgentAckQueuer(), 1000);
    wal.open();


    // // Phase I - (20 events arrive, 3 groups make it)
    for (int i = 0; i < count; i++) {

View Full Code Here

        + rpt + "\") ] } }  ";
    Context ctx = new ReportTestingContext();
    EventSink es = FlumeBuilder.buildSink(ctx, snk);


    NaiveFileWALDeco wal = new NaiveFileWALDeco(ctx, es, node.getWalManager(),
        new TimeTrigger(new ProcessTagger(), 100000), node.getAckChecker()
            .getAgentAckQueuer(), 1000);
    wal.open();


    // // Phase I - (19 events arrive, 3 groups make it)
    for (int i = 0; i < count; i++) {

View Full Code Here

    Context ctx = new ReportTestingContext();
    EventSink es = FlumeBuilder.buildSink(ctx, snk);


    // Big delay values so that test has to force different actions.
    NaiveFileWALDeco wal = new NaiveFileWALDeco(ctx, es, node.getWalManager(),
        new TimeTrigger(new ProcessTagger(), 100000), node.getAckChecker()
            .getAgentAckQueuer(), 1000);
    wal.open();


    // Phase I (20 message sent, 3 groups good, 1 group bad)
    for (int i = 0; i < count; i++) {

View Full Code Here

0 1 2 3 4

TOP

Related Classes of com.cloudera.flume.handlers.rolling.ProcessTagger

com.cloudera.flume.agent.diskfailover.DiskFailoverDeco

com.cloudera.flume.agent.diskfailover.TestDiskFailoverDeco

com.cloudera.flume.agent.diskfailover.TestDiskFailoverManager

com.cloudera.flume.agent.durability.TestNaiveFileWALDeco

com.cloudera.flume.agent.durability.TestNaiveFileWALManager

com.cloudera.flume.agent.durability.TestNaiveFileWALManagerConcurrently

com.cloudera.flume.agent.TestEndToEndAckFailureRecovery

com.cloudera.flume.collector.CollectorSink

com.cloudera.flume.collector.TestCollectorSink

java.util.regex.Matcher

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.