Package com.cloudera.flume.reporter.aggregator

Examples of com.cloudera.flume.reporter.aggregator.CounterSink


    }

    for (int i = 0; i < interval_size.length; i++) {
      mem.open();
      int sz = interval_size[i];
      CounterSink cnt = new CounterSink("null");
      EventSink res = new IntervalSampler<CounterSink>(cnt, sz);
      EventUtil.dumpAll(mem, res);
      b.mark("interval", sz);

      res.close();
      b.mark("count", cnt.getCount());
    }
    b.done();
  }
View Full Code Here


    }

    for (int i = 0; i < probs.length; i++) {
      mem.open();
      double prob = probs[i];
      CounterSink cnt = new CounterSink("null");
      EventSink res = new ProbabilitySampler<CounterSink>(cnt, prob);
      EventUtil.dumpAll(mem, res);
      b.mark("probability", prob);

      res.close();
      b.mark("count", cnt.getCount());
    }
    b.done();
  }
View Full Code Here

    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    CounterSink snk = new CounterSink("counter");
    EventUtil.dumpAll(mem, snk);
    b.mark(snk.getName() + " done", snk.getCount());

    b.done();
  }
View Full Code Here

  @Test
  public void testFailOverSink() throws IOException, InterruptedException {
    MockClock mock = new MockClock(0);
    Clock.setClock(mock);

    CounterSink primary = new CounterSink("primary");
    CounterSink secondary = new CounterSink("backup");
    ExceptionTwiddleDecorator<CounterSink> twiddle = new ExceptionTwiddleDecorator<CounterSink>(
        primary);
    BackOffFailOverSink failsink = new BackOffFailOverSink(twiddle, secondary,
        100, 10000); // 100 ms
    // initial
    // backoff,
    // 10000ms max
    // backoff
    failsink.open();

    Event e = new EventImpl("event".getBytes());
    // two successful appends to primary.
    failsink.append(e);
    failsink.append(e);
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(2, primary.getCount());
    Assert.assertEquals(0, secondary.getCount());

    mock.forward(100);
    twiddle.setAppendOk(false); // go to fail over.
    failsink.append(e); // primary fails and automatically go to 2ndary
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(1, failsink.getFails()); // one attempt on primary
    // failed.
    Assert.assertEquals(2, primary.getCount()); // same as before,
    Assert.assertEquals(1, secondary.getCount()); // message went to the
    // secondary

    mock.forward(50);
    failsink.append(e); // skip primary and just go to 2ndary
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(1, failsink.getFails()); // still only one attempt on
    // primary
    Assert.assertEquals(2, primary.getCount()); // same as before,
    Assert.assertEquals(2, secondary.getCount()); // message went to the
    // secondary

    mock.forward(50);
    failsink.append(e); // after this fails backoff is now 200
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(2, failsink.getFails()); // try primary
    Assert.assertEquals(0, primary.getCount()); // resets because primary
                                                // restarted (and still fails)
    Assert.assertEquals(3, secondary.getCount()); // but failover to secondary

    mock.forward(200);
    failsink.append(e); // should go to 2ndary, after this fails backoff is now
    // 400
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(3, failsink.getFails());
    Assert.assertEquals(0, primary.getCount());
    Assert.assertEquals(4, secondary.getCount());

    twiddle.setAppendOk(true);
    failsink.append(e); // even through primary is ok, we are backing off
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(3, failsink.getFails());
    Assert.assertEquals(0, primary.getCount());
    Assert.assertEquals(5, secondary.getCount());

    mock.forward(400);
    failsink.append(e); // now that the backoff has expired, we retry the
    // primary and succeed
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(3, failsink.getFails());
    Assert.assertEquals(1, primary.getCount());
    Assert.assertEquals(5, secondary.getCount());

    // this should succeed, with the counts being equal in primary and
    // secondary.
    failsink.close();
  }
View Full Code Here

  public void testFailTimeout() throws IOException, InterruptedException {
    System.out.println("===========================");
    MockClock mock = new MockClock(0);
    Clock.setClock(mock);

    CounterSink primary = new CounterSink("primary");
    CounterSink secondary = new CounterSink("backup");
    ExceptionTwiddleDecorator<CounterSink> twiddle = new ExceptionTwiddleDecorator<CounterSink>(
        primary);
    BackOffFailOverSink failsink = new BackOffFailOverSink(twiddle, secondary,
        100, 1000); // 100 ms
    // initial
    // backoff,
    // 10000ms max
    // backoff
    failsink.open();

    Event e = new EventImpl("event".getBytes());

    mock.forward(100);
    twiddle.setAppendOk(false); // go to fail over.
    failsink.append(e);
    failsink.append(e);
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());

    mock.forward(100);
    failsink.append(e);
    failsink.append(e);
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());

    mock.forward(200);
    failsink.append(e);
    failsink.append(e);
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());

    mock.forward(400);
    failsink.append(e);
    failsink.append(e);
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(4, failsink.getFails());
    Assert.assertEquals(0, primary.getCount());
    Assert.assertEquals(8, secondary.getCount());

    mock.forward(800);
    failsink.append(e);
    failsink.append(e);
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(5, failsink.getFails());
    Assert.assertEquals(0, primary.getCount());
    Assert.assertEquals(10, secondary.getCount());

    // without capping there would be no new fail here bug still the
    // twelve on the secondary count.
    mock.forward(1000);
    failsink.append(e);
    failsink.append(e);
    System.out.println(mock);
    System.out.printf("pri: %4d sec: %4d fail: %4d\n", primary.getCount(),
        secondary.getCount(), failsink.getFails());
    Assert.assertEquals(6, failsink.getFails());
    Assert.assertEquals(0, primary.getCount());
    Assert.assertEquals(12, secondary.getCount());

  }
View Full Code Here

    snk.append(e);
    snk.append(e);
    snk.append(e);

    snk.close();
    CounterSink priCnt = (CounterSink) ReportManager.get().getReportable("pri");
    CounterSink secCnt = (CounterSink) ReportManager.get().getReportable("sec");
    // these are timing based, may fail.
    Assert.assertEquals(3, priCnt.getCount());
    Assert.assertEquals(2, secCnt.getCount());
  }
View Full Code Here

    snk.append(e);
    snk.append(e);
    snk.append(e);

    snk.close();
    CounterSink priCnt = (CounterSink) ReportManager.get().getReportable("pri");
    CounterSink secCnt = (CounterSink) ReportManager.get().getReportable("sec");
    // these are timing based, may fail.
    Assert.assertEquals(1, priCnt.getCount());
    Assert.assertEquals(4, secCnt.getCount());

  }
View Full Code Here

    }
    fw.close();

    assertTrue(done.await(10, TimeUnit.SECONDS));

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("count");
    assertEquals(count, ctr.getCount());
  }
View Full Code Here

    // This sleep is necessary to make sure the file is not moved before tail
    // sees it.
    Clock.sleep(2000);
    f.renameTo(f2);

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("count");
    LOG.info("counted " + ctr.getCount());

    FileWriter fw2 = new FileWriter(f);
    for (int i = count / 2; i < count; i++) {
      fw2.append("Line " + i + "\n");
      fw2.flush();
    }
    fw2.close();

    done.await();

    ctr = (CounterSink) ReportManager.get().getReportable("count");
    assertEquals(count, ctr.getCount());
  }
View Full Code Here

    fw.close();
    fw2.close();

    assertTrue("Test timed out", done.await(30, TimeUnit.SECONDS));

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("count");
    LOG.info("events in file1: " + log1 + " events in file2: " + log2);
    assertEquals(count, ctr.getCount());

  }
View Full Code Here

TOP

Related Classes of com.cloudera.flume.reporter.aggregator.CounterSink

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.