Package com.cloudera.flume.core

Examples of com.cloudera.flume.core.CompositeSink


        public void run() {
          start.countDown();
          try {
            EventSource src = new NoNlASCIISynthSource(count, 100);
            Context ctx = new ReportTestingContext();
            EventSink snk = new CompositeSink(ctx, "counter(\"total." + idx
                + "\")");
            // use the same wal, but different counter.

            snk = new NaiveFileWALDeco(ctx, snk, wal, new TimeTrigger(
                new ProcessTagger(), 1000000), new AckListener.Empty(), 1000000);

            start.await();

            // allow for contention on the open call.
            src.open();
            snk.open();

            EventUtil.dumpAll(src, snk);
            src.close();
            snk.close();
          } catch (Exception e) {
            LOG.error("failure", e);
            // fail("e");
          } finally {
            done.countDown();
View Full Code Here


  }

  @Test
  public void testFactoryBuild() throws FlumeSpecException, IOException,
      InterruptedException {
    EventSink snk = new CompositeSink(new Context(),
        "ganglia(\"localhost\", \"foo\", \"int\")");
    for (int i = 0; i < 10; i++) {
      snk.open();
      snk.append(new EventImpl("".getBytes()));
      snk.close();
    }

  }
View Full Code Here

    ReportManager.get().clear();
    String spec = "agentE2EChain(\"foo:123\",\"bar\",\"baz\")";

    LOG.info("waled failchain: " + spec);
    String node = NetUtils.localhost();
    new CompositeSink(new LogicalNodeContext(node, node), spec);
  }
View Full Code Here

  public void testWALChainBadContext() throws FlumeSpecException {
    ReportManager.get().clear();
    String spec = "agentE2EChain(\"foo:123\",\"bar\",\"baz\")";

    LOG.info("waled failchain: " + spec);
    new CompositeSink(new Context(), spec);
  }
View Full Code Here

  public void testBEChain() throws FlumeSpecException {
    ReportManager.get().clear();
    String spec = "agentBEChain(\"foo:123\",\"bar\",\"baz\")";

    LOG.info("waled failchain: " + spec);
    new CompositeSink(new Context(), spec);
  }
View Full Code Here

        LogicalNodeContext.testingContext(), c2);
    c2Src.open();

    // create agentBEChain sink
    String spec = "agentBEChain(\"localhost:1234\", \"localhost:1235\")";
    EventSink snk = new CompositeSink(new Context(), spec);
    snk.open();

    Event e1 = new EventImpl("test 1".getBytes());
    Event e2 = new EventImpl("test 2".getBytes());
    Event e3 = new EventImpl("test 3".getBytes());
    Event e4 = new EventImpl("test 4".getBytes());

    // Everything is on and we send some messages
    snk.append(e1);
    Clock.sleep(100);
    LOG.info(c1Src.getMetrics().toString());
    assertEquals(1,
        (long) c1Src.getMetrics().getLongMetric(ThriftEventSource.A_ENQUEUED));
    c1Src.next();
    c1Src.close();

    // Killed the first of the chain, should go to backup
    // the number of events lost here is not consistent after close. this
    // seems time based, and the first two seem to be lost
    snk.append(e1);
    Clock.sleep(20);
    snk.append(e2);
    Clock.sleep(20);
    snk.append(e3);
    Clock.sleep(20);
    snk.append(e4);
    Clock.sleep(20);

    LOG.info(c2Src.getMetrics().toString());
    assertEquals(2,
        (long) c2Src.getMetrics().getLongMetric(ThriftEventSource.A_ENQUEUED));
    // 2 lost in network buffer, but two received in backup.  yay.
    c2Src.next();
    c2Src.next();
    c2Src.close();

    // all thrift sinks are closed now, we should loss messages
    snk.append(e1); // lost
    Clock.sleep(20);
    snk.append(e2); // lost
    Clock.sleep(20);
    snk.append(e3); // lost
    Clock.sleep(20);
    snk.append(e4); // lost
    Clock.sleep(20);

    // re-open desination 1.
    c1Src.open();
    snk.append(e1);
    Clock.sleep(20);
    snk.append(e2);
    Clock.sleep(20);
    c1Src.next();
    c1Src.close();
    LOG.info(c1Src.getMetrics().toString());
    // 2 events from prevoius + 1 from new open
    // first one fails on reopen but next succeeds
    assertEquals(2 + 1,
        (long) c2Src.getMetrics().getLongMetric(ThriftEventSource.A_ENQUEUED));
    snk.close();
  }
View Full Code Here

    ReportManager.get().clear();
    String spec = "agentDFOChain(\"foo:123\",\"bar\",\"baz\")";

    LOG.info("waled failchain: " + spec);
    String node = NetUtils.localhost();
    new CompositeSink(new LogicalNodeContext(node, node), spec);
  }
View Full Code Here

  public void testDFOChainBadContext() throws FlumeSpecException {
    ReportManager.get().clear();
    String spec = "agentDFOChain(\"foo:123\",\"bar\",\"baz\")";

    LOG.info("waled failchain: " + spec);
    new CompositeSink(new Context(), spec);
  }
View Full Code Here

    c2Src.open();

    // create agentDFOChain sink
    File tmpDir = FileUtil.mktempdir();
    String spec = "agentDFOChain(\"localhost:1234\", \"localhost:1235\")";
    CompositeSink snk = new CompositeSink(new LogicalNodeContext(
        tmpDir.getName(), tmpDir.getName()), spec);
    snk.open();

    Event e1 = new EventImpl("test 1".getBytes());
    Event e2 = new EventImpl("test 2".getBytes());
    Event e3 = new EventImpl("test 3".getBytes());
    Event e4 = new EventImpl("test 4".getBytes());

    // Everything is on and we send some messages.
    snk.append(e1);
    Clock.sleep(100);
    LOG.info(c1Src.getMetrics().toString());
    assertEquals(1,
        (long) c1Src.getMetrics().getLongMetric(ThriftEventSource.A_ENQUEUED));
    // it got through, yay.
    c1Src.next();
    c1Src.close();

    // Killed the first of the chain, should go to backup
    // the number of events lost here is not consistent after close. this
    // seems time based, and the first two seem to be lost
    snk.append(e1);
    Clock.sleep(20);
    snk.append(e2);
    Clock.sleep(20);
    snk.append(e3);
    Clock.sleep(20);
    snk.append(e4);
    Clock.sleep(20);
      
    LOG.info(c2Src.getMetrics().toString());
    assertEquals(2,
        (long) c2Src.getMetrics().getLongMetric(ThriftEventSource.A_ENQUEUED));
    // 2 lost in network buffer, but two received in backup. yay.
    c2Src.next();
    c2Src.next();
    c2Src.close();

    // all thrift sinks are closed now, we should end up in dfo
    snk.append(e1); // lost in thrift sink buffer
    Clock.sleep(20);
    snk.append(e2); // lost in thrift sink buffer
    Clock.sleep(20);
    snk.append(e3); // written
    Clock.sleep(20);
    snk.append(e4); // written
    Clock.sleep(20);
    LOG.info(snk.getMetrics().toText());
    assertEquals(
        2,
        (long) ReportUtil.getFlattenedReport(snk).getLongMetric(
            "backup.DiskFailover.NaiveDiskFailover.writingEvts"));

    // re-open destination 1
    c1Src.open();
    snk.append(e1);
    Clock.sleep(20);
    snk.append(e2);
    Clock.sleep(20);
    c1Src.next();

    // get handle to roller in dfo log roller to provide data
    AgentFailChainSink afcs = (AgentFailChainSink) snk.getSink();
    BackOffFailOverSink bofos = (BackOffFailOverSink) ((CompositeSink) afcs.snk)
        .getSink();
    DiskFailoverDeco dfo = (DiskFailoverDeco) bofos.getBackup();
    DiskFailoverManager dfm = dfo.getFailoverManager();
    RollSink dfoWriter = dfo.getDFOWriter();
View Full Code Here

    List<String> names = Arrays.asList("first", "second", "third", "fourth",
        "fifth");
    String body = "{ lazyOpen => { intervalFlakeyAppend(2) => accumulator(\"%s\")}}";
    String spec = FailoverChainManager.genAvailableSinkSpec(body, names);
    System.out.println(spec);
    EventSink snk = new CompositeSink(new ReportTestingContext(), spec);

    LOG.info(snk.getMetrics().toText());

    snk.open();
    EventSource src = MemorySinkSource.cannedData("test is a test", 31);
    src.open();
    EventUtil.dumpAll(src, snk);

    int[] ans = { 16, 8, 4, 2, 1 };
View Full Code Here

TOP

Related Classes of com.cloudera.flume.core.CompositeSink

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.