Examples of com.cloudera.flume.core.CompositeSink

com.cloudera.flume.core.CompositeSink
This sink takes a data flow spec string as a constructor argument. It parses the construction and instantiates the sink. During execution, all the sink API calls are forwarded to the instantiated sink. Currently, there no builder method because this should only be used by sinks/decorators that need to embed sinks specs as arguments. In the future there may be due to different spec syntax.

  };


  protected EventSink newSink(Context ctx) throws IOException {
    try {
      // TODO (jon) add roll-specific context information.
      return new CompositeSink(ctx, fspec);
    } catch (FlumeSpecException e) {
      // check done prior to construction.
      throw new IllegalArgumentException("This should never happen:"
          + e.getMessage());
    }

View Full Code Here

        }
        EventSink rptSink = new NullSink();
        if (argv.length >= 3) {
          String rptSpec = argv[2];
          try {
            rptSink = new CompositeSink(ctx, rptSpec);
          } catch (FlumeSpecException e) {
            LOG.debug("failed to parse rpt spec", e);
            throw new IllegalArgumentException(e.getMessage());
          }
        }

View Full Code Here

        String name = argv[0];
        EventSink rptSink = new NullSink();
        if (argv.length >= 2) {
          String rptSpec = argv[1];
          try {
            rptSink = new CompositeSink(context, rptSpec);
          } catch (FlumeSpecException e) {
            LOG.debug("failed to parse rpt spec", e);
            throw new IllegalArgumentException(e.getMessage());
          }
        }

View Full Code Here

        "{lazyOpen => counter(\"%s\")}");
    LOG.info(agent1);
    assertEquals(
        "failChain(\"{lazyOpen => counter(\\\"%s\\\")}\",\"collector 2\",\"collector 3\",\"collector 1\")",
        agent1);
    CompositeSink snk1 = new CompositeSink(new Context(), agent1);
    snk1.open();
    snk1.close();


    String agent2 = am.getFailChainSinkSpec("agent2",
        "{lazyOpen => counter(\"%s\")}");
    LOG.info(agent2);
    assertEquals(
        "failChain(\"{lazyOpen => counter(\\\"%s\\\")}\",\"collector 1\",\"collector 4\",\"collector 5\")",
        agent2);
    CompositeSink snk2 = new CompositeSink(new Context(), agent2);
    snk2.open();
    snk2.close();


    String agent3 = am.getFailChainSinkSpec("agent3",
        "{lazyOpen => counter(\"%s\")}");
    LOG.info(agent3);
    assertEquals(
        "failChain(\"{lazyOpen => counter(\\\"%s\\\")}\",\"collector 1\",\"collector 2\",\"collector 5\")",
        agent3);
    CompositeSink snk3 = new CompositeSink(new Context(), agent3);
    snk3.open();
    snk3.close();


  }

View Full Code Here

   * node.
   */
  synchronized public void spawn(String name, String src, String snk)
      throws IOException, FlumeSpecException {
    Context ctx = new LogicalNodeContext(physicalNode, name);
    spawn(ctx, name, FlumeBuilder.buildSource(ctx, src), new CompositeSink(ctx,
        snk));
  }

View Full Code Here

    for (int i = list.size() - 1; i >= 0; i--) {
      String s = list.get(i);
      // this should be a composite sink.
      String failoverSpec = String.format(spec, s);
      LOG.debug("failover spec is : " + failoverSpec);
      EventSink tsnk = new CompositeSink(context, failoverSpec);
      if (cur == null) {
        cur = tsnk;
        continue;
      }
      cur = new BackOffFailOverSink(tsnk, cur, policy);

View Full Code Here

      @Override
      public EventSink newSink(Context ctx) throws IOException {
        String tag = tagger.newTag();
        EventSink drain;
        try {
          drain = new CompositeSink(ctx, snkSpec);
        } catch (FlumeSpecException e) {
          throw new IOException("Unable to instantiate '" + snkSpec + "'", e);
        }
        return new RollDetectDeco(drain, tag);
      }

View Full Code Here

    switch (rel) {
    case E2E: {
      String chains = AgentFailChainSink.genE2EChain(thriftlist
          .toArray(new String[0]));
      LOG.info("Setting failover chain to  " + chains);
      snk = new CompositeSink(context, chains);


      break;
    }
    case DFO: {
      String chains = AgentFailChainSink.genDfoChain(thriftlist
          .toArray(new String[0]));
      LOG.info("Setting failover chain to  " + chains);
      snk = new CompositeSink(context, chains);
      break;
    }
    case BE: {
      thriftlist.add("null");
      String chains = AgentFailChainSink.genBestEffortChain(thriftlist
          .toArray(new String[0]));
      LOG.info("Setting failover chain to  " + chains);
      snk = new CompositeSink(context, chains);
      break;
    }
    default: {
      throw new FlumeSpecException("Unknown reliability " + rel);
    }

View Full Code Here

    final CountDownLatch done = new CountDownLatch(threads);
    final NaiveFileWALManager wal = new NaiveFileWALManager(dir);
    wal.open();


    Context ctx = new ReportTestingContext();
    EventSink cntsnk = new CompositeSink(ctx, "counter(\"total\")");
    // use the same wal, but different counter.
    final EventSink snk = new NaiveFileWALDeco(ctx, cntsnk, wal,
        new TimeTrigger(new ProcessTagger(), 1000000), new AckListener.Empty(),
        1000000);
    snk.open();

View Full Code Here

        public void run() {
          start.countDown();
          try {
            EventSource src = new NoNlASCIISynthSource(count, 100);
            Context ctx = new ReportTestingContext();
            EventSink snk = new CompositeSink(ctx, "counter(\"total." + idx
                + "\")");
            // use the same wal, but different counter.
            snk = new NaiveFileWALDeco(ctx, snk, wal, new TimeTrigger(
                new ProcessTagger(), 1000000), new AckListener.Empty(), 1000000);
            src.open();
            snk.open();


            start.await();


            EventUtil.dumpAll(src, snk);
            src.close();
            snk.close();
          } catch (Exception e) {
            LOG.error("failure", e);
            // fail("e");
          } finally {
            done.countDown();

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of com.cloudera.flume.core.CompositeSink

com.cloudera.flume.agent.AgentFailChainSink

com.cloudera.flume.agent.durability.TestNaiveFileWALManagerConcurrently

com.cloudera.flume.agent.LogicalNodeManager

com.cloudera.flume.agent.TestAgentFailChainSink

com.cloudera.flume.collector.CollectorSink

com.cloudera.flume.conf.Context

com.cloudera.flume.handlers.debug.BenchmarkReportDecorator

com.cloudera.flume.handlers.debug.BloomCheckDecorator

com.cloudera.flume.handlers.rolling.RollSink

com.cloudera.flume.handlers.rolling.TestRollRollTags

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.