Examples of com.cloudera.flume.core.CompositeSink

com.cloudera.flume.core.CompositeSink
This sink takes a data flow spec string as a constructor argument. It parses the construction and instantiates the sink. During execution, all the sink API calls are forwarded to the instantiated sink. Currently, there no builder method because this should only be used by sinks/decorators that need to embed sinks specs as arguments. In the future there may be due to different spec syntax.

    File f = File.createTempFile("temp", ".tmp");
    f.deleteOnExit();
    File f2 = File.createTempFile("moved", ".tmp");
    f2.delete();
    f2.deleteOnExit();
    final CompositeSink snk = new CompositeSink(new ReportTestingContext(),
        "{ delay(50) => counter(\"count\") }");
    final EventSource src = TailSource.builder().build(f.getAbsolutePath());
    final CountDownLatch done = new CountDownLatch(1);
    final int count = 30;
    runDriver(src, snk, done, count);

View Full Code Here

      InterruptedException {
    File f = File.createTempFile("multitemp1", ".tmp");
    f.deleteOnExit();
    File f2 = File.createTempFile("multitemp2", ".tmp");
    f2.deleteOnExit();
    final CompositeSink snk = new CompositeSink(new ReportTestingContext(),
        "{ delay(50) => counter(\"count\") }");
    final EventSource src = TailSource.multiTailBuilder().build(
        f.getAbsolutePath(), f2.getAbsolutePath());
    final CountDownLatch done = new CountDownLatch(1);
    final int count = 60;

View Full Code Here

      fw.append("Line " + i + "\n");
      fw.flush();
    }
    fw.close();


    final CompositeSink snk = new CompositeSink(new ReportTestingContext(),
        "{ delay(50) => counter(\"count\") }");
    // Test start from end.
    final TailSource src = (TailSource) TailSource.builder().build(
        f.getAbsolutePath(), "true");
    final CountDownLatch done = new CountDownLatch(1);

View Full Code Here

      @Override
      public EventSink newSink(Context ctx) throws IOException {
        String tag = tagger.newTag();
        EventSink drain;
        try {
          drain = new CompositeSink(ctx, snkSpec);
        } catch (FlumeSpecException e) {
          throw new IOException("Unable to instantiate '" + snkSpec + "'", e);
        }
        return new RollDetectDeco(drain, tag);
      }

View Full Code Here

  @Test
  public void testTailPermissionDenied() throws IOException,
      FlumeSpecException, InterruptedException {
    File f;
    final EventSource eventSource;
    final CompositeSink eventSink;
    final AtomicBoolean workerFailed;
    Thread workerThread;
    FileWriter writer;
    long sleepTime;
    long eventCount;


    f = FileUtil.createTempFile("temp", ".tmp");
    f.setReadable(false, false);


    f.deleteOnExit();


    eventSource = TailSource.builder().build(
        LogicalNodeContext.testingContext(), f.getAbsolutePath());
    eventSink = new CompositeSink(new ReportTestingContext(),
        "{ delay(50) => counter(\"count\") }");
    workerFailed = new AtomicBoolean(false);
    workerThread = new Thread() {


      @Override
      public void run() {
        try {
          eventSource.open();
          eventSink.open();


          EventUtil.dumpN(10, eventSource, eventSink);
          Clock.sleep(500);
          eventSource.close();
          eventSink.close();
        } catch (Exception e) {
          LOG.error("Unexpected exception", e);
        }
      }

View Full Code Here

  @Test
  public void testTailSource() throws IOException, FlumeSpecException,
      InterruptedException {
    File f = FileUtil.createTempFile("temp", ".tmp");
    f.deleteOnExit();
    final CompositeSink snk = new CompositeSink(new ReportTestingContext(),
        "{ delay(50) => counter(\"count\") }");
    final EventSource src = TailSource.builder().build(
        LogicalNodeContext.testingContext(), f.getAbsolutePath());
    final CountDownLatch done = new CountDownLatch(1);
    final int count = 30;

View Full Code Here

    File f = FileUtil.createTempFile("temp", ".tmp");
    f.deleteOnExit();
    File f2 = FileUtil.createTempFile("moved", ".tmp");
    f2.delete();
    f2.deleteOnExit();
    final CompositeSink snk = new CompositeSink(new ReportTestingContext(),
        "{ delay(50) => counter(\"count\") }");
    final EventSource src = TailSource.builder().build(
        LogicalNodeContext.testingContext(), f.getAbsolutePath());
    final CountDownLatch done = new CountDownLatch(1);
    final int count = 30;

View Full Code Here

      InterruptedException {
    File f = FileUtil.createTempFile("multitemp1", ".tmp");
    f.deleteOnExit();
    File f2 = FileUtil.createTempFile("multitemp2", ".tmp");
    f2.deleteOnExit();
    final CompositeSink snk = new CompositeSink(new ReportTestingContext(),
        "{ delay(50) => counter(\"count\") }");
    final EventSource src = TailSource.multiTailBuilder().build(
        LogicalNodeContext.testingContext(), f.getAbsolutePath(),
        f2.getAbsolutePath());
    final CountDownLatch done = new CountDownLatch(1);

View Full Code Here

      fw.append("Line " + i + "\n");
      fw.flush();
    }
    fw.close();


    final CompositeSink snk = new CompositeSink(new ReportTestingContext(),
        "{ delay(50) => counter(\"count\") }");
    // Test start from end.
    final TailSource src = (TailSource) TailSource.builder().build(
        LogicalNodeContext.testingContext(), f.getAbsolutePath(), "true");
    final CountDownLatch done = new CountDownLatch(1);

View Full Code Here

  };


  protected EventSink newSink(Context ctx) throws IOException {
    try {
      // TODO (jon) add roll-specific context information.
      return new CompositeSink(ctx, fspec);
    } catch (FlumeSpecException e) {
      // check done prior to construction.
      throw new IllegalArgumentException("This should never happen:"
          + e.getMessage());
    }

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of com.cloudera.flume.core.CompositeSink

com.cloudera.flume.agent.AgentFailChainSink

com.cloudera.flume.agent.durability.TestNaiveFileWALManagerConcurrently

com.cloudera.flume.agent.LogicalNodeManager

com.cloudera.flume.agent.TestAgentFailChainSink

com.cloudera.flume.collector.CollectorSink

com.cloudera.flume.conf.Context

com.cloudera.flume.handlers.debug.BenchmarkReportDecorator

com.cloudera.flume.handlers.debug.BloomCheckDecorator

com.cloudera.flume.handlers.rolling.RollSink

com.cloudera.flume.handlers.rolling.TestRollRollTags

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.