Examples of org.apache.flume.Channel

org.apache.flume.Channel

A channel connects a {@link Source} to a {@link Sink}. The source acts as producer while the sink acts as a consumer of events. The channel itself is the buffer between the two.

A channel exposes a {@link Transaction} interface that can be used byits clients to ensure atomic {@linkplain #put(Event) put} and{@linkplain #take() take} semantics.This is necessary to guarantee single hop reliability between agents. For instance, a source will successfully produce an {@linkplain Event event}if and only if that event can be committed to the source's associated channel. Similarly, a sink will consume an event if and only if its respective endpoint can accept the event. The extent of transaction support varies for different channel implementations ranging from strong to best-effort semantics.

Channels are associated with unique {@linkplain NamedComponent names} thatcan be used for separating configuration and working namespaces.

Channels must be thread safe, protecting any internal invariants as no guarantees are given as to when and by how many sources/sinks they may be simultaneously accessed by.
@see org.apache.flume.Source @see org.apache.flume.Sink @see org.apache.flume.Transaction

    context.put("hdfs.rollCount", String.valueOf(rollCount));
    context.put("hdfs.batchSize", String.valueOf(batchSize));


    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();


    // push the event batches into channel
    for (i = 1; i < numBatches; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= batchSize; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
            String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);
        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();

View Full Code Here

    context.put("hdfs.batchSize", String.valueOf(batchSize));
    context.put("hdfs.useLocalTimeStamp", String.valueOf(true));


    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.setBucketClock(clk);
    sink.start();


    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();


    // push the event batches into channel
    for (i = 1; i < numBatches; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= batchSize; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
          String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);
        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();

View Full Code Here

    context.put("hdfs.rollCount", String.valueOf(rollCount));
    context.put("hdfs.batchSize", String.valueOf(batchSize));


    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();
    // push the event batches into channel
    for (int i = 1; i < 4; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (int j = 1; j <= batchSize; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
            String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);
        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
      }
      txn.commit();
      txn.close();


      // execute sink to process the events

View Full Code Here

    context.put("hdfs.batchSize", String.valueOf(batchSize));
    context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);


    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();


    List<String> bodies = Lists.newArrayList();
    // push the event batches into channel
    for (i = 1; i < numBatches; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= batchSize; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
            String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);


        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        // inject fault
        if ((totalEvents % 30) == 1) {
          event.getHeaders().put("fault-once", "");
        }
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();

View Full Code Here

    context.put("hdfs.batchSize", String.valueOf(batchSize));
    context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
    context.put("hdfs.callTimeout", Long.toString(1000));
    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();


    // push the event batches into channel
    for (i = 0; i < numBatches; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= batchSize; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
            String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);
        event.getHeaders().put("slow", "1500");
        event.setBody(("Test." + i + "." + j).getBytes());
        channel.put(event);
      }
      txn.commit();
      txn.close();


      // execute sink to process the events

View Full Code Here

    context.put("hdfs.batchSize", String.valueOf(batchSize));
    context.put("hdfs.fileType", HDFSTestWriterFactory.TestSequenceFileType);
    context.put("hdfs.appendTimeout", String.valueOf(appendTimeout));
    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();
    // push the event batches into channel
    for (i = 0; i < numBatches; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= batchSize; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
            String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);
        event.getHeaders().put("slow", "1500");
        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();

View Full Code Here

    context.put("hdfs.rollInterval", "0");
    context.put("hdfs.batchSize", "2");
    context.put("hdfs.idleTimeout", "1");
    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Transaction txn = channel.getTransaction();
    txn.begin();
    for(int i=0; i < 10; i++) {
      Event event = new SimpleEvent();
      event.setBody(("test event " + i).getBytes());
      channel.put(event);
    }
    txn.commit();
    txn.close();


    sink.process();

View Full Code Here

    MockFileSystem mockFs = new MockFileSystem(fs, 3);


    Context context = getContextForRetryTests();
    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.setMockFs(mockFs);
    HDFSWriter hdfsWriter = new MockDataStream(mockFs);
    hdfsWriter.configure(context);
    sink.setMockWriter(hdfsWriter);
    sink.start();


    // push the event batches into channel
    for (int i = 0; i < 2; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      Map<String, String> hdr = Maps.newHashMap();
      hdr.put("retryHeader", "v1");


      channel.put(EventBuilder.withBody("random".getBytes(), hdr));
      txn.commit();
      txn.close();


      // execute sink to process the events
      sink.process();
    }
    // push the event batches into channel
    for (int i = 0; i < 2; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      Map<String, String> hdr = Maps.newHashMap();
      hdr.put("retryHeader", "v2");
      channel.put(EventBuilder.withBody("random".getBytes(), hdr));
      txn.commit();
      txn.close();
      // execute sink to process the events
      sink.process();
    }

View Full Code Here

  }
  
  @Override
  public Status process() throws EventDeliveryException {
    Status status = Status.READY;
    Channel channel = getChannel();
    Transaction transaction = channel.getTransaction();


    try {
      transaction.begin();
      createConnection();


      Event event = channel.take();


      if (event == null) {
        counterGroup.incrementAndGet("event.empty");
        status = Status.BACKOFF;
      } else {

View Full Code Here

   * Ensure the file is open. Serialize the data and write it to the file on
   * HDFS. <br/>
   * This method is not thread safe.
   */
  public Status process() throws EventDeliveryException {
    Channel channel = getChannel();
    Transaction transaction = channel.getTransaction();
    List<BucketWriter> writers = Lists.newArrayList();
    transaction.begin();
    try {
      int txnEventCount = 0;
      for (txnEventCount = 0; txnEventCount < batchSize; txnEventCount++) {
        Event event = channel.take();
        if (event == null) {
          break;
        }


        // reconstruct the path name by substituting place holders

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.flume.Channel

com.github.kristofa.flume.ZipkinMetricsSink

com.github.kristofa.flume.ZipkinSpanCollectorSink

com.salesforce.phoenix.flume.sink.PhoenixSink

com.salesforce.phoenix.flume.TestPhoenixSink

com.salesforce.phoenix.flume.TestRegexEventSerializer

org.apache.flume.channel.AbstractChannelSelector

org.apache.flume.channel.DefaultChannelFactory

org.apache.flume.channel.MockChannel

org.apache.flume.channel.MultiplexingChannelSelector

org.apache.flume.channel.ReplicatingChannelSelector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.