Examples of org.apache.flume.Channel

org.apache.flume.Channel

A channel connects a {@link Source} to a {@link Sink}. The source acts as producer while the sink acts as a consumer of events. The channel itself is the buffer between the two.

A channel exposes a {@link Transaction} interface that can be used byits clients to ensure atomic {@linkplain #put(Event) put} and{@linkplain #take() take} semantics.This is necessary to guarantee single hop reliability between agents. For instance, a source will successfully produce an {@linkplain Event event}if and only if that event can be committed to the source's associated channel. Similarly, a sink will consume an event if and only if its respective endpoint can accept the event. The extent of transaction support varies for different channel implementations ranging from strong to best-effort semantics.

Channels are associated with unique {@linkplain NamedComponent names} thatcan be used for separating configuration and working namespaces.

Channels must be thread safe, protecting any internal invariants as no guarantees are given as to when and by how many sources/sinks they may be simultaneously accessed by.
@see org.apache.flume.Source @see org.apache.flume.Sink @see org.apache.flume.Transaction

    testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
    AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
    Configurables.configure(sink, ctx);
    //Reset the context to a higher batchSize
    ctx.put("batchSize", "100");
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    for(int i = 0; i < 3; i++){
      Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
      channel.put(e);
    }
    tx.commit();
    tx.close();
    sink.process();
    HTable table = new HTable(testUtility.getConfiguration(), tableName);

View Full Code Here

   * is open - extract the pay-load and append to HDFS file <br />
   * WARNING: NOT THREAD SAFE
   */
  @Override
  public Status process() throws EventDeliveryException {
    Channel channel = getChannel();
    Transaction transaction = channel.getTransaction();
    List<BucketWriter> writers = Lists.newArrayList();
    transaction.begin();
    try {
      Event event = null;
      int txnEventCount = 0;
      for (txnEventCount = 0; txnEventCount < txnEventMax; txnEventCount++) {
        event = channel.take();
        if (event == null) {
          break;
        }


        // reconstruct the path name by substituting place holders

View Full Code Here

      throw new EventDeliveryException("Sink was never opened. " +
          "Please fix the configuration.");
    }
    AtomicBoolean txnFail = new AtomicBoolean(false);
    Status status = Status.READY;
    Channel channel = getChannel();
    txn = channel.getTransaction();
    txn.begin();
    List<PutRequest> actions = new LinkedList<PutRequest>();
    List<AtomicIncrementRequest> increments =
        new LinkedList<AtomicIncrementRequest>();
    for(int i = 0; i < batchSize; i++){
      Event event = channel.take();
      if(event == null){
        status = Status.BACKOFF;
        counterGroup.incrementAndGet("channel.underflow");
        break;
      } else {

View Full Code Here

  }


  @Override
  public Status process() throws EventDeliveryException {
    Status status = Status.READY;
    Channel channel = getChannel();
    Transaction txn = channel.getTransaction();
    List<Row> actions = new LinkedList<Row>();
    List<Increment> incs = new LinkedList<Increment>();
    txn.begin();
    for(long i = 0; i < batchSize; i++) {
      Event event = channel.take();
      if(event == null){
        status = Status.BACKOFF;
        counterGroup.incrementAndGet("channel.underflow");
        break;
      } else {

View Full Code Here

        throw new EventDeliveryException("Failed to open file "
            + pathController.getCurrentFile() + " while delivering event", e);
      }
    }


    Channel channel = getChannel();
    Transaction transaction = channel.getTransaction();
    Event event = null;
    Status result = Status.READY;


    try {
      transaction.begin();
      event = channel.take();


      if (event != null) {
        serializer.write(event);


        /*

View Full Code Here


  @Override
  public Status process() throws EventDeliveryException {
    Status status = Status.READY;


    Channel channel = getChannel();
    Transaction transaction = channel.getTransaction();
    Event event = null;


    try {
      transaction.begin();
      int i = 0;
      for (i = 0; i < batchSize; i++) {
        event = channel.take();
        if(event == null) {
          status = Status.BACKOFF;
          break;
        }
      }

View Full Code Here

  private List<Channel> getChannelListFromNames(String channels,
      Map<String, Channel> channelNameMap){
    List<Channel> configuredChannels = new ArrayList<Channel>();
    String[] chNames = channels.split(" ");
    for (String name : chNames) {
      Channel ch = channelNameMap.get(name);
      if (ch != null) {
        configuredChannels.add(ch);
      } else {
        throw new FlumeException("Selector channel not found: "
            + name);

View Full Code Here

    context.put("hdfs.writeFormat", "Text");
    context.put("hdfs.fileType", "DataStream");


    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();


    // push the event batches into channel to roll twice
    for (i = 1; i <= rollCount*2/txnMax; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= txnMax; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();

View Full Code Here

  @Test
  public void testEmptyChannelResultsInStatusBackoff()
      throws InterruptedException, LifecycleException, EventDeliveryException {
    LOG.debug("Starting...");
    Context context = new Context();
    Channel channel = new MemoryChannel();
    context.put("hdfs.path", testPath);
    context.put("keep-alive", "0");
    Configurables.configure(sink, context);
    Configurables.configure(channel, context);
    sink.setChannel(channel);

View Full Code Here

    context.put("hdfs.writeFormat", "Text");
    context.put("hdfs.fileType", "DataStream");


    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();


    // push the event batches into channel
    for (i = 1; i < 4; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= txnMax; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
            String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);
        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.flume.Channel

com.github.kristofa.flume.ZipkinMetricsSink

com.github.kristofa.flume.ZipkinSpanCollectorSink

com.salesforce.phoenix.flume.sink.PhoenixSink

com.salesforce.phoenix.flume.TestPhoenixSink

com.salesforce.phoenix.flume.TestRegexEventSerializer

org.apache.flume.channel.AbstractChannelSelector

org.apache.flume.channel.DefaultChannelFactory

org.apache.flume.channel.MockChannel

org.apache.flume.channel.MultiplexingChannelSelector

org.apache.flume.channel.ReplicatingChannelSelector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.