Examples of org.apache.flume.Channel

org.apache.flume.Channel

A channel connects a {@link Source} to a {@link Sink}. The source acts as producer while the sink acts as a consumer of events. The channel itself is the buffer between the two.

A channel exposes a {@link Transaction} interface that can be used byits clients to ensure atomic {@linkplain #put(Event) put} and{@linkplain #take() take} semantics.This is necessary to guarantee single hop reliability between agents. For instance, a source will successfully produce an {@linkplain Event event}if and only if that event can be committed to the source's associated channel. Similarly, a sink will consume an event if and only if its respective endpoint can accept the event. The extent of transaction support varies for different channel implementations ranging from strong to best-effort semantics.

Channels are associated with unique {@linkplain NamedComponent names} thatcan be used for separating configuration and working namespaces.

Channels must be thread safe, protecting any internal invariants as no guarantees are given as to when and by how many sources/sinks they may be simultaneously accessed by.
@see org.apache.flume.Source @see org.apache.flume.Sink @see org.apache.flume.Transaction

    ctx.put("batchSize", "2");
    HBaseSink sink = new HBaseSink(conf);
    Configurables.configure(sink, ctx);
    //Reset the context to a higher batchSize
    ctx.put("batchSize", "100");
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    sink.setChannel(channel);


    logger.info("Writing data into channel");
    Transaction tx = channel.getTransaction();
    tx.begin();
    for(int i = 0; i < 3; i++){
      Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
      channel.put(e);
    }
    tx.commit();
    tx.close();


    logger.info("Starting sink and processing events");

View Full Code Here

    ctx.put("batchSize", "2");
    HBaseSink sink = new HBaseSink(conf);
    Configurables.configure(sink, ctx);
    //Reset the context to a higher batchSize
    ctx.put("batchSize", "100");
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    for(int i = 0; i < 3; i++){
      Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
      channel.put(e);
    }
    tx.commit();
    tx.close();
    sink.process();
    HTable table = new HTable(conf, tableName);

View Full Code Here

    ctx.put("batchSize", "1");


    HBaseSink sink = new HBaseSink(conf);
    Configurables.configure(sink, ctx);
    // Reset the context to a higher batchSize
    Channel channel = spy(new MemoryChannel());
    Configurables.configure(channel, new Context());
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + 0));
    channel.put(e);
    tx.commit();
    tx.close();
    doThrow(new ChannelException("Mock Exception")).when(channel).take();
    try {
      sink.process();

View Full Code Here


    HBaseSink sink = new HBaseSink(conf);
    Configurables.configure(sink, ctx);
    // Reset the context to a higher batchSize
    ctx.put("batchSize", "100");
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + 0));
    channel.put(e);
    tx.commit();
    tx.close();
    try {
      MockSimpleHbaseEventSerializer.throwException = true;
      sink.process();

View Full Code Here

      conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
        HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT));


    HBaseSink sink = new HBaseSink();
    Configurables.configure(sink, tmpContext);
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, ctx);
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    for(int i = 0; i < 3; i++){
      Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
      channel.put(e);
    }
    tx.commit();
    tx.close();
    Status status = Status.READY;
    while(status != Status.BACKOFF){

View Full Code Here

    expectedCounts.put("r2:c3", 63L);
    HBaseSink.DebugIncrementsCallback cb = new CoalesceValidator(expectedCounts);


    HBaseSink sink = new HBaseSink(testUtility.getConfiguration(), cb);
    Configurables.configure(sink, ctx);
    Channel channel = createAndConfigureMemoryChannel(sink);


    List<Event> events = Lists.newLinkedList();
    generateEvents(events, expectedCounts);
    putEvents(channel, events);

View Full Code Here

    expectedCounts.put("r1:c1", 10L);
    HBaseSink.DebugIncrementsCallback cb = new CoalesceValidator(expectedCounts);


    HBaseSink sink = new HBaseSink(testUtility.getConfiguration(), cb);
    Configurables.configure(sink, ctx);
    Channel channel = createAndConfigureMemoryChannel(sink);


    List<Event> events = Lists.newLinkedList();
    generateEvents(events, expectedCounts);
    putEvents(channel, events);

View Full Code Here

  public void testBatchAware() throws EventDeliveryException {
    logger.info("Running testBatchAware()");
    initContextForIncrementHBaseSerializer();
    HBaseSink sink = new HBaseSink(testUtility.getConfiguration());
    Configurables.configure(sink, ctx);
    Channel channel = createAndConfigureMemoryChannel(sink);


    sink.start();
    int batchCount = 3;
    for (int i = 0; i < batchCount; i++) {
      sink.process();

View Full Code Here

      }
    }
  }


  private Channel createAndConfigureMemoryChannel(HBaseSink sink) {
    Channel channel = new MemoryChannel();
    Context channelCtx = new Context();
    channelCtx.put("capacity", String.valueOf(1000L));
    channelCtx.put("transactionCapacity", String.valueOf(1000L));
    Configurables.configure(channel, channelCtx);
    sink.setChannel(channel);
    channel.start();
    return channel;
  }

View Full Code Here

    ctx.put("columnFamily", columnFamily);
    ctx.put("serializer", SimpleHbaseEventSerializer.class.getName());


    HBaseSink sink = new HBaseSink(conf);
    Configurables.configure(sink, ctx);
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    Event e = EventBuilder.withBody(
            Bytes.toBytes(valBase));
    channel.put(e);
    tx.commit();
    tx.close();


    sink.process();
    sink.stop();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.flume.Channel

com.github.kristofa.flume.ZipkinMetricsSink

com.github.kristofa.flume.ZipkinSpanCollectorSink

com.salesforce.phoenix.flume.sink.PhoenixSink

com.salesforce.phoenix.flume.TestPhoenixSink

com.salesforce.phoenix.flume.TestRegexEventSerializer

org.apache.flume.channel.AbstractChannelSelector

org.apache.flume.channel.DefaultChannelFactory

org.apache.flume.channel.MockChannel

org.apache.flume.channel.MultiplexingChannelSelector

org.apache.flume.channel.ReplicatingChannelSelector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.