Examples of org.apache.flume.Channel

org.apache.flume.Channel

A channel connects a {@link Source} to a {@link Sink}. The source acts as producer while the sink acts as a consumer of events. The channel itself is the buffer between the two.

A channel exposes a {@link Transaction} interface that can be used byits clients to ensure atomic {@linkplain #put(Event) put} and{@linkplain #take() take} semantics.This is necessary to guarantee single hop reliability between agents. For instance, a source will successfully produce an {@linkplain Event event}if and only if that event can be committed to the source's associated channel. Similarly, a sink will consume an event if and only if its respective endpoint can accept the event. The extent of transaction support varies for different channel implementations ranging from strong to best-effort semantics.

Channels are associated with unique {@linkplain NamedComponent names} thatcan be used for separating configuration and working namespaces.

Channels must be thread safe, protecting any internal invariants as no guarantees are given as to when and by how many sources/sinks they may be simultaneously accessed by.
@see org.apache.flume.Source @see org.apache.flume.Sink @see org.apache.flume.Transaction


    testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
    deleteTable = true;
    AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
    Configurables.configure(sink, tmpctx);
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, tmpctx);
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    Event e = EventBuilder.withBody(
            Bytes.toBytes(valBase));
    channel.put(e);
    tx.commit();
    tx.close();
    Assert.assertFalse(sink.isConfNull());
    sink.process();
    sink.stop();

View Full Code Here

  public void testOneEvent() throws Exception {
    testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
    deleteTable = true;
    AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
    Configurables.configure(sink, ctx);
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, ctx);
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    Event e = EventBuilder.withBody(
        Bytes.toBytes(valBase));
    channel.put(e);
    tx.commit();
    tx.close();
    Assert.assertFalse(sink.isConfNull());
    sink.process();
    sink.stop();

View Full Code Here

  public void testThreeEvents() throws Exception {
    testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
    deleteTable = true;
    AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
    Configurables.configure(sink, ctx);
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, ctx);
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    for(int i = 0; i < 3; i++){
      Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
      channel.put(e);
    }
    tx.commit();
    tx.close();
    Assert.assertFalse(sink.isConfNull());
    sink.process();

View Full Code Here

    testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
    deleteTable = true;
    AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration(),
      true, false);
    Configurables.configure(sink, ctx);
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, ctx);
    sink.setChannel(channel);
    channel.start();
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    for(int i = 0; i < 3; i++){
      Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
      channel.put(e);
    }
    tx.commit();
    tx.close();
    Assert.assertFalse(sink.isConfNull());
    sink.process();

View Full Code Here

    ctx.put("batchSize", "2");
    AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
    Configurables.configure(sink, ctx);
    //Reset the context to a higher batchSize
    ctx.put("batchSize", "100");
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, ctx);
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    for(int i = 0; i < 3; i++){
      Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
      channel.put(e);
    }
    tx.commit();
    tx.close();
    int count = 0;
    Status status = Status.READY;

View Full Code Here

    // Restore the original serializer
    ctx.put("serializer", SimpleAsyncHbaseEventSerializer.class.getName());
    //Restore the no coalescing behavior
    ctx.put(HBaseSinkConfigurationConstants.CONFIG_COALESCE_INCREMENTS,
      "false");
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, ctx);
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    for (int i = 0; i < 4; i++) {
      for (int j = 0; j < 3; j++) {
        Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
        channel.put(e);
      }
    }
    tx.commit();
    tx.close();
    int count = 0;

View Full Code Here

    Configurables.configure(sink, ctx);
    // Reset context to values usable by other tests.
    ctx.put(HBaseSinkConfigurationConstants.ZK_QUORUM, null);
    ctx.put(HBaseSinkConfigurationConstants.ZK_ZNODE_PARENT,null);
    ctx.put("batchSize", "100");
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, ctx);
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    for(int i = 0; i < 3; i++){
      Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
      channel.put(e);
    }
    tx.commit();
    tx.close();
    int count = 0;
    Status status = Status.READY;

View Full Code Here

    ctx.put("batchSize", "2");
    AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
    Configurables.configure(sink, ctx);
    //Reset the context to a higher batchSize
    ctx.put("batchSize", "100");
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, ctx);
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    for(int i = 0; i < 3; i++){
      Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
      channel.put(e);
    }
    tx.commit();
    tx.close();
    sink.process();
    Assert.assertFalse(sink.isConfNull());

View Full Code Here

    deleteTable = false;
    AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
    Configurables.configure(sink, ctx);
    //Reset the context to a higher batchSize
    ctx.put("batchSize", "100");
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, ctx);
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    for(int i = 0; i < 3; i++){
      Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
      channel.put(e);
    }
    tx.commit();
    tx.close();
    sink.process();
    Assert.assertFalse(sink.isConfNull());

View Full Code Here

    ctx.put("batchSize", "2");
    HBaseSink sink = new HBaseSink(conf);
    Configurables.configure(sink, ctx);
    //Reset the context to a higher batchSize
    ctx.put("batchSize", "100");
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    for(int i = 0; i < 3; i++){
      Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
      channel.put(e);
    }
    tx.commit();
    tx.close();
    int count = 0;
    while(sink.process() != Status.BACKOFF){

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.flume.Channel

com.github.kristofa.flume.ZipkinMetricsSink

com.github.kristofa.flume.ZipkinSpanCollectorSink

com.salesforce.phoenix.flume.sink.PhoenixSink

com.salesforce.phoenix.flume.TestPhoenixSink

com.salesforce.phoenix.flume.TestRegexEventSerializer

org.apache.flume.channel.AbstractChannelSelector

org.apache.flume.channel.DefaultChannelFactory

org.apache.flume.channel.MockChannel

org.apache.flume.channel.MultiplexingChannelSelector

org.apache.flume.channel.ReplicatingChannelSelector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.