Examples of org.apache.flume.channel.PseudoTxnMemoryChannel

org.apache.flume.channel.PseudoTxnMemoryChannel

A capacity-capped {@link Channel} implementation that supports in-memorybuffering and delivery of events.

This channel is appropriate for best effort delivery of events where high throughput is favored over data durability. To be clear, this channel offers absolutely no guarantee of event delivery in the face of (any) component failure.

TODO: Discuss guarantees, corner cases re: potential data loss (e.g. consumer begins a tx, takes events, and gets SIGKILL before rollback).

Configuration options

Parameter	Description	Unit / Type	Default
`capacity`	The in-memory capacity of this channel. Store up to `capacity` events before refusing new events.	events / int	50
`keep-alive`	The amount of time (seconds) to wait for an event before returning `null` on {@link #take()}.	seconds / int	3

Metrics

TODO

    context.put("sink.directory", tmpDir.getPath());
    context.put("sink.rollInterval", "1");


    Configurables.configure(sink, context);


    Channel channel = new PseudoTxnMemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    for (int i = 0; i < 10; i++) {
      Event event = new SimpleEvent();


      event.setBody(("Test event " + i).getBytes());


      channel.put(event);
      sink.process();


      Thread.sleep(500);
    }

View Full Code Here

    context.put("sink.directory", tmpDir.getPath());
    context.put("sink.rollInterval", "0");


    Configurables.configure(sink, context);


    Channel channel = new PseudoTxnMemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    for (int i = 0; i < 10; i++) {
      Event event = new SimpleEvent();


      event.setBody(("Test event " + i).getBytes());


      channel.put(event);
      sink.process();


      Thread.sleep(500);
    }

View Full Code Here


  @Test
  public void testProcess() throws InterruptedException, LifecycleException,
      EventDeliveryException {


    Channel channel = new PseudoTxnMemoryChannel();
    Context context = new Context();


    context.put("logicalNode.name", "test");


    Configurables.configure(source, context);
    Configurables.configure(channel, context);


    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);


    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);


    source.setChannelProcessor(new ChannelProcessor(rcs));


    for (long i = 0; i < 100; i++) {
      source.process();
      Event event = channel.take();


      Assert.assertArrayEquals(String.valueOf(i).getBytes(),
          new String(event.getBody()).getBytes());
    }
  }

View Full Code Here

  public void testBatchProcessWithLifeCycle() throws InterruptedException, LifecycleException,
      EventDeliveryException {


    int batchSize = 10;


    Channel channel = new PseudoTxnMemoryChannel();
    Context context = new Context();


    context.put("logicalNode.name", "test");
    context.put("batchSize", Integer.toString(batchSize));


    Configurables.configure(source, context);
    Configurables.configure(channel, context);


    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);


    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);


    source.setChannelProcessor(new ChannelProcessor(rcs));


    source.start();


    for (long i = 0; i < 100; i++) {
      source.process();


      for (long j = batchSize; j > 0; j--) {
        Event event = channel.take();
        String expectedVal = String.valueOf(((i+1)*batchSize)-j);
        String resultedVal = new String(event.getBody());
        Assert.assertTrue("Expected " + expectedVal + " is not equals to " +
            resultedVal, expectedVal.equals(resultedVal));
      }

View Full Code Here


  @Test
  public void testLifecycle() throws InterruptedException,
      EventDeliveryException {


    Channel channel = new PseudoTxnMemoryChannel();
    Context context = new Context();


    context.put("logicalNode.name", "test");


    Configurables.configure(source, context);
    Configurables.configure(channel, context);


    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);


    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);


    source.setChannelProcessor(new ChannelProcessor(rcs));


    source.start();


    for (long i = 0; i < 100; i++) {
      source.process();
      Event event = channel.take();


      Assert.assertArrayEquals(String.valueOf(i).getBytes(),
          new String(event.getBody()).getBytes());
    }
    source.stop();

View Full Code Here


  @Test
  public void testProcess() throws InterruptedException, LifecycleException,
      EventDeliveryException {


    Channel channel = new PseudoTxnMemoryChannel();
    Context context = new Context();


    context.put("logicalNode.name", "test");


    Configurables.configure(source, context);
    Configurables.configure(channel, context);


    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);


    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);


    source.setChannelProcessor(new ChannelProcessor(rcs));


    for (long i = 0; i < 100; i++) {
      source.process();
      Event event = channel.take();


      Assert.assertArrayEquals(String.valueOf(i).getBytes(),
          new String(event.getBody()).getBytes());
    }
  }

View Full Code Here


  @Test
  public void testLifecycle() throws InterruptedException,
      EventDeliveryException {


    Channel channel = new PseudoTxnMemoryChannel();
    Context context = new Context();


    context.put("logicalNode.name", "test");


    Configurables.configure(source, context);
    Configurables.configure(channel, context);


    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);


    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);


    source.setChannelProcessor(new ChannelProcessor(rcs));


    source.start();


    for (long i = 0; i < 100; i++) {
      source.process();
      Event event = channel.take();


      Assert.assertArrayEquals(String.valueOf(i).getBytes(),
          new String(event.getBody()).getBytes());
    }

View Full Code Here

   */
  @Test
  public void testAppend() throws InterruptedException, LifecycleException,
      EventDeliveryException {


    Channel channel = new PseudoTxnMemoryChannel();
    Context context = new Context();
    Configurables.configure(channel, context);
    Configurables.configure(sink, context);


    sink.setChannel(channel);
    sink.start();


    for (int i = 0; i < 10; i++) {
      Event event = EventBuilder.withBody(("Test " + i).getBytes());


      channel.put(event);
      sink.process();
    }


    sink.stop();
  }

View Full Code Here

    context.put("sink.directory", tmpDir.getPath());
    context.put("sink.rollInterval", "1");


    Configurables.configure(sink, context);


    Channel channel = new PseudoTxnMemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    for (int i = 0; i < 10; i++) {
      Event event = new SimpleEvent();


      event.setBody(("Test event " + i).getBytes());


      channel.put(event);
      sink.process();


      Thread.sleep(500);
    }

View Full Code Here

    context.put("sink.rollInterval", "1");
    context.put("sink.batchSize", "1");


    Configurables.configure(sink, context);


    Channel channel = new PseudoTxnMemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    for (int i = 0; i < 10; i++) {
      Event event = new SimpleEvent();


      event.setBody(("Test event " + i).getBytes());


      channel.put(event);
      sink.process();


      Thread.sleep(500);
    }

View Full Code Here

0 1

TOP

Related Classes of org.apache.flume.channel.PseudoTxnMemoryChannel

org.apache.flume.Event

org.apache.flume.instrumentation.ChannelCounter

org.apache.flume.sink.TestLoggerSink

org.apache.flume.sink.TestRollingFileSink

org.apache.flume.source.TestSequenceGeneratorSource

org.apache.flume.ChannelException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.