Examples of org.apache.flume.channel.MemoryChannel$StampedEvent

org.apache.flume.channel.MemoryChannel
Memory channel that with full transaction support Uses transaction object for each thread (source and sink) attached to channel. The events are stored in the thread safe Dequeue. * The put and take are directly executed in the common queue. Channel has a marker for the last committed event in order to avoid sink reading uncommitted data. The transactions keep track of the actions to perform undo when rolled back.

    context.put("hdfs.rollCount", String.valueOf(rollCount));
    context.put("hdfs.batchSize", String.valueOf(batchSize));


    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();
    // push the event batches into channel
    for (int i = 1; i < 4; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (int j = 1; j <= txnMax; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
            String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);
        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
      }
      txn.commit();
      txn.close();


      // execute sink to process the events

View Full Code Here

    context.put("hdfs.batchSize", String.valueOf(batchSize));
    context.put("hdfs.fileType", HDFSBadWriterFactory.BadSequenceFileType);


    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();


    List<String> bodies = Lists.newArrayList();
    // push the event batches into channel
    for (i = 1; i < numBatches; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= txnMax; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
            String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);


        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        // inject fault
        if ((totalEvents % 30) == 1) {
          event.getHeaders().put("fault-once", "");
        }
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();

View Full Code Here

    context.put("hdfs.batchSize", String.valueOf(batchSize));
    context.put("hdfs.fileType", HDFSBadWriterFactory.BadSequenceFileType);


    Configurables.configure(sink, context);


    MemoryChannel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();
    // push the event batches into channel
    for (i = 1; i < numBatches; i++) {
      channel.getTransaction().begin();
      try {
        for (j = 1; j <= txnMax; j++) {
          Event event = new SimpleEvent();
          eventDate.clear();
          eventDate.set(2011, i, i, i, 0); // yy mm dd
          event.getHeaders().put("timestamp",
              String.valueOf(eventDate.getTimeInMillis()));
          event.getHeaders().put("hostname", "Host" + i);
          String body = "Test." + i + "." + j;
          event.setBody(body.getBytes());
          bodies.add(body);
          // inject fault
          event.getHeaders().put("fault-until-reopen", "");
          channel.put(event);
        }
        channel.getTransaction().commit();
      } finally {
        channel.getTransaction().close();
      }
      LOG.info("execute sink to process the events: " + sink.process());
    }
    LOG.info("clear any events pending due to errors: " + sink.process());
    sink.stop();

View Full Code Here

    context.put("hdfs.batchSize", String.valueOf(batchSize));
    context.put("hdfs.fileType", HDFSBadWriterFactory.BadSequenceFileType);
    context.put("hdfs.callTimeout", Long.toString(1000));
    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();


    // push the event batches into channel
    for (i = 0; i < numBatches; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= txnMax; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
            String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);
        event.getHeaders().put("slow", "1500");
        event.setBody(("Test." + i + "." + j).getBytes());
        channel.put(event);
      }
      txn.commit();
      txn.close();


      // execute sink to process the events

View Full Code Here

    context.put("hdfs.batchSize", String.valueOf(batchSize));
    context.put("hdfs.fileType", HDFSBadWriterFactory.BadSequenceFileType);
    context.put("hdfs.appendTimeout", String.valueOf(appendTimeout));
    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();
    // push the event batches into channel
    for (i = 0; i < numBatches; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= txnMax; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
            String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);
        event.getHeaders().put("slow", "1500");
        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();

View Full Code Here

    tmpDir = new File("/tmp/flume-rfs-" + System.currentTimeMillis() + "-"
        + Thread.currentThread().getId());


    sink = new RollingFileSink();


    sink.setChannel(new MemoryChannel());


    tmpDir.mkdirs();
  }

View Full Code Here

  }


  @Before
  public void setUp() {
    source = new ThriftLegacySource();
    channel = new MemoryChannel();


    Configurables.configure(channel, new Context());


    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);

View Full Code Here

  private static final int TEST_SYSLOG_PORT = 14455;


  @Before
  public void setUp() {
    source = new SyslogUDPSource(); //SyslogTcpSource();
    channel = new MemoryChannel();


    Configurables.configure(channel, new Context());


    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);

View Full Code Here


  @Test
  public void testProcess() throws InterruptedException, LifecycleException,
  EventDeliveryException, IOException {


    Channel channel = new MemoryChannel();
    Context context = new Context();


    context.put("command", "cat /etc/passwd");
    context.put("keep-alive", "1");
    context.put("capacity", "1000");
    context.put("transactionCapacity", "1000");
    Configurables.configure(source, context);
    Configurables.configure(channel, context);


    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(Lists.newArrayList(channel));


    source.setChannelProcessor(new ChannelProcessor(rcs));


    source.start();
    Transaction transaction = channel.getTransaction();


    transaction.begin();
    Event event;


    FileOutputStream outputStream = new FileOutputStream(
        "/tmp/flume-execsource." + Thread.currentThread().getId());


    while ((event = channel.take()) != null) {
      outputStream.write(event.getBody());
      outputStream.write('\n');
    }


    outputStream.close();

View Full Code Here


  @Test
  public void testRestart() throws InterruptedException, LifecycleException,
  EventDeliveryException, IOException {


    Channel channel = new MemoryChannel();
    Context context = new Context();


    context.put(ExecSourceConfigurationConstants.CONFIG_RESTART_THROTTLE, "10");
    context.put(ExecSourceConfigurationConstants.CONFIG_RESTART, "true");


    context.put("command", "echo flume");
    Configurables.configure(source, context);
    Configurables.configure(channel, context);


    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(Lists.newArrayList(channel));


    source.setChannelProcessor(new ChannelProcessor(rcs));


    source.start();
    Transaction transaction = channel.getTransaction();


    transaction.begin();


    long start = System.currentTimeMillis();


    for(int i = 0; i < 5; i++) {
      Event event = channel.take();
      assertNotNull(event);
      assertNotNull(event.getBody());
      assertEquals("flume", new String(event.getBody(), Charsets.UTF_8));
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.flume.channel.MemoryChannel$StampedEvent

com.salesforce.phoenix.flume.TestPhoenixSink

com.salesforce.phoenix.flume.TestRegexEventSerializer

org.apache.flume.clients.log4jappender.TestLoadBalancingLog4jAppender

org.apache.flume.clients.log4jappender.TestLog4jAppender

org.apache.flume.clients.log4jappender.TestLog4jAppenderWithAvro

org.apache.flume.Event

org.apache.flume.instrumentation.ChannelCounter

org.apache.flume.interceptor.TestCensoringInterceptor

org.apache.flume.node.TestAbstractLogicalNodeManager

org.apache.flume.node.TestDefaultLogicalNodeManager

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.