Examples of org.apache.flume.Channel

org.apache.flume.Channel

A channel connects a {@link Source} to a {@link Sink}. The source acts as producer while the sink acts as a consumer of events. The channel itself is the buffer between the two.

A channel exposes a {@link Transaction} interface that can be used byits clients to ensure atomic {@linkplain #put(Event) put} and{@linkplain #take() take} semantics.This is necessary to guarantee single hop reliability between agents. For instance, a source will successfully produce an {@linkplain Event event}if and only if that event can be committed to the source's associated channel. Similarly, a sink will consume an event if and only if its respective endpoint can accept the event. The extent of transaction support varies for different channel implementations ranging from strong to best-effort semantics.

Channels are associated with unique {@linkplain NamedComponent names} thatcan be used for separating configuration and working namespaces.

Channels must be thread safe, protecting any internal invariants as no guarantees are given as to when and by how many sources/sinks they may be simultaneously accessed by.
@see org.apache.flume.Source @see org.apache.flume.Sink @see org.apache.flume.Transaction

    setUp("deflate", 6);


    boolean bound = false;


    AvroSource source;
    Channel sourceChannel;
    int selectedPort;


    source = new AvroSource();
    sourceChannel = new MemoryChannel();


    Configurables.configure(sourceChannel, new Context());


    List<Channel> channels = new ArrayList<Channel>();
    channels.add(sourceChannel);


    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);


    source.setChannelProcessor(new ChannelProcessor(rcs));


    Context context = new Context();
    context.put("port", port.toString());
    context.put("bind", hostname);
    context.put("threads", "50");
    context.put("compression-type", "deflate");
    context.put("ssl", String.valueOf(true));
    context.put("keystore", "src/test/resources/server.p12");
    context.put("keystore-password", "password");
    context.put("keystore-type", "PKCS12");


    Configurables.configure(source, context);


    source.start();


    Assert
        .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
            source, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START,
        source.getLifecycleState());




    Event event = EventBuilder.withBody("Hello avro",
        Charset.forName("UTF8"));


    context = new Context();


    context.put("hostname", hostname);
    context.put("port", String.valueOf(port));
    context.put("ssl", String.valueOf(true));
    context.put("trust-all-certs", String.valueOf(true));
    context.put("batch-size", String.valueOf(2));
    context.put("connect-timeout", String.valueOf(2000L));
    context.put("request-timeout", String.valueOf(3000L));
    context.put("compression-type", "deflate");
    context.put("compression-level", Integer.toString(6));


    Configurables.configure(sink, context);
    sink.start();


    Transaction sickTransaction = channel.getTransaction();


    sickTransaction.begin();
    for (int i = 0; i < 10; i++) {
      channel.put(event);
    }
    sickTransaction.commit();
    sickTransaction.close();


    for (int i = 0; i < 5; i++) {
      Sink.Status status = sink.process();
      logger.debug("Calling Process " + i + " times:" + status);
      Assert.assertEquals(Sink.Status.READY, status);
    }


    sink.stop();


    Transaction sourceTransaction = sourceChannel.getTransaction();
    sourceTransaction.begin();


    Event sourceEvent = sourceChannel.take();
    Assert.assertNotNull(sourceEvent);
    Assert.assertEquals("Channel contained our event", "Hello avro",
        new String(sourceEvent.getBody()));
    sourceTransaction.commit();
    sourceTransaction.close();

View Full Code Here

    }


    boolean bound = false;


    AvroSource source;
    Channel sourceChannel;
    int selectedPort;


    source = new AvroSource();
    sourceChannel = new MemoryChannel();


    Configurables.configure(sourceChannel, new Context());


    List<Channel> channels = new ArrayList<Channel>();
    channels.add(sourceChannel);


    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);


    source.setChannelProcessor(new ChannelProcessor(rcs));


    Context context = new Context();
    context.put("port", port.toString());
    context.put("bind", hostname);
    context.put("threads", "50");
    if (serverEnableCompression) {
      context.put("compression-type", "deflate");
    } else {
      context.put("compression-type", "none");
    }


    Configurables.configure(source, context);


    source.start();


    Assert
        .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
            source, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START,
        source.getLifecycleState());




    Event event = EventBuilder.withBody("Hello avro",
        Charset.forName("UTF8"));


    sink.start();


    Transaction sickTransaction = channel.getTransaction();


    sickTransaction.begin();
    for (int i = 0; i < 10; i++) {
      channel.put(event);
    }
    sickTransaction.commit();
    sickTransaction.close();


    for (int i = 0; i < 5; i++) {
      Sink.Status status = sink.process();
      logger.debug("Calling Process " + i + " times:" + status);
      Assert.assertEquals(Sink.Status.READY, status);
    }


    sink.stop();




    Transaction sourceTransaction = sourceChannel.getTransaction();
    sourceTransaction.begin();


    Event sourceEvent = sourceChannel.take();
    Assert.assertNotNull(sourceEvent);
    Assert.assertEquals("Channel contained our event", "Hello avro",
        new String(sourceEvent.getBody()));
    sourceTransaction.commit();
    sourceTransaction.close();

View Full Code Here

   * Basic test to exercise multiple-port parsing.
   */
  @Test
  public void testMultiplePorts() throws IOException, ParseException {
    MultiportSyslogTCPSource source = new MultiportSyslogTCPSource();
    Channel channel = new MemoryChannel();


    Context channelContext = new Context();
    channelContext.put("capacity", String.valueOf(2000));
    channelContext.put("transactionCapacity", String.valueOf(2000));
    Configurables.configure(channel, channelContext);


    List<Channel> channels = Lists.newArrayList();
    channels.add(channel);


    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);


    source.setChannelProcessor(new ChannelProcessor(rcs));
    Context context = new Context();
    StringBuilder ports = new StringBuilder();
    for (int i = 0; i < 1000; i++) {
      ports.append(String.valueOf(BASE_TEST_SYSLOG_PORT + i)).append(" ");
    }
    context.put(SyslogSourceConfigurationConstants.CONFIG_PORTS,
        ports.toString().trim());
    source.configure(context);
    source.start();


    Socket syslogSocket;
    for (int i = 0; i < 1000 ; i++) {
      syslogSocket = new Socket(
              InetAddress.getLocalHost(), BASE_TEST_SYSLOG_PORT + i);
      syslogSocket.getOutputStream().write(getEvent(i));
      syslogSocket.close();
    }


    List<Event> channelEvents = new ArrayList<Event>();
    Transaction txn = channel.getTransaction();
    txn.begin();
    for (int i = 0; i < 1000; i++) {
      Event e = channel.take();
      if (e == null) {
        throw new NullPointerException("Event is null");
      }
      channelEvents.add(e);
    }

View Full Code Here

    source.configure(context);


    Map<String, String> channelContext = new HashMap();
    channelContext.put("capacity", "1000000");
    channelContext.put("keep-alive", "0"); // for faster tests
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context(channelContext));


    Sink sink = new LoggerSink();
    sink.setChannel(channel);
    sink.start();

View Full Code Here

    }


    Map<String, String> channelContext = new HashMap();
    channelContext.put("capacity", "1000000");
    channelContext.put("keep-alive", "0"); // for faster tests
    Channel channel = new MemoryChannel();
    channel.setName(channel.getClass().getName() + SEQ_NUM.getAndIncrement());
    Configurables.configure(channel, new Context(channelContext));
 
    class MySolrSink extends MorphlineSolrSink {
      public MySolrSink(MorphlineHandlerImpl indexer) {
        super(indexer);

View Full Code Here

      this.writer = newWriter(login, target);
      this.lastRolledMs = System.currentTimeMillis();
      LOG.info("Rolled writer for " + getName());
    }


    Channel channel = getChannel();
    Transaction transaction = null;
    try {
      long processedEvents = 0;


      transaction = channel.getTransaction();
      transaction.begin();
      for (; processedEvents < batchSize; processedEvents += 1) {
        Event event = channel.take();
        if (event == null) {
          // no events available in the channel
          break;
        }

View Full Code Here

     * Components which have a ComponentConfiguration object
     */
    for (String chName : channelNames) {
      ComponentConfiguration comp = compMap.get(chName);
      if(comp != null) {
        Channel channel = getOrCreateChannel(channelsNotReused,
            comp.getComponentName(), comp.getType());
        try {
          Configurables.configure(channel, comp);
          channelComponentMap.put(comp.getComponentName(),
              new ChannelComponent(channel));
          LOGGER.info("Created channel " + chName);
        } catch (Exception e) {
          String msg = String.format("Channel %s has been removed due to an " +
              "error during configuration", chName);
          LOGGER.error(msg, e);
        }
      }
    }
    /*
     * Components which DO NOT have a ComponentConfiguration object
     * and use only Context
     */
    for (String chName : channelNames) {
      Context context = agentConf.getChannelContext().get(chName);
      if(context != null){
        Channel channel =
            getOrCreateChannel(channelsNotReused, chName, context.getString(
                BasicConfigurationConstants.CONFIG_TYPE));
        try {
          Configurables.configure(channel, context);
          channelComponentMap.put(chName, new ChannelComponent(channel));

View Full Code Here

        getClass(type);
    /*
     * Channel has requested a new instance on each re-configuration
     */
    if(channelClass.isAnnotationPresent(Disposable.class)) {
      Channel channel = channelFactory.create(name, type);
      channel.setName(name);
      return channel;
    }
    Map<String, Channel> channelMap = channelCache.get(channelClass);
    if (channelMap == null) {
      channelMap = new HashMap<String, Channel>();
      channelCache.put(channelClass, channelMap);
    }
    Channel channel = channelMap.get(name);
    if(channel == null) {
      channel = channelFactory.create(name, type);
      channel.setName(name);
      channelMap.put(name, channel);
    }
    channelsNotReused.get(channelClass).remove(name);
    return channel;
  }

View Full Code Here


  @Override
  public Status process() throws EventDeliveryException {
    int batchSize = getMaxBatchSize();
    long batchEndTime = System.currentTimeMillis() + getMaxBatchDurationMillis();
    Channel myChannel = getChannel();
    Transaction txn = myChannel.getTransaction();
    txn.begin();
    boolean isMorphlineTransactionCommitted = true;
    try {
      int numEventsTaken = 0;
      handler.beginTransaction();
      isMorphlineTransactionCommitted = false;


      // repeatedly take and process events from the Flume queue
      for (int i = 0; i < batchSize; i++) {
        Event event = myChannel.take();
        if (event == null) {
          break;
        }
        sinkCounter.incrementEventDrainAttemptCount();
        numEventsTaken++;
        LOGGER.debug("Flume event: {}", event);
        //StreamEvent streamEvent = createStreamEvent(event);
        handler.process(event);
        if (System.currentTimeMillis() >= batchEndTime) {
          break;
        }
      }


      // update metrics
      if (numEventsTaken == 0) {
        sinkCounter.incrementBatchEmptyCount();
      }
      if (numEventsTaken < batchSize) {
        sinkCounter.incrementBatchUnderflowCount();
      } else {
        sinkCounter.incrementBatchCompleteCount();
      }
      handler.commitTransaction();
      isMorphlineTransactionCommitted = true;
      txn.commit();
      sinkCounter.addToEventDrainSuccessCount(numEventsTaken);
      return numEventsTaken == 0 ? Status.BACKOFF : Status.READY;
    } catch (Throwable t) {
      // Ooops - need to rollback and back off
      LOGGER.error("Morphline Sink " + getName() + ": Unable to process event from channel " + myChannel.getName()
            + ". Exception follows.", t);
      try {
        if (!isMorphlineTransactionCommitted) {
          handler.rollbackTransaction();
        }

View Full Code Here

  }


  @Test
  public void shouldIndexOneEvent() throws Exception {
    Configurables.configure(fixture, new Context(parameters));
    Channel channel = bindAndStartChannel(fixture);


    Transaction tx = channel.getTransaction();
    tx.begin();
    Event event = EventBuilder.withBody("event #1 or 1".getBytes());
    channel.put(event);
    tx.commit();
    tx.close();


    fixture.process();
    fixture.stop();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.flume.Channel

com.github.kristofa.flume.ZipkinMetricsSink

com.github.kristofa.flume.ZipkinSpanCollectorSink

com.salesforce.phoenix.flume.sink.PhoenixSink

com.salesforce.phoenix.flume.TestPhoenixSink

com.salesforce.phoenix.flume.TestRegexEventSerializer

org.apache.flume.channel.AbstractChannelSelector

org.apache.flume.channel.DefaultChannelFactory

org.apache.flume.channel.MockChannel

org.apache.flume.channel.MultiplexingChannelSelector

org.apache.flume.channel.ReplicatingChannelSelector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.