Examples of org.apache.flume.Sink

org.apache.flume.Sink

A sink is connected to a Channel and consumes its contents, sending them to a configured destination that may vary according to the sink type.

Sinks can be grouped together for various behaviors using SinkGroup and SinkProcessor. They are polled periodically by a SinkRunner via the processor

Sinks are associated with unique names that can be used for separating configuration and working namespaces.

While the {@link Sink#process()} call is guaranteed to only be accessedby a single thread, other calls may be concurrently accessed and should thus be protected.
@see org.apache.flume.Channel @see org.apache.flume.SinkProcessor @see org.apache.flume.SinkRunner

    if (sinkMap == null) {
      sinkMap = new HashMap<String, Sink>();
      sinks.put(sinkClass, sinkMap);
    }


    Sink sink = sinkMap.get(name);


    if (sink == null) {
      try {
        sink = sinkClass.newInstance();
        sink.setName(name);
        sinkMap.put(name,  sink);
      } catch (Exception ex) {
        // Clean up the sink map
        sinks.remove(sinkClass);
        throw new FlumeException("Unable to create sink: " + name

View Full Code Here

    channelContext.put("capacity", "1000000");
    channelContext.put("keep-alive", "0"); // for faster tests
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context(channelContext));


    Sink sink = new LoggerSink();
    sink.setChannel(channel);
    sink.start();
    DefaultSinkProcessor proc = new DefaultSinkProcessor();
    proc.setSinks(Collections.singletonList(sink));
    SinkRunner sinkRunner = new SinkRunner(proc);
    sinkRunner.start();


    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(Collections.singletonList(channel));
    ChannelProcessor chp = new ChannelProcessor(rcs);
    source.setChannelProcessor(chp);
    source.start();


    Thread.sleep(5000);
    source.stop();
    sinkRunner.stop();
    sink.stop();
  }

View Full Code Here

     */
    for (String sinkName : sinkNames) {
      ComponentConfiguration comp = compMap.get(sinkName);
      if(comp != null) {
        SinkConfiguration config = (SinkConfiguration) comp;
        Sink sink = sinkFactory.create(comp.getComponentName(),
            comp.getType());
        try {
          Configurables.configure(sink, config);
          ChannelComponent channelComponent = channelComponentMap.
              get(config.getChannel());
          if(channelComponent == null) {
            String msg = String.format("Sink %s is not connected to a " +
                "channel",  sinkName);
            throw new IllegalStateException(msg);
          }
          sink.setChannel(channelComponent.channel);
          sinks.put(comp.getComponentName(), sink);
          channelComponent.components.add(sinkName);
        } catch (Exception e) {
          String msg = String.format("Sink %s has been removed due to an " +
              "error during configuration", sinkName);
          LOGGER.error(msg, e);
        }
      }
    }
    /*
     * Components which DO NOT have a ComponentConfiguration object
     * and use only Context
     */
    Map<String, Context> sinkContexts = agentConf.getSinkContext();
    for (String sinkName : sinkNames) {
      Context context = sinkContexts.get(sinkName);
      if(context != null) {
        Sink sink = sinkFactory.create(sinkName, context.getString(
            BasicConfigurationConstants.CONFIG_TYPE));
        try {
          Configurables.configure(sink, context);
          ChannelComponent channelComponent = channelComponentMap.
              get(context.getString(BasicConfigurationConstants.CONFIG_CHANNEL));
          if(channelComponent == null) {
            String msg = String.format("Sink %s is not connected to a " +
                "channel",  sinkName);
            throw new IllegalStateException(msg);
          }
          sink.setChannel(channelComponent.channel);
          sinks.put(sinkName, sink);
          channelComponent.components.add(sinkName);
        } catch (Exception e) {
          String msg = String.format("Sink %s has been removed due to an " +
              "error during configuration", sinkName);

View Full Code Here

      ComponentConfiguration comp = compMap.get(groupName);
      if(comp != null) {
        SinkGroupConfiguration groupConf = (SinkGroupConfiguration) comp;
        List<Sink> groupSinks = new ArrayList<Sink>();
        for (String sink : groupConf.getSinks()) {
          Sink s = sinks.remove(sink);
          if (s == null) {
            String sinkUser = usedSinks.get(sink);
            if (sinkUser != null) {
              throw new InstantiationException(String.format(
                  "Sink %s of group %s already " +

View Full Code Here

    Preconditions.checkNotNull(name, "name");
    Preconditions.checkNotNull(type, "type");
    logger.info("Creating instance of sink: {}, type: {}", name, type);
    Class<? extends Sink> sinkClass = getClass(type);
    try {
      Sink sink = sinkClass.newInstance();
      sink.setName(name);
      return sink;
    } catch (Exception ex) {
      throw new FlumeException("Unable to create sink: " + name
          + ", type: " + type + ", class: " + sinkClass.getName(), ex);
    }

View Full Code Here

  public Status process() throws EventDeliveryException {
    Status status = null;


    Iterator<Sink> sinkIterator = selector.createSinkIterator();
    while (sinkIterator.hasNext()) {
      Sink sink = sinkIterator.next();
      try {
        status = sink.process();
        break;
      } catch (Exception ex) {
        LOGGER.warn("Sink failed to consume event. "
            + "Attempting next sink if available.", ex);
      }

View Full Code Here


    for (Map<String, Object> sinkDef : defs) {
      logger.debug("sink:{}", sinkDef);


      if (sinkDef.containsKey("type")) {
        Sink sink =
            getSinkFactory().create((String) sinkDef.get("type"));
        Channel channel = conf.getChannels().get(sinkDef.get("channel"));


        Context context = new Context();
        context.setParameters(sinkDef);


        Configurables.configure(sink, context);


        if (channel != null) {
          sink.setChannel(channel);
        } else {
          logger.warn(
              "No channel named {} - sink:{} is likely non-functional.", sink,
              sinkDef.get("channel"));
        }

View Full Code Here

   
   
    @Test
    public void testSinkCreation() {
        SinkFactory factory = new DefaultSinkFactory ();
        Sink sink = factory.create("PhoenixSink__", "org.apache.phoenix.flume.sink.PhoenixSink");
        Assert.assertNotNull(sink);
        Assert.assertTrue(PhoenixSink.class.isInstance(sink));
    }

View Full Code Here

    Preconditions.checkNotNull(name, "name");
    Preconditions.checkNotNull(type, "type");
    logger.info("Creating instance of sink: {}, type: {}", name, type);
    Class<? extends Sink> sinkClass = getClass(type);
    try {
      Sink sink = sinkClass.newInstance();
      sink.setName(name);
      return sink;
    } catch (Exception ex) {
      throw new FlumeException("Unable to create sink: " + name
          + ", type: " + type + ", class: " + sinkClass.getName(), ex);
    }

View Full Code Here

  public Status process() throws EventDeliveryException {
    Status status = null;


    Iterator<Sink> sinkIterator = selector.createSinkIterator();
    while (sinkIterator.hasNext()) {
      Sink sink = sinkIterator.next();
      try {
        status = sink.process();
        break;
      } catch (Exception ex) {
        selector.informSinkFailed(sink);
        LOGGER.warn("Sink failed to consume event. "
            + "Attempting next sink if available.", ex);

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.flume.Sink

com.salesforce.phoenix.flume.TestPhoenixSink

org.apache.flume.conf.file.JsonFileConfigurationProvider

org.apache.flume.conf.properties.PropertiesFileConfigurationProvider

org.apache.flume.node.AbstractConfigurationProvider

org.apache.flume.node.TestAbstractLogicalNodeManager

org.apache.flume.sink.DefaultSinkFactory

org.apache.flume.sink.elasticsearch.TestElasticSearchSinkCreation

org.apache.flume.sink.hbase.TestHBaseSinkCreation

org.apache.flume.sink.LoadBalancingSinkProcessor

org.apache.flume.sink.TestDefaultSinkFactory

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.