Examples of org.apache.flume.Sink

org.apache.flume.Sink

A sink is connected to a Channel and consumes its contents, sending them to a configured destination that may vary according to the sink type.

Sinks can be grouped together for various behaviors using SinkGroup and SinkProcessor. They are polled periodically by a SinkRunner via the processor

Sinks are associated with unique names that can be used for separating configuration and working namespaces.

While the {@link Sink#process()} call is guaranteed to only be accessedby a single thread, other calls may be concurrently accessed and should thus be protected.
@see org.apache.flume.Channel @see org.apache.flume.SinkProcessor @see org.apache.flume.SinkRunner

    Configurables.configure(channel, new Context());


    Source source = new SequenceGeneratorSource();
    source.setChannel(channel);


    Sink sink = new NullSink();
    sink.setChannel(channel);


    nodeManager.add(SourceRunner.forSource(source));
    nodeManager.add(SinkRunner.forSink(sink));


    for (int i = 0; i < 10; i++) {

View Full Code Here


    if (!sinkRegistry.containsKey(name)) {
      return null;
    }


    Sink sink = null;


    try {
      sink = sinkRegistry.get(name).newInstance();
    } catch (IllegalAccessException e) {
      throw new InstantiationException("Unable to create sink " + name

View Full Code Here


    Assert.assertEquals(1, sinkFactory.getSinkNames().size());


    Assert.assertEquals("null", sinkFactory.getSinkNames().iterator().next());


    Sink sink = sinkFactory.create("null");


    Assert.assertNotNull("Factory returned a null sink", sink);
    Assert.assertTrue("Source isn't an instance of NullSink",
        sink instanceof NullSink);

View Full Code Here

    if (sinkMap == null) {
      sinkMap = new HashMap<String, Sink>();
      sinks.put(sinkClass, sinkMap);
    }


    Sink sink = sinkMap.get(name);


    if (sink == null) {
      try {
        sink = sinkClass.newInstance();
        sink.setName(name);
        sinkMap.put(name,  sink);
      } catch (Exception ex) {
        // Clean up the sink map
        sinks.remove(sinkClass);
        throw new FlumeException("Unable to create sink: " + name

View Full Code Here

  public Status process() throws EventDeliveryException {
    Status status = null;


    Iterator<Sink> sinkIterator = selector.createSinkIterator();
    while (sinkIterator.hasNext()) {
      Sink sink = sinkIterator.next();
      try {
        status = sink.process();
        break;
      } catch (Exception ex) {
        LOGGER.warn("Sink failed to consume event. "
            + "Attempting next sink if available.", ex);
      }

View Full Code Here

        Map<String, Sink> sinks = new HashMap<String, Sink>();
        for (String sinkName : sinkNames) {
            ComponentConfiguration comp = compMap.get(sinkName);
            if (comp != null) {
                SinkConfiguration config = (SinkConfiguration) comp;
                Sink sink = sinkFactory.create(comp.getComponentName(), comp.getType());


                Configurables.configure(sink, config);


                sink.setChannel(conf.getChannels().get(config.getChannel()));
                sinks.put(comp.getComponentName(), sink);
            }
        }


        Map<String, Context> sinkContexts = agentConf.getSinkContext();
        for (String sinkName : sinkNames) {
            Context context = sinkContexts.get(sinkName);
            if (context != null) {
                Sink sink = sinkFactory.create(sinkName, context.getString(BasicConfigurationConstants.CONFIG_TYPE));
                Configurables.configure(sink, context);


                sink.setChannel(conf.getChannels().get(context.getString(BasicConfigurationConstants.CONFIG_CHANNEL)));
                sinks.put(sinkName, sink);
            }
        }


        loadSinkGroups(agentConf, sinks, conf);

View Full Code Here

            if (comp != null) {
                SinkGroupConfiguration groupConf = (SinkGroupConfiguration) comp;
                List<String> groupSinkList = groupConf.getSinks();
                List<Sink> groupSinks = new ArrayList<Sink>();
                for (String sink : groupSinkList) {
                    Sink s = sinks.remove(sink);
                    if (s == null) {
                        String sinkUser = usedSinks.get(sink);
                        if (sinkUser != null) {
                            throw new ConfigurationException(String.format(
                                "Sink %s of group %s already in use by group %s", sink, groupName, sinkUser));

View Full Code Here

        final Map<String, Sink> sinks = new HashMap<String, Sink>();
        for (final String sinkName : sinkNames) {
            final ComponentConfiguration comp = compMap.get(sinkName);
            if (comp != null) {
                final SinkConfiguration config = (SinkConfiguration) comp;
                final Sink sink = sinkFactory.create(comp.getComponentName(), comp.getType());


                Configurables.configure(sink, config);


                sink.setChannel(conf.getChannels().get(config.getChannel()));
                sinks.put(comp.getComponentName(), sink);
            }
        }


        final Map<String, Context> sinkContexts = agentConf.getSinkContext();
        for (final String sinkName : sinkNames) {
            final Context context = sinkContexts.get(sinkName);
            if (context != null) {
                final Sink sink = sinkFactory.create(sinkName,
                    context.getString(BasicConfigurationConstants.CONFIG_TYPE));
                Configurables.configure(sink, context);


                sink.setChannel(conf.getChannels().get(context.getString(BasicConfigurationConstants.CONFIG_CHANNEL)));
                sinks.put(sinkName, sink);
            }
        }


        loadSinkGroups(agentConf, sinks, conf);

View Full Code Here

            if (comp != null) {
                final SinkGroupConfiguration groupConf = (SinkGroupConfiguration) comp;
                final List<String> groupSinkList = groupConf.getSinks();
                final List<Sink> groupSinks = new ArrayList<Sink>();
                for (final String sink : groupSinkList) {
                    final Sink s = sinks.remove(sink);
                    if (s == null) {
                        final String sinkUser = usedSinks.get(sink);
                        if (sinkUser != null) {
                            throw new ConfigurationException(String.format(
                                "Sink %s of group %s already in use by group %s", sink, groupName, sinkUser));

View Full Code Here

        Map<String, Sink> sinks = new HashMap<String, Sink>();
        for (String sinkName : sinkNames) {
            ComponentConfiguration comp = compMap.get(sinkName);
            if (comp != null) {
                SinkConfiguration config = (SinkConfiguration) comp;
                Sink sink = sinkFactory.create(comp.getComponentName(), comp.getType());


                Configurables.configure(sink, config);


                sink.setChannel(conf.getChannels().get(config.getChannel()));
                sinks.put(comp.getComponentName(), sink);
            }
        }


        Map<String, Context> sinkContexts = agentConf.getSinkContext();
        for (String sinkName : sinkNames) {
            Context context = sinkContexts.get(sinkName);
            if (context != null) {
                Sink sink = sinkFactory.create(sinkName, context.getString(BasicConfigurationConstants.CONFIG_TYPE));
                Configurables.configure(sink, context);


                sink.setChannel(conf.getChannels().get(context.getString(BasicConfigurationConstants.CONFIG_CHANNEL)));
                sinks.put(sinkName, sink);
            }
        }


        loadSinkGroups(agentConf, sinks, conf);

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.flume.Sink

com.salesforce.phoenix.flume.TestPhoenixSink

org.apache.flume.conf.file.JsonFileConfigurationProvider

org.apache.flume.conf.properties.PropertiesFileConfigurationProvider

org.apache.flume.node.AbstractConfigurationProvider

org.apache.flume.node.TestAbstractLogicalNodeManager

org.apache.flume.sink.DefaultSinkFactory

org.apache.flume.sink.elasticsearch.TestElasticSearchSinkCreation

org.apache.flume.sink.hbase.TestHBaseSinkCreation

org.apache.flume.sink.LoadBalancingSinkProcessor

org.apache.flume.sink.TestDefaultSinkFactory

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.