Examples of org.apache.flume.Source

org.apache.flume.Source

A source generates {@plainlink Event events} and calls methods on theconfigured {@link ChannelProcessor} to persist those events into theconfigured {@linkplain Channel channels}.

Sources are associated with unique {@linkplain NamedComponent names} that canbe used for separating configuration and working namespaces.

No guarantees are given regarding thread safe access.
@see org.apache.flume.Channel @see org.apache.flume.Sink

    } else if(state == State.NEW) {
      throw new IllegalStateException("Cannot be started before being " +
          "configured");
    }
    doStart();
    Source source = Preconditions.checkNotNull(sourceRunner.getSource(),
        "Source runner returned null source");
    if(source instanceof EmbeddedSource) {
      embeddedSource = (EmbeddedSource)source;
    } else {
      throw new IllegalStateException("Unknown source type: " + source.
          getClass().getName());
    }
    state = State.STARTED;
  }

View Full Code Here

    for (String sourceName : sourceNames) {
      ComponentConfiguration comp = compMap.get(sourceName);
      if(comp != null) {
        SourceConfiguration config = (SourceConfiguration) comp;


        Source source = sourceFactory.create(comp.getComponentName(),
            comp.getType());
        try {
          Configurables.configure(source, config);
          Set<String> channelNames = config.getChannels();
          List<Channel> sourceChannels = new ArrayList<Channel>();
          for (String chName : channelNames) {
            ChannelComponent channelComponent = channelComponentMap.get(chName);
            if(channelComponent != null) {
              sourceChannels.add(channelComponent.channel);
            }
          }
          if(sourceChannels.isEmpty()) {
            String msg = String.format("Source %s is not connected to a " +
                "channel",  sourceName);
            throw new IllegalStateException(msg);
          }
          ChannelSelectorConfiguration selectorConfig =
              config.getSelectorConfiguration();


          ChannelSelector selector = ChannelSelectorFactory.create(
              sourceChannels, selectorConfig);


          ChannelProcessor channelProcessor = new ChannelProcessor(selector);
          Configurables.configure(channelProcessor, config);


          source.setChannelProcessor(channelProcessor);
          sourceRunnerMap.put(comp.getComponentName(),
              SourceRunner.forSource(source));
          for(Channel channel : sourceChannels) {
            ChannelComponent channelComponent = Preconditions.
                checkNotNull(channelComponentMap.get(channel.getName()),
                    String.format("Channel %s", channel.getName()));
            channelComponent.components.add(sourceName);
          }
        } catch (Exception e) {
          String msg = String.format("Source %s has been removed due to an " +
              "error during configuration", sourceName);
          LOGGER.error(msg, e);
        }
      }
    }
    /*
     * Components which DO NOT have a ComponentConfiguration object
     * and use only Context
     */
    Map<String, Context> sourceContexts = agentConf.getSourceContext();
    for (String sourceName : sourceNames) {
      Context context = sourceContexts.get(sourceName);
      if(context != null){
        Source source =
            sourceFactory.create(sourceName,
                context.getString(BasicConfigurationConstants.CONFIG_TYPE));
        try {
          Configurables.configure(source, context);
          List<Channel> sourceChannels = new ArrayList<Channel>();
          String[] channelNames = context.getString(
              BasicConfigurationConstants.CONFIG_CHANNELS).split("\\s+");
          for (String chName : channelNames) {
            ChannelComponent channelComponent = channelComponentMap.get(chName);
            if(channelComponent != null) {
              sourceChannels.add(channelComponent.channel);
            }
          }
          if(sourceChannels.isEmpty()) {
            String msg = String.format("Source %s is not connected to a " +
                "channel",  sourceName);
            throw new IllegalStateException(msg);
          }
          Map<String, String> selectorConfig = context.getSubProperties(
              BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);


          ChannelSelector selector = ChannelSelectorFactory.create(
              sourceChannels, selectorConfig);


          ChannelProcessor channelProcessor = new ChannelProcessor(selector);
          Configurables.configure(channelProcessor, context);
          source.setChannelProcessor(channelProcessor);
          sourceRunnerMap.put(sourceName,
              SourceRunner.forSource(source));
          for(Channel channel : sourceChannels) {
            ChannelComponent channelComponent = Preconditions.
                checkNotNull(channelComponentMap.get(channel.getName()),

View Full Code Here

  }


  @Test
  public void testDuplicateCreate()  {


    Source avroSource1 = sourceFactory.create("avroSource1", "avro");
    Source avroSource2 = sourceFactory.create("avroSource2", "avro");


    Assert.assertNotNull(avroSource1);
    Assert.assertNotNull(avroSource2);
    Assert.assertNotSame(avroSource1, avroSource2);
    Assert.assertTrue(avroSource1 instanceof AvroSource);
    Assert.assertTrue(avroSource2 instanceof AvroSource);


    Source s1 = sourceFactory.create("avroSource1", "avro");
    Source s2 = sourceFactory.create("avroSource2", "avro");


    Assert.assertNotSame(avroSource1, s1);
    Assert.assertNotSame(avroSource2, s2);


  }

View Full Code Here


  }


  private void verifySourceCreation(String name, String type,
      Class<?> typeClass) throws Exception {
    Source src = sourceFactory.create(name, type);
    Assert.assertNotNull(src);
    Assert.assertTrue(typeClass.isInstance(src));
  }

View Full Code Here

        for (final String sourceName : sources) {
            final ComponentConfiguration comp = compMap.get(sourceName);
            if (comp != null) {
                final SourceConfiguration config = (SourceConfiguration) comp;


                final Source source = sourceFactory.create(comp.getComponentName(), comp.getType());


                Configurables.configure(source, config);
                final Set<String> channelNames = config.getChannels();
                final List<Channel> channels = new ArrayList<Channel>();
                for (final String chName : channelNames) {
                    channels.add(conf.getChannels().get(chName));
                }


                final ChannelSelectorConfiguration selectorConfig = config.getSelectorConfiguration();


                final ChannelSelector selector = ChannelSelectorFactory.create(channels, selectorConfig);


                final ChannelProcessor channelProcessor = new ChannelProcessor(selector);
                Configurables.configure(channelProcessor, config);


                source.setChannelProcessor(channelProcessor);
                conf.getSourceRunners().put(comp.getComponentName(), SourceRunner.forSource(source));
            }
        }
        final Map<String, Context> sourceContexts = agentConf.getSourceContext();


        for (final String src : sources) {
            final Context context = sourceContexts.get(src);
            if (context != null) {
                final Source source = sourceFactory.create(src,
                    context.getString(BasicConfigurationConstants.CONFIG_TYPE));
                final List<Channel> channels = new ArrayList<Channel>();
                Configurables.configure(source, context);
                final String[] channelNames =
                    context.getString(BasicConfigurationConstants.CONFIG_CHANNELS).split("\\s+");
                for (final String chName : channelNames) {
                    channels.add(conf.getChannels().get(chName));
                }


                final Map<String, String> selectorConfig = context.getSubProperties(
                    BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);


                final ChannelSelector selector = ChannelSelectorFactory.create(channels, selectorConfig);


                final ChannelProcessor channelProcessor = new ChannelProcessor(selector);
                Configurables.configure(channelProcessor, context);


                source.setChannelProcessor(channelProcessor);
                conf.getSourceRunners().put(src, SourceRunner.forSource(source));
            }
        }
    }

View Full Code Here

  public void testLifecycle() throws LifecycleException, InterruptedException {


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());


    Source generatorSource = new SequenceGeneratorSource();
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);


    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);


    generatorSource.setChannelProcessor(new ChannelProcessor(rcs));


    Sink nullSink = new NullSink();
    nullSink.setChannel(channel);


    nodeManager.add(SourceRunner.forSource(generatorSource));

View Full Code Here

  InterruptedException {


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());


    Source source = new SequenceGeneratorSource();
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);


    source.setChannelProcessor(new ChannelProcessor(rcs));


    Sink sink = new NullSink();
    sink.setChannel(channel);


    nodeManager.add(SourceRunner.forSource(source));

View Full Code Here

    if (sourceMap == null) {
      sourceMap = new HashMap<String, Source>();
      sources.put(sourceClass, sourceMap);
    }


    Source source = sourceMap.get(name);


    if (source == null) {
      try {
        source = sourceClass.newInstance();
        source.setName(name);
        sourceMap.put(name, source);
      } catch (Exception ex) {
        // Clean up the source map
        sources.remove(sourceClass);
        throw new FlumeException("Unable to create source: " + name

View Full Code Here

  }


  @Test
  public void testDuplicateCreate()  {


    Source avroSource1 = sourceFactory.create("avroSource1", "avro");
    Source avroSource2 = sourceFactory.create("avroSource2", "avro");


    Assert.assertNotNull(avroSource1);
    Assert.assertNotNull(avroSource2);
    Assert.assertNotSame(avroSource1, avroSource2);
    Assert.assertTrue(avroSource1 instanceof AvroSource);
    Assert.assertTrue(avroSource2 instanceof AvroSource);


    Source s1 = sourceFactory.create("avroSource1", "avro");
    Source s2 = sourceFactory.create("avroSource2", "avro");


    Assert.assertSame(avroSource1, s1);
    Assert.assertSame(avroSource2, s2);


  }

View Full Code Here


  }


  private void verifySourceCreation(String name, String type,
      Class<?> typeClass) throws Exception {
    Source src = sourceFactory.create(name, type);
    Assert.assertNotNull(src);
    Assert.assertTrue(typeClass.isInstance(src));
  }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.flume.Source

org.apache.flume.agent.embedded.EmbeddedAgent

org.apache.flume.conf.file.JsonFileConfigurationProvider

org.apache.flume.conf.properties.PropertiesFileConfigurationProvider

org.apache.flume.node.AbstractConfigurationProvider

org.apache.flume.node.TestAbstractLogicalNodeManager

org.apache.flume.source.DefaultSourceFactory

org.apache.flume.source.EventDrivenSourceRunner

org.apache.flume.source.jms.TestJMSSourceCreation

org.apache.flume.source.PollableSourceRunner

org.apache.flume.source.TestDefaultSourceFactory

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.