Examples of org.apache.flume.Source

org.apache.flume.Source

A source generates {@plainlink Event events} and calls methods on theconfigured {@link ChannelProcessor} to persist those events into theconfigured {@linkplain Channel channels}.

Sources are associated with unique {@linkplain NamedComponent names} that canbe used for separating configuration and working namespaces.

No guarantees are given regarding thread safe access.
@see org.apache.flume.Channel @see org.apache.flume.Sink

        MockSource.class);
  }


  @Test
  public void testSourceRegistry() throws Exception {
    Source s1 = sourceFactory.create("s1", "avro");
    Map<Class<?>, Map<String, Source>> sr =
        ((DefaultSourceFactory) sourceFactory).getRegistryClone();


    Assert.assertEquals(1, sr.size());


    Map<String, Source> srMap = sr.get(AvroSource.class);
    Assert.assertNotNull(srMap);
    Assert.assertEquals(1, srMap.size());


    Source src = srMap.get("s1");
    Assert.assertNotNull(src);
    Assert.assertSame(s1, src);
  }

View Full Code Here

    for (ComponentConfiguration comp : agentConf.getSources()) {
      Context context = new Context();


      Map<String, String> componentConfig = comp.getConfiguration();


      Source source = getSourceFactory().create(comp.getComponentName(),
          componentConfig.get("type"));


      for (Entry<String, String> entry : comp.getConfiguration().entrySet()) {
        context.put(entry.getKey(), entry.getValue());
      }


      Configurables.configure(source, context);


      String channelNames = comp.getConfiguration().get("channels");
      List<Channel> channels = new ArrayList<Channel>();


      for (String chName : channelNames.split(" ")) {
        channels.add(conf.getChannels().get(chName));
      }


      Map<String,String> selectorConfig = comp.getSubconfiguration("selector");


      ChannelSelector selector = ChannelSelectorFactory.create(
          channels, selectorConfig);


      ChannelProcessor channelProcessor = new ChannelProcessor(selector);


      source.setChannelProcessor(channelProcessor);
      conf.getSourceRunners().put(comp.getComponentName(),
          SourceRunner.forSource(source));
    }
  }

View Full Code Here

    Preconditions.checkNotNull(name, "name");
    Preconditions.checkNotNull(type, "type");
    logger.info("Creating instance of source {}, type {}", name, type);
    Class<? extends Source> sourceClass = getClass(type);
    try {
      Source source = sourceClass.newInstance();
      source.setName(name);
      return source;
    } catch (Exception ex) {
      throw new FlumeException("Unable to create source: " + name
          +", type: " + type + ", class: " + sourceClass.getName(), ex);
    }

View Full Code Here

    lifecycleState = LifecycleState.IDLE;
  }


  @Override
  public void start() {
    Source source = getSource();
    ChannelProcessor cp = source.getChannelProcessor();
    cp.initialize();
    source.start();
    lifecycleState = LifecycleState.START;
  }

View Full Code Here

    lifecycleState = LifecycleState.START;
  }


  @Override
  public void stop() {
    Source source = getSource();
    source.stop();
    ChannelProcessor cp = source.getChannelProcessor();
    cp.close();
    lifecycleState = LifecycleState.STOP;
  }

View Full Code Here

          "Interrupted while waiting for polling runner to stop. Please report this.",
          e);
      Thread.currentThread().interrupt();
    }


    Source source = getSource();
    source.stop();
    ChannelProcessor cp = source.getChannelProcessor();
    cp.close();


    lifecycleState = LifecycleState.STOP;
  }

View Full Code Here

    for (String sourceName : sources) {
      ComponentConfiguration comp = compMap.get(sourceName);
      if(comp != null) {
        SourceConfiguration config = (SourceConfiguration) comp;


        Source source = getSourceFactory().create(comp.getComponentName(),
            comp.getType());


        Configurables.configure(source, config);
        Set<String> channelNames = config.getChannels();
        List<Channel> channels = new ArrayList<Channel>();
        for (String chName : channelNames) {
          channels.add(conf.getChannels().get(chName));
        }


        ChannelSelectorConfiguration selectorConfig =
            config.getSelectorConfiguration();


        ChannelSelector selector = ChannelSelectorFactory.create(
            channels, selectorConfig);


        ChannelProcessor channelProcessor = new ChannelProcessor(selector);
        Configurables.configure(channelProcessor, config);


        source.setChannelProcessor(channelProcessor);
        conf.getSourceRunners().put(comp.getComponentName(),
            SourceRunner.forSource(source));
      }
    }
    Map<String, Context> sourceContexts = agentConf.getSourceContext();


    for (String src : sources) {
      Context context = sourceContexts.get(src);
      if(context != null){
        Source source =
            getSourceFactory().create(src,
                context.getString(BasicConfigurationConstants.CONFIG_TYPE));
        List<Channel> channels = new ArrayList<Channel>();
        Configurables.configure(source, context);
        String[] channelNames = context.getString(
            BasicConfigurationConstants.CONFIG_CHANNELS).split("\\s+");
        for (String chName : channelNames) {
          channels.add(conf.getChannels().get(chName));
        }


        Map<String, String> selectorConfig = context.getSubProperties(
            BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);


        ChannelSelector selector = ChannelSelectorFactory.create(
            channels, selectorConfig);


        ChannelProcessor channelProcessor = new ChannelProcessor(selector);
        Configurables.configure(channelProcessor, context);


        source.setChannelProcessor(channelProcessor);
        conf.getSourceRunners().put(src,
            SourceRunner.forSource(source));


      }
    }

View Full Code Here

    Assert.assertEquals(1, sourceFactory.getSourceNames().size());


    Assert
        .assertEquals("seq", sourceFactory.getSourceNames().iterator().next());


    Source source = sourceFactory.create("seq");


    Assert.assertNotNull("Factory returned a null source", source);
    Assert.assertTrue("Source isn't an instance of SequenceGeneratorSource",
        source instanceof SequenceGeneratorSource);

View Full Code Here

      throws InstantiationException {


    for (ComponentConfiguration comp : agentConf.getSources()) {
      Context context = new Context();


      Source source = getSourceFactory().create(
          comp.getConfiguration().get("type"));


      for (Entry<String, String> entry : comp.getConfiguration().entrySet()) {
        context.put(entry.getKey(), entry.getValue());
      }


      Configurables.configure(source, context);


      if (comp.getConfiguration().get("channels").contains(",")) {
        // then create a fanout if there are channel for this source
        source.setChannel(getChannelFactory().createFanout(
            comp.getConfiguration().get("channels"),conf.getChannels()));
      } else {
        source.setChannel(conf.getChannels().get(
            comp.getConfiguration().get("channels")));        
      }
      conf.getSourceRunners().put(comp.getComponentName(),
          SourceRunner.forSource(source));
    }

View Full Code Here

  public void testLifecycle() throws LifecycleException, InterruptedException {


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());


    Source generatorSource = new SequenceGeneratorSource();
    generatorSource.setChannel(channel);


    Sink nullSink = new NullSink();
    nullSink.setChannel(channel);


    nodeManager.add(SourceRunner.forSource(generatorSource));

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.flume.Source

org.apache.flume.agent.embedded.EmbeddedAgent

org.apache.flume.conf.file.JsonFileConfigurationProvider

org.apache.flume.conf.properties.PropertiesFileConfigurationProvider

org.apache.flume.node.AbstractConfigurationProvider

org.apache.flume.node.TestAbstractLogicalNodeManager

org.apache.flume.source.DefaultSourceFactory

org.apache.flume.source.EventDrivenSourceRunner

org.apache.flume.source.jms.TestJMSSourceCreation

org.apache.flume.source.PollableSourceRunner

org.apache.flume.source.TestDefaultSourceFactory

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.