Package org.apache.flume.conf.source

Examples of org.apache.flume.conf.source.SourceConfiguration


      }
      Set<String> sourceSet =
          new HashSet<String>(Arrays.asList(sources.split("\\s+")));
      Map<String, Context> newContextMap = new HashMap<String, Context>();
      Iterator<String> iter = sourceSet.iterator();
      SourceConfiguration srcConf = null;
      /*
       * The logic for the following code:
       *
       * Is it a known component?
       *  -Yes: Get the SourceType and set the string name of that to
       *        config and set configSpecified to true.
       *  -No.Look for config type for the given component:
       *      -Config Found:
       *        Set config to the type mentioned, set configSpecified to true
       *      -No Config found:
       *        Set config to OTHER, configSpecified to false,
       *        do basic validation. Leave the context in the
       *        contextMap to process later. Setting it to other returns
       *        a vanilla configuration(Source/Sink/Channel Configuration),
       *        which does basic syntactic validation. This object is not
       *        put into the map, so the context is retained which can be
       *        picked up - this is meant for older classes which don't
       *        implement ConfigurableComponent.
       */
      while (iter.hasNext()) {
        String sourceName = iter.next();
        Context srcContext = sourceContextMap.get(sourceName);
        String config = null;
        boolean configSpecified = false;
        if (srcContext != null) {
          SourceType srcType = getKnownSource(srcContext.getString(
              BasicConfigurationConstants.CONFIG_TYPE));
          if (srcType == null) {
            config = srcContext.getString(
                BasicConfigurationConstants.CONFIG_CONFIG);
            if (config == null || config.isEmpty()) {
              config = "OTHER";
            } else {
              configSpecified = true;
            }
          } else {
            config = srcType.toString().toUpperCase();
            configSpecified = true;
          }
          try {
            // Possible reason the configuration can fail here:
            // Old component is configured directly using Context
            srcConf =
                (SourceConfiguration) ComponentConfigurationFactory.create(
                    sourceName, config, ComponentType.SOURCE);
            if (srcConf != null) {
              srcConf.configure(srcContext);
              Set<String> channels = new HashSet<String>();
              if (srcConf.getChannels() != null) {
                channels.addAll(srcConf.getChannels());
              }
              channels.retainAll(channelSet);
              if(channels.isEmpty()){
                throw new ConfigurationException(
                        "No Channels configured for " + sourceName);
              }
              srcContext.put(BasicConfigurationConstants.CONFIG_CHANNELS,
                      this.getSpaceDelimitedList(channels));
            }
            if ((configSpecified && srcConf.isNotFoundConfigClass()) ||
                !configSpecified) {
              newContextMap.put(sourceName, srcContext);
            } else if (configSpecified){
              sourceConfigMap.put(sourceName, srcConf);
            }
            if (srcConf != null) errorList.addAll(srcConf.getErrors());
          } catch (ConfigurationException e) {
            if (srcConf != null) errorList.addAll(srcConf.getErrors());
            iter.remove();
            logger.warn("Removed " + sourceName + " due to " + e.getMessage());
          }
        } else {
          iter.remove();
View Full Code Here


     * Components which have a ComponentConfiguration object
     */
    for (String sourceName : sourceNames) {
      ComponentConfiguration comp = compMap.get(sourceName);
      if(comp != null) {
        SourceConfiguration config = (SourceConfiguration) comp;

        Source source = sourceFactory.create(comp.getComponentName(),
            comp.getType());
        try {
          Configurables.configure(source, config);
          Set<String> channelNames = config.getChannels();
          List<Channel> sourceChannels = new ArrayList<Channel>();
          for (String chName : channelNames) {
            ChannelComponent channelComponent = channelComponentMap.get(chName);
            if(channelComponent != null) {
              sourceChannels.add(channelComponent.channel);
            }
          }
          if(sourceChannels.isEmpty()) {
            String msg = String.format("Source %s is not connected to a " +
                "channel",  sourceName);
            throw new IllegalStateException(msg);
          }
          ChannelSelectorConfiguration selectorConfig =
              config.getSelectorConfiguration();

          ChannelSelector selector = ChannelSelectorFactory.create(
              sourceChannels, selectorConfig);

          ChannelProcessor channelProcessor = new ChannelProcessor(selector);
View Full Code Here

      }
      Set<String> sourceSet =
          new HashSet<String>(Arrays.asList(sources.split("\\s+")));
      Map<String, Context> newContextMap = new HashMap<String, Context>();
      Iterator<String> iter = sourceSet.iterator();
      SourceConfiguration srcConf = null;
      /*
       * The logic for the following code:
       *
       * Is it a known component?
       *  -Yes: Get the SourceType and set the string name of that to
       *        config and set configSpecified to true.
       *  -No.Look for config type for the given component:
       *      -Config Found:
       *        Set config to the type mentioned, set configSpecified to true
       *      -No Config found:
       *        Set config to OTHER, configSpecified to false,
       *        do basic validation. Leave the context in the
       *        contextMap to process later. Setting it to other returns
       *        a vanilla configuration(Source/Sink/Channel Configuration),
       *        which does basic syntactic validation. This object is not
       *        put into the map, so the context is retained which can be
       *        picked up - this is meant for older classes which don't
       *        implement ConfigurableComponent.
       */
      while (iter.hasNext()) {
        String sourceName = iter.next();
        Context srcContext = sourceContextMap.get(sourceName);
        String config = null;
        boolean configSpecified = false;
        if (srcContext != null) {
          SourceType srcType = getKnownSource(srcContext.getString(
              BasicConfigurationConstants.CONFIG_TYPE));
          if (srcType == null) {
            config = srcContext.getString(
                BasicConfigurationConstants.CONFIG_CONFIG);
            if (config == null || config.isEmpty()) {
              config = "OTHER";
            } else {
              configSpecified = true;
            }
          } else {
            config = srcType.toString().toUpperCase(Locale.ENGLISH);
            configSpecified = true;
          }
          try {
            // Possible reason the configuration can fail here:
            // Old component is configured directly using Context
            srcConf =
                (SourceConfiguration) ComponentConfigurationFactory.create(
                    sourceName, config, ComponentType.SOURCE);
            if (srcConf != null) {
              srcConf.configure(srcContext);
              Set<String> channels = new HashSet<String>();
              if (srcConf.getChannels() != null) {
                channels.addAll(srcConf.getChannels());
              }
              channels.retainAll(channelSet);
              if(channels.isEmpty()){
                throw new ConfigurationException(
                        "No Channels configured for " + sourceName);
              }
              srcContext.put(BasicConfigurationConstants.CONFIG_CHANNELS,
                      this.getSpaceDelimitedList(channels));
            }
            if ((configSpecified && srcConf.isNotFoundConfigClass()) ||
                !configSpecified) {
              newContextMap.put(sourceName, srcContext);
            } else if (configSpecified){
              sourceConfigMap.put(sourceName, srcConf);
            }
            if (srcConf != null) errorList.addAll(srcConf.getErrors());
          } catch (ConfigurationException e) {
            if (srcConf != null) errorList.addAll(srcConf.getErrors());
            iter.remove();
            logger.warn("Could not configure source  " + sourceName
                + " due to: " + e.getMessage(), e);
          }
        } else {
View Full Code Here

      }
      Set<String> sourceSet =
          new HashSet<String>(Arrays.asList(sources.split("\\s+")));
      Map<String, Context> newContextMap = new HashMap<String, Context>();
      Iterator<String> iter = sourceSet.iterator();
      SourceConfiguration srcConf = null;
      /*
       * The logic for the following code:
       *
       * Is it a known component?
       *  -Yes: Get the SourceType and set the string name of that to
       *        config and set configSpecified to true.
       *  -No.Look for config type for the given component:
       *      -Config Found:
       *        Set config to the type mentioned, set configSpecified to true
       *      -No Config found:
       *        Set config to OTHER, configSpecified to false,
       *        do basic validation. Leave the context in the
       *        contextMap to process later. Setting it to other returns
       *        a vanilla configuration(Source/Sink/Channel Configuration),
       *        which does basic syntactic validation. This object is not
       *        put into the map, so the context is retained which can be
       *        picked up - this is meant for older classes which don't
       *        implement ConfigurableComponent.
       */
      while (iter.hasNext()) {
        String sourceName = iter.next();
        Context srcContext = sourceContextMap.get(sourceName);
        String config = null;
        boolean configSpecified = false;
        if (srcContext != null) {
          SourceType srcType = getKnownSource(srcContext.getString(
              BasicConfigurationConstants.CONFIG_TYPE));
          if (srcType == null) {
            config = srcContext.getString(
                BasicConfigurationConstants.CONFIG_CONFIG);
            if (config == null || config.isEmpty()) {
              config = "OTHER";
            } else {
              configSpecified = true;
            }
          } else {
            config = srcType.toString().toUpperCase();
            configSpecified = true;
          }
          try {
            // Possible reason the configuration can fail here:
            // Old component is configured directly using Context
            srcConf =
                (SourceConfiguration) ComponentConfigurationFactory.create(
                    sourceName, config, ComponentType.SOURCE);
            if (srcConf != null) {
              srcConf.configure(srcContext);
              Set<String> channels = new HashSet<String>();
              if (srcConf.getChannels() != null) {
                channels.addAll(srcConf.getChannels());
              }
              channels.retainAll(channelSet);
            }
            if ((configSpecified && srcConf.isNotFoundConfigClass()) ||
                !configSpecified) {
              newContextMap.put(sourceName, srcContext);
            } else if (configSpecified){
              sourceConfigMap.put(sourceName, srcConf);
            }
            if (srcConf != null) errorList.addAll(srcConf.getErrors());
          } catch (ConfigurationException e) {
            if (srcConf != null) errorList.addAll(srcConf.getErrors());
            iter.remove();
            logger.warn("Removed " + sourceName + " due to " + e.getMessage());
          }
        } else {
          iter.remove();
View Full Code Here

     * Components which have a ComponentConfiguration object
     */
    for (String sourceName : sourceNames) {
      ComponentConfiguration comp = compMap.get(sourceName);
      if(comp != null) {
        SourceConfiguration config = (SourceConfiguration) comp;

        Source source = sourceFactory.create(comp.getComponentName(),
            comp.getType());
        try {
          Configurables.configure(source, config);
          Set<String> channelNames = config.getChannels();
          List<Channel> sourceChannels = new ArrayList<Channel>();
          for (String chName : channelNames) {
            ChannelComponent channelComponent = channelComponentMap.get(chName);
            if(channelComponent != null) {
              sourceChannels.add(channelComponent.channel);
            }
          }
          if(sourceChannels.isEmpty()) {
            String msg = String.format("Source %s is not connected to a " +
                "channel",  sourceName);
            throw new IllegalStateException(msg);
          }
          ChannelSelectorConfiguration selectorConfig =
              config.getSelectorConfiguration();

          ChannelSelector selector = ChannelSelectorFactory.create(
              sourceChannels, selectorConfig);

          ChannelProcessor channelProcessor = new ChannelProcessor(selector);
View Full Code Here

        final Set<String> sources = agentConf.getSourceSet();
        final Map<String, ComponentConfiguration> compMap = agentConf.getSourceConfigMap();
        for (final String sourceName : sources) {
            final ComponentConfiguration comp = compMap.get(sourceName);
            if (comp != null) {
                final SourceConfiguration config = (SourceConfiguration) comp;

                final Source source = sourceFactory.create(comp.getComponentName(), comp.getType());

                Configurables.configure(source, config);
                final Set<String> channelNames = config.getChannels();
                final List<Channel> channels = new ArrayList<Channel>();
                for (final String chName : channelNames) {
                    channels.add(conf.getChannels().get(chName));
                }

                final ChannelSelectorConfiguration selectorConfig = config.getSelectorConfiguration();

                final ChannelSelector selector = ChannelSelectorFactory.create(channels, selectorConfig);

                final ChannelProcessor channelProcessor = new ChannelProcessor(selector);
                Configurables.configure(channelProcessor, config);
View Full Code Here

      }
      Set<String> sourceSet =
          new HashSet<String>(Arrays.asList(sources.split("\\s+")));
      Map<String, Context> newContextMap = new HashMap<String, Context>();
      Iterator<String> iter = sourceSet.iterator();
      SourceConfiguration srcConf = null;
      /*
       * The logic for the following code:
       *
       * Is it a known component?
       *  -Yes: Get the SourceType and set the string name of that to
       *        config and set configSpecified to true.
       *  -No.Look for config type for the given component:
       *      -Config Found:
       *        Set config to the type mentioned, set configSpecified to true
       *      -No Config found:
       *        Set config to OTHER, configSpecified to false,
       *        do basic validation. Leave the context in the
       *        contextMap to process later. Setting it to other returns
       *        a vanilla configuration(Source/Sink/Channel Configuration),
       *        which does basic syntactic validation. This object is not
       *        put into the map, so the context is retained which can be
       *        picked up - this is meant for older classes which don't
       *        implement ConfigurableComponent.
       */
      while (iter.hasNext()) {
        String sourceName = iter.next();
        Context srcContext = sourceContextMap.get(sourceName);
        String config = null;
        boolean configSpecified = false;
        if (srcContext != null) {
          SourceType srcType = getKnownSource(srcContext.getString(
              BasicConfigurationConstants.CONFIG_TYPE));
          if (srcType == null) {
            config = srcContext.getString(
                BasicConfigurationConstants.CONFIG_CONFIG);
            if (config == null || config.isEmpty()) {
              config = "OTHER";
            } else {
              configSpecified = true;
            }
          } else {
            config = srcType.toString().toUpperCase();
            configSpecified = true;
          }
          try {
            // Possible reason the configuration can fail here:
            // Old component is configured directly using Context
            srcConf =
                (SourceConfiguration) ComponentConfigurationFactory.create(
                    sourceName, config, ComponentType.SOURCE);
            if (srcConf != null) {
              srcConf.configure(srcContext);
              Set<String> channels = new HashSet<String>();
              if (srcConf.getChannels() != null) {
                channels.addAll(srcConf.getChannels());
              }
              channels.retainAll(channelSet);
              if(channels.isEmpty()){
                throw new ConfigurationException(
                        "No Channels configured for " + sourceName);
              }
              srcContext.put(BasicConfigurationConstants.CONFIG_CHANNELS,
                      this.getSpaceDelimitedList(channels));
            }
            if ((configSpecified && srcConf.isNotFoundConfigClass()) ||
                !configSpecified) {
              newContextMap.put(sourceName, srcContext);
            } else if (configSpecified){
              sourceConfigMap.put(sourceName, srcConf);
            }
            if (srcConf != null) errorList.addAll(srcConf.getErrors());
          } catch (ConfigurationException e) {
            if (srcConf != null) errorList.addAll(srcConf.getErrors());
            iter.remove();
            logger.warn("Could not configure source  " + sourceName
                + " due to: " + e.getMessage(), e);
          }
        } else {
View Full Code Here

    Map<String, ComponentConfiguration> compMap =
        agentConf.getSourceConfigMap();
    for (String sourceName : sources) {
      ComponentConfiguration comp = compMap.get(sourceName);
      if(comp != null) {
        SourceConfiguration config = (SourceConfiguration) comp;

        Source source = getSourceFactory().create(comp.getComponentName(),
            comp.getType());

        Configurables.configure(source, config);
        Set<String> channelNames = config.getChannels();
        List<Channel> channels = new ArrayList<Channel>();
        for (String chName : channelNames) {
          channels.add(conf.getChannels().get(chName));
        }

        ChannelSelectorConfiguration selectorConfig =
            config.getSelectorConfiguration();

        ChannelSelector selector = ChannelSelectorFactory.create(
            channels, selectorConfig);

        ChannelProcessor channelProcessor = new ChannelProcessor(selector);
View Full Code Here

        Set<String> sources = agentConf.getSourceSet();
        Map<String, ComponentConfiguration> compMap = agentConf.getSourceConfigMap();
        for (String sourceName : sources) {
            ComponentConfiguration comp = compMap.get(sourceName);
            if (comp != null) {
                SourceConfiguration config = (SourceConfiguration) comp;

                Source source = sourceFactory.create(comp.getComponentName(), comp.getType());

                Configurables.configure(source, config);
                Set<String> channelNames = config.getChannels();
                List<Channel> channels = new ArrayList<Channel>();
                for (String chName : channelNames) {
                    channels.add(conf.getChannels().get(chName));
                }

                ChannelSelectorConfiguration selectorConfig = config.getSelectorConfiguration();

                ChannelSelector selector = ChannelSelectorFactory.create(channels, selectorConfig);

                ChannelProcessor channelProcessor = new ChannelProcessor(selector);
                Configurables.configure(channelProcessor, config);
View Full Code Here

        final Set<String> sources = agentConf.getSourceSet();
        final Map<String, ComponentConfiguration> compMap = agentConf.getSourceConfigMap();
        for (final String sourceName : sources) {
            final ComponentConfiguration comp = compMap.get(sourceName);
            if (comp != null) {
                final SourceConfiguration config = (SourceConfiguration) comp;

                final Source source = sourceFactory.create(comp.getComponentName(), comp.getType());

                Configurables.configure(source, config);
                final Set<String> channelNames = config.getChannels();
                final List<Channel> channels = new ArrayList<Channel>();
                for (final String chName : channelNames) {
                    channels.add(conf.getChannels().get(chName));
                }

                final ChannelSelectorConfiguration selectorConfig = config.getSelectorConfiguration();

                final ChannelSelector selector = ChannelSelectorFactory.create(channels, selectorConfig);

                final ChannelProcessor channelProcessor = new ChannelProcessor(selector);
                Configurables.configure(channelProcessor, config);
View Full Code Here

TOP

Related Classes of org.apache.flume.conf.source.SourceConfiguration

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.