Package org.apache.flume.conf.sink

Examples of org.apache.flume.conf.sink.SinkConfiguration


    private Set<String> validateSinks(Set<String> channelSet) {
      // Preconditions.checkArgument(channelSet != null && channelSet.size() >
      // 0);
      Map<String, Context> newContextMap = new HashMap<String, Context>();
      Set<String> sinkSet;
      SinkConfiguration sinkConf = null;
      if (sinks == null || sinks.isEmpty()) {
        logger.warn("Agent configuration for '" + agentName
            + "' has no sinks.");
        errorList.add(new FlumeConfigurationError(agentName,
            BasicConfigurationConstants.CONFIG_SINKS,
            FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
            ErrorOrWarning.WARNING));
        return new HashSet<String>();
      } else {
        sinkSet =
            new HashSet<String>(Arrays.asList(sinks.split("\\s+")));
      }
      Iterator<String> iter = sinkSet.iterator();
      /*
       * The logic for the following code:
       *
       * Is it a known component?
       *  -Yes: Get the SinkType and set the string name of that to
       *        config and set configSpecified to true.
       *  -No.Look for config type for the given component:
       *      -Config Found:
       *        Set config to the type mentioned, set configSpecified to true
       *      -No Config found:
       *        Set config to OTHER, configSpecified to false,
       *        do basic validation. Leave the context in the
       *        contextMap to process later. Setting it to other returns
       *        a vanilla configuration(Source/Sink/Channel Configuration),
       *        which does basic syntactic validation. This object is not
       *        put into the map, so the context is retained which can be
       *        picked up - this is meant for older classes which don't
       *        implement ConfigurableComponent.
       */
      while (iter.hasNext()) {
        String sinkName = iter.next();
        Context sinkContext = sinkContextMap.get(sinkName.trim());
        if (sinkContext == null) {
          iter.remove();
          logger.warn("no context for sink" + sinkName);
          errorList.add(new FlumeConfigurationError(agentName, sinkName,
              FlumeConfigurationErrorType.CONFIG_ERROR, ErrorOrWarning.ERROR));
        } else {
          String config = null;
          boolean configSpecified = false;
          SinkType sinkType = getKnownSink(sinkContext.getString(
              BasicConfigurationConstants.CONFIG_TYPE));
          if (sinkType == null) {
            config = sinkContext.getString(
                BasicConfigurationConstants.CONFIG_CONFIG);
            if (config == null || config.isEmpty()) {
              config = "OTHER";
            } else{
              configSpecified = true;
            }
          } else {
            config = sinkType.toString().toUpperCase();
            configSpecified = true;
          }
          try {
            logger.debug("Creating sink: " + sinkName + " using " + config);

            sinkConf =
                (SinkConfiguration) ComponentConfigurationFactory.create(
                    sinkName, config, ComponentType.SINK);
            if (sinkConf != null) {
              sinkConf.configure(sinkContext);

            }
            if(!channelSet.contains(sinkConf.getChannel())){
              throw new ConfigurationException("Channel " +
                      sinkConf.getChannel() + " not in active set.");
            }
            if ((configSpecified && sinkConf.isNotFoundConfigClass()) ||
                !configSpecified) {
              newContextMap.put(sinkName, sinkContext);
            } else if (configSpecified) {
              sinkConfigMap.put(sinkName, sinkConf);
            }
            if (sinkConf != null) errorList.addAll(sinkConf.getErrors());
          } catch (ConfigurationException e) {
            iter.remove();
            if (sinkConf != null) errorList.addAll(sinkConf.getErrors());
            logger.warn("Configuration for : " + sinkName
                    + " has errors, and will be removed: ", e);
          }
        }
        // Filter out any sinks that have invalid channel
View Full Code Here


     * Components which have a ComponentConfiguration object
     */
    for (String sinkName : sinkNames) {
      ComponentConfiguration comp = compMap.get(sinkName);
      if(comp != null) {
        SinkConfiguration config = (SinkConfiguration) comp;
        Sink sink = sinkFactory.create(comp.getComponentName(),
            comp.getType());
        try {
          Configurables.configure(sink, config);
          ChannelComponent channelComponent = channelComponentMap.
              get(config.getChannel());
          if(channelComponent == null) {
            String msg = String.format("Sink %s is not connected to a " +
                "channel",  sinkName);
            throw new IllegalStateException(msg);
          }
View Full Code Here

    private Set<String> validateSinks(Set<String> channelSet) {
      // Preconditions.checkArgument(channelSet != null && channelSet.size() >
      // 0);
      Map<String, Context> newContextMap = new HashMap<String, Context>();
      Set<String> sinkSet;
      SinkConfiguration sinkConf = null;
      if (sinks == null || sinks.isEmpty()) {
        logger.warn("Agent configuration for '" + agentName
            + "' has no sinks.");
        errorList.add(new FlumeConfigurationError(agentName,
            BasicConfigurationConstants.CONFIG_SINKS,
            FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
            ErrorOrWarning.WARNING));
        return new HashSet<String>();
      } else {
        sinkSet =
            new HashSet<String>(Arrays.asList(sinks.split("\\s+")));
      }
      Iterator<String> iter = sinkSet.iterator();
      /*
       * The logic for the following code:
       *
       * Is it a known component?
       *  -Yes: Get the SinkType and set the string name of that to
       *        config and set configSpecified to true.
       *  -No.Look for config type for the given component:
       *      -Config Found:
       *        Set config to the type mentioned, set configSpecified to true
       *      -No Config found:
       *        Set config to OTHER, configSpecified to false,
       *        do basic validation. Leave the context in the
       *        contextMap to process later. Setting it to other returns
       *        a vanilla configuration(Source/Sink/Channel Configuration),
       *        which does basic syntactic validation. This object is not
       *        put into the map, so the context is retained which can be
       *        picked up - this is meant for older classes which don't
       *        implement ConfigurableComponent.
       */
      while (iter.hasNext()) {
        String sinkName = iter.next();
        Context sinkContext = sinkContextMap.get(sinkName.trim());
        if (sinkContext == null) {
          iter.remove();
          logger.warn("no context for sink" + sinkName);
          errorList.add(new FlumeConfigurationError(agentName, sinkName,
              FlumeConfigurationErrorType.CONFIG_ERROR, ErrorOrWarning.ERROR));
        } else {
          String config = null;
          boolean configSpecified = false;
          SinkType sinkType = getKnownSink(sinkContext.getString(
              BasicConfigurationConstants.CONFIG_TYPE));
          if (sinkType == null) {
            config = sinkContext.getString(
                BasicConfigurationConstants.CONFIG_CONFIG);
            if (config == null || config.isEmpty()) {
              config = "OTHER";
            } else{
              configSpecified = true;
            }
          } else {
            config = sinkType.toString().toUpperCase(Locale.ENGLISH);
            configSpecified = true;
          }
          try {
            logger.debug("Creating sink: " + sinkName + " using " + config);

            sinkConf =
                (SinkConfiguration) ComponentConfigurationFactory.create(
                    sinkName, config, ComponentType.SINK);
            if (sinkConf != null) {
              sinkConf.configure(sinkContext);

            }
            if(!channelSet.contains(sinkConf.getChannel())){
              throw new ConfigurationException("Channel " +
                      sinkConf.getChannel() + " not in active set.");
            }
            if ((configSpecified && sinkConf.isNotFoundConfigClass()) ||
                !configSpecified) {
              newContextMap.put(sinkName, sinkContext);
            } else if (configSpecified) {
              sinkConfigMap.put(sinkName, sinkConf);
            }
            if (sinkConf != null) errorList.addAll(sinkConf.getErrors());
          } catch (ConfigurationException e) {
            iter.remove();
            if (sinkConf != null) errorList.addAll(sinkConf.getErrors());
            logger.warn("Could not configure sink  " + sinkName
                + " due to: " + e.getMessage(), e);
          }
        }
        // Filter out any sinks that have invalid channel
View Full Code Here

    assertFalse(serializer.configuredWithContext);
    factory.configure(new Context());
    assertTrue(serializer.configuredWithContext);

    assertFalse(serializer.configuredWithComponentConfiguration);
    factory.configure(new SinkConfiguration("name"));
    assertTrue(serializer.configuredWithComponentConfiguration);
  }
View Full Code Here

    private Set<String> validateSinks(Set<String> channelSet) {
      // Preconditions.checkArgument(channelSet != null && channelSet.size() >
      // 0);
      Map<String, Context> newContextMap = new HashMap<String, Context>();
      Set<String> sinkSet;
      SinkConfiguration sinkConf = null;
      if (sinks == null || sinks.isEmpty()) {
        logger.warn("Agent configuration for '" + agentName
            + "' has no sinks.");
        errorList.add(new FlumeConfigurationError(agentName,
            BasicConfigurationConstants.CONFIG_SINKS,
            FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
            ErrorOrWarning.WARNING));
        return new HashSet<String>();
      } else {
        sinkSet =
            new HashSet<String>(Arrays.asList(sinks.split("\\s+")));
      }
      Iterator<String> iter = sinkSet.iterator();
      /*
       * The logic for the following code:
       *
       * Is it a known component?
       *  -Yes: Get the SinkType and set the string name of that to
       *        config and set configSpecified to true.
       *  -No.Look for config type for the given component:
       *      -Config Found:
       *        Set config to the type mentioned, set configSpecified to true
       *      -No Config found:
       *        Set config to OTHER, configSpecified to false,
       *        do basic validation. Leave the context in the
       *        contextMap to process later. Setting it to other returns
       *        a vanilla configuration(Source/Sink/Channel Configuration),
       *        which does basic syntactic validation. This object is not
       *        put into the map, so the context is retained which can be
       *        picked up - this is meant for older classes which don't
       *        implement ConfigurableComponent.
       */
      while (iter.hasNext()) {
        String sinkName = iter.next();
        Context sinkContext = sinkContextMap.get(sinkName.trim());
        if (sinkContext == null) {
          iter.remove();
          logger.warn("no context for sink" + sinkName);
          errorList.add(new FlumeConfigurationError(agentName, sinkName,
              FlumeConfigurationErrorType.CONFIG_ERROR, ErrorOrWarning.ERROR));
        } else {
          String config = null;
          boolean configSpecified = false;
          SinkType sinkType = getKnownSink(sinkContext.getString(
              BasicConfigurationConstants.CONFIG_TYPE));
          if (sinkType == null) {
            config = sinkContext.getString(
                BasicConfigurationConstants.CONFIG_CONFIG);
            if (config == null || config.isEmpty()) {
              config = "OTHER";
            } else{
              configSpecified = true;
            }
          } else {
            config = sinkType.toString().toUpperCase();
            configSpecified = true;
          }
          try {
            logger.debug("Creating sink: " + sinkName + " using " + config);

            sinkConf =
                (SinkConfiguration) ComponentConfigurationFactory.create(
                    sinkName, config, ComponentType.SINK);
            if (sinkConf != null) {
              sinkConf.configure(sinkContext);

            }
            if ((configSpecified && sinkConf.isNotFoundConfigClass()) ||
                !configSpecified) {
              newContextMap.put(sinkName, sinkContext);
            } else if (configSpecified) {
              sinkConfigMap.put(sinkName, sinkConf);
            }
            if (sinkConf != null) errorList.addAll(sinkConf.getErrors());
          } catch (ConfigurationException e) {
            iter.remove();
            if (sinkConf != null) errorList.addAll(sinkConf.getErrors());
            logger.warn("Configuration empty for: " + sinkName + ".Removed.");

          }
        }
        // Filter out any sinks that have invalid channel
View Full Code Here

    assertFalse(serializer.configuredWithContext);
    factory.configure(new Context());
    assertTrue(serializer.configuredWithContext);

    assertFalse(serializer.configuredWithComponentConfiguration);
    factory.configure(new SinkConfiguration("name"));
    assertTrue(serializer.configuredWithComponentConfiguration);
  }
View Full Code Here

     * Components which have a ComponentConfiguration object
     */
    for (String sinkName : sinkNames) {
      ComponentConfiguration comp = compMap.get(sinkName);
      if(comp != null) {
        SinkConfiguration config = (SinkConfiguration) comp;
        Sink sink = sinkFactory.create(comp.getComponentName(),
            comp.getType());
        try {
          Configurables.configure(sink, config);
          ChannelComponent channelComponent = channelComponentMap.
              get(config.getChannel());
          if(channelComponent == null) {
            String msg = String.format("Sink %s is not connected to a " +
                "channel",  sinkName);
            throw new IllegalStateException(msg);
          }
View Full Code Here

        final Map<String, ComponentConfiguration> compMap = agentConf.getSinkConfigMap();
        final Map<String, Sink> sinks = new HashMap<String, Sink>();
        for (final String sinkName : sinkNames) {
            final ComponentConfiguration comp = compMap.get(sinkName);
            if (comp != null) {
                final SinkConfiguration config = (SinkConfiguration) comp;
                final Sink sink = sinkFactory.create(comp.getComponentName(), comp.getType());

                Configurables.configure(sink, config);

                sink.setChannel(conf.getChannels().get(config.getChannel()));
                sinks.put(comp.getComponentName(), sink);
            }
        }

        final Map<String, Context> sinkContexts = agentConf.getSinkContext();
View Full Code Here

    assertFalse(serializer.configuredWithContext);
    factory.configure(new Context());
    assertTrue(serializer.configuredWithContext);

    assertFalse(serializer.configuredWithComponentConfiguration);
    factory.configure(new SinkConfiguration("name"));
    assertTrue(serializer.configuredWithComponentConfiguration);
  }
View Full Code Here

    private Set<String> validateSinks(Set<String> channelSet) {
      // Preconditions.checkArgument(channelSet != null && channelSet.size() >
      // 0);
      Map<String, Context> newContextMap = new HashMap<String, Context>();
      Set<String> sinkSet;
      SinkConfiguration sinkConf = null;
      if (sinks == null || sinks.isEmpty()) {
        logger.warn("Agent configuration for '" + agentName
            + "' has no sinks.");
        errorList.add(new FlumeConfigurationError(agentName,
            BasicConfigurationConstants.CONFIG_SINKS,
            FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
            ErrorOrWarning.WARNING));
        return new HashSet<String>();
      } else {
        sinkSet =
            new HashSet<String>(Arrays.asList(sinks.split("\\s+")));
      }
      Iterator<String> iter = sinkSet.iterator();
      /*
       * The logic for the following code:
       *
       * Is it a known component?
       *  -Yes: Get the SinkType and set the string name of that to
       *        config and set configSpecified to true.
       *  -No.Look for config type for the given component:
       *      -Config Found:
       *        Set config to the type mentioned, set configSpecified to true
       *      -No Config found:
       *        Set config to OTHER, configSpecified to false,
       *        do basic validation. Leave the context in the
       *        contextMap to process later. Setting it to other returns
       *        a vanilla configuration(Source/Sink/Channel Configuration),
       *        which does basic syntactic validation. This object is not
       *        put into the map, so the context is retained which can be
       *        picked up - this is meant for older classes which don't
       *        implement ConfigurableComponent.
       */
      while (iter.hasNext()) {
        String sinkName = iter.next();
        Context sinkContext = sinkContextMap.get(sinkName.trim());
        if (sinkContext == null) {
          iter.remove();
          logger.warn("no context for sink" + sinkName);
          errorList.add(new FlumeConfigurationError(agentName, sinkName,
              FlumeConfigurationErrorType.CONFIG_ERROR, ErrorOrWarning.ERROR));
        } else {
          String config = null;
          boolean configSpecified = false;
          SinkType sinkType = getKnownSink(sinkContext.getString(
              BasicConfigurationConstants.CONFIG_TYPE));
          if (sinkType == null) {
            config = sinkContext.getString(
                BasicConfigurationConstants.CONFIG_CONFIG);
            if (config == null || config.isEmpty()) {
              config = "OTHER";
            } else{
              configSpecified = true;
            }
          } else {
            config = sinkType.toString().toUpperCase();
            configSpecified = true;
          }
          try {
            logger.debug("Creating sink: " + sinkName + " using " + config);

            sinkConf =
                (SinkConfiguration) ComponentConfigurationFactory.create(
                    sinkName, config, ComponentType.SINK);
            if (sinkConf != null) {
              sinkConf.configure(sinkContext);

            }
            if(!channelSet.contains(sinkConf.getChannel())){
              throw new ConfigurationException("Channel " +
                      sinkConf.getChannel() + " not in active set.");
            }
            if ((configSpecified && sinkConf.isNotFoundConfigClass()) ||
                !configSpecified) {
              newContextMap.put(sinkName, sinkContext);
            } else if (configSpecified) {
              sinkConfigMap.put(sinkName, sinkConf);
            }
            if (sinkConf != null) errorList.addAll(sinkConf.getErrors());
          } catch (ConfigurationException e) {
            iter.remove();
            if (sinkConf != null) errorList.addAll(sinkConf.getErrors());
            logger.warn("Could not configure sink  " + sinkName
                + " due to: " + e.getMessage(), e);
          }
        }
        // Filter out any sinks that have invalid channel
View Full Code Here

TOP

Related Classes of org.apache.flume.conf.sink.SinkConfiguration

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.