Package org.apache.flume

Examples of org.apache.flume.Context


  }

  @Test
  public void testSourceDoesNotDieOnFullChannel() throws Exception {

    Context chContext = new Context();
    chContext.put("capacity", "2");
    chContext.put("transactionCapacity", "2");
    chContext.put("keep-alive", "0");
    channel.stop();
    Configurables.configure(channel, chContext);

    channel.start();
    Context context = new Context();
    File f1 = new File(tmpDir.getAbsolutePath() + "/file1");

    Files.write("file1line1\nfile1line2\nfile1line3\nfile1line4\n" +
      "file1line5\nfile1line6\nfile1line7\nfile1line8\n",
      f1, Charsets.UTF_8);


    context.put(SpoolDirectorySourceConfigurationConstants.SPOOL_DIRECTORY,
      tmpDir.getAbsolutePath());

    context.put(SpoolDirectorySourceConfigurationConstants.BATCH_SIZE, "2");
    Configurables.configure(source, context);
    source.setBackOff(false);
    source.start();

    // Wait for the source to read enough events to fill up the channel.
View Full Code Here


    Map<String, String> channelContext = new HashMap();
    channelContext.put("capacity", "1000000");
    channelContext.put("keep-alive", "0"); // for faster tests
    Channel channel = new MemoryChannel();
    channel.setName(channel.getClass().getName() + SEQ_NUM.getAndIncrement());
    Configurables.configure(channel, new Context(channelContext));
    class MySolrSink extends MorphlineSolrSink {
      public MySolrSink(MorphlineHandlerImpl indexer) {
        super(indexer);
      }
    }
   
    int batchSize = SEQ_NUM2.incrementAndGet() % 2 == 0 ? 100 : 1;
    DocumentLoader testServer = new SolrServerDocumentLoader(solrServer, batchSize);
    MorphlineContext solrMorphlineContext = new SolrMorphlineContext.Builder()
      .setDocumentLoader(testServer)
      .setExceptionHandler(new FaultTolerance(false, false, SolrServerException.class.getName()))
      .setMetricRegistry(new MetricRegistry()).build();
   
    MorphlineHandlerImpl impl = new MorphlineHandlerImpl();
    impl.setMorphlineContext(solrMorphlineContext);
   
    class MySolrLocator extends SolrLocator { // trick to access protected ctor
      public MySolrLocator(MorphlineContext indexer) {
        super(indexer);
      }
    }

    SolrLocator locator = new MySolrLocator(solrMorphlineContext);
    locator.setSolrHomeDir(testSolrHome + "/collection1");
    String str1 = "SOLR_LOCATOR : " + locator.toString();
    //File solrLocatorFile = new File("target/test-classes/test-morphlines/solrLocator.conf");
    //String str1 = Files.toString(solrLocatorFile, Charsets.UTF_8);
    File morphlineFile = new File("target/test-classes/test-morphlines/solrCellDocumentTypes.conf");
    String str2 = Files.toString(morphlineFile, Charsets.UTF_8);
    tmpFile = File.createTempFile("morphline", ".conf");
    tmpFile.deleteOnExit();
    Files.write(str1 + "\n" + str2, tmpFile, Charsets.UTF_8);   
    context.put("morphlineFile", tmpFile.getPath());

    impl.configure(new Context(context));
    sink = new MySolrSink(impl);
    sink.setName(sink.getClass().getName() + SEQ_NUM.getAndIncrement());
    sink.configure(new Context(context));
    sink.setChannel(channel);
    sink.start();
   
    source = new EmbeddedSource(sink);   
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(Collections.singletonList(channel));
    ChannelProcessor chp = new ChannelProcessor(rcs);
    Context chpContext = new Context();
    chpContext.put("interceptors", "uuidinterceptor");
    chpContext.put("interceptors.uuidinterceptor.type", UUIDInterceptor.Builder.class.getName());
    chp.configure(chpContext);
    source.setChannelProcessor(chp);
   
    deleteAllDocuments();
  }
View Full Code Here

  public void setUp() throws UnknownHostException {
    localhost = InetAddress.getByName("127.0.0.1");
    source = new AvroSource();
    channel = new MemoryChannel();

    Configurables.configure(channel, new Context());

    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);

    ChannelSelector rcs = new ReplicatingChannelSelector();
View Full Code Here

  public void testLifecycle() throws InterruptedException {
    boolean bound = false;

    for (int i = 0; i < 100 && !bound; i++) {
      try {
        Context context = new Context();

        context.put("port", String.valueOf(selectedPort = 41414 + i));
        context.put("bind", "0.0.0.0");

        Configurables.configure(source, context);

        source.start();
        bound = true;
View Full Code Here

       *        picked up - this is meant for older classes which don't
       *        implement ConfigurableComponent.
       */
      while (iter.hasNext()) {
        String channelName = iter.next();
        Context channelContext = channelContextMap.get(channelName);
        // Context exists in map.
        if (channelContext != null) {
          // Get the configuration object for the channel:
          ChannelType chType = getKnownChannel(channelContext.getString(
                  BasicConfigurationConstants.CONFIG_TYPE));
          boolean configSpecified = false;
          String config = null;
          // Not a known channel - cannot do specific validation to this channel
          if (chType == null) {
            config = channelContext.getString
                (BasicConfigurationConstants.CONFIG_CONFIG);
            if (config == null || config.isEmpty()) {
              config = "OTHER";
            } else {
              configSpecified = true;
View Full Code Here

  private void doRequest(boolean serverEnableCompression, boolean clientEnableCompression, int compressionLevel) throws InterruptedException, IOException {
    boolean bound = false;

    for (int i = 0; i < 100 && !bound; i++) {
      try {
        Context context = new Context();
        context.put("port", String.valueOf(selectedPort = 41414 + i));
        context.put("bind", "0.0.0.0");
        context.put("threads", "50");
        if (serverEnableCompression) {
          context.put("compression-type", "deflate");
        } else {
          context.put("compression-type", "none");
        }

        Configurables.configure(source, context);

        source.start();
View Full Code Here

       *        picked up - this is meant for older classes which don't
       *        implement ConfigurableComponent.
       */
      while (iter.hasNext()) {
        String sourceName = iter.next();
        Context srcContext = sourceContextMap.get(sourceName);
        String config = null;
        boolean configSpecified = false;
        if (srcContext != null) {
          SourceType srcType = getKnownSource(srcContext.getString(
              BasicConfigurationConstants.CONFIG_TYPE));
          if (srcType == null) {
            config = srcContext.getString(
                BasicConfigurationConstants.CONFIG_CONFIG);
            if (config == null || config.isEmpty()) {
              config = "OTHER";
            } else {
              configSpecified = true;
            }
          } else {
            config = srcType.toString().toUpperCase(Locale.ENGLISH);
            configSpecified = true;
          }
          try {
            // Possible reason the configuration can fail here:
            // Old component is configured directly using Context
            srcConf =
                (SourceConfiguration) ComponentConfigurationFactory.create(
                    sourceName, config, ComponentType.SOURCE);
            if (srcConf != null) {
              srcConf.configure(srcContext);
              Set<String> channels = new HashSet<String>();
              if (srcConf.getChannels() != null) {
                channels.addAll(srcConf.getChannels());
              }
              channels.retainAll(channelSet);
              if(channels.isEmpty()){
                throw new ConfigurationException(
                        "No Channels configured for " + sourceName);
              }
              srcContext.put(BasicConfigurationConstants.CONFIG_CHANNELS,
                      this.getSpaceDelimitedList(channels));
            }
            if ((configSpecified && srcConf.isNotFoundConfigClass()) ||
                !configSpecified) {
              newContextMap.put(sourceName, srcContext);
View Full Code Here

        .trim();
    if(JMSSourceConfiguration.CONVERTER_TYPE_DEFAULT.
        equalsIgnoreCase(converterClassName)) {
      converterClassName = DefaultJMSMessageConverter.Builder.class.getName();
    }
    Context converterContext = new Context(context.
        getSubProperties(JMSSourceConfiguration.CONVERTER + "."));
    try {
      @SuppressWarnings("rawtypes")
      Class clazz = Class.forName(converterClassName);
      boolean isBuilder = JMSMessageConverter.Builder.class
View Full Code Here

        } catch (Exception e) {
          //Not a known type, use FQCN
          klass = (Class<? extends MonitorService>) Class.forName(monitorType);
        }
        this.monitorServer = klass.newInstance();
        Context context = new Context();
        for (String key : keys) {
          if (key.startsWith(CONF_MONITOR_PREFIX)) {
            context.put(key.substring(CONF_MONITOR_PREFIX.length()),
                    systemProps.getProperty(key));
          }
        }
        monitorServer.configure(context);
        monitorServer.start();
View Full Code Here

  }

  @Test
  public void testKafkaConsumerProperties() {
    Context context = new Context();
    context.put("kafka.auto.commit.enable", "override.default.autocommit");
    context.put("kafka.fake.property", "kafka.property.value");
    context.put("kafka.zookeeper.connect","bad-zookeeper-list");
    context.put("zookeeperConnect","real-zookeeper-list");
    Properties kafkaProps = KafkaSourceUtil.getKafkaProperties(context);

    //check that we have defaults set
    assertEquals(
            kafkaProps.getProperty(KafkaSourceConstants.GROUP_ID),
View Full Code Here

TOP

Related Classes of org.apache.flume.Context

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.