Package kafka.consumer

Examples of kafka.consumer.ConsumerConfig

40
41
42
43
44
45
46
47
48
49
50
    addDocumentedKafkaProps(context,props);
    return props;
  }

  public static ConsumerConnector getConsumer(Properties kafkaProps) {
    ConsumerConfig consumerConfig =
            new ConsumerConfig(kafkaProps);
    ConsumerConnector consumer =
            Consumer.createJavaConsumerConnector(consumerConfig);
    return consumer;
  }
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
    @Override
    protected void doStart() throws Exception {
        super.doStart();
        log.info("Starting Kafka consumer");

        consumer = kafka.consumer.Consumer.createJavaConsumerConnector(new ConsumerConfig(getProps()));

        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
        topicCountMap.put(endpoint.getTopic(), endpoint.getConsumerStreams());
        Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap);
        List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(endpoint.getTopic());
View Full Code Here
31
32
33
34
35
36
37
38
    props.put("groupid", KafkaProperties.groupId);
    props.put("zk.sessiontimeout.ms", "400");
    props.put("zk.synctime.ms", "200");
    props.put("autocommit.interval.ms", "1000");

    return new ConsumerConfig(props);

  }
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
    @Override
    protected void doStart() throws Exception {
        super.doStart();
        log.info("Starting Kafka consumer");

        consumer = kafka.consumer.Consumer.createJavaConsumerConnector(new ConsumerConfig(getProps()));

        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
        topicCountMap.put(endpoint.getTopic(), endpoint.getConsumerStreams());
        Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap);
        List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(endpoint.getTopic());
View Full Code Here
63
64
65
66
67
68
69
        props.put("group.id", KafkaConstants.DEFAULT_GROUP);
        props.put("zookeeper.session.timeout.ms", "400");
        props.put("zookeeper.sync.time.ms", "200");
        props.put("auto.commit.interval.ms", "1000");

        kafkaConsumer = kafka.consumer.Consumer.createJavaConsumerConnector(new ConsumerConfig(props));
    }
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
    @Override
    protected void doStart() throws Exception {
        super.doStart();
        log.info("Starting Kafka consumer");

        consumer = kafka.consumer.Consumer.createJavaConsumerConnector(new ConsumerConfig(getProps()));

        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
        topicCountMap.put(endpoint.getTopic(), endpoint.getConsumerStreams());
        Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap);
        List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(endpoint.getTopic());
View Full Code Here
178
179
180
181
182
183
184
185
186
187
188
        // Default auto commit interval is 60 seconds. Reduce to 1 second to minimize message duplication
        // if something breaks.
        props.put("auto.commit.interval.ms", "1000");

        final int numThreads = configuration.getInt(CK_THREADS);
        final ConsumerConfig consumerConfig = new ConsumerConfig(props);
        cc = Consumer.createJavaConsumerConnector(consumerConfig);

        final TopicFilter filter = new Whitelist(configuration.getString(CK_TOPIC_FILTER));

        final List<KafkaStream<byte[], byte[]>> streams = cc.createMessageStreamsByFilter(filter, numThreads);
View Full Code Here
118
119
120
121
122
123
124
        }
        if (mConfig.getFetchMessageMaxBytes() != null && !mConfig.getFetchMessageMaxBytes().isEmpty()) {
            props.put("fetch.message.max.bytes", mConfig.getFetchMessageMaxBytes());
        }

        return new ConsumerConfig(props);
    }
View Full Code Here
60
61
62
63
64
65
66
    props.put("zookeeper.session.timeout.ms", "1000");
    props.put("zookeeper.sync.time.ms", "200");
    props.put("auto.commit.interval.ms", "1000");
    props.put("auto.offset.reset", "smallest");
    props.put("consumer.timeout.ms","1000");
    return new ConsumerConfig(props);
  }
View Full Code Here
190
191
192
193
194
195
196
        props.setProperty("consumer.timeout.ms",
                String.valueOf(configuration.getTimeout().toMilliseconds()));
        props.setProperty("rebalance.retries.max",
                String.valueOf(configuration.getRebalanceRetries()));

        return new ConsumerConfig(props);
    }
View Full Code Here
TOP

Related Classes of kafka.consumer.ConsumerConfig

  • com.cloudera.oryx.kafka.util.ConsumeData
  • com.cloudera.oryx.lambda.serving.ModelManagerListener
  • com.cloudera.oryx.lambda.speed.SpeedLayer
  • com.datasift.dropwizard.kafka.KafkaConsumerFactory
  • com.jointhegrid.ironcount.IronIntegrationTest
  • com.jointhegrid.ironcount.manager.WorkerThread
  • com.netflix.suro.sink.kafka.TestKafkaSink
  • com.netflix.suro.sink.kafka.TestKafkaSinkV2
  • com.pinterest.secor.reader.MessageReader
  • com.senseidb.gateway.kafka.KafkaStreamDataProvider
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.