Package kafka.consumer

Examples of kafka.consumer.ConsumerConfig

116
117
118
119
120
121
122
123
124
125
126
  public void run(){
    status=WorkerThreadStatus.INIT;
    props = new Properties();
    props.put("groupid", workload.consumerGroup);
    props.put("zk.connect", workload.zkConnect);
    config = new ConsumerConfig(props);
    consumerConnector = Consumer.createJavaConsumerConnector(config);
   
    try {
      handler = (MessageHandler) new ICURLClassLoader().getClassLoader(workload)
              .loadClass(this.workload.messageHandlerName).newInstance();
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
    producerProps = new Properties();
    producerProps.put("serializer.class", "kafka.serializer.StringEncoder");
    producerProps.put("zk.connect", "localhost:8888");

    consumerProps.setProperty("groupid", "group1");
    consumerConfig = new ConsumerConfig(consumerProps);
    producerConfig = new ProducerConfig(producerProps);

    brokerProps.setProperty("topic.partition.count.map", topic+":2"+",map:2,reduce:2");
    brokerProps.setProperty("num.partitions", "10");
    brokerProps.setProperty("brokerid", "1");
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
            return Response.status(400)
                    .entity(new String[]{"Undefined topic"})
                    .build();

        Properties props = consumerCfg.asProperties(timeout);
        ConsumerConfig config = new ConsumerConfig(props);
        ConsumerConnector connector = Consumer.createJavaConsumerConnector(config);

        Map<String, Integer> streamCounts = Collections.singletonMap(topic, 1);
        Map<String, List<KafkaStream<byte[], byte[]>>> streams = connector.createMessageStreams(streamCounts);
        KafkaStream<byte[], byte[]> stream = streams.get(topic).get(0);
View Full Code Here
53
54
55
56
57
58
59
60
61
  @Override
  public CloseableIterator<Pair<String,String>> iterator() {
    Properties consumerProps = new Properties();
    consumerProps.setProperty("group.id", "OryxGroup-ConsumeData");
    consumerProps.setProperty("zookeeper.connect", "localhost:" + zkPort);
    ConsumerConfig consumerConfig = new ConsumerConfig(consumerProps);
    ConsumerConnector consumer = Consumer.createJavaConsumerConnector(consumerConfig);
    return new ConsumeDataIterator(topic, consumer);
  }
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
    public void open() {

        // these consumers use ZooKeeper for commit, offset and segment consumption tracking
        // TODO: consider using SimpleConsumer the same way the Hadoop consumer job does to avoid ZK dependency
        // TODO: use the task details from TopologyContext in the normal open method
        ConsumerConfig consumerConfig = new ConsumerConfig(kafkaProperties);
        consumerConnector = Consumer.createJavaConsumerConnector(consumerConfig);

        // consumer with just one thread since the real parallelism is handled by Storm already
        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
        topicCountMap.put(topic, new Integer(1));
View Full Code Here
376
377
378
379
380
381
382
383
    for (int i = 0; i < keyValues.length; i += 2) {
      String key = keyValues[i];
      String value = keyValues[i + 1];
      props.put(key, value);
    }
    ConsumerConfig config = new ConsumerConfig(props);
    return Consumer.createJavaConsumerConnector(config);
  }
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
        Properties props = new Properties();
        props.put("zookeeper.connect", cluster.getZkConnectString());
        props.put("group.id", "foo");
        props.put("rebalance.backoff.ms", "2000");
        props.put("rebalance.max.retries", "2000");
        ConsumerConfig config = new ConsumerConfig(props);


        CuratorFramework curator = CuratorFrameworkFactory.newClient(cluster.getZkConnectString(), new RetryUntilElapsed(1000, 100));
        curator.start();
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
      props.put(key, _kafkaConfig.getProperty(key));
    }

    logger.info("Kafka properties: " + props);

    ConsumerConfig consumerConfig = new ConsumerConfig(props);
    _consumerConnector = Consumer.createJavaConsumerConnector(consumerConfig);

    Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    for (String topic : _topics)
    {
View Full Code Here
43
44
45
46
47
48
49
50
    props.put("groupid", KafkaProperties.groupId);
    props.put("zk.sessiontimeout.ms", "400");
    props.put("zk.synctime.ms", "200");
    props.put("autocommit.interval.ms", "1000");

    return new ConsumerConfig(props);

  }
View Full Code Here
46
47
48
49
50
51
52
53
    props.put("group.id", KafkaProperties.groupId);
    props.put("zookeeper.session.timeout.ms", "400");
    props.put("zookeeper.sync.time.ms", "200");
    props.put("auto.commit.interval.ms", "1000");

    return new ConsumerConfig(props);

  }
View Full Code Here
TOP

Related Classes of kafka.consumer.ConsumerConfig

  • com.cloudera.oryx.kafka.util.ConsumeData
  • com.cloudera.oryx.lambda.serving.ModelManagerListener
  • com.cloudera.oryx.lambda.speed.SpeedLayer
  • com.datasift.dropwizard.kafka.KafkaConsumerFactory
  • com.jointhegrid.ironcount.IronIntegrationTest
  • com.jointhegrid.ironcount.manager.WorkerThread
  • com.netflix.suro.sink.kafka.TestKafkaSink
  • com.netflix.suro.sink.kafka.TestKafkaSinkV2
  • com.pinterest.secor.reader.MessageReader
  • com.senseidb.gateway.kafka.KafkaStreamDataProvider
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.