Examples of ConsumerConfig

Examples of kafka.consumer.ConsumerConfig

71
72
73
74
75
76
77
78
79
80
81
    producerProps = new Properties();
    producerProps.put("serializer.class", "kafka.serializer.StringEncoder");
    producerProps.put("zk.connect", "localhost:8888");

    consumerProps.setProperty("groupid", "group1");
    consumerConfig = new ConsumerConfig(consumerProps);
    producerConfig = new ProducerConfig(producerProps);

    brokerProps.setProperty("topic.partition.count.map", topic+":2"+",map:2,reduce:2");
    brokerProps.setProperty("num.partitions", "10");
    brokerProps.setProperty("brokerid", "1");
View Full Code Here

Examples of kafka.consumer.ConsumerConfig

71
72
73
74
75
76
77
78
79
80
81
            return Response.status(400)
                    .entity(new String[]{"Undefined topic"})
                    .build();

        Properties props = consumerCfg.asProperties(timeout);
        ConsumerConfig config = new ConsumerConfig(props);
        ConsumerConnector connector = Consumer.createJavaConsumerConnector(config);

        Map<String, Integer> streamCounts = Collections.singletonMap(topic, 1);
        Map<String, List<KafkaStream<byte[], byte[]>>> streams = connector.createMessageStreams(streamCounts);
        KafkaStream<byte[], byte[]> stream = streams.get(topic).get(0);
View Full Code Here

Examples of kafka.consumer.ConsumerConfig

53
54
55
56
57
58
59
60
61
  @Override
  public CloseableIterator<Pair<String,String>> iterator() {
    Properties consumerProps = new Properties();
    consumerProps.setProperty("group.id", "OryxGroup-ConsumeData");
    consumerProps.setProperty("zookeeper.connect", "localhost:" + zkPort);
    ConsumerConfig consumerConfig = new ConsumerConfig(consumerProps);
    ConsumerConnector consumer = Consumer.createJavaConsumerConnector(consumerConfig);
    return new ConsumeDataIterator(topic, consumer);
  }
View Full Code Here

Examples of kafka.consumer.ConsumerConfig

118
119
120
121
122
123
124
125
126
127
128
    public void open() {

        // these consumers use ZooKeeper for commit, offset and segment consumption tracking
        // TODO: consider using SimpleConsumer the same way the Hadoop consumer job does to avoid ZK dependency
        // TODO: use the task details from TopologyContext in the normal open method
        ConsumerConfig consumerConfig = new ConsumerConfig(kafkaProperties);
        consumerConnector = Consumer.createJavaConsumerConnector(consumerConfig);

        // consumer with just one thread since the real parallelism is handled by Storm already
        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
        topicCountMap.put(topic, new Integer(1));
View Full Code Here

Examples of kafka.consumer.ConsumerConfig

376
377
378
379
380
381
382
383
    for (int i = 0; i < keyValues.length; i += 2) {
      String key = keyValues[i];
      String value = keyValues[i + 1];
      props.put(key, value);
    }
    ConsumerConfig config = new ConsumerConfig(props);
    return Consumer.createJavaConsumerConnector(config);
  }
View Full Code Here

Examples of kafka.consumer.ConsumerConfig

102
103
104
105
106
107
108
109
110
111
112
        Properties props = new Properties();
        props.put("zookeeper.connect", cluster.getZkConnectString());
        props.put("group.id", "foo");
        props.put("rebalance.backoff.ms", "2000");
        props.put("rebalance.max.retries", "2000");
        ConsumerConfig config = new ConsumerConfig(props);


        CuratorFramework curator = CuratorFrameworkFactory.newClient(cluster.getZkConnectString(), new RetryUntilElapsed(1000, 100));
        curator.start();
View Full Code Here

Examples of kafka.consumer.ConsumerConfig

171
172
173
174
175
176
177
178
179
180
181
      props.put(key, _kafkaConfig.getProperty(key));
    }

    logger.info("Kafka properties: " + props);

    ConsumerConfig consumerConfig = new ConsumerConfig(props);
    _consumerConnector = Consumer.createJavaConsumerConnector(consumerConfig);

    Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    for (String topic : _topics)
    {
View Full Code Here

Examples of kafka.consumer.ConsumerConfig

43
44
45
46
47
48
49
50
    props.put("groupid", KafkaProperties.groupId);
    props.put("zk.sessiontimeout.ms", "400");
    props.put("zk.synctime.ms", "200");
    props.put("autocommit.interval.ms", "1000");

    return new ConsumerConfig(props);

  }
View Full Code Here

Examples of kafka.consumer.ConsumerConfig

46
47
48
49
50
51
52
53
    props.put("group.id", KafkaProperties.groupId);
    props.put("zookeeper.session.timeout.ms", "400");
    props.put("zookeeper.sync.time.ms", "200");
    props.put("auto.commit.interval.ms", "1000");

    return new ConsumerConfig(props);

  }
View Full Code Here

Examples of kafka.consumer.ConsumerConfig

85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
            props.put("auto.offset.reset", "smallest");
            props.put("queuedchunks.max", "1000");
            props.put("backoff.increment.ms", "1500");
            props.put("consumer.timeout.ms", "120000");

            consumerConfig = new ConsumerConfig(props);
            consumer = kafka.consumer.Consumer.createJavaConsumerConnector(consumerConfig);

            //Certain properties in done consumer are different.
            props.remove("consumer.timeout.ms");
            props.put("group.id", m_groupId + "-done");
            //Use higher autocommit interval as we read only 1 row and then real consumer follows for long time.
            props.put("auto.commit.interval.ms", "10000");
            doneConsumerConfig = new ConsumerConfig(props);
            doneConsumer = kafka.consumer.Consumer.createJavaConsumerConnector(doneConsumerConfig);
        }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.