package net.scharrenbach.kafka;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.serializer.Decoder;
/*
* #%L
* Kafka Run
* %%
* Copyright (C) 2013 Thomas Scharrenbach
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
/**
* <p>
* Simple kafka consumer that consumes messages from stdin.
* </p>
*
* @author Thomas Scharrenbach
* @version 0.8.0
* @since 0.8.0
*
*/
public class KafkaSimpleConsumer<K, V> extends java.lang.Thread {
private Properties _properties;
private ConsumerConnector _consumer;
private String _topic;
private List<KafkaStream<K, V>> _kafkaStreamsList;
private Decoder<K> _keyDecoder;
private Decoder<V> _valueDecoder;
private int _numberOfStreams;
private KafkaMessageProcessorFactory<K, V> _kafkaMessageProcessorFactory;
private final List<Thread> _kafkaMessageProcessorThreadList;
//
//
//
/**
* <p>
* This class is not yet fully implemented so it must not be instantiated.
* </p>
*
* @author Thomas Scharrenbach
* @version 0.8.0
* @since 0.8.0
*/
public KafkaSimpleConsumer(String topic, int numberOfStreams,
KafkaMessageProcessorFactory<K, V> kafkaMessageProcessorFactory,
Decoder<K> keyDecoder, Decoder<V> valueDecoder) {
_properties = KafkaProperties.createConsumerDefaultProperties();
_topic = topic;
_numberOfStreams = numberOfStreams;
_kafkaMessageProcessorFactory = kafkaMessageProcessorFactory;
_kafkaMessageProcessorThreadList = new ArrayList<Thread>();
_keyDecoder = keyDecoder;
_valueDecoder = valueDecoder;
}
/**
* <p>
*
* </p>
*
* @author Thomas Scharrenbach
* @version 0.8.0
* @since 0.8.0
*/
public void init() {
// Clear the list of threads.
_kafkaMessageProcessorThreadList.clear();
// Create the Kafka consumer config.
final ConsumerConfig kafkaConsumerConfig = new ConsumerConfig(
getProperties());
// Connect to Kafka via the high-level consumer API.
_consumer = kafka.consumer.Consumer
.createJavaConsumerConnector(kafkaConsumerConfig);
// Create list of streams for the topic of this consumer.
final Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
topicCountMap.put(getTopic(), getNumberOfStreams());
// Extract the Kafka streams.
_kafkaStreamsList = _consumer.createMessageStreams(topicCountMap,
getKeyDecoder(), getValueDecoder()).get(getTopic());
for (final KafkaStream<K, V> kafkaStream : _kafkaStreamsList) {
final KafkaConsumerRunnable<K, V> kafkaRunnable = new KafkaConsumerRunnable<K, V>(
kafkaStream, _kafkaMessageProcessorFactory);
_kafkaMessageProcessorThreadList.add(new Thread(kafkaRunnable) {
@Override
public void interrupt() {
super.interrupt();
kafkaRunnable.setInterrupted(true);
}
});
}
}
/**
* <p>
* Start all threads.
* </p>
*/
public void start() {
super.start();
init();
for (Thread kafkaMessageProcessorThread : _kafkaMessageProcessorThreadList) {
kafkaMessageProcessorThread.start();
}
}
//
//
//
/**
* <p>
* Note: this class references the specified properties object. Setting
* properties overwrites the reference directly. Please use a copy of your
* properties object in order not exclude unwanted side-effects.
* </p>
*
* @author Thomas Scharrenbach
* @version 0.8.0
* @since 0.8.0
*
* @param properties
*/
public void setProperties(Properties properties) {
if (properties == null) {
throw new NullPointerException();
}
_properties = properties;
}
/**
* <p>
* Note: this class references a properties object. Any changes to the
* original object will be reflected to the reference this class holds.
* Please use a copy of your properties object in order not exclude unwanted
* side-effects.
* </p>
*
* @author Thomas Scharrenbach
* @version 0.8.0
* @since 0.8.0
*
*
* @return
*/
public Properties getProperties() {
return _properties;
}
/**
* @author Thomas Scharrenbach
* @version 0.8.0
* @since 0.8.0
* @return
*/
public void setTopic(String topic) {
_topic = topic;
}
/**
* @author Thomas Scharrenbach
* @version 0.8.0
* @since 0.8.0
* @return
*/
public String getTopic() {
return _topic;
}
public List<KafkaStream<K, V>> getKafkaStreamsList() {
return new ArrayList<KafkaStream<K, V>>(_kafkaStreamsList);
}
public void setKeyDecoder(Decoder<K> keyDecoder) {
_keyDecoder = keyDecoder;
}
public Decoder<K> getKeyDecoder() {
return _keyDecoder;
}
public void setValueDecoder(Decoder<V> valueDecoder) {
_valueDecoder = valueDecoder;
}
public Decoder<V> getValueDecoder() {
return _valueDecoder;
}
public void setNumberOfStreams(int numberOfStreams) {
_numberOfStreams = numberOfStreams;
}
public int getNumberOfStreams() {
return _numberOfStreams;
}
}