Package kafka.javaapi.consumer

Examples of kafka.javaapi.consumer.SimpleConsumer

63
64
65
66
67
68
69
70
71
72
73
        }

       
        public Map emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, int partition, Map lastMeta) {
            try {
                SimpleConsumer consumer = _connections.getConsumer(partition);
                return KafkaUtils.emitPartitionBatchNew(_config, partition, consumer, attempt, collector, lastMeta, _topologyInstanceId);
            } catch(FailedFetchException e) {
                LOG.warn("Failed to fetch from partition " + partition);
                if(lastMeta==null) {
                    return null;
View Full Code Here
20
21
22
23
24
25
26
27
28
29
    public SimpleConsumer getConsumer(int partition) {
        int hostIndex = partition / hosts.partitionsPerHost;
        if(!_kafka.containsKey(hostIndex)) {
            HostPort hp = hosts.hosts.get(hostIndex);
            _kafka.put(hostIndex, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes));

        }
        return _kafka.get(hostIndex);
    }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
        _input = input;
        _request = new KafkaETLRequest(input.trim());
       
        // read data from queue
        URI uri = _request.getURI();
        _consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), _timeout, _bufferSize);
       
        // get available offset range
        _offsetRange = getOffsetRange();
        System.out.println("Connected to node " + uri
                + " beginning reading at offset " + _offsetRange[0]
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
  public static void main(String[] args)
  {
   
    generateData();
    SimpleConsumer simpleConsumer = new SimpleConsumer(KafkaProperties.kafkaServerURL,
                                                       KafkaProperties.kafkaServerPort,
                                                       KafkaProperties.connectionTimeOut,
                                                       KafkaProperties.kafkaProducerBufferSize);

    System.out.println("Testing single fetch");
    FetchRequest req = new FetchRequest(KafkaProperties.topic2, 0, 0L, 100);
    ByteBufferMessageSet messageSet = simpleConsumer.fetch(req);
    printMessages(messageSet);

    System.out.println("Testing single multi-fetch");
    req = new FetchRequest(KafkaProperties.topic2, 0, 0L, 100);
    List<FetchRequest> list = new ArrayList<FetchRequest>();
    list.add(req);
    req = new FetchRequest(KafkaProperties.topic3, 0, 0L, 100);
    list.add(req);
    MultiFetchResponse response = simpleConsumer.multifetch(list);
    int fetchReq = 0;
    for (ByteBufferMessageSet resMessageSet : response )
    {
      System.out.println("Response from fetch request no: " + ++fetchReq);
      printMessages(resMessageSet);
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
  public SimplePerfConsumer(String topic, String kafkaServerURL, int kafkaServerPort,
                            int kafkaProducerBufferSize, int connectionTimeOut, int reconnectInterval,
                            int fetchSize, String name, int numParts)
  {
    super(name);
    simpleConsumer = new SimpleConsumer(kafkaServerURL,
                                        kafkaServerPort,
                                        connectionTimeOut,
                                        kafkaProducerBufferSize);
    this.topic = topic;
    this.fetchSize = fetchSize;
View Full Code Here
46
47
48
49
50
51
52
  public MessageConsumer(KafkaSpoutConfig config, Host broker) {
    this.config = config;
    this.broker = broker;
    partitions = new HashSet<Partition>();
    consumer = new SimpleConsumer(broker.getHost(), broker.getPort(), config.socketTimeoutMs,config.socketReceiveBufferBytes, config.clientId);
  }
View Full Code Here
TOP

Related Classes of kafka.javaapi.consumer.SimpleConsumer

  • co.cask.cdap.logging.kafka.KafkaConsumer
  • co.gridport.kafka.hadoop.KafkaInputRecordReader
  • com.alibaba.jstorm.kafka.MessageConsumer
  • com.facebook.presto.kafka.KafkaRecordSet$KafkaRecordCursor
  • com.facebook.presto.kafka.KafkaSimpleConsumerManager$SimpleConsumerCacheLoader
  • com.facebook.presto.kafka.KafkaSplitManager
  • com.linkedin.camus.etl.kafka.common.EtlRequest
  • com.linkedin.camus.etl.kafka.common.KafkaReader
  • com.linkedin.camus.etl.kafka.mapred.EtlInputFormat
  • com.netflix.suro.sink.kafka.TestKafkaSink
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.