Examples of SimpleConsumer

Examples of kafka.consumer.SimpleConsumer

27
28
29
30
31
32
33
34
35
36
37
  public SimplePerfConsumer(String topic, String kafkaServerURL, int kafkaServerPort,
                            int kafkaProducerBufferSize, int connectionTimeOut, int reconnectInterval,
                            int fetchSize, String name, int numParts)
  {
    simpleConsumer = new SimpleConsumer(kafkaServerURL,
                                        kafkaServerPort,
                                        connectionTimeOut,
                                        kafkaProducerBufferSize);
    this.topic = topic;
    this.fetchSize = fetchSize;
View Full Code Here

Examples of kafka.consumer.SimpleConsumer

119
120
121
122
123
124
125
126
127
128
129
          input
              + " : invalid node id in offset file--no host information found");
    URI node = _nodes.get(_nodeId);

    // read data from queue
    _consumer = new SimpleConsumer(node.getHost(), node.getPort(),
        _soTimeout, _bufferSize);

    // get available offset range
    _offsetRange = getOffsetRange(_consumer, topic, _partition, input);
View Full Code Here

Examples of kafka.consumer.SimpleConsumer

56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
  public static void main(String[] args)
  {
   
    generateData();
    SimpleConsumer simpleConsumer = new SimpleConsumer(KafkaProperties.kafkaServerURL,
                                                       KafkaProperties.kafkaServerPort,
                                                       KafkaProperties.connectionTimeOut,
                                                       KafkaProperties.kafkaProducerBufferSize);

    System.out.println("Testing single fetch");
    FetchRequest req = new FetchRequest(KafkaProperties.topic2, 0, 0L, 100);
    ByteBufferMessageSet messageSet = simpleConsumer.fetch(req);
    printMessages(messageSet);

    System.out.println("Testing single multi-fetch");
    req = new FetchRequest(KafkaProperties.topic2, 0, 0L, 100);
    List<FetchRequest> list = new ArrayList<FetchRequest>();
    list.add(req);
    req = new FetchRequest(KafkaProperties.topic3, 0, 0L, 100);
    list.add(req);
    MultiFetchResponse response = simpleConsumer.multifetch(list);
    int fetchReq = 0;
    while(response.hasNext())
    {
      System.out.println("Response from fetch request no: " + ++fetchReq);
      messageSet = response.next();
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

106
107
108
109
110
111
112
113
114
115
116
                long totalLatestEmittedOffset = 0;
                HashMap ret = new HashMap();
                if (_partitions != null && _partitions.size() == _partitionToOffset.size()) {
                    for (Map.Entry<Partition, Long> e : _partitionToOffset.entrySet()) {
                        Partition partition = e.getKey();
                        SimpleConsumer consumer = _connections.getConnection(partition);
                        if (consumer == null) {
                            LOG.warn("partitionToOffset contains partition not found in _connections. Stale partition data?");
                            return null;
                        }
                        long latestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.LatestTime());
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

50
51
52
53
54
55
56
57
    }

    public static SimpleConsumer getKafkaConsumer(KafkaTestBroker broker) {
        BrokerHosts brokerHosts = getBrokerHosts(broker);
        KafkaConfig kafkaConfig = new KafkaConfig(brokerHosts, TOPIC);
        SimpleConsumer simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient");
        return simpleConsumer;
    }
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

55
56
57
58
59
60
61
62
63
64
65
        return register(broker, partition.partition);
    }

    public SimpleConsumer register(Broker host, int partition) {
        if (!_connections.containsKey(host)) {
            _connections.put(host, new ConnectionInfo(new SimpleConsumer(host.host, host.port, _config.socketTimeoutMs, _config.bufferSizeBytes, _config.clientId)));
        }
        ConnectionInfo info = _connections.get(host);
        info.partitions.add(partition);
        return info.consumer;
    }
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

36
37
38
39
40
41
42
43
44
45
    }

    public SimpleConsumer getConsumer(int partition) {
        if (!_kafka.containsKey(partition)) {
            Broker hp = hosts.getPartitionInformation().getBrokerFor(partition);
            _kafka.put(partition, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes, _config.clientId));

        }
        return _kafka.get(partition);
    }
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

66
67
68
69
70
71
72
73
74
75
        _kafkaMaxFetchLatencyMetric = context.registerMetric("kafkaFetchMax", new MaxMetric(), _config.metricsTimeBucketSizeInSecs);
    }


    private Map failFastEmitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map lastMeta) {
        SimpleConsumer consumer = _connections.register(partition);
        Map ret = doEmitNewPartitionBatch(consumer, partition, collector, lastMeta);
        _kafkaOffsetMetric.setLatestEmittedOffset(partition, (Long) ret.get("offset"));
        return ret;
    }
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

147
148
149
150
151
152
153
154
155
156
157
     */
    private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map meta) {
        LOG.info("re-emitting batch, attempt " + attempt);
        String instanceId = (String) meta.get("instanceId");
        if (!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) {
            SimpleConsumer consumer = _connections.register(partition);
            long offset = (Long) meta.get("offset");
            long nextOffset = (Long) meta.get("nextOffset");
            ByteBufferMessageSet msgs = fetchMessages(consumer, partition, offset);
            for (MessageAndOffset msg : msgs) {
                if (offset == nextOffset) {
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

52
53
54
55
56
57
58
        broker = new KafkaTestBroker();
        GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation();
        globalPartitionInformation.addPartition(0, Broker.fromString(broker.getBrokerConnectionString()));
        brokerHosts = new StaticHosts(globalPartitionInformation);
        config = new KafkaConfig(brokerHosts, "testTopic");
        simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient");
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.