Package kafka.javaapi.consumer

Examples of kafka.javaapi.consumer.SimpleConsumer

106
107
108
109
110
111
112
113
114
115
116
                long totalLatestEmittedOffset = 0;
                HashMap ret = new HashMap();
                if (_partitions != null && _partitions.size() == _partitionToOffset.size()) {
                    for (Map.Entry<Partition, Long> e : _partitionToOffset.entrySet()) {
                        Partition partition = e.getKey();
                        SimpleConsumer consumer = _connections.getConnection(partition);
                        if (consumer == null) {
                            LOG.warn("partitionToOffset contains partition not found in _connections. Stale partition data?");
                            return null;
                        }
                        long latestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.LatestTime());
View Full Code Here
50
51
52
53
54
55
56
57
    }

    public static SimpleConsumer getKafkaConsumer(KafkaTestBroker broker) {
        BrokerHosts brokerHosts = getBrokerHosts(broker);
        KafkaConfig kafkaConfig = new KafkaConfig(brokerHosts, TOPIC);
        SimpleConsumer simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient");
        return simpleConsumer;
    }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
        return register(broker, partition.partition);
    }

    public SimpleConsumer register(Broker host, int partition) {
        if (!_connections.containsKey(host)) {
            _connections.put(host, new ConnectionInfo(new SimpleConsumer(host.host, host.port, _config.socketTimeoutMs, _config.bufferSizeBytes, _config.clientId)));
        }
        ConnectionInfo info = _connections.get(host);
        info.partitions.add(partition);
        return info.consumer;
    }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
    }

    public SimpleConsumer getConsumer(int partition) {
        if (!_kafka.containsKey(partition)) {
            Broker hp = hosts.getPartitionInformation().getBrokerFor(partition);
            _kafka.put(partition, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes, _config.clientId));

        }
        return _kafka.get(partition);
    }
View Full Code Here
66
67
68
69
70
71
72
73
74
75
        _kafkaMaxFetchLatencyMetric = context.registerMetric("kafkaFetchMax", new MaxMetric(), _config.metricsTimeBucketSizeInSecs);
    }


    private Map failFastEmitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map lastMeta) {
        SimpleConsumer consumer = _connections.register(partition);
        Map ret = doEmitNewPartitionBatch(consumer, partition, collector, lastMeta);
        _kafkaOffsetMetric.setLatestEmittedOffset(partition, (Long) ret.get("offset"));
        return ret;
    }
View Full Code Here
147
148
149
150
151
152
153
154
155
156
157
     */
    private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map meta) {
        LOG.info("re-emitting batch, attempt " + attempt);
        String instanceId = (String) meta.get("instanceId");
        if (!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) {
            SimpleConsumer consumer = _connections.register(partition);
            long offset = (Long) meta.get("offset");
            long nextOffset = (Long) meta.get("nextOffset");
            ByteBufferMessageSet msgs = fetchMessages(consumer, partition, offset);
            for (MessageAndOffset msg : msgs) {
                if (offset == nextOffset) {
View Full Code Here
52
53
54
55
56
57
58
        broker = new KafkaTestBroker();
        GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation();
        globalPartitionInformation.addPartition(0, Broker.fromString(broker.getBrokerConnectionString()));
        brokerHosts = new StaticHosts(globalPartitionInformation);
        config = new KafkaConfig(brokerHosts, "testTopic");
        simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient");
    }
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
82
    @Test(expected = FailedFetchException.class)
    public void brokerIsDown() throws Exception {
        int port = broker.getPort();
        broker.shutdown();
        SimpleConsumer simpleConsumer = new SimpleConsumer("localhost", port, 100, 1024, "testClient");
        try {
            KafkaUtils.fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), OffsetRequest.LatestTime());
        } finally {
            simpleConsumer.close();
        }
    }
View Full Code Here
58
59
60
61
62
63
64
65
        ZkHosts hosts = new ZkHosts(connectionString);
        hosts.refreshFreqSecs = 1;
        spoutConfig = new SpoutConfig(hosts, "topic", "/test", "id");
        Map conf = buildZookeeperConfig(server);
        state = new ZkState(conf);
        simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient");
        when(dynamicPartitionConnections.register(any(Broker.class), anyInt())).thenReturn(simpleConsumer);
    }
View Full Code Here
78
79
80
81
82
83
84
    private void setupKafkaConsumer() {
        GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation();
        globalPartitionInformation.addPartition(0, Broker.fromString(broker.getBrokerConnectionString()));
        BrokerHosts brokerHosts = new StaticHosts(globalPartitionInformation);
        kafkaConfig = new KafkaConfig(brokerHosts, TEST_TOPIC);
        simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient");
    }
View Full Code Here
TOP

Related Classes of kafka.javaapi.consumer.SimpleConsumer

  • co.cask.cdap.logging.kafka.KafkaConsumer
  • co.gridport.kafka.hadoop.KafkaInputRecordReader
  • com.alibaba.jstorm.kafka.MessageConsumer
  • com.facebook.presto.kafka.KafkaRecordSet$KafkaRecordCursor
  • com.facebook.presto.kafka.KafkaSimpleConsumerManager$SimpleConsumerCacheLoader
  • com.facebook.presto.kafka.KafkaSplitManager
  • com.linkedin.camus.etl.kafka.common.EtlRequest
  • com.linkedin.camus.etl.kafka.common.KafkaReader
  • com.linkedin.camus.etl.kafka.mapred.EtlInputFormat
  • com.netflix.suro.sink.kafka.TestKafkaSink
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.