Package kafka.javaapi.consumer

Examples of kafka.javaapi.consumer.SimpleConsumer

89
90
91
92
93
94
95
96
97
98
99
        @Override
        public SimpleConsumer load(HostAddress host)
                throws Exception
        {
            log.info("Creating new Consumer for %s", host);
            return new SimpleConsumer(host.getHostText(),
                    host.getPort(),
                    Ints.checkedCast(kafkaConnectorConfig.getKafkaConnectTimeout().toMillis()),
                    Ints.checkedCast(kafkaConnectorConfig.getKafkaBufferSize().toBytes()),
                    format("presto-kafka-%s-%s", connectorId, nodeManager.getCurrentNode().getNodeIdentifier()));
        }
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
                long totalLatestEmittedOffset = 0;
                HashMap ret = new HashMap();
                if (_partitions != null && _partitions.size() == _partitionToOffset.size()) {
                    for (Map.Entry<Partition, Long> e : _partitionToOffset.entrySet()) {
                        Partition partition = e.getKey();
                        SimpleConsumer consumer = _connections.getConnection(partition);
                        if (consumer == null) {
                            LOG.warn("partitionToOffset contains partition not found in _connections. Stale partition data?");
                            return null;
                        }
                        long latestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.LatestTime());
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
        }

        @Override
        public Map emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map lastMeta) {
            try {
                SimpleConsumer consumer = _connections.register(partition);
                Map ret = KafkaUtils.emitPartitionBatchNew(_config, consumer, partition, collector, lastMeta, _topologyInstanceId, _topologyName, _kafkaMeanFetchLatencyMetric, _kafkaMaxFetchLatencyMetric);
                _kafkaOffsetMetric.setLatestEmittedOffset(partition, (Long)ret.get("offset"));
                return ret;
            } catch(FailedFetchException e) {
                LOG.warn("Failed to fetch from partition " + partition);
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
                long totalLatestEmittedOffset = 0;
                HashMap ret = new HashMap();
                if(_partitions != null && _partitions.size() == _partitionToOffset.size()) {
                    for(Map.Entry<GlobalPartitionId, Long> e : _partitionToOffset.entrySet()) {
                        GlobalPartitionId partition = e.getKey();
                        SimpleConsumer consumer = _connections.getConnection(partition);
                        if(consumer == null) {
                            LOG.warn("partitionToOffset contains partition not found in _connections. Stale partition data?");
                            return null;
                        }
                        long latestTimeOffset = consumer.getOffsetsBefore(_topic, partition.partition, OffsetRequest.LatestTime(), 1)[0];
                        if(latestTimeOffset == 0) {
                            LOG.warn("No data found in Kafka Partition " + partition.getId());
                            return null;
                        }
                        long latestEmittedOffset = (Long)e.getValue();
View Full Code Here
28
29
30
31
32
33
34
35
36
37
38
        return register(id.host, id.partition);
    }
   
    public SimpleConsumer register(HostPort host, int partition) {
        if(!_connections.containsKey(host)) {
            _connections.put(host, new ConnectionInfo(new SimpleConsumer(host.host, host.port, _config.socketTimeoutMs, _config.bufferSizeBytes)));
        }
        ConnectionInfo info = _connections.get(host);
        info.partitions.add(partition);
        return info.consumer;
    }
View Full Code Here
21
22
23
24
25
26
27
28
29
30
    public SimpleConsumer getConsumer(int partition) {
        int hostIndex = partition / hosts.partitionsPerHost;
        if(!_kafka.containsKey(hostIndex)) {
            HostPort hp = hosts.hosts.get(hostIndex);
            _kafka.put(hostIndex, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes));

        }
        return _kafka.get(hostIndex);
    }
View Full Code Here
75
76
77
78
79
80
81
82
83
84
            _kafkaMaxFetchLatencyMetric = context.registerMetric("kafkaFetchMax", new MaxMetric(), 60);
        }
       
        @Override
        public Map emitPartitionBatchNew(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map lastMeta) {
            SimpleConsumer consumer = _connections.register(partition);
            Map ret = KafkaUtils.emitPartitionBatchNew(_config, consumer, partition, collector, lastMeta, _topologyInstanceId, _topologyName, _kafkaMeanFetchLatencyMetric, _kafkaMaxFetchLatencyMetric);
            _kafkaOffsetMetric.setLatestEmittedOffset(partition, (Long)ret.get("offset"));
            return ret;
        }
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
        @Override
        public void emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map meta) {
            String instanceId = (String) meta.get("instanceId");
            if(!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) {
                SimpleConsumer consumer = _connections.register(partition);
                long offset = (Long) meta.get("offset");
                long nextOffset = (Long) meta.get("nextOffset");
                long start = System.nanoTime();
                ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition.partition, offset, _config.fetchSizeBytes));
                long end = System.nanoTime();
                long millis = (end - start) / 1000000;
                _kafkaMeanFetchLatencyMetric.update(millis);
                _kafkaMaxFetchLatencyMetric.update(millis);
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
        _input = input;
        _request = new KafkaETLRequest(input.trim());
       
        // read data from queue
        URI uri = _request.getURI();
        _consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), _timeout, _bufferSize);
       
        // get available offset range
        _offsetRange = getOffsetRange();
        System.out.println("Connected to node " + uri
                + " beginning reading at offset " + _offsetRange[0]
View Full Code Here
113
114
115
116
117
118
119
120
121
   */
  private CacheLoader<BrokerInfo, SimpleConsumer> createConsumerLoader() {
    return new CacheLoader<BrokerInfo, SimpleConsumer>() {
      @Override
      public SimpleConsumer load(BrokerInfo key) throws Exception {
        return new SimpleConsumer(key.getHost(), key.getPort(), SO_TIMEOUT, FETCH_SIZE, "simple-kafka-client");
      }
    };
  }
View Full Code Here
TOP

Related Classes of kafka.javaapi.consumer.SimpleConsumer

  • co.cask.cdap.logging.kafka.KafkaConsumer
  • co.gridport.kafka.hadoop.KafkaInputRecordReader
  • com.alibaba.jstorm.kafka.MessageConsumer
  • com.facebook.presto.kafka.KafkaRecordSet$KafkaRecordCursor
  • com.facebook.presto.kafka.KafkaSimpleConsumerManager$SimpleConsumerCacheLoader
  • com.facebook.presto.kafka.KafkaSplitManager
  • com.linkedin.camus.etl.kafka.common.EtlRequest
  • com.linkedin.camus.etl.kafka.common.KafkaReader
  • com.linkedin.camus.etl.kafka.mapred.EtlInputFormat
  • com.netflix.suro.sink.kafka.TestKafkaSink
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.