Examples of SimpleConsumer

Examples of kafka.javaapi.consumer.SimpleConsumer

301
302
303
304
305
306
307
308
309
310
311
312
313
                        .addFetch(split.getTopicName(), split.getPartitionId(), cursorOffset, KAFKA_READ_BUFFER_SIZE)
                        .build();

                // TODO - this should look at the actual node this is running on and prefer
                // that copy if running locally. - look into NodeInfo
                SimpleConsumer consumer = consumerManager.getConsumer(split.getNodes().get(0));

                FetchResponse fetchResponse = consumer.fetch(req);
                if (fetchResponse.hasError()) {
                    short errorCode = fetchResponse.errorCode(split.getTopicName(), split.getPartitionId());
                    log.warn("Fetch response has error: %d", errorCode);
                    throw new PrestoException(KafkaErrorCode.KAFKA_SPLIT_ERROR.toErrorCode(), "could not fetch data from Kafka, error code is '" + errorCode + "'");
                }
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
        KafkaTableHandle kafkaTableHandle = handleResolver.convertTableHandle(tableHandle);

        List<HostAddress> nodes = new ArrayList<>(kafkaConnectorConfig.getNodes());
        Collections.shuffle(nodes);

        SimpleConsumer simpleConsumer = consumerManager.getConsumer(nodes.get(0));

        try {
            TopicMetadataRequest topicMetadataRequest = new TopicMetadataRequest(ImmutableList.of(kafkaTableHandle.getTopicName()));
            TopicMetadataResponse topicMetadataResponse = simpleConsumer.send(topicMetadataRequest);

            ImmutableList.Builder<ConnectorPartition> builder = ImmutableList.builder();

            for (TopicMetadata metadata : topicMetadataResponse.topicsMetadata()) {
                for (PartitionMetadata part : metadata.partitionsMetadata()) {
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

124
125
126
127
128
129
130
131
132
133
134
        for (ConnectorPartition cp : partitions) {
            checkState(cp instanceof KafkaPartition, "Found an unknown partition type: %s", cp.getClass().getSimpleName());
            KafkaPartition partition = (KafkaPartition) cp;

            SimpleConsumer leaderConsumer = consumerManager.getConsumer(partition.getPartitionLeader());
            // Kafka contains a reverse list of "end - start" pairs for the splits

            long[] offsets = findAllOffsets(leaderConsumer, partition);

            for (int i = offsets.length - 1; i > 0; i--) {
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

89
90
91
92
93
94
95
96
97
98
99
        @Override
        public SimpleConsumer load(HostAddress host)
                throws Exception
        {
            log.info("Creating new Consumer for %s", host);
            return new SimpleConsumer(host.getHostText(),
                    host.getPort(),
                    Ints.checkedCast(kafkaConnectorConfig.getKafkaConnectTimeout().toMillis()),
                    Ints.checkedCast(kafkaConnectorConfig.getKafkaBufferSize().toBytes()),
                    format("presto-kafka-%s-%s", connectorId, nodeManager.getCurrentNode().getNodeIdentifier()));
        }
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

106
107
108
109
110
111
112
113
114
115
116
                long totalLatestEmittedOffset = 0;
                HashMap ret = new HashMap();
                if (_partitions != null && _partitions.size() == _partitionToOffset.size()) {
                    for (Map.Entry<Partition, Long> e : _partitionToOffset.entrySet()) {
                        Partition partition = e.getKey();
                        SimpleConsumer consumer = _connections.getConnection(partition);
                        if (consumer == null) {
                            LOG.warn("partitionToOffset contains partition not found in _connections. Stale partition data?");
                            return null;
                        }
                        long latestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.LatestTime());
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

90
91
92
93
94
95
96
97
98
99
100
        }

        @Override
        public Map emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map lastMeta) {
            try {
                SimpleConsumer consumer = _connections.register(partition);
                Map ret = KafkaUtils.emitPartitionBatchNew(_config, consumer, partition, collector, lastMeta, _topologyInstanceId, _topologyName, _kafkaMeanFetchLatencyMetric, _kafkaMaxFetchLatencyMetric);
                _kafkaOffsetMetric.setLatestEmittedOffset(partition, (Long)ret.get("offset"));
                return ret;
            } catch(FailedFetchException e) {
                LOG.warn("Failed to fetch from partition " + partition);
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
                long totalLatestEmittedOffset = 0;
                HashMap ret = new HashMap();
                if(_partitions != null && _partitions.size() == _partitionToOffset.size()) {
                    for(Map.Entry<GlobalPartitionId, Long> e : _partitionToOffset.entrySet()) {
                        GlobalPartitionId partition = e.getKey();
                        SimpleConsumer consumer = _connections.getConnection(partition);
                        if(consumer == null) {
                            LOG.warn("partitionToOffset contains partition not found in _connections. Stale partition data?");
                            return null;
                        }
                        long latestTimeOffset = consumer.getOffsetsBefore(_topic, partition.partition, OffsetRequest.LatestTime(), 1)[0];
                        if(latestTimeOffset == 0) {
                            LOG.warn("No data found in Kafka Partition " + partition.getId());
                            return null;
                        }
                        long latestEmittedOffset = (Long)e.getValue();
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

28
29
30
31
32
33
34
35
36
37
38
        return register(id.host, id.partition);
    }
   
    public SimpleConsumer register(HostPort host, int partition) {
        if(!_connections.containsKey(host)) {
            _connections.put(host, new ConnectionInfo(new SimpleConsumer(host.host, host.port, _config.socketTimeoutMs, _config.bufferSizeBytes)));
        }
        ConnectionInfo info = _connections.get(host);
        info.partitions.add(partition);
        return info.consumer;
    }
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

21
22
23
24
25
26
27
28
29
30
    public SimpleConsumer getConsumer(int partition) {
        int hostIndex = partition / hosts.partitionsPerHost;
        if(!_kafka.containsKey(hostIndex)) {
            HostPort hp = hosts.hosts.get(hostIndex);
            _kafka.put(hostIndex, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes));

        }
        return _kafka.get(hostIndex);
    }
View Full Code Here

Examples of kafka.javaapi.consumer.SimpleConsumer

75
76
77
78
79
80
81
82
83
84
            _kafkaMaxFetchLatencyMetric = context.registerMetric("kafkaFetchMax", new MaxMetric(), 60);
        }
       
        @Override
        public Map emitPartitionBatchNew(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map lastMeta) {
            SimpleConsumer consumer = _connections.register(partition);
            Map ret = KafkaUtils.emitPartitionBatchNew(_config, consumer, partition, collector, lastMeta, _topologyInstanceId, _topologyName, _kafkaMeanFetchLatencyMetric, _kafkaMaxFetchLatencyMetric);
            _kafkaOffsetMetric.setLatestEmittedOffset(partition, (Long)ret.get("offset"));
            return ret;
        }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.