Package kafka.api

Examples of kafka.api.FetchRequest

158
159
160
161
162
163
164
165
166
167
168
    public static ByteBufferMessageSet fetchMessages(KafkaConfig config, SimpleConsumer consumer, Partition partition, long offset) throws UpdateOffsetException {
        ByteBufferMessageSet msgs = null;
        String topic = config.topic;
        int partitionId = partition.partition;
        FetchRequestBuilder builder = new FetchRequestBuilder();
        FetchRequest fetchRequest = builder.addFetch(topic, partitionId, offset, config.fetchSizeBytes).
                clientId(config.clientId).maxWait(config.fetchMaxWait).build();
        FetchResponse fetchResponse;
        try {
            fetchResponse = consumer.fetch(fetchRequest);
        } catch (Exception e) {
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
        ByteBufferMessageSet msgs = null;
        String topic = config.topic;
        int partitionId = partition.partition;
        for (int errors = 0; errors < 2 && msgs == null; errors++) {
            FetchRequestBuilder builder = new FetchRequestBuilder();
            FetchRequest fetchRequest = builder.addFetch(topic, partitionId, offset, config.fetchSizeBytes).
                    clientId(config.clientId).build();
            FetchResponse fetchResponse;
            try {
                fetchResponse = consumer.fetch(fetchRequest);
            } catch (Exception e) {
View Full Code Here
294
295
296
297
298
299
300
301
302
303
304
        private void openFetchRequest()
        {
            if (messageAndOffsetIterator == null) {
                log.debug("Fetching %d bytes from offset %d (%d - %d). %d messages read so far", KAFKA_READ_BUFFER_SIZE, cursorOffset, split.getStart(), split.getEnd(), totalMessages);
                FetchRequest req = new FetchRequestBuilder()
                        .clientId("presto-worker-" + Thread.currentThread().getName())
                        .addFetch(split.getTopicName(), split.getPartitionId(), cursorOffset, KAFKA_READ_BUFFER_SIZE)
                        .build();

                // TODO - this should look at the actual node this is running on and prefer
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
        ByteBufferMessageSet msgs = null;
        String topic = config.topic;
        int partitionId = partition.partition;
        for (int errors = 0; errors < 2 && msgs == null; errors++) {
            FetchRequestBuilder builder = new FetchRequestBuilder();
            FetchRequest fetchRequest = builder.addFetch(topic, partitionId, offset, config.fetchSizeBytes).
                    clientId(config.clientId).maxWait(config.fetchMaxWait).build();
            FetchResponse fetchResponse;
            try {
                fetchResponse = consumer.fetch(fetchRequest);
            } catch (Exception e) {
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
             offset = consumer.getOffsetsBefore(config.topic, partition.partition, startTime, 1)[0];
         }
         ByteBufferMessageSet msgs;
         try {
            long start = System.nanoTime();
            msgs = consumer.fetch(new FetchRequest(config.topic, partition.partition, offset, config.fetchSizeBytes));
            long end = System.nanoTime();
            long millis = (end - start) / 1000000;
            meanMetric.update(millis);
            maxMetric.update(millis);
         } catch(Exception e) {
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
    private void fill() {
        //LOG.info("Fetching from Kafka: " + _consumer.host() + ":" + _partition.partition + " from offset " + _emittedToOffset);
        long start = System.nanoTime();
        ByteBufferMessageSet msgs = _consumer.fetch(
                new FetchRequest(
                    _spoutConfig.topic,
                    _partition.partition,
                    _emittedToOffset,
                    _spoutConfig.fetchSizeBytes));
        long end = System.nanoTime();
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
            if(!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) {
                SimpleConsumer consumer = _connections.register(partition);
                long offset = (Long) meta.get("offset");
                long nextOffset = (Long) meta.get("nextOffset");
                long start = System.nanoTime();
                ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition.partition, offset, _config.fetchSizeBytes));
                long end = System.nanoTime();
                long millis = (end - start) / 1000000;
                _kafkaMeanFetchLatencyMetric.update(millis);
                _kafkaMaxFetchLatencyMetric.update(millis);
View Full Code Here
147
148
149
150
151
152
153
154
155
156
157
158
    }
   
    public boolean fetchMore () throws IOException {
        if (!hasMore()) return false;
       
        FetchRequest fetchRequest =
            new FetchRequest(_request.getTopic(), _request.getPartition(), _offset, _bufferSize);
        List<FetchRequest> array = new ArrayList<FetchRequest>();
        array.add(fetchRequest);

        long tempTime = System.currentTimeMillis();
        _response = _consumer.multifetch(array);
View Full Code Here
409
410
411
412
413
414
415
416
417
418
419
    /**
     * Makes a call to kafka to fetch messages.
     */
    private FetchResponse fetchMessages(SimpleConsumer consumer, long offset) {
      FetchRequest request = new FetchRequestBuilder()
        .clientId(consumer.clientId())
        .addFetch(topicPart.getTopic(), topicPart.getPartition(), offset, FETCH_SIZE)
        .maxWait(MAX_WAIT)
        .build();
      return consumer.fetch(request);
View Full Code Here
287
288
289
290
291
292
293
294
295
296
297
        private void openFetchRequest()
        {
            if (messageAndOffsetIterator == null) {
                log.debug("Fetching %d bytes from offset %d (%d - %d). %d messages read so far", KAFKA_READ_BUFFER_SIZE, cursorOffset, split.getStart(), split.getEnd(), totalMessages);
                FetchRequest req = new FetchRequestBuilder()
                        .clientId("presto-worker-" + Thread.currentThread().getName())
                        .addFetch(split.getTopicName(), split.getPartitionId(), cursorOffset, KAFKA_READ_BUFFER_SIZE)
                        .build();

                // TODO - this should look at the actual node this is running on and prefer
View Full Code Here
TOP

Related Classes of kafka.api.FetchRequest

  • co.cask.cdap.logging.kafka.KafkaConsumer
  • co.gridport.kafka.hadoop.KafkaInputRecordReader
  • com.alibaba.jstorm.kafka.MessageConsumer
  • com.facebook.presto.kafka.KafkaRecordSet$KafkaRecordCursor
  • kafka.etl.KafkaETLContext
  • kafka.etl.KafkaETLMapper
  • kafka.examples.SimpleConsumerDemo
  • kafka.perf.consumer.SimplePerfConsumer
  • org.apache.twill.internal.kafka.client.SimpleKafkaConsumer$ConsumerThread
  • storm.kafka.KafkaUtils
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.