Examples of TopicPartition

Examples of org.apache.kafka.common.TopicPartition

220
221
222
223
224
225
226
227
228
229
230
        Map<TopicPartition, RecordBatch> batches = (Map<TopicPartition, RecordBatch>) response.request().attachment();
        // if we have a response, parse it
        if (response.hasResponse()) {
            ProduceResponse produceResponse = new ProduceResponse(response.responseBody());
            for (Map.Entry<TopicPartition, ProduceResponse.PartitionResponse> entry : produceResponse.responses().entrySet()) {
                TopicPartition tp = entry.getKey();
                ProduceResponse.PartitionResponse partResp = entry.getValue();
                Errors error = Errors.forCode(partResp.errorCode);
                RecordBatch batch = batches.get(tp);
                completeBatch(batch, error, partResp.baseOffset, correlationId, now);
            }
View Full Code Here

Examples of org.apache.kafka.common.TopicPartition

287
288
289
290
291
292
293
294
295
296
297
     */
    private ClientRequest produceRequest(long now, int destination, short acks, int timeout, List<RecordBatch> batches) {
        Map<TopicPartition, ByteBuffer> produceRecordsByPartition = new HashMap<TopicPartition, ByteBuffer>(batches.size());
        Map<TopicPartition, RecordBatch> recordsByPartition = new HashMap<TopicPartition, RecordBatch>(batches.size());
        for (RecordBatch batch : batches) {
            TopicPartition tp = batch.topicPartition;
            ByteBuffer recordsBuffer = batch.records.buffer();
            recordsBuffer.flip();
            produceRecordsByPartition.put(tp, recordsBuffer);
            recordsByPartition.put(tp, batch);
        }
View Full Code Here

Examples of org.apache.kafka.common.TopicPartition

103
104
105
106
107
108
109
110
111
                Struct partitionResponse = (Struct) partitionResponseObj;
                int partition = partitionResponse.getInt(PARTITION_KEY_NAME);
                long offset = partitionResponse.getLong(FETCH_OFFSET_KEY_NAME);
                int maxBytes = partitionResponse.getInt(MAX_BYTES_KEY_NAME);
                PartitionData partitionData = new PartitionData(offset, maxBytes);
                fetchData.put(new TopicPartition(topic, partition), partitionData);
            }
        }
    }
View Full Code Here

Examples of org.apache.kafka.common.TopicPartition

202
203
204
205
206
207
208
209
210
211
212
        long nextReadyCheckDelayMs = Long.MAX_VALUE;
        boolean unknownLeadersExist = false;

        boolean exhausted = this.free.queued() > 0;
        for (Map.Entry<TopicPartition, Deque<RecordBatch>> entry : this.batches.entrySet()) {
            TopicPartition part = entry.getKey();
            Deque<RecordBatch> deque = entry.getValue();

            Node leader = cluster.leaderFor(part);
            if (leader == null) {
                unknownLeadersExist = true;
View Full Code Here

Examples of org.apache.kafka.common.TopicPartition

274
275
276
277
278
279
280
281
282
283
284
            List<RecordBatch> ready = new ArrayList<RecordBatch>();
            /* to make starvation less likely this loop doesn't start at 0 */
            int start = drainIndex = drainIndex % parts.size();
            do {
                PartitionInfo part = parts.get(drainIndex);
                Deque<RecordBatch> deque = dequeFor(new TopicPartition(part.topic(), part.partition()));
                if (deque != null) {
                    synchronized (deque) {
                        RecordBatch first = deque.peekFirst();
                        if (first != null) {
                            if (size + first.records.sizeInBytes() > maxSize && !ready.isEmpty()) {
View Full Code Here

Examples of org.apache.kafka.common.TopicPartition

93
94
95
96
97
98
99
100
101
                Struct partitionResponse = (Struct) partitionResponseObj;
                int partition = partitionResponse.getInt(PARTITION_KEY_NAME);
                long timestamp = partitionResponse.getLong(TIMESTAMP_KEY_NAME);
                int maxNumOffsets = partitionResponse.getInt(MAX_NUM_OFFSETS_KEY_NAME);
                PartitionData partitionData = new PartitionData(timestamp, maxNumOffsets);
                offsetData.put(new TopicPartition(topic, partition), partitionData);
            }
        }
    }
View Full Code Here

Examples of org.apache.kafka.common.TopicPartition

72
73
74
75
76
77
78
79
80
81
            for (Object partResponse : topicRespStruct.getArray("partition_responses")) {
                Struct partRespStruct = (Struct) partResponse;
                int partition = partRespStruct.getInt("partition");
                short errorCode = partRespStruct.getShort("error_code");
                long offset = partRespStruct.getLong("base_offset");
                TopicPartition tp = new TopicPartition(topic, partition);
                responses.put(tp, new PartitionResponse(errorCode, offset));
            }
        }
    }
View Full Code Here

Examples of org.apache.kafka.common.TopicPartition

231
232
233
234
235
236
237
238
239
240
241
            // first make sure the metadata for the topic is available
            waitOnMetadata(record.topic(), this.metadataFetchTimeoutMs);
            int partition = partitioner.partition(record, metadata.fetch());
            int serializedSize = Records.LOG_OVERHEAD + Record.recordSize(record.key(), record.value());
            ensureValidRecordSize(serializedSize);
            TopicPartition tp = new TopicPartition(record.topic(), partition);
            log.trace("Sending record {} with callback {} to topic {} partition {}", record, callback, record.topic(), partition);
            RecordAccumulator.RecordAppendResult result = accumulator.append(tp, record.key(), record.value(), compressionType, callback);
            if (result.batchIsFull || result.newBatchCreated) {
                log.trace("Waking up the sender since topic {} partition {} is either full or getting a new batch", record.topic(), partition);
                this.sender.wakeup();
View Full Code Here

Examples of org.apache.kafka.common.TopicPartition

90
91
92
93
94
95
96
97
98
                int partition = partitionResponse.getInt(PARTITION_KEY_NAME);
                long offset = partitionResponse.getLong(COMMIT_OFFSET_KEY_NAME);
                String metadata = partitionResponse.getString(METADATA_KEY_NAME);
                short errorCode = partitionResponse.getShort(ERROR_CODE_KEY_NAME);
                PartitionData partitionData = new PartitionData(offset, metadata, errorCode);
                responseData.put(new TopicPartition(topic, partition), partitionData);
            }
        }
    }
View Full Code Here

Examples of org.apache.kafka.common.TopicPartition

104
105
106
107
108
109
110
111
112
113
114
        int partition = 0;
        if (this.cluster.partitionsForTopic(record.topic()) != null)
            partition = partitioner.partition(record, this.cluster);
        ProduceRequestResult result = new ProduceRequestResult();
        FutureRecordMetadata future = new FutureRecordMetadata(result, 0);
        TopicPartition topicPartition = new TopicPartition(record.topic(), partition);
        long offset = nextOffset(topicPartition);
        Completion completion = new Completion(topicPartition, offset, new RecordMetadata(topicPartition, 0, offset), result, callback);
        this.sent.add(record);
        if (autoComplete)
            completion.complete(null);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.