Examples of com.pinterest.secor.common.TopicPartition

com.pinterest.secor.common.TopicPartition
Topic partition describes a kafka message topic-partition pair. @author Pawel Garbacki (pawel@pinterest.com)

                    for (int i = 0; i < num_partitions; i++) {
                        OffsetTracker offsetTracker = consumer
                                .getOffsetTracker();
                        long val = (offsetTracker == null) ? -1
                                : offsetTracker
                                        .getLastSeenOffset(new TopicPartition(
                                                topic, i)) + 1;


                        System.out.println("topic:" + topic + " partition:" + i
                                + " secor offset:" + val + " elapsed:"
                                + (System.currentTimeMillis() - startMillis));
                        Long lastOffset = lastOffsets.get(new TopicPartition(
                                topic, i));
                        if (lastOffset != null && lastOffset == val) {
                            lastOffsets.remove(new TopicPartition(topic, i));
                        }
                    }
                }
            }

View Full Code Here

        for (String topic : topics) {
            for (int i = 0; i < num_partitions; i++) {
                TopicAndPartition topicAndPartition = new TopicAndPartition(
                        topic, i);
                SimpleConsumer consumer = mKafkaClient
                        .createConsumer(new TopicPartition(topic, i));
                Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
                requestInfo.put(topicAndPartition,
                        new PartitionOffsetRequestInfo(-1, 1));
                kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
                        requestInfo, kafka.api.OffsetRequest.CurrentVersion(),
                        "TestPerformance");
                OffsetResponse response = consumer.getOffsetsBefore(request);
                if (response.hasError()) {
                    System.out
                            .println("Error fetching data Offset Data the Broker. Reason: "
                                    + response.errorCode(topic, i));
                    return null;
                }
                long[] offsets = response.offsets(topic, i);
                System.out.println("Topic: " + topic + " partition: " + i
                        + " offset: " + offsets[0]);
                lastOffsets.put(new TopicPartition(topic, i), offsets[0]);
            }
        }
        return lastOffsets;
    }

View Full Code Here

            ZookeeperConnector zookeeperConnector = new ZookeeperConnector(config);
            String topic = commandLine.getOptionValue("topic");
            if (commandLine.hasOption("partition")) {
                int partition =
                    ((Number) commandLine.getParsedOptionValue("partition")).intValue();
                TopicPartition topicPartition = new TopicPartition(topic, partition);
                zookeeperConnector.deleteCommittedOffsetPartitionCount(topicPartition);
            } else {
                zookeeperConnector.deleteCommittedOffsetTopicCount(topic);
            }
        } catch (Throwable t) {

View Full Code Here

        assert hasNext();
        RateLimitUtil.acquire();
        MessageAndMetadata<byte[], byte[]> kafkaMessage = mIterator.next();
        Message message = new Message(kafkaMessage.topic(), kafkaMessage.partition(),
                                      kafkaMessage.offset(), kafkaMessage.message());
        TopicPartition topicPartition = new TopicPartition(message.getTopic(),
                                                           message.getKafkaPartition());
        updateAccessTime(topicPartition);
        // Skip already committed messages.
        long committedOffsetCount = mOffsetTracker.getTrueCommittedOffsetCount(topicPartition);
        LOG.debug("read message" + message);

View Full Code Here

        String topicPrefix = getTopicPrefix();
        String[] paths = FileUtil.listRecursively(topicPrefix);
        for (String path : paths) {
            if (!path.endsWith("/_SUCCESS")) {
                LogFilePath logFilePath = new LogFilePath(prefix, path);
                TopicPartition topicPartition = new TopicPartition(logFilePath.getTopic(),
                    logFilePath.getKafkaPartition());
                SortedMap<Long, HashSet<LogFilePath>> offsetToFiles =
                    mTopicPartitionToOffsetToFiles.get(topicPartition);
                if (offsetToFiles == null) {
                    offsetToFiles = new TreeMap<Long, HashSet<LogFilePath>>();

View Full Code Here

                if (offset >= toOffset && toOffset == Long.MAX_VALUE) {
                    lastOffset = offset;
                }
            }
            if (firstOffset != -2) {
                TopicPartition topicPartition = (TopicPartition) entry.getKey();
                offsetToFiles = offsetToFiles.subMap(firstOffset, lastOffset);
                mTopicPartitionToOffsetToFiles.put(topicPartition, offsetToFiles);
            }
        }
    }

View Full Code Here

                        Long.toString(offset) + " || " + offset + " == " + logFilePath.getOffset();
                    messageCount += getMessageCount(logFilePath);
                    offset = logFilePath.getOffset();
                }
                if (previousOffset != -2 && offset - previousOffset != previousMessageCount) {
                    TopicPartition topicPartition = (TopicPartition) entry.getKey();
                    throw new RuntimeException("Message count of " + previousMessageCount +
                                               " in topic " + topicPartition.getTopic() +
                                               " partition " + topicPartition.getPartition() +
                                               " does not agree with adjacent offsets " +
                                               previousOffset + " and " + offset);
                }
                previousOffset = offset;
                previousMessageCount = messageCount;

View Full Code Here


        Iterator iterator = mTopicPartitionToOffsetToFiles.entrySet().iterator();
        while (iterator.hasNext()) {
            TreeSet<Long> offsets = new TreeSet<Long>();
            Map.Entry entry = (Map.Entry) iterator.next();
            TopicPartition topicPartition = (TopicPartition) entry.getKey();
            SortedMap<Long, HashSet<LogFilePath>> offsetToFiles =
                    (SortedMap<Long, HashSet<LogFilePath>>) entry.getValue();
            for (HashSet<LogFilePath> logFilePaths : offsetToFiles.values()) {
                for (LogFilePath logFilePath : logFilePaths) {
                    getOffsets(logFilePath, offsets);
                }
            }
            long lastOffset = -2;
            for (Long offset : offsets) {
                if (lastOffset != -2) {
                    assert lastOffset + 1 == offset: Long.toString(offset) + " + 1 == " + offset +
                        " for topic " + topicPartition.getTopic() + " partition " +
                        topicPartition.getPartition();
                }
                lastOffset = offset;
            }
        }
    }

View Full Code Here

                LOG.info("skipping topic " + topic);
                continue;
            }
            List<Integer> partitions = mZookeeperConnector.getCommittedOffsetPartitions(topic);
            for (Integer partition : partitions) {
                TopicPartition topicPartition = new TopicPartition(topic, partition);
                Message committedMessage = mKafkaClient.getCommittedMessage(topicPartition);
                long committedOffset = - 1;
                long committedTimestampMillis = -1;
                if (committedMessage == null) {
                    LOG.warn("no committed message found in topic " + topic + " partition " +

View Full Code Here

TOP

Related Classes of com.pinterest.secor.common.TopicPartition

com.pinterest.secor.main.ZookeeperClientMain

com.pinterest.secor.performance.PerformanceTest

com.pinterest.secor.reader.MessageReader

com.pinterest.secor.tools.LogFileVerifier

com.pinterest.secor.tools.ProgressMonitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.