Package kafka.javaapi

Examples of kafka.javaapi.OffsetRequest

69
70
71
72
73
74
75
76
77
78
79
    public static long getOffset(SimpleConsumer consumer, String topic, int partition, long startOffsetTime) {
        TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
        OffsetRequest request = new OffsetRequest(
                requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());

        long[] offsets = consumer.getOffsetsBefore(request).offsets(topic, partition);
        if (offsets.length > 0) {
            return offsets[0];
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
    public static long getOffset(SimpleConsumer consumer, String topic, int partition, long startOffsetTime) {
        TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
        OffsetRequest request = new OffsetRequest(
                requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());

        long[] offsets = consumer.getOffsetsBefore(request).offsets(topic, partition);
        if (offsets.length > 0) {
            return offsets[0];
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
                new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
                kafka.api.OffsetRequest.LatestTime(), 1));
        final String clientName = getClientName(topicPartition);
        OffsetRequest request = new OffsetRequest(requestInfo,
                                                  kafka.api.OffsetRequest.CurrentVersion(),
                                                  clientName);
        OffsetResponse response = consumer.getOffsetsBefore(request);

        if (response.hasError()) {
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
        // more than Integer.MAX_VALUE-1 segments.
        //
        // This also assumes that the lowest value returned will be the first segment available. So if segments have been dropped off, this value
        // should not be 0.
        PartitionOffsetRequestInfo partitionOffsetRequestInfo = new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), Integer.MAX_VALUE);
        OffsetRequest offsetRequest = new OffsetRequest(ImmutableMap.of(topicAndPartition, partitionOffsetRequestInfo), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
        OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest);

        if (offsetResponse.hasError()) {
            short errorCode = offsetResponse.errorCode(partition.getTopicName(), partition.getPartitionIdAsInt());
            log.warn("Offset response has error: %d", errorCode);
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
    public static long getOffset(SimpleConsumer consumer, String topic, int partition, long startOffsetTime) {
        TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
        OffsetRequest request = new OffsetRequest(
                requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());

        long[] offsets = consumer.getOffsetsBefore(request).offsets(topic, partition);
        if (offsets.length > 0) {
            return offsets[0];
View Full Code Here
149
150
151
152
153
154
155
156
157
158
159
      LOG.warn("Failed to talk to any broker. Default offset to 0 for {}", topicPart);
      return 0L;
    }

    // Fire offset request
    OffsetRequest request = new OffsetRequest(ImmutableMap.of(
      new TopicAndPartition(topicPart.getTopic(), topicPart.getPartition()),
      new PartitionOffsetRequestInfo(timestamp, 1)
    ), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());

    OffsetResponse response = consumer.getOffsetsBefore(request);
View Full Code Here
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
        TopicAndPartition topicAndPartition = new TopicAndPartition(_request.getTopic(), _request.getPartition());
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
                new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1));
        OffsetRequest request = new OffsetRequest(
            requestInfo, kafka.api.OffsetRequest.CurrentVersion(), kafka.api.OffsetRequest.DefaultClientId());
        long[] startOffsets = _consumer.getOffsetsBefore(request).offsets(_request.getTopic(), _request.getPartition());
        if (startOffsets.length != 1)
            throw new IOException("input:" + _input + " Expect one smallest offset but get "
                                            + startOffsets.length);
        range[0] = startOffsets[0];
       
        requestInfo.clear();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
        request = new OffsetRequest(
            requestInfo, kafka.api.OffsetRequest.CurrentVersion(), kafka.api.OffsetRequest.DefaultClientId());
        long[] endOffsets = _consumer.getOffsetsBefore(request).offsets(_request.getTopic(), _request.getPartition());
        if (endOffsets.length != 1)
            throw new IOException("input:" + _input + " Expect one latest offset but get "
                                            + endOffsets.length);
View Full Code Here
257
258
259
260
261
262
263
264
265
266
267
                    1024 * 1024, "hadoop-etl");
            Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
            offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(
                    kafka.api.OffsetRequest.EarliestTime(), 1));
            OffsetResponse response = consumer
                    .getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest
                            .CurrentVersion(), "hadoop-etl"));
            long[] endOffset = response.offsets(topic, partition);
            consumer.close();
            this.earliestOffset = endOffset[0];
            return endOffset[0];
View Full Code Here
291
292
293
294
295
296
297
298
299
300
301
        SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000,
                1024 * 1024, "hadoop-etl");
        Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(
                time, 1));
        OffsetResponse response = consumer.getOffsetsBefore(new OffsetRequest(offsetInfo,
                kafka.api.OffsetRequest.CurrentVersion(),"hadoop-etl"));
        long[] endOffset = response.offsets(topic, partition);
        consumer.close();
        if(endOffset.length == 0)
        {
View Full Code Here
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
        earliestOffsetInfo.put(topicAndPartition,
            partitionEarliestOffsetRequestInfo);
      }

      OffsetResponse latestOffsetResponse = consumer
          .getOffsetsBefore(new OffsetRequest(latestOffsetInfo,
              kafka.api.OffsetRequest.CurrentVersion(), CamusJob
                  .getKafkaClientName(context)));
      OffsetResponse earliestOffsetResponse = consumer
          .getOffsetsBefore(new OffsetRequest(earliestOffsetInfo,
              kafka.api.OffsetRequest.CurrentVersion(), CamusJob
                  .getKafkaClientName(context)));
      consumer.close();
      for (TopicAndPartition topicAndPartition : topicAndPartitions) {
        long latestOffset = latestOffsetResponse.offsets(
View Full Code Here
TOP

Related Classes of kafka.javaapi.OffsetRequest

  • co.cask.cdap.logging.kafka.KafkaConsumer
  • com.alibaba.jstorm.kafka.MessageConsumer
  • com.facebook.presto.kafka.KafkaSplitManager
  • com.linkedin.camus.etl.kafka.common.EtlRequest
  • com.linkedin.camus.etl.kafka.mapred.EtlInputFormat
  • com.pinterest.secor.common.KafkaClient
  • kafka.etl.KafkaETLContext
  • org.apache.twill.internal.kafka.client.SimpleKafkaConsumer
  • storm.kafka.KafkaUtils
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.