Package kafka.javaapi.message

Examples of kafka.javaapi.message.ByteBufferMessageSet

123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
    }

    private void fill() {
        //LOG.info("Fetching from Kafka: " + _consumer.host() + ":" + _partition.partition + " from offset " + _emittedToOffset);
        long start = System.nanoTime();
        ByteBufferMessageSet msgs = _consumer.fetch(
                new FetchRequest(
                    _spoutConfig.topic,
                    _partition.partition,
                    _emittedToOffset,
                    _spoutConfig.fetchSizeBytes));
        long end = System.nanoTime();
        long millis = (end - start) / 1000000;
        _fetchAPILatencyMax.update(millis);
        _fetchAPILatencyMean.update(millis);
        _fetchAPICallCount.incr();
        _fetchAPIMessageCount.incrBy(msgs.underlying().size());

        int numMessages = msgs.underlying().size();
        if(numMessages>0) {
          LOG.info("Fetched " + numMessages + " messages from Kafka: " + _consumer.host() + ":" + _partition.partition);
        }
        for(MessageAndOffset msg: msgs) {
            _pending.add(_emittedToOffset);
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
            if(!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) {
                SimpleConsumer consumer = _connections.register(partition);
                long offset = (Long) meta.get("offset");
                long nextOffset = (Long) meta.get("nextOffset");
                long start = System.nanoTime();
                ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition.partition, offset, _config.fetchSizeBytes));
                long end = System.nanoTime();
                long millis = (end - start) / 1000000;
                _kafkaMeanFetchLatencyMetric.update(millis);
                _kafkaMaxFetchLatencyMetric.update(millis);
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
  }

  protected void sendMsgList()
  {
    if (msgList.size() > 0) {
      ByteBufferMessageSet msgSet = new ByteBufferMessageSet(kafka.message.NoCompressionCodec$.MODULE$, msgList);
      producer.send(topic, msgSet);
      msgList.clear();
      totalSize = 0;
    }
  }
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
147
        boolean gotNext = get(key, value);

        if(_response != null) {

            while ( !gotNext && _respIterator.hasNext()) {
                ByteBufferMessageSet msgSet = _respIterator.next();
                if ( hasError(msgSet)) return false;
                _messageIt =  (Iterator<MessageAndOffset>) msgSet.iterator();
                gotNext = get(key, value);
            }
        }
        return gotNext;
    }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
      Message message = new Message(bytes);
      list.add(message);
    }
    // send events
    System.out.println(" send " + list.size() + " " + _topic + " count events to " + _uri);
    _producer.send(_topic, new ByteBufferMessageSet(kafka.message.NoCompressionCodec$.MODULE$, list));

    // close the producer
    _producer.close();
   
    // generate offset files
View Full Code Here
366
367
368
369
370
371
372
373
374
375
376
            consumers.refresh(consumerEntry.getKey());
            consumerEntry = null;
            continue;
          }

          ByteBufferMessageSet messages = response.messageSet(topicPart.getTopic(), topicPart.getPartition());
          if (sleepIfEmpty(messages)) {
            continue;
          }

          // Call the callback
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
148
        boolean gotNext = get(key, value);

        if(_response != null) {

            while ( !gotNext && _respIterator.hasNext()) {
                ByteBufferMessageSet msgSet = _respIterator.next();
                _messageIt = msgSet.iterator();
                gotNext = get(key, value);
            }
        }
        return gotNext;
    }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
                                                       KafkaProperties.connectionTimeOut,
                                                       KafkaProperties.kafkaProducerBufferSize);

    System.out.println("Testing single fetch");
    FetchRequest req = new FetchRequest(KafkaProperties.topic2, 0, 0L, 100);
    ByteBufferMessageSet messageSet = simpleConsumer.fetch(req);
    printMessages(messageSet);

    System.out.println("Testing single multi-fetch");
    req = new FetchRequest(KafkaProperties.topic2, 0, 0L, 100);
    List<FetchRequest> list = new ArrayList<FetchRequest>();
View Full Code Here
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
        log.info("Error Code generated : "
            + fetchResponse.errorCode(kafkaRequest.getTopic(),
                kafkaRequest.getPartition()));
        return false;
      } else {
        ByteBufferMessageSet messageBuffer = fetchResponse.messageSet(
            kafkaRequest.getTopic(), kafkaRequest.getPartition());
        lastFetchTime = (System.currentTimeMillis() - tempTime);
        log.debug("Time taken to fetch : "
            + (lastFetchTime / 1000) + " seconds");
        log.debug("The size of the ByteBufferMessageSet returned is : " + messageBuffer.sizeInBytes());
        int skipped = 0;
        totalFetchTime += lastFetchTime;
        messageIter = messageBuffer.iterator();
        //boolean flag = false;
        Iterator<MessageAndOffset> messageIter2 = messageBuffer
            .iterator();
        MessageAndOffset message = null;
        while (messageIter2.hasNext()) {
          message = messageIter2.next();
          if (message.offset() < currentOffset) {
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
   * @param offset message offset to start.
   * @param callback callback to handle the messages fetched.
   * @return number of messages fetched.
   */
  public int fetchMessages(long offset, Callback callback) throws OffsetOutOfRangeException {
    ByteBufferMessageSet messageSet = fetchMessageSet(offset);
    int msgCount = 0;
    for (MessageAndOffset msg : messageSet) {
      ++msgCount;
      callback.handle(msg.offset(), msg.message().payload());
    }
View Full Code Here
TOP

Related Classes of kafka.javaapi.message.ByteBufferMessageSet

  • co.cask.cdap.logging.kafka.KafkaConsumer
  • com.alibaba.jstorm.kafka.MessageConsumer
  • com.alibaba.jstorm.kafka.PartitionConsumer
  • com.linkedin.camus.etl.kafka.common.KafkaReader
  • kafka.bridge.hadoop.KafkaRecordWriter
  • kafka.etl.impl.DataGenerator
  • kafka.etl.KafkaETLContext
  • kafka.examples.SimpleConsumerDemo
  • kafka.perf.producer.Producer
  • org.apache.twill.internal.kafka.client.SimpleKafkaConsumer$ConsumerThread
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.