Package kafka.message

Examples of kafka.message.MessageAndOffset.message()

83
84
85
86
87
88
89
90
91
92
93
    public static boolean verifyMessage(String key, String message, KafkaTestBroker broker, SimpleConsumer simpleConsumer) {
        long lastMessageOffset = KafkaUtils.getOffset(simpleConsumer, TestUtils.TOPIC, 0, OffsetRequest.LatestTime()) - 1;
        ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(TestUtils.getKafkaConfig(broker), simpleConsumer,
                new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), lastMessageOffset);
        MessageAndOffset messageAndOffset = messageAndOffsets.iterator().next();
        Message kafkaMessage = messageAndOffset.message();
        ByteBuffer messageKeyBuffer = kafkaMessage.key();
        String keyString = null;
        String messageString = new String(Utils.toByteArray(kafkaMessage.payload()));
        if (messageKeyBuffer != null) {
            keyString = new String(Utils.toByteArray(messageKeyBuffer));
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
    private boolean verifyMessage(String key, String message) {
        long lastMessageOffset = KafkaUtils.getOffset(simpleConsumer, kafkaConfig.topic, 0, OffsetRequest.LatestTime()) - 1;
        ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(kafkaConfig, simpleConsumer,
                new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), lastMessageOffset);
        MessageAndOffset messageAndOffset = messageAndOffsets.iterator().next();
        Message kafkaMessage = messageAndOffset.message();
        ByteBuffer messageKeyBuffer = kafkaMessage.key();
        String keyString = null;
        String messageString = new String(Utils.toByteArray(kafkaMessage.payload()));
        if (messageKeyBuffer != null) {
            keyString = new String(Utils.toByteArray(messageKeyBuffer));
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
    private boolean verifyMessage(String key, String message) {
        long lastMessageOffset = KafkaUtils.getOffset(simpleConsumer, kafkaConfig.topic, 0, OffsetRequest.LatestTime()) - 1;
        ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(kafkaConfig, simpleConsumer,
                new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), lastMessageOffset);
        MessageAndOffset messageAndOffset = messageAndOffsets.iterator().next();
        Message kafkaMessage = messageAndOffset.message();
        ByteBuffer messageKeyBuffer = kafkaMessage.key();
        String keyString = null;
        String messageString = new String(Utils.toByteArray(kafkaMessage.payload()));
        if (messageKeyBuffer != null) {
            keyString = new String(Utils.toByteArray(messageKeyBuffer));
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
            throw new RuntimeException("Error fetching offset data. Reason: " +
                    response.errorCode(topicPartition.getTopic(), topicPartition.getPartition()));
        }
        MessageAndOffset messageAndOffset = response.messageSet(
                topicPartition.getTopic(), topicPartition.getPartition()).iterator().next();
        ByteBuffer payload = messageAndOffset.message().payload();
        byte[] payloadBytes = new byte[payload.limit()];
        payload.get(payloadBytes);
        return new Message(topicPartition.getTopic(), topicPartition.getPartition(),
                messageAndOffset.offset(), payloadBytes);
    }
View Full Code Here
190
191
192
193
194
195
196
197
198
199
200
   
    protected boolean get(KafkaETLKey key, BytesWritable value) throws IOException {
        if (_messageIt != null && _messageIt.hasNext()) {
            MessageAndOffset msgAndOffset = _messageIt.next();
           
            ByteBuffer buf = msgAndOffset.message().payload();
            int origSize = buf.remaining();
            byte[] bytes = new byte[origSize];
            buf.get(bytes, buf.position(), origSize);
            value.set(bytes, 0, origSize);
           
View Full Code Here
196
197
198
199
200
201
202
203
204
205
206
            int origSize = buf.remaining();
            byte[] bytes = new byte[origSize];
            buf.get(bytes, buf.position(), origSize);
            value.set(bytes, 0, origSize);
           
            key.set(_index, _offset, msgAndOffset.message().checksum());
           
            _offset = msgAndOffset.offset();  //increase offset
            _count ++;  //increase count
           
            return true;
View Full Code Here
466
467
468
469
470
471
472
473
474
475
476
                        msgOffset, offset.get(), topicPart);
              continue;
            }

            offset.set(message.nextOffset());
            fetchedMessage.setPayload(message.message().payload());
            fetchedMessage.setNextOffset(offset.get());

            return fetchedMessage;
          }
          return endOfData();
View Full Code Here
194
195
196
197
198
199
200
201
202
203
204
   
    protected boolean get(KafkaETLKey key, BytesWritable value) throws IOException {
        if (_messageIt != null && _messageIt.hasNext()) {
            MessageAndOffset messageAndOffset = _messageIt.next();
           
            ByteBuffer buf = messageAndOffset.message().buffer();
            int origSize = buf.remaining();
            byte[] bytes = new byte[origSize];
          buf.get(bytes, buf.position(), origSize);
            value.set(bytes, 0, origSize);
           
View Full Code Here
200
201
202
203
204
205
206
207
208
209
210
            int origSize = buf.remaining();
            byte[] bytes = new byte[origSize];
          buf.get(bytes, buf.position(), origSize);
            value.set(bytes, 0, origSize);
           
            key.set(_index, _offset, messageAndOffset.message().checksum());
           
            _offset = messageAndOffset.nextOffset();  //increase offset
            _count ++;  //increase count
           
            return true;
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
   */
  public boolean getNext(EtlKey key, BytesWritable payload ,BytesWritable pKey) throws IOException {
    if (hasNext()) {

      MessageAndOffset msgAndOffset = messageIter.next();
      Message message = msgAndOffset.message();

      ByteBuffer buf = message.payload();
      int origSize = buf.remaining();
      byte[] bytes = new byte[origSize];
      buf.get(bytes, buf.position(), origSize);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.