Package kafka.message

Examples of kafka.message.Message

83
84
85
86
87
88
89
90
91
92
93
94
95
96
    public static boolean verifyMessage(String key, String message, KafkaTestBroker broker, SimpleConsumer simpleConsumer) {
        long lastMessageOffset = KafkaUtils.getOffset(simpleConsumer, TestUtils.TOPIC, 0, OffsetRequest.LatestTime()) - 1;
        ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(TestUtils.getKafkaConfig(broker), simpleConsumer,
                new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), lastMessageOffset);
        MessageAndOffset messageAndOffset = messageAndOffsets.iterator().next();
        Message kafkaMessage = messageAndOffset.message();
        ByteBuffer messageKeyBuffer = kafkaMessage.key();
        String keyString = null;
        String messageString = new String(Utils.toByteArray(kafkaMessage.payload()));
        if (messageKeyBuffer != null) {
            keyString = new String(Utils.toByteArray(messageKeyBuffer));
        }
        assertEquals(key, keyString);
        assertEquals(message, messageString);
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
161
162
163
    private boolean verifyMessage(String key, String message) {
        long lastMessageOffset = KafkaUtils.getOffset(simpleConsumer, kafkaConfig.topic, 0, OffsetRequest.LatestTime()) - 1;
        ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(kafkaConfig, simpleConsumer,
                new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), lastMessageOffset);
        MessageAndOffset messageAndOffset = messageAndOffsets.iterator().next();
        Message kafkaMessage = messageAndOffset.message();
        ByteBuffer messageKeyBuffer = kafkaMessage.key();
        String keyString = null;
        String messageString = new String(Utils.toByteArray(kafkaMessage.payload()));
        if (messageKeyBuffer != null) {
            keyString = new String(Utils.toByteArray(messageKeyBuffer));
        }
        assertEquals(key, keyString);
        assertEquals(message, messageString);
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
144
145
146
    private boolean verifyMessage(String key, String message) {
        long lastMessageOffset = KafkaUtils.getOffset(simpleConsumer, kafkaConfig.topic, 0, OffsetRequest.LatestTime()) - 1;
        ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(kafkaConfig, simpleConsumer,
                new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), lastMessageOffset);
        MessageAndOffset messageAndOffset = messageAndOffsets.iterator().next();
        Message kafkaMessage = messageAndOffset.message();
        ByteBuffer messageKeyBuffer = kafkaMessage.key();
        String keyString = null;
        String messageString = new String(Utils.toByteArray(kafkaMessage.payload()));
        if (messageKeyBuffer != null) {
            keyString = new String(Utils.toByteArray(messageKeyBuffer));
        }
        assertEquals(key, keyString);
        assertEquals(message, messageString);
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
    @Test
    public void testCreatingKafkaExchangeSetsHeaders() throws URISyntaxException {
        KafkaEndpoint endpoint = new KafkaEndpoint("kafka:localhost", "localhost", new KafkaComponent());

        Message message = new Message("mymessage".getBytes(), "somekey".getBytes());
        DefaultDecoder decoder = new DefaultDecoder(null);
        MessageAndMetadata<byte[], byte[]> mm =
                new MessageAndMetadata<byte[], byte[]>("topic", 4, message, 56, decoder, decoder);

        Exchange exchange = endpoint.createKafkaExchange(mm);
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
      }

      @Override
      public InputRow nextRow()
      {
        final Message message = iter.next().message();

        if (message == null) {
          return null;
        }

        return parseMessage(message);
      }

      public InputRow parseMessage(Message message)
      {
        return theParser.parse(message.payload());
      }

      @Override
      public Runnable commit()
      {
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
  }

  @Override
  public void write(NullWritable key, BytesWritable value) throws IOException, InterruptedException
  {
    Message msg = new Message(value.getBytes());
    msgList.add(msg);
    totalSize += msg.size();

    if (totalSize > queueSize)
      sendMsgList();
  }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
       
        
        byte[] bytes = KafkaETLUtils.getBytes(val);
       
        //check the checksum of message
        Message message = new Message(bytes);
        long checksum = key.getChecksum();
        if (checksum != message.checksum())
            throw new IOException ("Invalid message checksum "
                                            + message.checksum() + ". Expected " + key + ".");
        Text data = getData (message);
        _count ++;
          
        collector.collect(new LongWritable (_count), data);
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
    List<Message> list = new ArrayList<Message>();
    for (int i = 0; i < _count; i++) {
      Long timestamp = RANDOM.nextLong();
      if (timestamp < 0) timestamp = -timestamp;
      byte[] bytes = timestamp.toString().getBytes("UTF8");
      Message message = new Message(bytes);
      list.add(message);
    }
    // send events
    System.out.println(" send " + list.size() + " " + _topic + " count events to " + _uri);
    _producer.send(_topic, new ByteBufferMessageSet(kafka.message.NoCompressionCodec$.MODULE$, list));
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
        // if (!consumerIterator.hasNext()) {
        // Utils.sleep(50);
        // return;
        // }

        Message msg = consumerIterator.next();

        if (msg != null && msg.isValid()) {

            // get the trimmed byte array given the size of the message payload
            ByteBuffer byteBuffer = msg.payload();
            byte[] bytes = new byte[byteBuffer.remaining()];
            byteBuffer.get(bytes);

            emit(bytes);
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
    while(true){
      String line = reader.readLine();
      if (line==null) break;
      count++;
      System.out.println(count+" msgs pushed.");
      Message m = new Message(line.getBytes(DefaultJsonDataSourceFilter.UTF8));
        KeyedMessage<String,Message> msg = new KeyedMessage<String,Message>(topic,m);
      msgList.add(msg);

      if (msgList.size()>batchSize){
        kafkaProducer.send(msgList);
View Full Code Here
TOP

Related Classes of kafka.message.Message

  • co.gridport.kafka.hadoop.KafkaInputRecordReader
  • com.linkedin.camus.etl.kafka.common.KafkaReader
  • com.linkedin.camus.etl.kafka.mapred.EtlRecordReader
  • com.senseidb.gateway.perf.PrimeKafkaUtil
  • io.druid.firehose.kafka.KafkaSevenFirehoseFactory
  • kafka.bridge.hadoop.KafkaRecordWriter
  • kafka.etl.impl.DataGenerator
  • kafka.etl.impl.SimpleKafkaETLMapper
  • kafka.etl.KafkaETLMapper
  • kafka.etl.KafkaETLReducer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.