Examples of org.apache.kafka.clients.producer.ProducerRecord

Package org.apache.kafka.clients.producer

Examples of org.apache.kafka.clients.producer.ProducerRecord

org.apache.kafka.clients.producer.ProducerRecord
A key/value pair to be sent to Kafka. This consists of a topic name to which the record is being sent, an optional partition number, and an optional key and value.
If a valid partition number is specified that partition will be used when sending the record. If no partition is specified but a key is present a partition will be chosen using a hash of the key. If neither key nor partition is present a partition will be assigned in a round-robin fashion.

    }


    @Override
    protected void innerProcess(CrawlURI curi) throws InterruptedException {
        byte[] message = buildMessage(curi);
        ProducerRecord producerRecord = new ProducerRecord(getTopic(), message);
        kafkaProducer().send(producerRecord, new KafkaResultCallback(curi));
    }

View Full Code Here

        KafkaProducer producer = new KafkaProducer(props);


        /* setup perf test */
        byte[] payload = new byte[recordSize];
        Arrays.fill(payload, (byte) 1);
        ProducerRecord record = new ProducerRecord(topicName, payload);
        long sleepTime = NS_PER_SEC / throughput;
        long sleepDeficitNs = 0;
        Stats stats = new Stats(numRecords, 5000);
        for (int i = 0; i < numRecords; i++) {
            long sendStart = System.currentTimeMillis();

View Full Code Here


    @Test
    public void testUserSuppliedPartitioning() {
        assertEquals("If the user supplies a partition we should use it.",
                     0,
                     partitioner.partition(new ProducerRecord("test", 0, key, value), cluster));
    }

View Full Code Here

                     partitioner.partition(new ProducerRecord("test", 0, key, value), cluster));
    }


    @Test
    public void testKeyPartitionIsStable() {
        int partition = partitioner.partition(new ProducerRecord("test", key, value), cluster);
        assertEquals("Same key should yield same partition",
                     partition,
                     partitioner.partition(new ProducerRecord("test", key, "value2".getBytes()), cluster));
    }

View Full Code Here

                     partitioner.partition(new ProducerRecord("test", key, "value2".getBytes()), cluster));
    }


    @Test
    public void testRoundRobinIsStable() {
        int startPart = partitioner.partition(new ProducerRecord("test", value), cluster);
        for (int i = 1; i <= 100; i++) {
            int partition = partitioner.partition(new ProducerRecord("test", value), cluster);
            assertEquals("Should yield a different partition each call with round-robin partitioner",
                partition, (startPart + i) % 2);
      }
    }

View Full Code Here

    }


    @Test
    public void testRoundRobinWithDownNode() {
        for (int i = 0; i < partitions.size(); i++) {
            int part = partitioner.partition(new ProducerRecord("test", value), cluster);
            assertTrue("We should never choose a leader-less node in round robin", part >= 0 && part < 2);
        }
    }

View Full Code Here

    private String topic = "topic";


    @Test
    public void testAutoCompleteMock() throws Exception {
        MockProducer producer = new MockProducer(true);
        ProducerRecord record = new ProducerRecord(topic, "key".getBytes(), "value".getBytes());
        Future<RecordMetadata> metadata = producer.send(record);
        assertTrue("Send should be immediately complete", metadata.isDone());
        assertFalse("Send should be successful", isError(metadata));
        assertEquals("Offset should be 0", 0, metadata.get().offset());
        assertEquals(topic, metadata.get().topic());

View Full Code Here

    }


    @Test
    public void testManualCompletion() throws Exception {
        MockProducer producer = new MockProducer(false);
        ProducerRecord record1 = new ProducerRecord("topic", "key1".getBytes(), "value1".getBytes());
        ProducerRecord record2 = new ProducerRecord("topic", "key2".getBytes(), "value2".getBytes());
        Future<RecordMetadata> md1 = producer.send(record1);
        assertFalse("Send shouldn't have completed", md1.isDone());
        Future<RecordMetadata> md2 = producer.send(record2);
        assertFalse("Send shouldn't have completed", md2.isDone());
        assertTrue("Complete the first request", producer.completeNext());

View Full Code Here


                    // calculate the kafka partition, with backward compatibility with old kafka producer
                    int numPartitions = producer.partitionsFor(topic).size();
                    int partition = partitioner.partition(m.getKey(), numPartitions);


                    ProducerRecord r = new ProducerRecord( topic,
                                                           partition,
                                                           null, // don't store the key
                                                           m.getPayload() );
                    log.trace( "Will send message to Kafka" );
                    long startTimeMs = System.currentTimeMillis();

View Full Code Here

TOP

Related Classes of org.apache.kafka.clients.producer.ProducerRecord

com.netflix.suro.sink.kafka.KafkaSinkV2

org.apache.kafka.clients.producer.MockProducerTest

org.apache.kafka.clients.producer.PartitionerTest

org.apache.kafka.clients.tools.ProducerPerformance

org.archive.modules.postprocessor.KafkaCrawlLogFeed

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.