+ " num_topics num_partitions num_records message_size");
System.exit(1);
}
Random rnd = new Random();
int num_topics = Integer.parseInt(args[0]);
SecorConfig config = SecorConfig.load();
String zkConfig = config.getZookeeperQuorum()
+ config.getKafkaZookeeperPath();
// create topics list
String perfTopicPrefix = config.getPerfTestTopicPrefix();
List<String> topics = Lists.newLinkedList();
for (int i = 0; i < num_topics; i++) {
topics.add(perfTopicPrefix + rnd.nextInt(9999));
}
int num_partitions = Integer.parseInt(args[1]);
// createTopics
createTopics(topics, num_partitions, zkConfig);
int numRecords = Integer.parseInt(args[2]);
Properties props = new Properties();
props.put("metadata.broker.list", "localhost:9092");
props.put("serializer.class", "kafka.serializer.StringEncoder");
props.put("request.required.acks", "1");
props.put("producer.type", "async");
ProducerConfig producerConfig = new ProducerConfig(props);
Producer<String, String> producer = new Producer<String, String>(
producerConfig);
long size = 0;
int message_size = Integer.parseInt(args[3]);
// produce messages
for (String topic : topics) {
for (long nEvents = 0; nEvents < numRecords; nEvents++) {
String ip = String.valueOf(nEvents % num_partitions);
byte[] payload = new byte[message_size];
Arrays.fill(payload, (byte) 1);
String msg = new String(payload, "UTF-8");
size += msg.length();
KeyedMessage<String, String> data = new KeyedMessage<String, String>(
topic, ip, msg);
producer.send(data);
}
}
producer.close();
RateLimitUtil.configure(config);
Map<TopicPartition, Long> lastOffsets = getTopicMetadata(topics,
num_partitions, config);
OstrichAdminService ostrichService = new OstrichAdminService(
config.getOstrichPort());
ostrichService.start();
FileUtil.configure(config);
LogFileDeleter logFileDeleter = new LogFileDeleter(config);
logFileDeleter.deleteOldLogs();
Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread thread, Throwable exception) {
exception.printStackTrace();
System.out.println("Thread " + thread + " failed:"
+ exception.getMessage());
System.exit(1);
}
};
System.out.println("starting " + config.getConsumerThreads()
+ " consumer threads");
System.out.println("Rate limit:" + config.getMessagesPerSecond());
LinkedList<Consumer> consumers = new LinkedList<Consumer>();
long startMillis = System.currentTimeMillis();
for (int i = 0; i < config.getConsumerThreads(); ++i) {
Consumer consumer = new Consumer(config);
consumer.setUncaughtExceptionHandler(handler);
consumers.add(consumer);
consumer.start();
}