/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
package org.apache.bookkeeper.benchmark;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.client.AsyncCallback.AddCallback;
import org.apache.bookkeeper.client.BookKeeper.DigestType;
import org.apache.bookkeeper.conf.ClientConfiguration;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This is a simple test program to compare the performance of writing to
* BookKeeper and to the local file system.
*
*/
public class TestClient {
private static final Logger LOG = LoggerFactory.getLogger(TestClient.class);
/**
* First says if entries should be written to BookKeeper (0) or to the local
* disk (1). Second parameter is an integer defining the length of a ledger entry.
* Third parameter is the number of writes.
*
* @param args
*/
public static void main(String[] args) throws ParseException {
Options options = new Options();
options.addOption("length", true, "Length of packets being written. Default 1024");
options.addOption("target", true, "Target medium to write to. Options are bk, fs & hdfs. Default fs");
options.addOption("runfor", true, "Number of seconds to run for. Default 60");
options.addOption("path", true, "Path to write to. fs & hdfs only. Default /foobar");
options.addOption("zkservers", true, "ZooKeeper servers, comma separated. bk only. Default localhost:2181.");
options.addOption("bkensemble", true, "BookKeeper ledger ensemble size. bk only. Default 3");
options.addOption("bkquorum", true, "BookKeeper ledger quorum size. bk only. Default 2");
options.addOption("bkthrottle", true, "BookKeeper throttle size. bk only. Default 10000");
options.addOption("sync", false, "Use synchronous writes with BookKeeper. bk only.");
options.addOption("numconcurrent", true, "Number of concurrently clients. Default 1");
options.addOption("timeout", true, "Number of seconds after which to give up");
options.addOption("help", false, "This message");
CommandLineParser parser = new PosixParser();
CommandLine cmd = parser.parse(options, args);
if (cmd.hasOption("help")) {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("TestClient <options>", options);
System.exit(-1);
}
int length = Integer.valueOf(cmd.getOptionValue("length", "1024"));
String target = cmd.getOptionValue("target", "fs");
long runfor = Long.valueOf(cmd.getOptionValue("runfor", "60")) * 1000;
StringBuilder sb = new StringBuilder();
while(length-- > 0) {
sb.append('a');
}
Timer timeouter = new Timer();
if (cmd.hasOption("timeout")) {
final long timeout = Long.valueOf(cmd.getOptionValue("timeout", "360")) * 1000;
timeouter.schedule(new TimerTask() {
public void run() {
System.err.println("Timing out benchmark after " + timeout + "ms");
System.exit(-1);
}
}, timeout);
}
BookKeeper bkc = null;
try {
int numFiles = Integer.valueOf(cmd.getOptionValue("numconcurrent", "1"));
int numThreads = Math.min(numFiles, 1000);
byte[] data = sb.toString().getBytes();
long runid = System.currentTimeMillis();
List<Callable<Long>> clients = new ArrayList<Callable<Long>>();
if (target.equals("bk")) {
String zkservers = cmd.getOptionValue("zkservers", "localhost:2181");
int bkensemble = Integer.valueOf(cmd.getOptionValue("bkensemble", "3"));
int bkquorum = Integer.valueOf(cmd.getOptionValue("bkquorum", "2"));
int bkthrottle = Integer.valueOf(cmd.getOptionValue("bkthrottle", "10000"));
ClientConfiguration conf = new ClientConfiguration();
conf.setThrottleValue(bkthrottle);
conf.setZkServers(zkservers);
bkc = new BookKeeper(conf);
List<LedgerHandle> handles = new ArrayList<LedgerHandle>();
for (int i = 0; i < numFiles; i++) {
handles.add(bkc.createLedger(bkensemble, bkquorum, DigestType.CRC32, new byte[] {'a', 'b'}));
}
for (int i = 0; i < numFiles; i++) {
clients.add(new BKClient(handles, data, runfor, cmd.hasOption("sync")));
}
} else if (target.equals("hdfs")) {
FileSystem fs = FileSystem.get(new Configuration());
LOG.info("Default replication for HDFS: {}", fs.getDefaultReplication());
List<FSDataOutputStream> streams = new ArrayList<FSDataOutputStream>();
for (int i = 0; i < numFiles; i++) {
String path = cmd.getOptionValue("path", "/foobar");
streams.add(fs.create(new Path(path + runid + "_" + i)));
}
for (int i = 0; i < numThreads; i++) {
clients.add(new HDFSClient(streams, data, runfor));
}
} else if (target.equals("fs")) {
List<FileOutputStream> streams = new ArrayList<FileOutputStream>();
for (int i = 0; i < numFiles; i++) {
String path = cmd.getOptionValue("path", "/foobar " + i);
streams.add(new FileOutputStream(path + runid + "_" + i));
}
for (int i = 0; i < numThreads; i++) {
clients.add(new FileClient(streams, data, runfor));
}
} else {
LOG.error("Unknown option: " + target);
throw new IllegalArgumentException("Unknown target " + target);
}
ExecutorService executor = Executors.newFixedThreadPool(numThreads);
long start = System.currentTimeMillis();
List<Future<Long>> results = executor.invokeAll(clients,
10, TimeUnit.MINUTES);
long end = System.currentTimeMillis();
long count = 0;
for (Future<Long> r : results) {
if (!r.isDone()) {
LOG.warn("Job didn't complete");
System.exit(2);
}
long c = r.get();
if (c == 0) {
LOG.warn("Task didn't complete");
}
count += c;
}
long time = end-start;
LOG.info("Finished processing writes (ms): {} TPT: {} op/s",
time, count/((double)time/1000));
executor.shutdown();
} catch (ExecutionException ee) {
LOG.error("Exception in worker", ee);
} catch (KeeperException ke) {
LOG.error("Error accessing zookeeper", ke);
} catch (BKException e) {
LOG.error("Error accessing bookkeeper", e);
} catch (IOException ioe) {
LOG.error("I/O exception during benchmark", ioe);
} catch (InterruptedException ie) {
LOG.error("Benchmark interrupted", ie);
} finally {
if (bkc != null) {
try {
bkc.close();
} catch (BKException bke) {
LOG.error("Error closing bookkeeper client", bke);
} catch (InterruptedException ie) {
LOG.warn("Interrupted closing bookkeeper client", ie);
}
}
}
timeouter.cancel();
}
static class HDFSClient implements Callable<Long> {
final List<FSDataOutputStream> streams;
final byte[] data;
final long time;
final Random r;
HDFSClient(List<FSDataOutputStream> streams, byte[] data, long time) {
this.streams = streams;
this.data = data;
this.time = time;
this.r = new Random(System.identityHashCode(this));
}
public Long call() {
try {
long count = 0;
long start = System.currentTimeMillis();
long stopat = start + time;
while(System.currentTimeMillis() < stopat) {
FSDataOutputStream stream = streams.get(r.nextInt(streams.size()));
synchronized(stream) {
stream.write(data);
stream.flush();
stream.hflush();
}
count++;
}
long time = (System.currentTimeMillis() - start);
LOG.info("Worker finished processing writes (ms): {} TPT: {} op/s",
time, count/((double)time/1000));
return count;
} catch(IOException ioe) {
LOG.error("Exception in worker thread", ioe);
return 0L;
}
}
}
static class FileClient implements Callable<Long> {
final List<FileOutputStream> streams;
final byte[] data;
final long time;
final Random r;
FileClient(List<FileOutputStream> streams, byte[] data, long time) {
this.streams = streams;
this.data = data;
this.time = time;
this.r = new Random(System.identityHashCode(this));
}
public Long call() {
try {
long count = 0;
long start = System.currentTimeMillis();
long stopat = start + time;
while(System.currentTimeMillis() < stopat) {
FileOutputStream stream = streams.get(r.nextInt(streams.size()));
synchronized(stream) {
stream.write(data);
stream.flush();
stream.getChannel().force(false);
}
count++;
}
long time = (System.currentTimeMillis() - start);
LOG.info("Worker finished processing writes (ms): {} TPT: {} op/s", time, count/((double)time/1000));
return count;
} catch(IOException ioe) {
LOG.error("Exception in worker thread", ioe);
return 0L;
}
}
}
static class BKClient implements Callable<Long>, AddCallback {
final List<LedgerHandle> handles;
final byte[] data;
final long time;
final Random r;
final boolean sync;
final AtomicLong success = new AtomicLong(0);
final AtomicLong outstanding = new AtomicLong(0);
BKClient(List<LedgerHandle> handles, byte[] data, long time, boolean sync) {
this.handles = handles;
this.data = data;
this.time = time;
this.r = new Random(System.identityHashCode(this));
this.sync = sync;
}
public Long call() {
try {
long start = System.currentTimeMillis();
long stopat = start + time;
while(System.currentTimeMillis() < stopat) {
LedgerHandle lh = handles.get(r.nextInt(handles.size()));
if (sync) {
lh.addEntry(data);
success.incrementAndGet();
} else {
lh.asyncAddEntry(data, this, null);
outstanding.incrementAndGet();
}
}
int ticks = 10; // don't wait for more than 10 seconds
while (outstanding.get() > 0 && ticks-- > 0) {
Thread.sleep(10);
}
long time = (System.currentTimeMillis() - start);
LOG.info("Worker finished processing writes (ms): {} TPT: {} op/s",
time, success.get()/((double)time/1000));
return success.get();
} catch (BKException e) {
LOG.error("Exception in worker thread", e);
return 0L;
} catch (InterruptedException ie) {
LOG.error("Exception in worker thread", ie);
return 0L;
}
}
@Override
public void addComplete(int rc, LedgerHandle lh, long entryId, Object ctx) {
if (rc == BKException.Code.OK) {
success.incrementAndGet();
}
outstanding.decrementAndGet();
}
}
}