*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* Script used evaluating HBase performance and scalability.  Runs a HBase

* client that steps through one of a set of hardcoded tests or 'experiments'

* (e.g. a random reads test, a random writes test, etc.). Pass on the

* command-line which test to run and how many clients are participating in

* this experiment. Run {@code PerformanceEvaluation --help} to obtain usage.

*

* <p>This class sets up and runs the evaluation programs described in

* Section 7, <i>Performance Evaluation</i>, of the <a

* href="http://labs.google.com/papers/bigtable.html">Bigtable</a>

* paper, pages 8-10.

*

* <p>By default, runs as a mapreduce job where each mapper runs a single test

* client. Can also run as a non-mapreduce, multithreaded application by

* specifying {@code --nomapred}. Each client does about 1GB of data, unless

* specified otherwise.

*/

  protected static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName());

  private static final ObjectMapper MAPPER = new ObjectMapper();

  static {

    MAPPER.configure(SORT_PROPERTIES_ALPHABETICALLY, true);

  }

  public static final String TABLE_NAME = "TestTable";

  public static final byte[] FAMILY_NAME = Bytes.toBytes("info");

  public static final byte[] QUALIFIER_NAME = Bytes.toBytes("data");

  public static final int DEFAULT_VALUE_LENGTH = 1000;

  public static final int ROW_LENGTH = 26;

  private static final int ONE_GB = 1024 * 1024 * 1000;

  private static final int DEFAULT_ROWS_PER_GB = ONE_GB / DEFAULT_VALUE_LENGTH;

  // TODO : should we make this configurable

  private static final int TAG_LENGTH = 256;

  private static final DecimalFormat FMT = new DecimalFormat("0.##");

  private static final MathContext CXT = MathContext.DECIMAL64;

  private static final BigDecimal MS_PER_SEC = BigDecimal.valueOf(1000);

  private static final BigDecimal BYTES_PER_MB = BigDecimal.valueOf(1024 * 1024);

  private static final TestOptions DEFAULT_OPTS = new TestOptions();

  private static Map<String, CmdDescriptor> COMMANDS = new TreeMap<String, CmdDescriptor>();

  private static final Path PERF_EVAL_DIR = new Path("performance_evaluation");

  static {

    addCommandDescriptor(RandomReadTest.class, "randomRead",

      "Run random read test");

    addCommandDescriptor(RandomSeekScanTest.class, "randomSeekScan",

      "Run random seek and scan 100 test");

    addCommandDescriptor(RandomScanWithRange10Test.class, "scanRange10",

      "Run random seek scan with both start and stop row (max 10 rows)");

    addCommandDescriptor(RandomScanWithRange100Test.class, "scanRange100",

      "Run random seek scan with both start and stop row (max 100 rows)");

    addCommandDescriptor(RandomScanWithRange1000Test.class, "scanRange1000",

      "Run random seek scan with both start and stop row (max 1000 rows)");

    addCommandDescriptor(RandomScanWithRange10000Test.class, "scanRange10000",

      "Run random seek scan with both start and stop row (max 10000 rows)");

    addCommandDescriptor(RandomWriteTest.class, "randomWrite",

      "Run random write test");

    addCommandDescriptor(SequentialReadTest.class, "sequentialRead",

      "Run sequential read test");

    addCommandDescriptor(SequentialWriteTest.class, "sequentialWrite",

      "Run sequential write test");

    addCommandDescriptor(ScanTest.class, "scan",

      "Run scan test (read every row)");

    addCommandDescriptor(FilteredScanTest.class, "filterScan",

      "Run scan test using a filter to find a specific row based on it's value " +

        "(make sure to use --rows=20)");

  }

  /**

   * Enum for map metrics.  Keep it out here rather than inside in the Map

   * inner-class so we can find associated properties.

   */

  protected static enum Counter {

    /** elapsed time */

    ELAPSED_TIME,

    /** number of rows */

    ROWS

  }

  protected static class RunResult implements Comparable<RunResult> {

    public RunResult(long duration, Histogram hist) {

      this.duration = duration;

      this.hist = hist;

    }

    public final long duration;

    public final Histogram hist;

    @Override

    public String toString() {

      return Long.toString(duration);

    }

    @Override public int compareTo(RunResult o) {

      return Long.compare(this.duration, o.duration);

    }

  }

  /**

   * Constructor

   * @param conf Configuration object

   */

  public PerformanceEvaluation(final Configuration conf) {

    super(conf);

  }

  protected static void addCommandDescriptor(Class<? extends Test> cmdClass,

      String name, String description) {

    CmdDescriptor cmdDescriptor = new CmdDescriptor(cmdClass, name, description);

    COMMANDS.put(name, cmdDescriptor);

  }

  /**

   * Implementations can have their status set.

   */

  interface Status {

    /**

     * Sets status

     * @param msg status message

     * @throws IOException

     */

    void setStatus(final String msg) throws IOException;

  }

  /**

   * MapReduce job that runs a performance evaluation client in each map task.

   */

  public static class EvaluationMapTask

      extends Mapper<LongWritable, Text, LongWritable, LongWritable> {

    /** configuration parameter name that contains the command */

    public final static String CMD_KEY = "EvaluationMapTask.command";

    /** configuration parameter name that contains the PE impl */

    public static final String PE_KEY = "EvaluationMapTask.performanceEvalImpl";

    private Class<? extends Test> cmd;

    @Override

    protected void setup(Context context) throws IOException, InterruptedException {

      this.cmd = forName(context.getConfiguration().get(CMD_KEY), Test.class);

      // this is required so that extensions of PE are instantiated within the

      // map reduce task...

      Class<? extends PerformanceEvaluation> peClass =

          forName(context.getConfiguration().get(PE_KEY), PerformanceEvaluation.class);

      try {

        peClass.getConstructor(Configuration.class).newInstance(context.getConfiguration());

      } catch (Exception e) {

        throw new IllegalStateException("Could not instantiate PE instance", e);

      }

    }

    private <Type> Class<? extends Type> forName(String className, Class<Type> type) {

      try {

        return Class.forName(className).asSubclass(type);

      } catch (ClassNotFoundException e) {

        throw new IllegalStateException("Could not find class for name: " + className, e);

      }

    }

    @Override

    protected void map(LongWritable key, Text value, final Context context)

           throws IOException, InterruptedException {

      Status status = new Status() {

        @Override

        public void setStatus(String msg) {

           context.setStatus(msg);

        }

      };

      ObjectMapper mapper = new ObjectMapper();

      TestOptions opts = mapper.readValue(value.toString(), TestOptions.class);

      Configuration conf = HBaseConfiguration.create(context.getConfiguration());

      final Connection con = ConnectionFactory.createConnection(conf);

      // Evaluation task

      RunResult result = PerformanceEvaluation.runOneClient(this.cmd, conf, con, opts, status);

      // Collect how much time the thing took. Report as map output and

      // to the ELAPSED_TIME counter.

      context.getCounter(Counter.ELAPSED_TIME).increment(result.duration);

      context.getCounter(Counter.ROWS).increment(opts.perClientRunRows);

      context.write(new LongWritable(opts.startRow), new LongWritable(result.duration));

      context.progress();

    }

  }

  /*

   * If table does not already exist, create. Also create a table when

   * {@code opts.presplitRegions} is specified or when the existing table's

   * region replica count doesn't match {@code opts.replicas}.

   */

  static boolean checkTable(Admin admin, TestOptions opts) throws IOException {

    TableName tableName = TableName.valueOf(opts.tableName);

    boolean needsDelete = false, exists = admin.tableExists(tableName);

    boolean isReadCmd = opts.cmdName.toLowerCase().contains("read")

      || opts.cmdName.toLowerCase().contains("scan");

    if (!exists && isReadCmd) {

      throw new IllegalStateException(

        "Must specify an existing table for read commands. Run a write command first.");

    }

    HTableDescriptor desc =

      exists ? admin.getTableDescriptor(TableName.valueOf(opts.tableName)) : null;

    byte[][] splits = getSplits(opts);

    // recreate the table when user has requested presplit or when existing

    // {RegionSplitPolicy,replica count} does not match requested.

    if ((exists && opts.presplitRegions != DEFAULT_OPTS.presplitRegions)

      || (!isReadCmd && desc != null && desc.getRegionSplitPolicyClassName() != opts.splitPolicy)

      || (!isReadCmd && desc != null && desc.getRegionReplication() != opts.replicas)) {

      needsDelete = true;

      // wait, why did it delete my table?!?

      LOG.debug(Objects.toStringHelper("needsDelete")

        .add("needsDelete", needsDelete)

        .add("isReadCmd", isReadCmd)

        .add("exists", exists)

        .add("desc", desc)

        .add("presplit", opts.presplitRegions)

        .add("splitPolicy", opts.splitPolicy)

        .add("replicas", opts.replicas));

    }

    // remove an existing table

    if (needsDelete) {

      if (admin.isTableEnabled(tableName)) {

        admin.disableTable(tableName);

      }

      admin.deleteTable(tableName);

    }

    // table creation is necessary

    if (!exists || needsDelete) {

      desc = getTableDescriptor(opts);

      if (splits != null) {

        if (LOG.isDebugEnabled()) {

          for (int i = 0; i < splits.length; i++) {

            LOG.debug(" split " + i + ": " + Bytes.toStringBinary(splits[i]));

          }

        }

      }

      admin.createTable(desc, splits);

      LOG.info("Table " + desc + " created");

    }

    return admin.tableExists(tableName);

  }

  /**

   * Create an HTableDescriptor from provided TestOptions.

   */

  protected static HTableDescriptor getTableDescriptor(TestOptions opts) {

    HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(opts.tableName));

    HColumnDescriptor family = new HColumnDescriptor(FAMILY_NAME);

    family.setDataBlockEncoding(opts.blockEncoding);

    family.setCompressionType(opts.compression);

    family.setBloomFilterType(opts.bloomType);

    if (opts.inMemoryCF) {

      family.setInMemory(true);

    }

    desc.addFamily(family);

    if (opts.replicas != DEFAULT_OPTS.replicas) {

      desc.setRegionReplication(opts.replicas);

    }

    if (opts.splitPolicy != DEFAULT_OPTS.splitPolicy) {

      desc.setRegionSplitPolicyClassName(opts.splitPolicy);

    }

    return desc;

  }

  /**

   * generates splits based on total number of rows and specified split regions

   */

  protected static byte[][] getSplits(TestOptions opts) {

    if (opts.presplitRegions == DEFAULT_OPTS.presplitRegions)

      return null;

    int numSplitPoints = opts.presplitRegions - 1;

    byte[][] splits = new byte[numSplitPoints][];

    int jump = opts.totalRows / opts.presplitRegions;

    for (int i = 0; i < numSplitPoints; i++) {

      int rowkey = jump * (1 + i);

      splits[i] = format(rowkey);

    }

    return splits;

  }

  /*

   * Run all clients in this vm each to its own thread.

   */

  static RunResult[] doLocalClients(final TestOptions opts, final Configuration conf)

      throws IOException, InterruptedException {

    final Class<? extends Test> cmd = determineCommandClass(opts.cmdName);

    assert cmd != null;

    Future<RunResult>[] threads = new Future[opts.numClientThreads];

    RunResult[] results = new RunResult[opts.numClientThreads];

    ExecutorService pool = Executors.newFixedThreadPool(opts.numClientThreads,

      new ThreadFactoryBuilder().setNameFormat("TestClient-%s").build());

    final Connection con = ConnectionFactory.createConnection(conf);

    for (int i = 0; i < threads.length; i++) {

      final int index = i;

      threads[i] = pool.submit(new Callable<RunResult>() {

        @Override

        public RunResult call() throws Exception {

          TestOptions threadOpts = new TestOptions(opts);

          if (threadOpts.startRow == 0) threadOpts.startRow = index * threadOpts.perClientRunRows;

          RunResult run = runOneClient(cmd, conf, con, threadOpts, new Status() {

            @Override

            public void setStatus(final String msg) throws IOException {

              LOG.info(msg);

            }

          });

          LOG.info("Finished " + Thread.currentThread().getName() + " in " + run.duration +

            "ms over " + threadOpts.perClientRunRows + " rows");

          return run;

        }

      });

    }

    pool.shutdown();

    for (int i = 0; i < threads.length; i++) {

      try {

        results[i] = threads[i].get();

      } catch (ExecutionException e) {

        throw new IOException(e.getCause());

      }

    }

    final String test = cmd.getSimpleName();

    LOG.info("[" + test + "] Summary of timings (ms): "

             + Arrays.toString(results));

    Arrays.sort(results);

    long total = 0;

    for (RunResult result : results) {

      total += result.duration;

    }

    LOG.info("[" + test + "]"

      + "\tMin: " + results[0] + "ms"

      + "\tMax: " + results[results.length - 1] + "ms"

      + "\tAvg: " + (total / results.length) + "ms");

    con.close();

    return results;

  }

  /*

   * Run a mapreduce job.  Run as many maps as asked-for clients.

   * Before we start up the job, write out an input file with instruction

   * per client regards which row they are to start on.

   * @param cmd Command to run.

   * @throws IOException

   */

  static Job doMapReduce(TestOptions opts, final Configuration conf)

      throws IOException, InterruptedException, ClassNotFoundException {

    final Class<? extends Test> cmd = determineCommandClass(opts.cmdName);

    assert cmd != null;

    Path inputDir = writeInputFile(conf, opts);

    conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());

    conf.set(EvaluationMapTask.PE_KEY, PerformanceEvaluation.class.getName());

    Job job = new Job(conf);

    job.setJarByClass(PerformanceEvaluation.class);

    job.setJobName("HBase Performance Evaluation - " + opts.cmdName);

    job.setInputFormatClass(NLineInputFormat.class);

    NLineInputFormat.setInputPaths(job, inputDir);

    // this is default, but be explicit about it just in case.

    NLineInputFormat.setNumLinesPerSplit(job, 1);

    job.setOutputKeyClass(LongWritable.class);

    job.setOutputValueClass(LongWritable.class);

    job.setMapperClass(EvaluationMapTask.class);

    job.setReducerClass(LongSumReducer.class);

    job.setNumReduceTasks(1);

    job.setOutputFormatClass(TextOutputFormat.class);

    TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs"));

    TableMapReduceUtil.addDependencyJars(job);

    TableMapReduceUtil.addDependencyJars(job.getConfiguration(),

      Histogram.class,     // yammer metrics

      ObjectMapper.class); // jackson-mapper-asl

    TableMapReduceUtil.initCredentials(job);

    job.waitForCompletion(true);

    return job;

  }

  /**

   * Per client, how many tasks will we run?  We divide number of rows by this number and have the

   * client do the resulting count in a map task.

   */

  static int TASKS_PER_CLIENT = 10;

  static String JOB_INPUT_FILENAME = "input.txt";

  /*

   * Write input file of offsets-per-client for the mapreduce job.

   * @param c Configuration

   * @return Directory that contains file written whose name is JOB_INPUT_FILENAME

   * @throws IOException

   */

  static Path writeInputFile(final Configuration c, final TestOptions opts) throws IOException {

    return writeInputFile(c, opts, new Path("."));

  }

  static Path writeInputFile(final Configuration c, final TestOptions opts, final Path basedir)

  throws IOException {

    SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");

    Path jobdir = new Path(new Path(basedir, PERF_EVAL_DIR), formatter.format(new Date()));

    Path inputDir = new Path(jobdir, "inputs");

    FileSystem fs = FileSystem.get(c);

    fs.mkdirs(inputDir);

    Path inputFile = new Path(inputDir, JOB_INPUT_FILENAME);

    PrintStream out = new PrintStream(fs.create(inputFile));

    // Make input random.

    Map<Integer, String> m = new TreeMap<Integer, String>();

    Hash h = MurmurHash.getInstance();

    int perClientRows = (opts.totalRows / opts.numClientThreads);

    try {

      for (int i = 0; i < TASKS_PER_CLIENT; i++) {

        for (int j = 0; j < opts.numClientThreads; j++) {

          TestOptions next = new TestOptions(opts);

          next.startRow = (j * perClientRows) + (i * (perClientRows/10));

          next.perClientRunRows = perClientRows / 10;

          String s = MAPPER.writeValueAsString(next);

          LOG.info("Client=" + j + ", maptask=" + i + ", input=" + s);

          int hash = h.hash(Bytes.toBytes(s));

          m.put(hash, s);

        }

      }

      for (Map.Entry<Integer, String> e: m.entrySet()) {

        out.println(e.getValue());

      }

    } finally {

      out.close();

    }

    return inputDir;

  }

  /**

   * Describes a command.

   */

  static class CmdDescriptor {

    private Class<? extends Test> cmdClass;

    private String name;

    private String description;

    CmdDescriptor(Class<? extends Test> cmdClass, String name, String description) {

      this.cmdClass = cmdClass;

      this.name = name;

      this.description = description;

    }

    public Class<? extends Test> getCmdClass() {

      return cmdClass;

    }

    public String getName() {

      return name;

    }

    public String getDescription() {

      return description;

    }

  }

  /**

   * Wraps up options passed to {@link org.apache.hadoop.hbase.PerformanceEvaluation}.

   * This makes tracking all these arguments a little easier.

   * NOTE: ADDING AN OPTION, you need to add a data member, a getter/setter (to make JSON

   * serialization of this TestOptions class behave), and you need to add to the clone constructor

   * below copying your new option from the 'that' to the 'this'.  Look for 'clone' below.

   */

  static class TestOptions {

    String cmdName = null;

    boolean nomapred = false;

    boolean filterAll = false;

    int startRow = 0;

    float size = 1.0f;

    int perClientRunRows = DEFAULT_ROWS_PER_GB;

    int numClientThreads = 1;

    int totalRows = DEFAULT_ROWS_PER_GB;

    int measureAfter = 0;

    float sampleRate = 1.0f;

    double traceRate = 0.0;

    String tableName = TABLE_NAME;

    boolean flushCommits = true;

    boolean writeToWAL = true;

    boolean autoFlush = false;

    boolean oneCon = false;

    boolean useTags = false;

    int noOfTags = 1;

    boolean reportLatency = false;

    int multiGet = 0;

    int randomSleep = 0;

    boolean inMemoryCF = false;

    int presplitRegions = 0;

    int replicas = HTableDescriptor.DEFAULT_REGION_REPLICATION;

    String splitPolicy = null;

    Compression.Algorithm compression = Compression.Algorithm.NONE;

    BloomType bloomType = BloomType.ROW;

    DataBlockEncoding blockEncoding = DataBlockEncoding.NONE;

    boolean valueRandom = false;

    boolean valueZipf = false;

    int valueSize = DEFAULT_VALUE_LENGTH;

    int period = (this.perClientRunRows / 10) == 0? perClientRunRows: perClientRunRows / 10;

    int cycles = 1;

    public TestOptions() {}

    /**

     * Clone constructor.

     * @param that Object to copy from.

     */

    public TestOptions(TestOptions that) {

      this.cmdName = that.cmdName;

      this.cycles = that.cycles;

      this.nomapred = that.nomapred;

      this.startRow = that.startRow;

      this.size = that.size;

      this.perClientRunRows = that.perClientRunRows;

      this.numClientThreads = that.numClientThreads;

      this.totalRows = that.totalRows;

      this.sampleRate = that.sampleRate;

      this.traceRate = that.traceRate;

      this.tableName = that.tableName;

      this.flushCommits = that.flushCommits;

      this.writeToWAL = that.writeToWAL;

      this.autoFlush = that.autoFlush;

      this.oneCon = that.oneCon;

      this.useTags = that.useTags;

      this.noOfTags = that.noOfTags;

      this.reportLatency = that.reportLatency;

      this.multiGet = that.multiGet;

      this.inMemoryCF = that.inMemoryCF;

      this.presplitRegions = that.presplitRegions;

      this.replicas = that.replicas;

      this.splitPolicy = that.splitPolicy;

      this.compression = that.compression;

      this.blockEncoding = that.blockEncoding;

      this.filterAll = that.filterAll;

      this.bloomType = that.bloomType;

      this.valueRandom = that.valueRandom;

      this.valueZipf = that.valueZipf;

      this.valueSize = that.valueSize;

      this.period = that.period;

      this.randomSleep = that.randomSleep;

      this.measureAfter = that.measureAfter;

    }

    public int getCycles() {

      return this.cycles;

    }

    public void setCycles(final int cycles) {

      this.cycles = cycles;

    }

    public boolean isValueZipf() {

      return valueZipf;

    }

    public void setValueZipf(boolean valueZipf) {

      this.valueZipf = valueZipf;

    }

    public String getCmdName() {

      return cmdName;

    }

    public void setCmdName(String cmdName) {

      this.cmdName = cmdName;

    }

    public int getRandomSleep() {

      return randomSleep;

    }

    public void setRandomSleep(int randomSleep) {

      this.randomSleep = randomSleep;

    }

    public int getReplicas() {

      return replicas;

    }

    public void setReplicas(int replicas) {

      this.replicas = replicas;

    }

    public String getSplitPolicy() {

      return splitPolicy;

    }

    public void setSplitPolicy(String splitPolicy) {

      this.splitPolicy = splitPolicy;

    }

    public void setNomapred(boolean nomapred) {

      this.nomapred = nomapred;

    }

    public void setFilterAll(boolean filterAll) {

      this.filterAll = filterAll;

    }

    public void setStartRow(int startRow) {

      this.startRow = startRow;

    }

    public void setSize(float size) {

      this.size = size;

    }

    public void setPerClientRunRows(int perClientRunRows) {

      this.perClientRunRows = perClientRunRows;

    }

    public void setNumClientThreads(int numClientThreads) {

      this.numClientThreads = numClientThreads;

    }

    public void setTotalRows(int totalRows) {

      this.totalRows = totalRows;

    }

    public void setSampleRate(float sampleRate) {

      this.sampleRate = sampleRate;

    }

    public void setTraceRate(double traceRate) {

      this.traceRate = traceRate;

    }

    public void setTableName(String tableName) {

      this.tableName = tableName;

    }

    public void setFlushCommits(boolean flushCommits) {

      this.flushCommits = flushCommits;

    }

    public void setWriteToWAL(boolean writeToWAL) {

      this.writeToWAL = writeToWAL;

    }

    public void setAutoFlush(boolean autoFlush) {

      this.autoFlush = autoFlush;

    }

    public void setOneCon(boolean oneCon) {

      this.oneCon = oneCon;

    }

    public void setUseTags(boolean useTags) {

      this.useTags = useTags;

    }

    public void setNoOfTags(int noOfTags) {

      this.noOfTags = noOfTags;

    }

    public void setReportLatency(boolean reportLatency) {

      this.reportLatency = reportLatency;

    }

    public void setMultiGet(int multiGet) {

      this.multiGet = multiGet;

    }

    public void setInMemoryCF(boolean inMemoryCF) {

      this.inMemoryCF = inMemoryCF;

    }

    public void setPresplitRegions(int presplitRegions) {

      this.presplitRegions = presplitRegions;

    }

    public void setCompression(Compression.Algorithm compression) {

      this.compression = compression;

    }

    public void setBloomType(BloomType bloomType) {

      this.bloomType = bloomType;

    }

    public void setBlockEncoding(DataBlockEncoding blockEncoding) {

      this.blockEncoding = blockEncoding;

    }

    public void setValueRandom(boolean valueRandom) {

      this.valueRandom = valueRandom;

    }

    public void setValueSize(int valueSize) {

      this.valueSize = valueSize;

    }

    public void setPeriod(int period) {

      this.period = period;

    }

    public boolean isNomapred() {

      return nomapred;

    }

    public boolean isFilterAll() {

      return filterAll;

    }

    public int getStartRow() {

      return startRow;

    }

    public float getSize() {

      return size;

    }

    public int getPerClientRunRows() {

      return perClientRunRows;

    }

    public int getNumClientThreads() {

      return numClientThreads;

    }

    public int getTotalRows() {

      return totalRows;

    }

    public float getSampleRate() {

      return sampleRate;

    }

    public double getTraceRate() {

      return traceRate;

    }

    public String getTableName() {

      return tableName;

    }

    public boolean isFlushCommits() {

      return flushCommits;

    }

    public boolean isWriteToWAL() {

      return writeToWAL;

    }

    public boolean isAutoFlush() {

      return autoFlush;

    }

    public boolean isUseTags() {

      return useTags;

    }

    public int getNoOfTags() {

      return noOfTags;

    }

    public boolean isReportLatency() {

      return reportLatency;

    }

    public int getMultiGet() {

      return multiGet;

    }

    public boolean isInMemoryCF() {

      return inMemoryCF;

    }

    public int getPresplitRegions() {

      return presplitRegions;

    }

    public Compression.Algorithm getCompression() {

      return compression;

    }

    public DataBlockEncoding getBlockEncoding() {

      return blockEncoding;

    }

    public boolean isValueRandom() {

      return valueRandom;

    }

    public int getValueSize() {

      return valueSize;

    }

    public int getPeriod() {

      return period;

    }

    public BloomType getBloomType() {

      return bloomType;

    }

    public boolean isOneCon() {

      return oneCon;

    }

    public int getMeasureAfter() {

      return measureAfter;

    }

    public void setMeasureAfter(int measureAfter) {

      this.measureAfter = measureAfter;

    }

  }

  /*

   * A test.

   * Subclass to particularize what happens per row.

   */

  static abstract class Test {

    // Below is make it so when Tests are all running in the one

    // jvm, that they each have a differently seeded Random.

    private static final Random randomSeed = new Random(System.currentTimeMillis());

    private static long nextRandomSeed() {

      return randomSeed.nextLong();

    }

    private final int everyN;

    protected final Random rand = new Random(nextRandomSeed());

    protected final Configuration conf;

    protected final TestOptions opts;

    private final Status status;

    private final Sampler<?> traceSampler;

    private final SpanReceiverHost receiverHost;

    protected Connection connection;

    protected Table table;

    private String testName;

    private Histogram latency;

    private Histogram valueSize;

    private RandomDistribution.Zipf zipf;

    /**

     * Note that all subclasses of this class must provide a public constructor

     * that has the exact same list of arguments.

     */

    Test(final Connection con, final TestOptions options, final Status status) {

      this.connection = con;

      this.conf = con ==  null? null: this.connection.getConfiguration();

      this.receiverHost = this.conf == null? null: SpanReceiverHost.getInstance(conf);

      this.opts = options;

      this.status = status;

      this.testName = this.getClass().getSimpleName();

      if (options.traceRate >= 1.0) {

        this.traceSampler = Sampler.ALWAYS;

      } else if (options.traceRate > 0.0) {

        this.traceSampler = new ProbabilitySampler(options.traceRate);

      } else {

        this.traceSampler = Sampler.NEVER;

      }

      everyN = (int) (opts.totalRows / (opts.totalRows * opts.sampleRate));

      if (options.isValueZipf()) {

        this.zipf = new RandomDistribution.Zipf(this.rand, 1, options.getValueSize(), 1.2);

      }

      LOG.info("Sampling 1 every " + everyN + " out of " + opts.perClientRunRows + " total rows.");

    }

    int getValueLength(final Random r) {

      if (this.opts.isValueRandom()) return Math.abs(r.nextInt() % opts.valueSize);

      else if (this.opts.isValueZipf()) return Math.abs(this.zipf.nextInt());

      else return opts.valueSize;

    }

    void updateValueSize(final Result [] rs) throws IOException {

      if (rs == null || !isRandomValueSize()) return;

      for (Result r: rs) updateValueSize(r);

    }

    void updateValueSize(final Result r) throws IOException {

      if (r == null || !isRandomValueSize()) return;

      int size = 0;

      for (CellScanner scanner = r.cellScanner(); scanner.advance();) {

        size += scanner.current().getValueLength();

      }

      updateValueSize(size);

    }

    void updateValueSize(final int valueSize) {

      if (!isRandomValueSize()) return;

      this.valueSize.update(valueSize);

    }

    String generateStatus(final int sr, final int i, final int lr) {

      return sr + "/" + i + "/" + lr + ", latency " + getShortLatencyReport() +

        (!isRandomValueSize()? "": ", value size " + getShortValueSizeReport());

    }

    boolean isRandomValueSize() {

      return opts.valueRandom;

    }

    protected int getReportingPeriod() {

      return opts.period;

    }

    /**

     * Populated by testTakedown. Only implemented by RandomReadTest at the moment.

     */

    public Histogram getLatency() {

      return latency;

    }

    void testSetup() throws IOException {

      if (!opts.oneCon) {

        this.connection = ConnectionFactory.createConnection(conf);

      }

      this.table = new HTable(TableName.valueOf(opts.tableName), connection);

      this.table.setAutoFlushTo(opts.autoFlush);

      latency = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));

      valueSize = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));

    }

    void testTakedown() throws IOException {

      reportLatency();

      reportValueSize();

      if (opts.flushCommits) {

        this.table.flushCommits();

      }

      table.close();

      if (!opts.oneCon) {

        connection.close();

      }

      receiverHost.closeReceivers();

    }

    /*

     * Run test

     * @return Elapsed time.

     * @throws IOException

     */

    long test() throws IOException, InterruptedException {

      testSetup();

      LOG.info("Timed test starting in thread " + Thread.currentThread().getName());

      final long startTime = System.nanoTime();

      try {

        testTimed();

      } finally {

        testTakedown();

      }

      return (System.nanoTime() - startTime) / 1000000;

    }

    /**

     * Provides an extension point for tests that don't want a per row invocation.

     */

    void testTimed() throws IOException, InterruptedException {

      int lastRow = opts.startRow + opts.perClientRunRows;