Package org.hbase.async.test

Source Code of org.hbase.async.test.TestIntegration

/*
* Copyright (C) 2012  The Async HBase Authors.  All rights reserved.
* This file is part of Async HBase.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*   - Redistributions of source code must retain the above copyright notice,
*     this list of conditions and the following disclaimer.
*   - Redistributions in binary form must reproduce the above copyright notice,
*     this list of conditions and the following disclaimer in the documentation
*     and/or other materials provided with the distribution.
*   - Neither the name of the StumbleUpon nor the names of its contributors
*     may be used to endorse or promote products derived from this software
*     without specific prior written permission.
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
package org.hbase.async.test;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.lang.reflect.Method;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;

import org.jboss.netty.logging.InternalLoggerFactory;
import org.jboss.netty.logging.Slf4JLoggerFactory;

import org.slf4j.Logger;

import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.Description;
import org.junit.runner.JUnitCore;
import org.junit.runner.Request;
import org.junit.runner.Result;
import org.junit.runner.notification.Failure;
import org.junit.runner.notification.RunListener;
import org.powermock.reflect.Whitebox;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;

import com.stumbleupon.async.Callback;
import com.stumbleupon.async.Deferred;
import com.stumbleupon.async.DeferredGroupException;

import org.hbase.async.AtomicIncrementRequest;
import org.hbase.async.BinaryComparator;
import org.hbase.async.BinaryPrefixComparator;
import org.hbase.async.BitComparator;
import org.hbase.async.Bytes;
import org.hbase.async.ColumnPrefixFilter;
import org.hbase.async.ColumnRangeFilter;
import org.hbase.async.CompareFilter.CompareOp;
import org.hbase.async.DeleteRequest;
import org.hbase.async.DependentColumnFilter;
import org.hbase.async.FamilyFilter;
import org.hbase.async.FilterList;
import org.hbase.async.GetRequest;
import org.hbase.async.HBaseClient;
import org.hbase.async.KeyRegexpFilter;
import org.hbase.async.KeyValue;
import org.hbase.async.NoSuchColumnFamilyException;
import org.hbase.async.PutRequest;
import org.hbase.async.QualifierFilter;
import org.hbase.async.RegexStringComparator;
import org.hbase.async.RowFilter;
import org.hbase.async.ScanFilter;
import org.hbase.async.Scanner;
import org.hbase.async.SubstringComparator;
import org.hbase.async.TableNotFoundException;
import org.hbase.async.TimestampsFilter;
import org.hbase.async.ValueFilter;

import org.hbase.async.test.Common;

/**
* Basic integration and regression tests for asynchbase.
*
* Requires a locally running HBase cluster.
*/
final public class TestIntegration {

  private static final Logger LOG = Common.logger(TestIntegration.class);

  private static final short FAST_FLUSH = 10;    // milliseconds
  private static final short SLOW_FLUSH = 1000// milliseconds
  /** Path to HBase home so we can run the HBase shell.  */
  private static final String HBASE_HOME;
  static {
    HBASE_HOME = System.getenv("HBASE_HOME");
    if (HBASE_HOME == null) {
      throw new RuntimeException("Please set the HBASE_HOME environment"
                                 + " variable.");
    }
    final File dir = new File(HBASE_HOME);
    if (!dir.isDirectory()) {
      throw new RuntimeException("No such directory: " + HBASE_HOME);
    }
  }

  /** Whether or not to truncate existing tables during tests.  */
  private static final boolean TRUNCATE =
    System.getenv("TEST_NO_TRUNCATE") == null;

  private static String table;
  private static String family;
  private static String[] args;
  private HBaseClient client;

  public static void main(final String[] args) throws Exception {
    preFlightTest(args);
    table = args[0];
    family = args[1];
    TestIntegration.args = args;
    LOG.info("Starting integration tests");
    final JUnitCore junit = new JUnitCore();
    final JunitListener listener = new JunitListener();
    junit.addListener(listener);
    final String singleTest = System.getenv("TEST_NAME");
    final Request req;
    if (singleTest != null) {
      req = Request.method(TestIntegration.class, singleTest);
    } else {
      req = Request.aClass(TestIntegration.class);
    }
    final Result result = junit.run(req);
    LOG.info("Ran " + result.getRunCount() + " tests in "
             + result.getRunTime() + "ms");
    if (!result.wasSuccessful()) {
      LOG.error(result.getFailureCount() + " tests failed: "
                + result.getFailures());
      System.exit(1);
    }
    LOG.info("All tests passed!");
  }

  @Before
  public void setUp() {
    client = Common.getOpt(TestIntegration.class, args);
  }

  @After
  public void tearDown() throws Exception {
    client.shutdown().join();
  }

  /** Ensures the table/family we use for our test exists. */
  private static void preFlightTest(final String[] args) throws Exception {
    final HBaseClient client = Common.getOpt(TestIncrementCoalescing.class,
                                             args);
    try {
      createOrTruncateTable(client, args[0], args[1]);
    } finally {
      client.shutdown().join();
    }
  }

  /** Creates or truncates the given table name. */
  private static void createOrTruncateTable(final HBaseClient client,
                                            final String table,
                                            final String family)
    throws Exception {
    try {
      client.ensureTableFamilyExists(table, family).join();
      truncateTable(table);
    } catch (TableNotFoundException e) {
      createTable(table, family);
      createOrTruncateTable(client, table, family)// Check again.
    }
  }

  /** Write a single thing to HBase and read it back. */
  @Test
  public void putRead() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    final double write_time = System.currentTimeMillis();
    final PutRequest put = new PutRequest(table, "k", family, "q", "val");
    final GetRequest get = new GetRequest(table, "k", family, "q");
    client.put(put).join();
    final ArrayList<KeyValue> kvs = client.get(get).join();
    assertSizeIs(1, kvs);
    final KeyValue kv = kvs.get(0);
    assertEq("k", kv.key());
    assertEq(family, kv.family());
    assertEq("q", kv.qualifier());
    assertEq("val", kv.value());
    final double kvts = kv.timestamp();
    assertEquals(write_time, kvts, 5000.0)// Within five seconds.
  }

  /** Write a single thing to HBase and read it back, delete it, read it. */
  @Test
  public void putReadDeleteRead() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    final PutRequest put = new PutRequest(table, "k", family, "q", "val");
    final GetRequest get = new GetRequest(table, "k", family, "q");
    client.put(put).join();
    final ArrayList<KeyValue> kvs = client.get(get).join();
    assertSizeIs(1, kvs);
    assertEq("val", kvs.get(0).value());
    final DeleteRequest del = new DeleteRequest(table, "k", family, "q");
    client.delete(del).join();
    final ArrayList<KeyValue> kvs2 = client.get(get).join();
    assertSizeIs(0, kvs2);
  }

  /**
   * Write two values to a HBase column and read them back,
   * delete one, and read back the other.
   */
  @Test
  public void putReadDeleteAtTimestamp() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    byte[] t = table.getBytes();
    byte[] k = "kprd@ts".getBytes();
    byte[] f = family.getBytes();
    // Make the qualifier unique to avoid running into HBASE-9879.
    byte[] q = ("q" + System.currentTimeMillis()
                + "-" + System.nanoTime()).getBytes();
    byte[] v1 = "val1".getBytes();
    byte[] v2 = "val2".getBytes();
    final PutRequest put1 = new PutRequest(t, k, f, q, v1, 100L);
    final PutRequest put2 = new PutRequest(t, k, f, q, v2, 200L);
    client.put(put1).join();
    client.put(put2).join();
    final GetRequest get = new GetRequest(t, k, f, q).maxVersions(2);
    final ArrayList<KeyValue> kvs = client.get(get).join();
    assertSizeIs(2, kvs);
    assertEq("val2", kvs.get(0).value());
    assertEq("val1", kvs.get(1).value());
    final DeleteRequest del = new DeleteRequest(t, k, f, q, 200L);
    del.setDeleteAtTimestampOnly(true);
    client.delete(del).join();
    final ArrayList<KeyValue> kvs2 = client.get(get).join();
    assertSizeIs(1, kvs2);
    assertEq("val1", kvs2.get(0).value());
  }

  /** Basic scan test. */
  @Test
  public void basicScan() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    final PutRequest put1 = new PutRequest(table, "s1", family, "q", "v1");
    final PutRequest put2 = new PutRequest(table, "s2", family, "q", "v2");
    final PutRequest put3 = new PutRequest(table, "s3", family, "q", "v3");
    Deferred.group(client.put(put1), client.put(put2),
                   client.put(put3)).join();
    // Scan the same 3 rows created above twice.
    for (int i = 0; i < 2; i++) {
      LOG.info("------------ iteration #" + i);
      final Scanner scanner = client.newScanner(table);
      scanner.setStartKey("s0");
      scanner.setStopKey("s9");
      // Callback class to keep scanning recursively.
      class cb implements Callback<Object, ArrayList<ArrayList<KeyValue>>> {
        private int n = 0;
        public Object call(final ArrayList<ArrayList<KeyValue>> rows) {
          if (rows == null) {
            return null;
          }
          n++;
          try {
            assertSizeIs(1, rows);
            final ArrayList<KeyValue> kvs = rows.get(0);
            final KeyValue kv = kvs.get(0);
            assertSizeIs(1, kvs);
            assertEq("s" + n, kv.key());
            assertEq("q", kv.qualifier());
            assertEq("v" + n, kv.value());
            return scanner.nextRows(1).addCallback(this);
          } catch (AssertionError e) {
            // Deferred doesn't catch Errors on purpose, so transform any
            // assertion failure into an Exception.
            throw new RuntimeException("Asynchronous failure", e);
          }
        }
      }
      try {
        scanner.nextRows(1).addCallback(new cb()).join();
      } finally {
        scanner.close().join();
      }
    }
  }

  /** Scan with multiple qualifiers. */
  @Test
  public void scanWithQualifiers() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    final PutRequest put1 = new PutRequest(table, "k", family, "a", "val1");
    final PutRequest put2 = new PutRequest(table, "k", family, "b", "val2");
    final PutRequest put3 = new PutRequest(table, "k", family, "c", "val3");
    Deferred.group(client.put(put1), client.put(put2),
                   client.put(put3)).join();
    final Scanner scanner = client.newScanner(table);
    scanner.setFamily(family);
    scanner.setQualifiers(new byte[][] { { 'a' }, { 'c' } });
    final ArrayList<ArrayList<KeyValue>> rows = scanner.nextRows(2).join();
    assertSizeIs(1, rows);
    final ArrayList<KeyValue> kvs = rows.get(0);
    assertSizeIs(2, kvs);
    assertEq("val1", kvs.get(0).value());
    assertEq("val3", kvs.get(1).value());
  }

  /** Write a few KVs and delete them in one batch */
  @Test
  public void multiDelete() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    final PutRequest put2 = new PutRequest(table, "mdk1", family, "q2", "val2");
    client.put(put2).join();
    final PutRequest put3 = new PutRequest(table, "mdk2", family, "q3", "val3");
    client.put(put3).join();
    final PutRequest put1 = new PutRequest(table, "mdk1", family, "q1", "val1");
    client.put(put1).join();
    final DeleteRequest del2 = new DeleteRequest(table, "mdk1", family, "q2");
    final DeleteRequest del3 = new DeleteRequest(table, "mdk2", family, "q3");
    final DeleteRequest del1 = new DeleteRequest(table, "mdk1", family, "q1");
    Deferred.group(client.delete(del2), client.delete(del3),
                   client.delete(del1)).join();
    GetRequest get = new GetRequest(table, "mdk1");
    ArrayList<KeyValue> kvs = client.get(get).join();
    assertSizeIs(0, kvs);
    get = new GetRequest(table, "mdk2");
    kvs = client.get(get).join();
    assertSizeIs(0, kvs);
  }

  /** Write a few KVs in different regions and delete them in one batch */
  @Test
  public void multiRegionMultiDelete() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    final String table1 = args[0] + "1";
    final String table2 = args[0] + "2";
    createOrTruncateTable(client, table1, family);
    createOrTruncateTable(client, table2, family);
    final PutRequest put2 = new PutRequest(table1, "mdk1", family, "q2", "val2");
    client.put(put2).join();
    final PutRequest put3 = new PutRequest(table1, "mdk2", family, "q3", "val3");
    client.put(put3).join();
    final PutRequest put1 = new PutRequest(table2, "mdk1", family, "q1", "val1");
    client.put(put1).join();
    final DeleteRequest del2 = new DeleteRequest(table1, "mdk1", family, "q2");
    final DeleteRequest del3 = new DeleteRequest(table1, "mdk2", family, "q3");
    final DeleteRequest del1 = new DeleteRequest(table2, "mdk1", family, "q1");
    Deferred.group(client.delete(del2), client.delete(del3),
                   client.delete(del1)).join();
    GetRequest get = new GetRequest(table1, "mdk1");
    ArrayList<KeyValue> kvs = client.get(get).join();
    assertSizeIs(0, kvs);
    get = new GetRequest(table1, "mdk2");
    kvs = client.get(get).join();
    assertSizeIs(0, kvs);
    get = new GetRequest(table2, "mdk1");
    kvs = client.get(get).join();
    assertSizeIs(0, kvs);
  }

  /** Attempt to write a column family that doesn't exist. */
  @Test
  public void putNonexistentFamily() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    final PutRequest put = new PutRequest(table, "k", family + family,
                                          "q", "val");
    try {
      client.put(put).join();
    } catch (NoSuchColumnFamilyException e) {
      assertEquals(put, e.getFailedRpc());
      return;
    }
    throw new AssertionError("Should never be here");
  }

  /** Send a bunch of edits with one that references a non-existent family. */
  @Test
  public void multiPutWithOneBadRpcInBatch() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    final PutRequest put1 = new PutRequest(table, "mk1", family, "m1", "mpb1");
    // The following edit is destined to a non-existent family.
    final PutRequest put2 = new PutRequest(table, "mk2", family + family,
                                           "m2", "mpb2");
    final PutRequest put3 = new PutRequest(table, "mk3", family, "m3", "mpb3");
    try {
      final ArrayList<Deferred<Object>> ds = new ArrayList<Deferred<Object>>(3);
      ds.add(client.put(put1));
      ds.add(client.put(put2));
      ds.add(client.put(put3));
      Deferred.groupInOrder(ds).join();
    } catch (DeferredGroupException e) {
      final ArrayList<Object> results = e.results();
      final Object res2 = results.get(1);
      if (!(res2 instanceof NoSuchColumnFamilyException)) {
        throw new AssertionError("res2 wasn't a NoSuchColumnFamilyException: "
                                 + res2);
      }
      assertEquals(put2, ((NoSuchColumnFamilyException) res2).getFailedRpc());
      final GetRequest get1 = new GetRequest(table, "mk1", family, "m1");
      ArrayList<KeyValue> kvs = client.get(get1).join();
      assertSizeIs(1, kvs);
      assertEq("mpb1", kvs.get(0).value());
      final GetRequest get2 = new GetRequest(table, "mk2", family, "m2");
      assertSizeIs(0, client.get(get2).join());
      final GetRequest get3 = new GetRequest(table, "mk3", family, "m3");
      kvs = client.get(get3).join();
      assertSizeIs(1, kvs);
      assertEq("mpb3", kvs.get(0).value());
      return;
    }
    throw new AssertionError("Should never be here");
  }

  /** Lots of buffered counter increments from multiple threads. */
  @Test
  public void bufferedIncrementStressTest() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    final byte[] table = TestIntegration.table.getBytes();
    final byte[] key1 = "cnt1".getBytes()// Spread the increments..
    final byte[] key2 = "cnt2".getBytes()// .. over these two counters.
    final byte[] family = TestIntegration.family.getBytes();
    final byte[] qual = { 'q' };
    final DeleteRequest del1 = new DeleteRequest(table, key1, family, qual);
    final DeleteRequest del2 = new DeleteRequest(table, key2, family, qual);
    Deferred.group(client.delete(del1), client.delete(del2)).join();

    final int nthreads = Runtime.getRuntime().availableProcessors() * 2;
    // The magic number comes from the limit on callbacks that Deferred
    // imposes.  We spread increments over two counters, hence the x 2.
    final int incr_per_thread = 8192 / nthreads * 2;
    final boolean[] successes = new boolean[nthreads];

    final class IncrementThread extends Thread {
      private final int num;
      public IncrementThread(final int num) {
        super("IncrementThread-" + num);
        this.num = num;
      }
      public void run() {
        try {
          doIncrements();
          successes[num] = true;
        } catch (Throwable e) {
          successes[num] = false;
          LOG.error("Uncaught exception", e);
        }
      }
      private void doIncrements() {
        for (int i = 0; i < incr_per_thread; i++) {
          final byte[] key = i % 2 == 0 ? key1 : key2;
          bufferIncrement(table, key, family, qual, 1);
        }
      }
    }

    final IncrementThread[] threads = new IncrementThread[nthreads];
    for (int i = 0; i < nthreads; i++) {
      threads[i] = new IncrementThread(i);
    }
    LOG.info("Starting to generate increments");
    for (int i = 0; i < nthreads; i++) {
      threads[i].start();
    }
    for (int i = 0; i < nthreads; i++) {
      threads[i].join();
    }

    LOG.info("Flushing all buffered increments.");
    client.flush().joinUninterruptibly();
    LOG.info("Done flushing all buffered increments.");

    // Check that we the counters have the expected value.
    final GetRequest[] gets = { mkGet(table, key1, family, qual),
                                mkGet(table, key2, family, qual) };
    for (final GetRequest get : gets) {
      final ArrayList<KeyValue> kvs = client.get(get).join();
      assertSizeIs(1, kvs);
      assertEquals(incr_per_thread * nthreads / 2,
                   Bytes.getLong(kvs.get(0).value()));
    }

    for (int i = 0; i < nthreads; i++) {
      assertEquals(true, successes[i])// Make sure no exception escaped.
    }
  }

  /** Increment coalescing with values too large to be coalesced. */
  @Test
  public void incrementCoalescingWithAmountsTooBig() throws Exception {
    client.setFlushInterval(SLOW_FLUSH);
    final byte[] table = TestIntegration.table.getBytes();
    final byte[] key = "cnt".getBytes();
    final byte[] family = TestIntegration.family.getBytes();
    final byte[] qual = { 'q' };
    final DeleteRequest del = new DeleteRequest(table, key, family, qual);
    del.setBufferable(false);
    client.delete(del).join();
    final long big = 1L << 48// Too big to be coalesced.
    final ArrayList<KeyValue> kvs = Deferred.group(
      bufferIncrement(table, key, family, qual, big),
      bufferIncrement(table, key, family, qual, big)
    ).addCallbackDeferring(new Callback<Deferred<ArrayList<KeyValue>>,
                                        ArrayList<Long>>() {
      public Deferred<ArrayList<KeyValue>> call(final ArrayList<Long> incs) {
        final GetRequest get = new GetRequest(table, key)
          .family(family).qualifier(qual);
        return client.get(get);
      }
    }).join();
    assertSizeIs(1, kvs);
    assertEquals(big + big, Bytes.getLong(kvs.get(0).value()));
    // Check we sent the right number of RPCs.
    assertEquals(2, client.stats().atomicIncrements());
  }

  /** Increment coalescing with large values that overflow. */
  @Test
  public void incrementCoalescingWithOverflowingAmounts() throws Exception {
    client.setFlushInterval(SLOW_FLUSH);
    final byte[] table = TestIntegration.table.getBytes();
    final byte[] key = "cnt".getBytes();
    final byte[] family = TestIntegration.family.getBytes();
    final byte[] qual = { 'q' };
    final DeleteRequest del = new DeleteRequest(table, key, family, qual);
    del.setBufferable(false);
    client.delete(del).join();
    final long big = 1L << 47;
    // First two RPCs can be coalesced.
    bufferIncrement(table, key, family, qual, big);
    bufferIncrement(table, key, family, qual, 1);
    // This one would cause an overflow, so will be sent as a separate RPC.
    // Overflow would happen because the max value is (1L << 48) - 1.
    bufferIncrement(table, key, family, qual, big);
    client.flush().joinUninterruptibly();
    final GetRequest get = new GetRequest(table, key)
      .family(family).qualifier(qual);
    final ArrayList<KeyValue> kvs = client.get(get).join();
    assertSizeIs(1, kvs);
    assertEquals(big + 1 + big, Bytes.getLong(kvs.get(0).value()));
    // Check we sent the right number of RPCs.
    assertEquals(2, client.stats().atomicIncrements());
  }

  /** Increment coalescing with negative values and underflows. */
  @Test
  public void incrementCoalescingWithUnderflowingAmounts() throws Exception {
    client.setFlushInterval(SLOW_FLUSH);
    final byte[] table = TestIntegration.table.getBytes();
    final byte[] key = "cnt".getBytes();
    final byte[] family = TestIntegration.family.getBytes();
    final byte[] qual = { 'q' };
    final DeleteRequest del = new DeleteRequest(table, key, family, qual);
    del.setBufferable(false);
    client.delete(del).join();
    final long big = -1L << 47;
    // First two RPCs can be coalesced.
    bufferIncrement(table, key, family, qual, big);
    bufferIncrement(table, key, family, qual, -1);
    // This one would cause an underflow, so will be sent as a separate RPC.
    // Overflow would happen because the max value is -1L << 48.
    bufferIncrement(table, key, family, qual, big);
    client.flush().joinUninterruptibly();
    final GetRequest get = new GetRequest(table, key)
      .family(family).qualifier(qual);
    final ArrayList<KeyValue> kvs = client.get(get).join();
    assertSizeIs(1, kvs);
    assertEquals(big - 1 + big, Bytes.getLong(kvs.get(0).value()));
    // Check we sent the right number of RPCs.
    assertEquals(2, client.stats().atomicIncrements());
  }

  /** Increment coalescing where the coalesced sum ends up being zero. */
  @Test
  public void incrementCoalescingWithZeroSumAmount() throws Exception {
    client.setFlushInterval(SLOW_FLUSH);
    final byte[] table = TestIntegration.table.getBytes();
    final byte[] key = "cnt".getBytes();
    final byte[] family = TestIntegration.family.getBytes();
    final byte[] qual = { 'q' };
    final DeleteRequest del = new DeleteRequest(table, key, family, qual);
    del.setBufferable(false);
    client.delete(del).join();
    // HBase 0.98 and up do not create a KV on atomic increment when the
    // increment amount is 0.  So let's first send an increment of some
    // arbitrary value, and then ensure that this value hasn't changed.
    long n = client.atomicIncrement(new AtomicIncrementRequest(table, key,
                                                               family, qual,
                                                               42)).join();
    assertEquals(42, n);
    bufferIncrement(table, key, family, qual, 1);
    bufferIncrement(table, key, family, qual, 2);
    bufferIncrement(table, key, family, qual, -3);
    client.flush().joinUninterruptibly();
    final GetRequest get = new GetRequest(table, key)
      .family(family).qualifier(qual);
    final ArrayList<KeyValue> kvs = client.get(get).join();
    assertSizeIs(1, kvs);
    assertEquals(42, Bytes.getLong(kvs.get(0).value()));
    // The sum was 0, but must have sent the increment anyway.
    // So in total we should have sent two increments, the initial one,
    // that sets the value to 42, and the one incrementing by zero.
    assertEquals(2, client.stats().atomicIncrements());
  }

  /** Helper method to create an atomic increment request.  */
  private Deferred<Long> bufferIncrement(final byte[] table,
                                         final byte[] key, final byte[] family,
                                         final byte[] qual, final long value) {
    return
      client.bufferAtomicIncrement(new AtomicIncrementRequest(table, key,
                                                              family, qual,
                                                              value));
  }

  /** Helper method to create a get request.  */
  private static GetRequest mkGet(final byte[] table, final byte[] key,
                                  final byte[] family, final byte[] qual) {
    return new GetRequest(table, key).family(family).qualifier(qual);
  }

  /** Test regexp-based row key filtering.  */
  @Test
  public void keyRegexpFilter() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    final PutRequest put1 = new PutRequest(table, "krf accept:by the filter",
                                           family, "q", "krfv1");
    final PutRequest put2 = new PutRequest(table, "krf filtered out",
                                           family, "q", "krfv2");
    final PutRequest put3 = new PutRequest(table, "krf this is Accepted too",
                                           family, "q", "krfv3");
    Deferred.group(client.put(put1), client.put(put2),
                   client.put(put3)).join();
    final Scanner scanner = client.newScanner(table);
    scanner.setFamily(family);
    scanner.setStartKey("krf ");
    scanner.setStopKey("krf!");
    scanner.setKeyRegexp("[Aa]ccept(ed)?");
    final ArrayList<ArrayList<KeyValue>> rows = scanner.nextRows().join();
    assertSizeIs(2, rows);
    ArrayList<KeyValue> kvs = rows.get(0);
    assertSizeIs(1, kvs);
    assertEq("krfv1", kvs.get(0).value());
    kvs = rows.get(1);
    assertSizeIs(1, kvs);
    assertEq("krfv3", kvs.get(0).value());
  }

  /** Simple column prefix filter tests.  */
  @Test
  public void columnPrefixFilter() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    // Keep only rows with a column qualifier that starts with "qa".
    final PutRequest put1 = new PutRequest(table, "cpf1", family, "qa1", "v1");
    final PutRequest put2 = new PutRequest(table, "cpf1", family, "qa2", "v2");
    final PutRequest put3 = new PutRequest(table, "cpf2", family, "qa3", "v3");
    final PutRequest put4 = new PutRequest(table, "cpf2", family, "qb4", "v4");
    Deferred.group(Deferred.group(client.put(put1), client.put(put2)),
                   Deferred.group(client.put(put3), client.put(put4))).join();
    final Scanner scanner = client.newScanner(table);
    scanner.setFamily(family);
    scanner.setStartKey("cpf1");
    scanner.setStopKey("cpf3");
    scanner.setFilter(new ColumnPrefixFilter("qa"));
    final ArrayList<ArrayList<KeyValue>> rows = scanner.nextRows().join();
    assertSizeIs(2, rows);
    ArrayList<KeyValue> kvs = rows.get(0);
    assertSizeIs(2, kvs);
    assertEq("v1", kvs.get(0).value());
    assertEq("v2", kvs.get(1).value());
    kvs = rows.get(1);
    assertSizeIs(1, kvs);
    assertEq("v3", kvs.get(0).value());
  }

  /** Simple column range filter tests.  */
  @Test
  public void columnRangeFilter() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    // Keep rows that have a qualifier in between "qb" (inclusive) and "qd4"
    // (exclusive).  So only v2 and v3 should be returned by the scanner.
    final PutRequest put1 = new PutRequest(table, "crf1", family, "qa1", "v1");
    final PutRequest put2 = new PutRequest(table, "crf1", family, "qb2", "v2");
    final PutRequest put3 = new PutRequest(table, "crf2", family, "qc3", "v3");
    final PutRequest put4 = new PutRequest(table, "crf2", family, "qd4", "v4");
    Deferred.group(Deferred.group(client.put(put1), client.put(put2)),
                   Deferred.group(client.put(put3), client.put(put4))).join();
    final Scanner scanner = client.newScanner(table);
    scanner.setFamily(family);
    scanner.setStartKey("crf1");
    scanner.setStopKey("crf3");
    scanner.setFilter(new ColumnRangeFilter("qb", true, "qd4", false));
    final ArrayList<ArrayList<KeyValue>> rows = scanner.nextRows().join();
    assertSizeIs(2, rows)// One KV from row "fl1" and one from "fl2".
    ArrayList<KeyValue> kvs = rows.get(0);
    assertSizeIs(1, kvs);
    assertEq("v2", kvs.get(0).value());
    kvs = rows.get(1);
    assertSizeIs(1, kvs);
    assertEq("v3", kvs.get(0).value());
  }

  @Test
  public void filterComparators() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    final PutRequest put1 = new PutRequest(table, "fc1", family, "a", "v1");
    final PutRequest put2 = new PutRequest(table, "fc2", family, "a", "v2");
    final PutRequest put3 = new PutRequest(table, "fc3", family, "a", "v3");
    Deferred.group(client.put(put1), client.put(put2), client.put(put3)).join();

    final Scanner binary_scanner = client.newScanner(table);
    binary_scanner.setStartKey("fc1");
    binary_scanner.setStopKey("fc4");
    binary_scanner.setFilter(
        new RowFilter(CompareOp.LESS, new BinaryComparator(Bytes.UTF8("fc2"))));
    final ArrayList<ArrayList<KeyValue>> binary_rows =
        binary_scanner.nextRows().join();
    assertSizeIs(1, binary_rows);
    assertSizeIs(1, binary_rows.get(0));
    assertEq("v1", binary_rows.get(0).get(0).value());

    final Scanner prefix_scanner = client.newScanner(table);
    prefix_scanner.setStartKey("fc1");
    prefix_scanner.setStopKey("fc4");
    prefix_scanner.setFilter(
        new RowFilter(CompareOp.GREATER_OR_EQUAL,
            new BinaryPrefixComparator(Bytes.UTF8("fc2"))));
    final ArrayList<ArrayList<KeyValue>> prefix_rows =
        prefix_scanner.nextRows().join();
    assertSizeIs(2, prefix_rows);
    assertSizeIs(1, prefix_rows.get(0));
    assertEq("v2", prefix_rows.get(0).get(0).value());
    assertSizeIs(1, prefix_rows.get(1));
    assertEq("v3", prefix_rows.get(1).get(0).value());

    final Scanner bit_scanner = client.newScanner(table);
    bit_scanner.setStartKey("fc1");
    bit_scanner.setStopKey("fc4");
    bit_scanner.setFilter(
        new RowFilter(CompareOp.EQUAL,
            new BitComparator(Bytes.UTF8("fc2"), BitComparator.BitwiseOp.XOR)));
    final ArrayList<ArrayList<KeyValue>> bit_rows =
        bit_scanner.nextRows().join();
    assertSizeIs(2, bit_rows);
    assertSizeIs(1, bit_rows.get(0));
    assertEq("v1", bit_rows.get(0).get(0).value());
    assertSizeIs(1, bit_rows.get(1));
    assertEq("v3", bit_rows.get(1).get(0).value());

    final Scanner regex_scanner = client.newScanner(table);
    regex_scanner.setStartKey("fc1");
    regex_scanner.setStopKey("fc4");
    regex_scanner.setFilter(
        new RowFilter(CompareOp.EQUAL, new RegexStringComparator("fc2")));
    final ArrayList<ArrayList<KeyValue>> regex_rows =
        regex_scanner.nextRows().join();
    assertSizeIs(1, regex_rows);
    assertSizeIs(1, regex_rows.get(0));
    assertEq("v2", regex_rows.get(0).get(0).value());

    final Scanner substring_scanner = client.newScanner(table);
    substring_scanner.setStartKey("fc1");
    substring_scanner.setStopKey("fc4");
    substring_scanner.setFilter(
        new RowFilter(CompareOp.EQUAL, new SubstringComparator("2")));
    final ArrayList<ArrayList<KeyValue>> substring_rows =
        substring_scanner.nextRows().join();
    assertSizeIs(1, substring_rows);
    assertSizeIs(1, substring_rows.get(0));
    assertEq("v2", substring_rows.get(0).get(0).value());
  }

  @Test
  public void compareFilters() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    final PutRequest put1 = new PutRequest(table, "cf1", family, "a", "v1");
    final PutRequest put2 = new PutRequest(table, "cf2", family, "b", "v2");
    final PutRequest put3 = new PutRequest(
        Bytes.UTF8(table),
        Bytes.UTF8("cf3"),
        Bytes.UTF8(family),
        Bytes.UTF8("c"),
        Bytes.UTF8("v3"),
        42);
    final PutRequest put4 = new PutRequest(
        Bytes.UTF8(table),
        Bytes.UTF8("cf3"),
        Bytes.UTF8(family),
        Bytes.UTF8("dep"),
        Bytes.UTF8("v4"),
        42);
    Deferred.group(
        Deferred.group(client.put(put1), client.put(put2)),
        Deferred.group(client.put(put3), client.put(put4))).join();

    final Scanner row_scanner = client.newScanner(table);
    row_scanner.setStartKey("cf1");
    row_scanner.setStopKey("cf4");
    row_scanner.setFilter(
        new RowFilter(CompareOp.NOT_EQUAL, new BinaryComparator(Bytes.UTF8("cf2"))));
    final ArrayList<ArrayList<KeyValue>> row_rows =
        row_scanner.nextRows().join();
    assertSizeIs(2, row_rows);
    assertSizeIs(1, row_rows.get(0));
    assertEq("v1", row_rows.get(0).get(0).value());
    assertSizeIs(2, row_rows.get(1));
    assertEq("v3", row_rows.get(1).get(0).value());
    assertEq("v4", row_rows.get(1).get(1).value());

    final Scanner family_scanner = client.newScanner(table);
    family_scanner.setFilter(
        new FamilyFilter(CompareOp.LESS_OR_EQUAL,
            new BinaryComparator(Bytes.UTF8("aSomeOtherFamily"))));
    final ArrayList<ArrayList<KeyValue>> family_rows =
        family_scanner.nextRows().join();
    assertNull(family_rows);

    final Scanner qualifier_scanner = client.newScanner(table);
    qualifier_scanner.setStartKey("cf1");
    qualifier_scanner.setStopKey("cf4");
    qualifier_scanner.setFilter(
        new QualifierFilter(CompareOp.GREATER,
            new BinaryComparator(Bytes.UTF8("b"))));
    final ArrayList<ArrayList<KeyValue>> qualifier_rows =
        qualifier_scanner.nextRows().join();
    assertSizeIs(1, qualifier_rows);
    assertSizeIs(2, qualifier_rows.get(0));
    assertEq("v3", qualifier_rows.get(0).get(0).value());
    assertEq("v4", qualifier_rows.get(0).get(1).value());

    final Scanner value_scanner = client.newScanner(table);
    value_scanner.setStartKey("cf1");
    value_scanner.setStopKey("cf4");
    value_scanner.setFilter(
        new ValueFilter(CompareOp.GREATER_OR_EQUAL,
            new BinaryComparator(Bytes.UTF8("v3"))));
    final ArrayList<ArrayList<KeyValue>> value_rows =
        value_scanner.nextRows().join();
    assertSizeIs(1, value_rows);
    assertSizeIs(2, value_rows.get(0));
    assertEq("v3", value_rows.get(0).get(0).value());
    assertEq("v4", value_rows.get(0).get(1).value());

    final Scanner dependent_scanner = client.newScanner(table);
    dependent_scanner.setMaxNumKeyValues(-1);
    dependent_scanner.setFilter(
        new DependentColumnFilter(Bytes.UTF8(family), Bytes.UTF8("dep")));
    final ArrayList<ArrayList<KeyValue>> dependent_rows =
        dependent_scanner.nextRows().join();
    assertSizeIs(1, dependent_rows);
    assertSizeIs(2, dependent_rows.get(0));
    assertEq("v3", dependent_rows.get(0).get(0).value());
    assertEq("v4", dependent_rows.get(0).get(1).value());

    final Scanner dependent_value_scanner = client.newScanner(table);
    dependent_value_scanner.setMaxNumKeyValues(-1);
    dependent_value_scanner.setFilter(
        new DependentColumnFilter(
            Bytes.UTF8(family),
            Bytes.UTF8("dep"),
            true,
            CompareOp.EQUAL,
            new BinaryComparator(Bytes.UTF8("v4"))));
    final ArrayList<ArrayList<KeyValue>> dependent_value_rows =
        dependent_value_scanner.nextRows().join();
    assertSizeIs(1, dependent_value_rows);
    assertSizeIs(1, dependent_value_rows.get(0));
    assertEq("v3", dependent_value_rows.get(0).get(0).value());
  }

  /** Simple column filter list tests.  */
  @Test
  public void filterList() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    // Keep rows that have both:
    //   - a row key that is exactly either "fl1" or "fl2".
    //   - a qualifier in between "qb" (inclusive) and "qd4" (exclusive).
    final ArrayList<ScanFilter> filters = new ArrayList<ScanFilter>(2);
    filters.add(new ColumnRangeFilter("qb", true, "qd4", false));
    filters.add(new KeyRegexpFilter("fl[12]$"));
    // Filtered out as we're looking due to qualifier being out of range:
    final PutRequest put1 = new PutRequest(table, "fl1", family, "qa1", "v1");
    // Kept by the filter:
    final PutRequest put2 = new PutRequest(table, "fl1", family, "qb2", "v2");
    // Filtered out because the row key doesn't match the regexp:
    final PutRequest put3 = new PutRequest(table, "fl1a", family, "qb3", "v3");
    // Kept by the filter:
    final PutRequest put4 = new PutRequest(table, "fl2", family, "qc4", "v4");
    // Filtered out because the qualifier is on the exclusive upper bound:
    final PutRequest put5 = new PutRequest(table, "fl2", family, "qd5", "v5");
    // Filtered out because the qualifier is past the upper bound:
    final PutRequest put6 = new PutRequest(table, "fl2", family, "qd6", "v6");
    Deferred.group(Deferred.group(client.put(put1), client.put(put2),
                                  client.put(put3)),
                   Deferred.group(client.put(put4), client.put(put5),
                                  client.put(put6))).join();
    final Scanner scanner = client.newScanner(table);
    scanner.setFamily(family);
    scanner.setStartKey("fl0");
    scanner.setStopKey("fl9");
    scanner.setFilter(new FilterList(filters));
    final ArrayList<ArrayList<KeyValue>> rows = scanner.nextRows().join();
    assertSizeIs(2, rows)// One KV from row "fl1" and one from "fl2".
    ArrayList<KeyValue> kvs = rows.get(0);
    assertSizeIs(1, kvs);   // KV from "fl1":
    assertEq("v2", kvs.get(0).value());
    kvs = rows.get(1);
    assertSizeIs(1, kvs);   // KV from "fl2":
    assertEq("v4", kvs.get(0).value());
  }

  /** Simple timestamps filter list tests.  */
  @Test
  public void timestampsFilter() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    final byte[] tableBytes = Bytes.UTF8(table);
    final byte[] familyBytes = Bytes.UTF8(family);
    final byte[] qualifier = Bytes.UTF8("q");
    final PutRequest put1 =
      new PutRequest(tableBytes, Bytes.UTF8("tf1"), familyBytes, qualifier, Bytes.UTF8("v1"), 1L);
    final PutRequest put2 =
      new PutRequest(tableBytes, Bytes.UTF8("tf2"), familyBytes, qualifier, Bytes.UTF8("v2"), 2L);
    final PutRequest put3 =
      new PutRequest(tableBytes, Bytes.UTF8("tf3"), familyBytes, qualifier, Bytes.UTF8("v3"), 3L);
    Deferred.group(client.put(put1), client.put(put2), client.put(put3)).join();
    final Scanner scanner = client.newScanner(table);
    scanner.setFamily(family);
    scanner.setStartKey(Bytes.UTF8("tf"));
    scanner.setStopKey(Bytes.UTF8("tf4"));
    scanner.setFilter(new TimestampsFilter(1L, 3L));
    final ArrayList<ArrayList<KeyValue>> rows = scanner.nextRows().join();
    assertSizeIs(2, rows);
    assertSizeIs(1, rows.get(0));
    assertEq("v1", rows.get(0).get(0).value());
    assertSizeIs(1, rows.get(1));
    assertEq("v3", rows.get(1).get(0).value());
  }

  @Test
  public void prefetchMeta() throws Exception {
    // Prefetch the metadata for a given table, then invasively probe the
    // region cache to demonstrate it is filled.
    client.prefetchMeta(table).join();

    Object region_info = Whitebox.invokeMethod(client, "getRegion",
                                               table.getBytes(),
                                               HBaseClient.EMPTY_ARRAY);
    assertNotNull(region_info);
  }

  /** Regression test for issue #2. */
  @Test
  public void regression2() throws Exception {
    try {
      final PutRequest put1 = new PutRequest(table, "k1", family, "q", "val1");
      final PutRequest put2 = new PutRequest(table, "k2", family, "q", "val2");
      LOG.info("Before calling put()");
      client.put(put1);
      client.put(put2);
      LOG.info("After calling put()");
    } finally {
      LOG.info("Before calling flush()");
      // Flushing immediately a cold client used to be troublesome because we
      // wouldn't do a good job at making sure that we can let the client do
      // the entire start-up dance (find ROOT, META, issue pending queries...).
      client.flush().join();
      LOG.info("After calling flush()");
      assertEquals(1, client.stats().numBatchedRpcSent());
    }
  }

  /** Regression test for issue #25. */
  @Test
  public void regression25() throws Exception {
    client.setFlushInterval(FAST_FLUSH);
    final String table1 = args[0] + "1";
    final String table2 = args[0] + "2";
    final String family = args[1];
    createOrTruncateTable(client, table1, family);
    createOrTruncateTable(client, table2, family);
    for (int i = 0; i < 2; i++) {
      final PutRequest put;
      final String key = 'k' + String.valueOf(i);
      if (i % 2 == 0) {
        put = new PutRequest(table1, key, family, "q", "v");
      } else {
        put = new PutRequest(table2, key, family, "q", "v");
      }
      final DeleteRequest delete = new DeleteRequest(put.table(), put.key());
      client.delete(delete);
      client.put(put);
    }
    client.flush().joinUninterruptibly();
  }

  /** Regression test for issue #40 (which was actually Netty bug #474). */
  @Test
  public void regression40() throws Exception {
    // Cause a META lookup first to avoid some DEBUG-level spam due to the
    // long key below.
    client.ensureTableFamilyExists(table, family).join();
    client.setFlushInterval(FAST_FLUSH);
    final byte[] table = TestIntegration.table.getBytes();
    // 980 was empirically found to be the minimum size with which
    // Netty bug #474 gets triggered.  Bug got fixed in Netty 3.5.8.
    final byte[] key = new byte[980];
    key[0] = 'k';
    key[1] = '4';
    key[2] = '0';
    key[key.length - 1] = '*';
    final byte[] family = TestIntegration.family.getBytes();
    final byte[] qual = { 'q' };
    final PutRequest put = new PutRequest(table, key, family, qual,
                                          new byte[0] /* empty */);
    final GetRequest get = new GetRequest(table, key);
    client.put(put).join();
    final ArrayList<KeyValue> kvs = client.get(get).join();
    assertSizeIs(1, kvs);
    KeyValue kv = kvs.get(0);
    assertEq("q", kv.qualifier());
    assertEq("", kv.value());
  }

  /** Regression test for issue #41. */
  @Test
  public void regression41() throws Exception {
    client.setFlushInterval(SLOW_FLUSH);
    final byte[] table = TestIntegration.table.getBytes();
    final byte[] key = "cnt".getBytes();
    final byte[] family = TestIntegration.family.getBytes();
    final byte[] qual = { 'q' };
    final DeleteRequest del = new DeleteRequest(table, key, family, qual);
    del.setBufferable(false);
    client.delete(del).join();
    final int iterations = 100000;
    for (int i = 0; i < iterations; i++) {
      bufferIncrement(table, key, family, qual, 1);
    }
    client.flush().joinUninterruptibly();
    final GetRequest get = new GetRequest(table, key)
      .family(family).qualifier(qual);
    final ArrayList<KeyValue> kvs = client.get(get).join();
    assertSizeIs(1, kvs);
    assertEquals(iterations, Bytes.getLong(kvs.get(0).value()));
  }

  private static <T> void assertSizeIs(final int size,
                                       final Collection<T> list) {
    final int actual = list.size();
    if (size != actual) {
      throw new AssertionError("List was expected to contain " + size
                 + " items but was found to contain " + actual + ": " + list);
    }
  }

  private static void assertEq(final String expect, final byte[] actual) {
    assertArrayEquals(expect.getBytes(), actual);
  }

  private static void createTable(final String table,
                                  final String family) throws Exception {
    LOG.info("Creating table " + table + " with family " + family);
    hbaseShell("create '" + table + "',"
               + " {NAME => '" + family + "', VERSIONS => 2}");
  }

  private static void truncateTable(final String table) throws Exception {
    if (!TRUNCATE) {
      return;
    }
    LOG.warn("Truncating table " + table + "...");
    for (int i = 3; i >= 0; i--) {
      LOG.warn(i + " Press Ctrl-C if you care about " + table);
      Thread.sleep(1000);
    }
    hbaseShell("truncate '" + table + '\'');
  }

  private static void hbaseShell(final String command) throws Exception {
    final ProcessBuilder pb = new ProcessBuilder();
    pb.command(HBASE_HOME + "/bin/hbase", "shell");
    pb.environment().remove("HBASE_HOME");
    LOG.info("Running HBase shell command: " + command);
    final Process shell = pb.start();
    try {
      final OutputStream stdin = shell.getOutputStream();
      stdin.write(command.getBytes());
      stdin.write('\n');
      stdin.flush()// Technically the JDK doesn't guarantee that close()
      stdin.close()// will flush(), so better do it explicitly to be safe.
      // Let's hope that the HBase shell doesn't print more than 4KB of shit
      // on stderr, otherwise we're getting deadlocked here.  Yeah seriously.
      // Dealing with subprocesses in Java is such a royal PITA.
      printLines("stdout", shell.getInputStream())// Confusing method name,
      printLines("stderr", shell.getErrorStream())// courtesy of !@#$% JDK.
      final int rv = shell.waitFor();
      if (rv != 0) {
        throw new RuntimeException("hbase shell returned " + rv);
      }
    } finally {
      shell.destroy()// Required by the fucking JDK, no matter what.
    }
  }

  private static void printLines(final String what, final InputStream in)
    throws Exception {
    final BufferedReader r = new BufferedReader(new InputStreamReader(in));
    String line;
    while ((line = r.readLine()) != null) {
      LOG.info('(' + what + ") " + line);
    }
  }

  private static final class JunitListener extends RunListener {
    @Override
    public void testStarted(final Description description) {
      LOG.info("Running test " + description.getMethodName());
    }

    @Override
    public void testFinished(final Description description) {
      LOG.info("Done running test " + description.getMethodName());
    }

    @Override
    public void testFailure(final Failure failure) {
      LOG.error("Test failed: " + failure.getDescription().getMethodName(),
                failure.getException());
    }

    @Override
    public void testIgnored(final Description description) {
      LOG.info("Test ignored: " + description.getMethodName());
    }
  }

}
TOP

Related Classes of org.hbase.async.test.TestIntegration

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.