* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* Facility for testing HBase. Replacement for

* old HBaseTestCase and HBaseClusterTestCase functionality.

* Create an instance and keep it around testing HBase.  This class is

* meant to be your one-stop shop for anything you might need testing.  Manages

* one cluster at a time only. Managed cluster can be an in-process

* {@link MiniHBaseCluster}, or a deployed cluster of type {@link DistributedHBaseCluster}.

* Not all methods work with the real cluster.

* Depends on log4j being on classpath and

* hbase-site.xml for logging and test-run configuration.  It does not set

* logging levels nor make changes to configuration parameters.

* <p>To preserve test data directories, pass the system property "hbase.testing.preserve.testdir"

* setting it to true.

*/

@InterfaceStability.Evolving

   private MiniZooKeeperCluster zkCluster = null;

  /**

   * The default number of regions per regionserver when creating a pre-split

   * table.

   */

  private static int DEFAULT_REGIONS_PER_SERVER = 5;

  /**

   * Set if we were passed a zkCluster.  If so, we won't shutdown zk as

   * part of general shutdown.

   */

  private boolean passedZkCluster = false;

  private MiniDFSCluster dfsCluster = null;

  private HBaseCluster hbaseCluster = null;

  private MiniMRCluster mrCluster = null;

  /** If there is a mini cluster running for this testing utility instance. */

  private boolean miniClusterRunning;

  private String hadoopLogDir;

  /** Directory (a subdirectory of dataTestDir) used by the dfs cluster if any */

  private File clusterTestDir = null;

  /** Directory on test filesystem where we put the data for this instance of

    * HBaseTestingUtility*/

  private Path dataTestDirOnTestFS = null;

  /**

   * System property key to get test directory value.

   * Name is as it is because mini dfs has hard-codings to put test data here.

   * It should NOT be used directly in HBase, as it's a property used in

   *  mini dfs.

   *  @deprecated can be used only with mini dfs

   */

  @Deprecated

  private static final String TEST_DIRECTORY_KEY = "test.build.data";

  /** Filesystem URI used for map-reduce mini-cluster setup */

  private static String FS_URI;

  /** A set of ports that have been claimed using {@link #randomFreePort()}. */

  private static final Set<Integer> takenRandomPorts = new HashSet<Integer>();

  /** Compression algorithms to use in parameterized JUnit 4 tests */

  public static final List<Object[]> COMPRESSION_ALGORITHMS_PARAMETERIZED =

    Arrays.asList(new Object[][] {

      { Compression.Algorithm.NONE },

      { Compression.Algorithm.GZ }

    });

  /** This is for unit tests parameterized with a two booleans. */

  public static final List<Object[]> BOOLEAN_PARAMETERIZED =

      Arrays.asList(new Object[][] {

          { new Boolean(false) },

          { new Boolean(true) }

      });

  /** This is for unit tests parameterized with a single boolean. */

  public static final List<Object[]> MEMSTORETS_TAGS_PARAMETRIZED = memStoreTSAndTagsCombination()  ;

  /** Compression algorithms to use in testing */

  public static final Compression.Algorithm[] COMPRESSION_ALGORITHMS ={

      Compression.Algorithm.NONE, Compression.Algorithm.GZ

    };

  /**

   * Create all combinations of Bloom filters and compression algorithms for

   * testing.

   */

  private static List<Object[]> bloomAndCompressionCombinations() {

    List<Object[]> configurations = new ArrayList<Object[]>();

    for (Compression.Algorithm comprAlgo :

         HBaseTestingUtility.COMPRESSION_ALGORITHMS) {

      for (BloomType bloomType : BloomType.values()) {

        configurations.add(new Object[] { comprAlgo, bloomType });

      }

    }

    return Collections.unmodifiableList(configurations);

  }

  /**

   * Create combination of memstoreTS and tags

   */

  private static List<Object[]> memStoreTSAndTagsCombination() {

    List<Object[]> configurations = new ArrayList<Object[]>();

    configurations.add(new Object[] { false, false });

    configurations.add(new Object[] { false, true });

    configurations.add(new Object[] { true, false });

    configurations.add(new Object[] { true, true });

    return Collections.unmodifiableList(configurations);

  }

  public static final Collection<Object[]> BLOOM_AND_COMPRESSION_COMBINATIONS =

      bloomAndCompressionCombinations();

  public HBaseTestingUtility() {

    this(HBaseConfiguration.create());

  }

  public HBaseTestingUtility(Configuration conf) {

    super(conf);

    // a hbase checksum verification failure will cause unit tests to fail

    ChecksumUtil.generateExceptionForChecksumFailureForTest(true);

  }

  /**

   * Create an HBaseTestingUtility where all tmp files are written to the local test data dir.

   * It is needed to properly base FSUtil.getRootDirs so that they drop temp files in the proper

   * test dir.  Use this when you aren't using an Mini HDFS cluster.

   * @return HBaseTestingUtility that use local fs for temp files.

   */

  public static HBaseTestingUtility createLocalHTU() {

    Configuration c = HBaseConfiguration.create();

    return createLocalHTU(c);

  }

  /**

   * Create an HBaseTestingUtility where all tmp files are written to the local test data dir.

   * It is needed to properly base FSUtil.getRootDirs so that they drop temp files in the proper

   * test dir.  Use this when you aren't using an Mini HDFS cluster.

   * @param c Configuration (will be modified)

   * @return HBaseTestingUtility that use local fs for temp files.

   */

  public static HBaseTestingUtility createLocalHTU(Configuration c) {

    HBaseTestingUtility htu = new HBaseTestingUtility(c);

    String dataTestDir = htu.getDataTestDir().toString();

    htu.getConfiguration().set(HConstants.HBASE_DIR, dataTestDir);

    LOG.debug("Setting " + HConstants.HBASE_DIR + " to " + dataTestDir);

    return htu;

  }

  /**

   * Returns this classes's instance of {@link Configuration}.  Be careful how

   * you use the returned Configuration since {@link HConnection} instances

   * can be shared.  The Map of HConnections is keyed by the Configuration.  If

   * say, a Connection was being used against a cluster that had been shutdown,

   * see {@link #shutdownMiniCluster()}, then the Connection will no longer

   * be wholesome.  Rather than use the return direct, its usually best to

   * make a copy and use that.  Do

   * <code>Configuration c = new Configuration(INSTANCE.getConfiguration());</code>

   * @return Instance of Configuration.

   */

  @Override

  public Configuration getConfiguration() {

    return super.getConfiguration();

  }

  public void setHBaseCluster(HBaseCluster hbaseCluster) {

    this.hbaseCluster = hbaseCluster;

  }

  /**

   * Home our data in a dir under {@link #DEFAULT_BASE_TEST_DIRECTORY}.

   * Give it a random name so can have many concurrent tests running if

   * we need to.  It needs to amend the {@link #TEST_DIRECTORY_KEY}

   * System property, as it's what minidfscluster bases

   * it data dir on.  Moding a System property is not the way to do concurrent

   * instances -- another instance could grab the temporary

   * value unintentionally -- but not anything can do about it at moment;

   * single instance only is how the minidfscluster works.

   *

   * We also create the underlying directory for

   *  hadoop.log.dir, mapred.local.dir and hadoop.tmp.dir, and set the values

   *  in the conf, and as a system property for hadoop.tmp.dir

   *

   * @return The calculated data test build directory, if newly-created.

   */

  @Override

  protected Path setupDataTestDir() {

    Path testPath = super.setupDataTestDir();

    if (null == testPath) {

      return null;

    }

    createSubDirAndSystemProperty(

      "hadoop.log.dir",

      testPath, "hadoop-log-dir");

    // This is defaulted in core-default.xml to /tmp/hadoop-${user.name}, but

    //  we want our own value to ensure uniqueness on the same machine

    createSubDirAndSystemProperty(

      "hadoop.tmp.dir",

      testPath, "hadoop-tmp-dir");

    // Read and modified in org.apache.hadoop.mapred.MiniMRCluster

    createSubDir(

      "mapred.local.dir",

      testPath, "mapred-local-dir");

    return testPath;

  }

  private void createSubDirAndSystemProperty(

    String propertyName, Path parent, String subDirName){

    String sysValue = System.getProperty(propertyName);

    if (sysValue != null) {

      // There is already a value set. So we do nothing but hope

      //  that there will be no conflicts

      LOG.info("System.getProperty(\""+propertyName+"\") already set to: "+

        sysValue + " so I do NOT create it in " + parent);

      String confValue = conf.get(propertyName);

      if (confValue != null && !confValue.endsWith(sysValue)){

       LOG.warn(

         propertyName + " property value differs in configuration and system: "+

         "Configuration="+confValue+" while System="+sysValue+

         " Erasing configuration value by system value."

       );

      }

      conf.set(propertyName, sysValue);

    } else {

      // Ok, it's not set, so we create it as a subdirectory

      createSubDir(propertyName, parent, subDirName);

      System.setProperty(propertyName, conf.get(propertyName));

    }

  }

  /**

   * @return Where to write test data on the test filesystem; Returns working directory

   * for the test filesystem by default

   * @see #setupDataTestDirOnTestFS()

   * @see #getTestFileSystem()

   */

  private Path getBaseTestDirOnTestFS() throws IOException {

    FileSystem fs = getTestFileSystem();

    return new Path(fs.getWorkingDirectory(), "test-data");

  }

  /**

   * @return Where the DFS cluster will write data on the local subsystem.

   * Creates it if it does not exist already.  A subdir of {@link #getBaseTestDir()}

   * @see #getTestFileSystem()

   */

  Path getClusterTestDir() {

    if (clusterTestDir == null){

      setupClusterTestDir();

    }

    return new Path(clusterTestDir.getAbsolutePath());

  }

  /**

   * Creates a directory for the DFS cluster, under the test data

   */

  private void setupClusterTestDir() {

    if (clusterTestDir != null) {

      return;

    }

    // Using randomUUID ensures that multiple clusters can be launched by

    //  a same test, if it stops & starts them

    Path testDir = getDataTestDir("dfscluster_" + UUID.randomUUID().toString());

    clusterTestDir = new File(testDir.toString()).getAbsoluteFile();

    // Have it cleaned up on exit

    boolean b = deleteOnExit();

    if (b) clusterTestDir.deleteOnExit();

    conf.set(TEST_DIRECTORY_KEY, clusterTestDir.getPath());

    LOG.info("Created new mini-cluster data directory: " + clusterTestDir + ", deleteOnExit=" + b);

  }

  /**

   * Returns a Path in the test filesystem, obtained from {@link #getTestFileSystem()}

   * to write temporary test data. Call this method after setting up the mini dfs cluster

   * if the test relies on it.

   * @return a unique path in the test filesystem

   */

  public Path getDataTestDirOnTestFS() throws IOException {

    if (dataTestDirOnTestFS == null) {

      setupDataTestDirOnTestFS();

    }

    return dataTestDirOnTestFS;

  }

  /**

   * Returns a Path in the test filesystem, obtained from {@link #getTestFileSystem()}

   * to write temporary test data. Call this method after setting up the mini dfs cluster

   * if the test relies on it.

   * @return a unique path in the test filesystem

   * @param subdirName name of the subdir to create under the base test dir

   */

  public Path getDataTestDirOnTestFS(final String subdirName) throws IOException {

    return new Path(getDataTestDirOnTestFS(), subdirName);

  }

  /**

   * Sets up a path in test filesystem to be used by tests

   */

  private void setupDataTestDirOnTestFS() throws IOException {

    if (dataTestDirOnTestFS != null) {

      LOG.warn("Data test on test fs dir already setup in "

          + dataTestDirOnTestFS.toString());

      return;

    }

    //The file system can be either local, mini dfs, or if the configuration