*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* This tests HBaseFsck's ability to detect reasons for inconsistent tables.

*/

  final static Log LOG = LogFactory.getLog(TestHBaseFsck.class);

  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();

  private final static Configuration conf = TEST_UTIL.getConfiguration();

  private final static String FAM_STR = "fam";

  private final static byte[] FAM = Bytes.toBytes(FAM_STR);

  private final static int REGION_ONLINE_TIMEOUT = 800;

  private static RegionStates regionStates;

  private static ExecutorService executorService;

  // for the instance, reset every test run

  private HTable tbl;

  private final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("A"),

    Bytes.toBytes("B"), Bytes.toBytes("C") };

  // one row per region.

  private final static byte[][] ROWKEYS= new byte[][] {

    Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"),

    Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") };

  @BeforeClass

  public static void setUpBeforeClass() throws Exception {

    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.handler.count", 2);

    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.metahandler.count", 2);

    TEST_UTIL.startMiniCluster(3);

    executorService = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 60, TimeUnit.SECONDS,

        new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("testhbck"));

    AssignmentManager assignmentManager =

      TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();

    regionStates = assignmentManager.getRegionStates();

  }

  @AfterClass

  public static void tearDownAfterClass() throws Exception {

    TEST_UTIL.shutdownMiniCluster();

  }

  @Test

  public void testHBaseFsck() throws Exception {

    assertNoErrors(doFsck(conf, false));

    String table = "tableBadMetaAssign";

    TEST_UTIL.createTable(Bytes.toBytes(table), FAM);

    // We created 1 table, should be fine

    assertNoErrors(doFsck(conf, false));

    // Now let's mess it up and change the assignment in hbase:meta to

    // point to a different region server

    HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getTableName(),

        executorService);

    Scan scan = new Scan();

    scan.setStartRow(Bytes.toBytes(table+",,"));

    ResultScanner scanner = meta.getScanner(scan);

    HRegionInfo hri = null;

    Result res = scanner.next();

    ServerName currServer =

      ServerName.parseFrom(res.getValue(HConstants.CATALOG_FAMILY,

          HConstants.SERVER_QUALIFIER));

    long startCode = Bytes.toLong(res.getValue(HConstants.CATALOG_FAMILY,

        HConstants.STARTCODE_QUALIFIER));

    for (JVMClusterUtil.RegionServerThread rs :

        TEST_UTIL.getHBaseCluster().getRegionServerThreads()) {

      ServerName sn = rs.getRegionServer().getServerName();

      // When we find a diff RS, change the assignment and break

      if (!currServer.getHostAndPort().equals(sn.getHostAndPort()) ||

          startCode != sn.getStartcode()) {

        Put put = new Put(res.getRow());

        put.setDurability(Durability.SKIP_WAL);

        put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,

          Bytes.toBytes(sn.getHostAndPort()));

        put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,

          Bytes.toBytes(sn.getStartcode()));

        meta.put(put);

        hri = HRegionInfo.getHRegionInfo(res);

        break;

      }

    }

    // Try to fix the data

    assertErrors(doFsck(conf, true), new ERROR_CODE[]{

        ERROR_CODE.SERVER_DOES_NOT_MATCH_META});

    TEST_UTIL.getHBaseCluster().getMaster()

      .getAssignmentManager().waitForAssignment(hri);

    // Should be fixed now

    assertNoErrors(doFsck(conf, false));

    // comment needed - what is the purpose of this line

    HTable t = new HTable(conf, Bytes.toBytes(table), executorService);

    ResultScanner s = t.getScanner(new Scan());

    s.close();

    t.close();

    scanner.close();

    meta.close();

  }

  @Test(timeout=180000)

  public void testFixAssignmentsWhenMETAinTransition() throws Exception {

    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();

    HBaseAdmin admin = null;

    try {

      admin = new HBaseAdmin(TEST_UTIL.getConfiguration());

      admin.closeRegion(cluster.getServerHoldingMeta(),

          HRegionInfo.FIRST_META_REGIONINFO);

    } finally {

      if (admin != null) {

        admin.close();

      }

    }

    regionStates.regionOffline(HRegionInfo.FIRST_META_REGIONINFO);

    MetaRegionTracker.deleteMetaLocation(cluster.getMaster().getZooKeeper());

    assertFalse(regionStates.isRegionOnline(HRegionInfo.FIRST_META_REGIONINFO));

    HBaseFsck hbck = doFsck(conf, true);

    assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.UNKNOWN, ERROR_CODE.NO_META_REGION,

        ERROR_CODE.NULL_META_REGION });

    assertNoErrors(doFsck(conf, false));

  }

  /**

   * Create a new region in META.

   */

  private HRegionInfo createRegion(Configuration conf, final HTableDescriptor

      htd, byte[] startKey, byte[] endKey)

      throws IOException {

    HTable meta = new HTable(conf, TableName.META_TABLE_NAME, executorService);

    HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);

    MetaEditor.addRegionToMeta(meta, hri);

    meta.close();

    return hri;

  }

  /**

   * Debugging method to dump the contents of meta.

   */

  private void dumpMeta(TableName tableName) throws IOException {

    List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(tableName);

    for (byte[] row : metaRows) {

      LOG.info(Bytes.toString(row));

    }

  }

  /**

   * This method is used to undeploy a region -- close it and attempt to

   * remove its state from the Master.

   */

  private void undeployRegion(HBaseAdmin admin, ServerName sn,

      HRegionInfo hri) throws IOException, InterruptedException {

    try {

      HBaseFsckRepair.closeRegionSilentlyAndWait(admin, sn, hri);

      if (!hri.isMetaTable()) {

        admin.offline(hri.getRegionName());

      }

    } catch (IOException ioe) {

      LOG.warn("Got exception when attempting to offline region "

          + Bytes.toString(hri.getRegionName()), ioe);

    }

  }

  /**

   * Delete a region from assignments, meta, or completely from hdfs.

   * @param unassign if true unassign region if assigned

   * @param metaRow  if true remove region's row from META

   * @param hdfs if true remove region's dir in HDFS

   */

  private void deleteRegion(Configuration conf, final HTableDescriptor htd,

      byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,

      boolean hdfs) throws IOException, InterruptedException {

    deleteRegion(conf, htd, startKey, endKey, unassign, metaRow, hdfs, false);

  }

  /**

   * Delete a region from assignments, meta, or completely from hdfs.

   * @param unassign if true unassign region if assigned

   * @param metaRow  if true remove region's row from META

   * @param hdfs if true remove region's dir in HDFS

   * @param regionInfoOnly if true remove a region dir's .regioninfo file

   */

  private void deleteRegion(Configuration conf, final HTableDescriptor htd,

      byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,

      boolean hdfs, boolean regionInfoOnly) throws IOException, InterruptedException {

    LOG.info("** Before delete:");

    dumpMeta(htd.getTableName());

    Map<HRegionInfo, ServerName> hris = tbl.getRegionLocations();

    for (Entry<HRegionInfo, ServerName> e: hris.entrySet()) {

      HRegionInfo hri = e.getKey();

      ServerName hsa = e.getValue();

      if (Bytes.compareTo(hri.getStartKey(), startKey) == 0

          && Bytes.compareTo(hri.getEndKey(), endKey) == 0) {

        LOG.info("RegionName: " +hri.getRegionNameAsString());

        byte[] deleteRow = hri.getRegionName();

        if (unassign) {

          LOG.info("Undeploying region " + hri + " from server " + hsa);

          undeployRegion(new HBaseAdmin(conf), hsa, hri);

        }

        if (regionInfoOnly) {

          LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());

          Path rootDir = FSUtils.getRootDir(conf);

          FileSystem fs = rootDir.getFileSystem(conf);

          Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),

              hri.getEncodedName());

          Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);

          fs.delete(hriPath, true);

        }

        if (hdfs) {

          LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());

          Path rootDir = FSUtils.getRootDir(conf);

          FileSystem fs = rootDir.getFileSystem(conf);

          Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),

              hri.getEncodedName());

          HBaseFsck.debugLsr(conf, p);

          boolean success = fs.delete(p, true);

          LOG.info("Deleted " + p + " sucessfully? " + success);

          HBaseFsck.debugLsr(conf, p);

        }

        if (metaRow) {

          HTable meta = new HTable(conf, TableName.META_TABLE_NAME, executorService);

          Delete delete = new Delete(deleteRow);

          meta.delete(delete);

        }

      }

      LOG.info(hri.toString() + hsa.toString());

    }

    TEST_UTIL.getMetaTableRows(htd.getTableName());

    LOG.info("*** After delete:");

    dumpMeta(htd.getTableName());

  }

  /**

   * Setup a clean table before we start mucking with it.

   *

   * @throws IOException

   * @throws InterruptedException

   * @throws KeeperException

   */

  HTable setupTable(TableName tablename) throws Exception {

    HTableDescriptor desc = new HTableDescriptor(tablename);

    HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));

    desc.addFamily(hcd); // If a table has no CF's it doesn't get checked

    TEST_UTIL.getHBaseAdmin().createTable(desc, SPLITS);

    tbl = new HTable(TEST_UTIL.getConfiguration(), tablename, executorService);

    List<Put> puts = new ArrayList<Put>();

    for (byte[] row : ROWKEYS) {

      Put p = new Put(row);

      p.add(FAM, Bytes.toBytes("val"), row);

      puts.add(p);

    }

    tbl.put(puts);

    tbl.flushCommits();

    return tbl;

  }

  /**

   * Counts the number of row to verify data loss or non-dataloss.

   */

  int countRows() throws IOException {

     Scan s = new Scan();

     ResultScanner rs = tbl.getScanner(s);

     int i = 0;

     while(rs.next() !=null) {

       i++;

     }

     return i;

  }

  /**

   * delete table in preparation for next test

   *

   * @param tablename

   * @throws IOException

   */

  void deleteTable(TableName tablename) throws IOException {

    HBaseAdmin admin = new HBaseAdmin(conf);

    admin.getConnection().clearRegionCache();

    if (admin.isTableEnabled(tablename)) {

      admin.disableTableAsync(tablename);

    }

    long totalWait = 0;

    long maxWait = 30*1000;

    long sleepTime = 250;

    while (!admin.isTableDisabled(tablename)) {

      try {

        Thread.sleep(sleepTime);

        totalWait += sleepTime;

        if (totalWait >= maxWait) {

          fail("Waited too long for table to be disabled + " + tablename);

        }

      } catch (InterruptedException e) {

        e.printStackTrace();

        fail("Interrupted when trying to disable table " + tablename);

      }

    }

    admin.deleteTable(tablename);

  }

  /**

   * This creates a clean table and confirms that the table is clean.

   */

  @Test

  public void testHBaseFsckClean() throws Exception {

    assertNoErrors(doFsck(conf, false));

    TableName table = TableName.valueOf("tableClean");

    try {

      HBaseFsck hbck = doFsck(conf, false);

      assertNoErrors(hbck);

      setupTable(table);

      assertEquals(ROWKEYS.length, countRows());

      // We created 1 table, should be fine

      hbck = doFsck(conf, false);

      assertNoErrors(hbck);

      assertEquals(0, hbck.getOverlapGroups(table).size());

      assertEquals(ROWKEYS.length, countRows());

    } finally {

      deleteTable(table);

    }

  }

  /**

   * Test thread pooling in the case where there are more regions than threads

   */

  @Test

  public void testHbckThreadpooling() throws Exception {

    TableName table =

        TableName.valueOf("tableDupeStartKey");

    try {

      // Create table with 4 regions

      setupTable(table);

      // limit number of threads to 1.

      Configuration newconf = new Configuration(conf);

      newconf.setInt("hbasefsck.numthreads", 1);

      assertNoErrors(doFsck(newconf, false));

      // We should pass without triggering a RejectedExecutionException

    } finally {

      deleteTable(table);

    }

  }

  @Test

  public void testHbckFixOrphanTable() throws Exception {

    TableName table = TableName.valueOf("tableInfo");

    FileSystem fs = null;

    Path tableinfo = null;

    try {

      setupTable(table);

      HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();

      Path hbaseTableDir = FSUtils.getTableDir(

          FSUtils.getRootDir(conf), table);

      fs = hbaseTableDir.getFileSystem(conf);

      FileStatus status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);

      tableinfo = status.getPath();

      fs.rename(tableinfo, new Path("/.tableinfo"));

      //to report error if .tableinfo is missing.

      HBaseFsck hbck = doFsck(conf, false);

      assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_TABLEINFO_FILE });

      // fix OrphanTable with default .tableinfo (htd not yet cached on master)

      hbck = doFsck(conf, true);

      assertNoErrors(hbck);

      status = null;

      status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);

      assertNotNull(status);

      HTableDescriptor htd = admin.getTableDescriptor(table);

      htd.setValue("NOT_DEFAULT", "true");

      admin.disableTable(table);

      admin.modifyTable(table, htd);

      admin.enableTable(table);

      fs.delete(status.getPath(), true);

      // fix OrphanTable with cache

      htd = admin.getTableDescriptor(table); // warms up cached htd on master

      hbck = doFsck(conf, true);

      assertNoErrors(hbck);

      status = null;

      status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);

      assertNotNull(status);

      htd = admin.getTableDescriptor(table);

      assertEquals(htd.getValue("NOT_DEFAULT"), "true");

    } finally {

      fs.rename(new Path("/.tableinfo"), tableinfo);

      deleteTable(table);

    }

  }

  /**

   * This create and fixes a bad table with regions that have a duplicate

   * start key

   */

  @Test

  public void testDupeStartKey() throws Exception {

    TableName table =

        TableName.valueOf("tableDupeStartKey");

    try {

      setupTable(table);

      assertNoErrors(doFsck(conf, false));

      assertEquals(ROWKEYS.length, countRows());

      // Now let's mess it up, by adding a region with a duplicate startkey

      HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(),

          Bytes.toBytes("A"), Bytes.toBytes("A2"));

      TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);

      TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()

          .waitForAssignment(hriDupe);

      ServerName server = regionStates.getRegionServerOfRegion(hriDupe);

      TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);

      HBaseFsck hbck = doFsck(conf, false);

      assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,

            ERROR_CODE.DUPE_STARTKEYS});

      assertEquals(2, hbck.getOverlapGroups(table).size());

      assertEquals(ROWKEYS.length, countRows()); // seems like the "bigger" region won.

      // fix the degenerate region.

      doFsck(conf,true);

      // check that the degenerate region is gone and no data loss

      HBaseFsck hbck2 = doFsck(conf,false);

      assertNoErrors(hbck2);

      assertEquals(0, hbck2.getOverlapGroups(table).size());

      assertEquals(ROWKEYS.length, countRows());

    } finally {

      deleteTable(table);

    }

  }

  /**

   * Get region info from local cluster.

   */

  Map<ServerName, List<String>> getDeployedHRIs(

      final HBaseAdmin admin) throws IOException {

    ClusterStatus status = admin.getClusterStatus();

    Collection<ServerName> regionServers = status.getServers();

    Map<ServerName, List<String>> mm =

        new HashMap<ServerName, List<String>>();

    HConnection connection = admin.getConnection();

    for (ServerName hsi : regionServers) {

      AdminProtos.AdminService.BlockingInterface server = connection.getAdmin(hsi);

      // list all online regions from this region server

      List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);

      List<String> regionNames = new ArrayList<String>();

      for (HRegionInfo hri : regions) {

        regionNames.add(hri.getRegionNameAsString());

      }

      mm.put(hsi, regionNames);

    }

    return mm;

  }

  /**

   * Returns the HSI a region info is on.

   */

  ServerName findDeployedHSI(Map<ServerName, List<String>> mm, HRegionInfo hri) {

    for (Map.Entry<ServerName,List <String>> e : mm.entrySet()) {

      if (e.getValue().contains(hri.getRegionNameAsString())) {

        return e.getKey();

      }

    }

    return null;

  }

  /**

   * This create and fixes a bad table with regions that have a duplicate

   * start key

   */

  @Test

  public void testDupeRegion() throws Exception {

    TableName table =

        TableName.valueOf("tableDupeRegion");

    try {

      setupTable(table);

      assertNoErrors(doFsck(conf, false));

      assertEquals(ROWKEYS.length, countRows());

      // Now let's mess it up, by adding a region with a duplicate startkey

      HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(),

          Bytes.toBytes("A"), Bytes.toBytes("B"));

      TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);

      TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()

          .waitForAssignment(hriDupe);

      ServerName server = regionStates.getRegionServerOfRegion(hriDupe);

      TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);

      // Yikes! The assignment manager can't tell between diff between two

      // different regions with the same start/endkeys since it doesn't

      // differentiate on ts/regionId!  We actually need to recheck

      // deployments!

      HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();

      while (findDeployedHSI(getDeployedHRIs(admin), hriDupe) == null) {

        Thread.sleep(250);

      }

      LOG.debug("Finished assignment of dupe region");

      // TODO why is dupe region different from dupe start keys?

      HBaseFsck hbck = doFsck(conf, false);

      assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,

            ERROR_CODE.DUPE_STARTKEYS});

      assertEquals(2, hbck.getOverlapGroups(table).size());

      assertEquals(ROWKEYS.length, countRows()); // seems like the "bigger" region won.

      // fix the degenerate region.

      doFsck(conf,true);

      // check that the degenerate region is gone and no data loss

      HBaseFsck hbck2 = doFsck(conf,false);

      assertNoErrors(hbck2);

      assertEquals(0, hbck2.getOverlapGroups(table).size());

      assertEquals(ROWKEYS.length, countRows());

    } finally {

      deleteTable(table);

    }

  }

  /**

   * This creates and fixes a bad table with regions that has startkey == endkey

   */

  @Test

  public void testDegenerateRegions() throws Exception {

    TableName table =

        TableName.valueOf("tableDegenerateRegions");

    try {

      setupTable(table);

      assertNoErrors(doFsck(conf,false));

      assertEquals(ROWKEYS.length, countRows());

      // Now let's mess it up, by adding a region with a duplicate startkey

      HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(),

          Bytes.toBytes("B"), Bytes.toBytes("B"));

      TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);

      TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()

          .waitForAssignment(hriDupe);

      ServerName server = regionStates.getRegionServerOfRegion(hriDupe);

      TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);

      HBaseFsck hbck = doFsck(conf,false);

      assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DEGENERATE_REGION,

          ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.DUPE_STARTKEYS});

      assertEquals(2, hbck.getOverlapGroups(table).size());

      assertEquals(ROWKEYS.length, countRows());

      // fix the degenerate region.

      doFsck(conf,true);

      // check that the degenerate region is gone and no data loss

      HBaseFsck hbck2 = doFsck(conf,false);

      assertNoErrors(hbck2);

      assertEquals(0, hbck2.getOverlapGroups(table).size());

      assertEquals(ROWKEYS.length, countRows());

    } finally {

      deleteTable(table);

    }

  }

  /**

   * This creates and fixes a bad table where a region is completely contained

   * by another region.

   */

  @Test

  public void testContainedRegionOverlap() throws Exception {

    TableName table =

        TableName.valueOf("tableContainedRegionOverlap");

    try {

      setupTable(table);

      assertEquals(ROWKEYS.length, countRows());

      // Mess it up by creating an overlap in the metadata

      HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(),

          Bytes.toBytes("A2"), Bytes.toBytes("B"));

      TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);

      TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()

          .waitForAssignment(hriOverlap);

      ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);

      TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);

      HBaseFsck hbck = doFsck(conf, false);

      assertErrors(hbck, new ERROR_CODE[] {

          ERROR_CODE.OVERLAP_IN_REGION_CHAIN });

      assertEquals(2, hbck.getOverlapGroups(table).size());

      assertEquals(ROWKEYS.length, countRows());

      // fix the problem.

      doFsck(conf, true);

      // verify that overlaps are fixed

      HBaseFsck hbck2 = doFsck(conf,false);

      assertNoErrors(hbck2);

      assertEquals(0, hbck2.getOverlapGroups(table).size());

      assertEquals(ROWKEYS.length, countRows());

    } finally {

       deleteTable(table);

    }

  }

  /**

   * This creates and fixes a bad table where an overlap group of

   * 3 regions. Set HBaseFsck.maxMerge to 2 to trigger sideline overlapped

   * region. Mess around the meta data so that closeRegion/offlineRegion

   * throws exceptions.

   */

  @Test

  public void testSidelineOverlapRegion() throws Exception {

    TableName table =

        TableName.valueOf("testSidelineOverlapRegion");

    try {

      setupTable(table);

      assertEquals(ROWKEYS.length, countRows());

      // Mess it up by creating an overlap

      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();

      HMaster master = cluster.getMaster();

      HRegionInfo hriOverlap1 = createRegion(conf, tbl.getTableDescriptor(),

        Bytes.toBytes("A"), Bytes.toBytes("AB"));

      master.assignRegion(hriOverlap1);

      master.getAssignmentManager().waitForAssignment(hriOverlap1);

      HRegionInfo hriOverlap2 = createRegion(conf, tbl.getTableDescriptor(),

        Bytes.toBytes("AB"), Bytes.toBytes("B"));

      master.assignRegion(hriOverlap2);

      master.getAssignmentManager().waitForAssignment(hriOverlap2);

      HBaseFsck hbck = doFsck(conf, false);

      assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.DUPE_STARTKEYS,

        ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.OVERLAP_IN_REGION_CHAIN});

      assertEquals(3, hbck.getOverlapGroups(table).size());

      assertEquals(ROWKEYS.length, countRows());

      // mess around the overlapped regions, to trigger NotServingRegionException

      Multimap<byte[], HbckInfo> overlapGroups = hbck.getOverlapGroups(table);

      ServerName serverName = null;

      byte[] regionName = null;

      for (HbckInfo hbi: overlapGroups.values()) {

        if ("A".equals(Bytes.toString(hbi.getStartKey()))

            && "B".equals(Bytes.toString(hbi.getEndKey()))) {

          regionName = hbi.getRegionName();

          // get an RS not serving the region to force bad assignment info in to META.

          int k = cluster.getServerWith(regionName);

          for (int i = 0; i < 3; i++) {

            if (i != k) {

              HRegionServer rs = cluster.getRegionServer(i);

              serverName = rs.getServerName();

              break;

            }

          }

          HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();

          HBaseFsckRepair.closeRegionSilentlyAndWait(admin,

            cluster.getRegionServer(k).getServerName(), hbi.getHdfsHRI());

          admin.offline(regionName);

          break;

        }

      }

      assertNotNull(regionName);

      assertNotNull(serverName);

      HTable meta = new HTable(conf, TableName.META_TABLE_NAME, executorService);

      Put put = new Put(regionName);

      put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,

        Bytes.toBytes(serverName.getHostAndPort()));

      meta.put(put);

      // fix the problem.

      HBaseFsck fsck = new HBaseFsck(conf);

      fsck.connect();

      fsck.setDisplayFullReport(); // i.e. -details

      fsck.setTimeLag(0);

      fsck.setFixAssignments(true);

      fsck.setFixMeta(true);

      fsck.setFixHdfsHoles(true);

      fsck.setFixHdfsOverlaps(true);

      fsck.setFixHdfsOrphans(true);

      fsck.setFixVersionFile(true);

      fsck.setSidelineBigOverlaps(true);

      fsck.setMaxMerge(2);

      fsck.onlineHbck();

      // verify that overlaps are fixed, and there are less rows

      // since one region is sidelined.

      HBaseFsck hbck2 = doFsck(conf,false);

      assertNoErrors(hbck2);

      assertEquals(0, hbck2.getOverlapGroups(table).size());

      assertTrue(ROWKEYS.length > countRows());

    } finally {

       deleteTable(table);

    }

  }

  /**

   * This creates and fixes a bad table where a region is completely contained

   * by another region, and there is a hole (sort of like a bad split)

   */

  @Test

  public void testOverlapAndOrphan() throws Exception {

    TableName table =

        TableName.valueOf("tableOverlapAndOrphan");

    try {

      setupTable(table);

      assertEquals(ROWKEYS.length, countRows());

      // Mess it up by creating an overlap in the metadata

      TEST_UTIL.getHBaseAdmin().disableTable(table);

      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),

          Bytes.toBytes("B"), true, true, false, true);

      TEST_UTIL.getHBaseAdmin().enableTable(table);