Configuration conf = HBaseConfiguration.create();
// Start the cluster
HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
log("Cluster started");
// Create a ZKW to use in the test
ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
// get all the master threads
List<MasterThread> masterThreads = cluster.getMasterThreads();
assertEquals(1, masterThreads.size());
// only one master thread, let's wait for it to be initialized
assertTrue(cluster.waitForActiveAndReadyMaster());
HMaster master = masterThreads.get(0).getMaster();
assertTrue(master.isActiveMaster());
assertTrue(master.isInitialized());
// disable load balancing on this master
master.balanceSwitch(false);
// create two tables in META, each with 10 regions
byte [] FAMILY = Bytes.toBytes("family");
byte [][] SPLIT_KEYS = new byte [][] {
new byte[0], Bytes.toBytes("aaa"), Bytes.toBytes("bbb"),
Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
Bytes.toBytes("iii"), Bytes.toBytes("jjj")
};
byte [] enabledTable = Bytes.toBytes("enabledTable");
HTableDescriptor htdEnabled = new HTableDescriptor(TableName.valueOf(enabledTable));
htdEnabled.addFamily(new HColumnDescriptor(FAMILY));
FileSystem filesystem = FileSystem.get(conf);
Path rootdir = FSUtils.getRootDir(conf);
FSTableDescriptors fstd = new FSTableDescriptors(filesystem, rootdir);
// Write the .tableinfo
fstd.createTableDescriptor(htdEnabled);
HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getTableName(), null, null);
createRegion(hriEnabled, rootdir, conf, htdEnabled);
List<HRegionInfo> enabledRegions = TEST_UTIL.createMultiRegionsInMeta(
TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS);
TableName disabledTable = TableName.valueOf("disabledTable");
HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
// Write the .tableinfo
fstd.createTableDescriptor(htdDisabled);
HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getTableName(), null, null);
createRegion(hriDisabled, rootdir, conf, htdDisabled);
List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
TableName tableWithMergingRegions = TableName.valueOf("tableWithMergingRegions");
TEST_UTIL.createTable(tableWithMergingRegions, FAMILY, new byte [][] {Bytes.toBytes("m")});
log("Regions in hbase:meta and namespace have been created");
// at this point we only expect 4 regions to be assigned out
// (catalogs and namespace, + 2 merging regions)
assertEquals(4, cluster.countServedRegions());
// Move merging regions to the same region server
AssignmentManager am = master.getAssignmentManager();
RegionStates regionStates = am.getRegionStates();
List<HRegionInfo> mergingRegions = regionStates.getRegionsOfTable(tableWithMergingRegions);
assertEquals(2, mergingRegions.size());
HRegionInfo a = mergingRegions.get(0);
HRegionInfo b = mergingRegions.get(1);
HRegionInfo newRegion = RegionMergeTransaction.getMergedRegionInfo(a, b);
ServerName mergingServer = regionStates.getRegionServerOfRegion(a);
ServerName serverB = regionStates.getRegionServerOfRegion(b);
if (!serverB.equals(mergingServer)) {
RegionPlan plan = new RegionPlan(b, serverB, mergingServer);
am.balance(plan);
assertTrue(am.waitForAssignment(b));
}
// Let's just assign everything to first RS
HRegionServer hrs = cluster.getRegionServer(0);
ServerName serverName = hrs.getServerName();
HRegionInfo closingRegion = enabledRegions.remove(0);
// we'll need some regions to already be assigned out properly on live RS
List<HRegionInfo> enabledAndAssignedRegions = new ArrayList<HRegionInfo>();
enabledAndAssignedRegions.add(enabledRegions.remove(0));
enabledAndAssignedRegions.add(enabledRegions.remove(0));
enabledAndAssignedRegions.add(closingRegion);
List<HRegionInfo> disabledAndAssignedRegions = new ArrayList<HRegionInfo>();
disabledAndAssignedRegions.add(disabledRegions.remove(0));
disabledAndAssignedRegions.add(disabledRegions.remove(0));
// now actually assign them
for (HRegionInfo hri : enabledAndAssignedRegions) {
master.assignmentManager.regionPlans.put(hri.getEncodedName(),
new RegionPlan(hri, null, serverName));
master.assignRegion(hri);
}
for (HRegionInfo hri : disabledAndAssignedRegions) {
master.assignmentManager.regionPlans.put(hri.getEncodedName(),
new RegionPlan(hri, null, serverName));
master.assignRegion(hri);
}
// wait for no more RIT
log("Waiting for assignment to finish");
ZKAssign.blockUntilNoRIT(zkw);
log("Assignment completed");
// Stop the master
log("Aborting master");
cluster.abortMaster(0);
cluster.waitOnMaster(0);
log("Master has aborted");
/*
* Now, let's start mocking up some weird states as described in the method
* javadoc.
*/
List<HRegionInfo> regionsThatShouldBeOnline = new ArrayList<HRegionInfo>();
List<HRegionInfo> regionsThatShouldBeOffline = new ArrayList<HRegionInfo>();
log("Beginning to mock scenarios");
// Disable the disabledTable in ZK
ZKTable zktable = new ZKTable(zkw);
zktable.setDisabledTable(disabledTable);
/*
* ZK = OFFLINE
*/
// Region that should be assigned but is not and is in ZK as OFFLINE
// Cause: This can happen if the master crashed after creating the znode but before sending the
// request to the region server
HRegionInfo region = enabledRegions.remove(0);
regionsThatShouldBeOnline.add(region);
ZKAssign.createNodeOffline(zkw, region, serverName);
/*
* ZK = CLOSING
*/
// Cause: Same as offline.
regionsThatShouldBeOnline.add(closingRegion);
ZKAssign.createNodeClosing(zkw, closingRegion, serverName);
/*
* ZK = CLOSED
*/
// Region of enabled table closed but not ack
//Cause: Master was down while the region server updated the ZK status.
region = enabledRegions.remove(0);
regionsThatShouldBeOnline.add(region);
int version = ZKAssign.createNodeClosing(zkw, region, serverName);
ZKAssign.transitionNodeClosed(zkw, region, serverName, version);
// Region of disabled table closed but not ack
region = disabledRegions.remove(0);
regionsThatShouldBeOffline.add(region);
version = ZKAssign.createNodeClosing(zkw, region, serverName);
ZKAssign.transitionNodeClosed(zkw, region, serverName, version);
/*
* ZK = OPENED
*/
// Region of enabled table was opened on RS
// Cause: as offline
region = enabledRegions.remove(0);
regionsThatShouldBeOnline.add(region);
ZKAssign.createNodeOffline(zkw, region, serverName);
ProtobufUtil.openRegion(hrs, region);
while (true) {
byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
RegionTransition rt = RegionTransition.parseFrom(bytes);
if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
break;
}
Thread.sleep(100);
}
// Region of disable table was opened on RS
// Cause: Master failed while updating the status for this region server.
region = disabledRegions.remove(0);
regionsThatShouldBeOffline.add(region);
ZKAssign.createNodeOffline(zkw, region, serverName);
ProtobufUtil.openRegion(hrs, region);
while (true) {
byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
RegionTransition rt = RegionTransition.parseFrom(bytes);
if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
break;
}
Thread.sleep(100);
}
/*
* ZK = MERGING
*/
// Regions of table of merging regions
// Cause: Master was down while merging was going on
RegionMergeTransaction.createNodeMerging(
zkw, newRegion, mergingServer, a, b);
/*
* ZK = NONE
*/
/*
* DONE MOCKING
*/
log("Done mocking data up in ZK");
// Start up a new master
log("Starting up a new master");
master = cluster.startMaster().getMaster();
log("Waiting for master to be ready");
cluster.waitForActiveAndReadyMaster();
log("Master is ready");
// Get new region states since master restarted
regionStates = master.getAssignmentManager().getRegionStates();
// Merging region should remain merging
assertTrue(regionStates.isRegionInState(a, State.MERGING));
assertTrue(regionStates.isRegionInState(b, State.MERGING));
assertTrue(regionStates.isRegionInState(newRegion, State.MERGING_NEW));
// Now remove the faked merging znode, merging regions should be
// offlined automatically, otherwise it is a bug in AM.
ZKAssign.deleteNodeFailSilent(zkw, newRegion);
// Failover should be completed, now wait for no RIT
log("Waiting for no more RIT");
ZKAssign.blockUntilNoRIT(zkw);
log("No more RIT in ZK, now doing final test verification");
// Grab all the regions that are online across RSs
Set<HRegionInfo> onlineRegions = new TreeSet<HRegionInfo>();
for (JVMClusterUtil.RegionServerThread rst :
cluster.getRegionServerThreads()) {
onlineRegions.addAll(ProtobufUtil.getOnlineRegions(rst.getRegionServer()));
}
// Now, everything that should be online should be online
for (HRegionInfo hri : regionsThatShouldBeOnline) {