public void testCorrectnessWhenMasterFailOver() throws Exception {
final byte[] TABLENAME = Bytes.toBytes("testCorrectnessWhenMasterFailOver");
final byte[] FAMILY = Bytes.toBytes("family");
final byte[][] SPLITKEYS = { Bytes.toBytes("b"), Bytes.toBytes("i") };
MiniHBaseCluster cluster = TESTUTIL.getHBaseCluster();
HTableDescriptor desc = new HTableDescriptor(TABLENAME);
desc.addFamily(new HColumnDescriptor(FAMILY));
HBaseAdmin hbaseAdmin = TESTUTIL.getHBaseAdmin();
hbaseAdmin.createTable(desc, SPLITKEYS);
assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
HTable table = new HTable(TESTUTIL.getConfiguration(), TABLENAME);
List<Put> puts = new ArrayList<Put>();
Put put1 = new Put(Bytes.toBytes("a"));
put1.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
Put put2 = new Put(Bytes.toBytes("h"));
put2.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
Put put3 = new Put(Bytes.toBytes("o"));
put3.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
puts.add(put1);
puts.add(put2);
puts.add(put3);
table.put(puts);
ResultScanner resultScanner = table.getScanner(new Scan());
int count = 0;
while (resultScanner.next() != null) {
count++;
}
resultScanner.close();
table.close();
assertEquals(3, count);
/* Starting test */
cluster.getConfiguration().setBoolean("TestingMaster.sleep", true);
cluster.getConfiguration().setInt("TestingMaster.sleep.duration", 10000);
/* NO.1 .META. region correctness */
// First abort master
abortMaster(cluster);
TestingMaster master = startMasterAndWaitUntilLogSplit(cluster);
// Second kill meta server
int metaServerNum = cluster.getServerWithMeta();
int rootServerNum = cluster.getServerWith(HRegionInfo.ROOT_REGIONINFO
.getRegionName());
HRegionServer metaRS = cluster.getRegionServer(metaServerNum);
LOG.debug("Killing metaRS and carryingRoot = "
+ (metaServerNum == rootServerNum));
metaRS.kill();
metaRS.join();
/*
* Sleep double time of TestingMaster.sleep.duration, so we can ensure that
* master has already assigned ROOTandMETA or is blocking on assigning
* ROOTandMETA
*/
Thread.sleep(10000 * 2);
waitUntilMasterIsInitialized(master);
// Third check whether data is correct in meta region
assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
/*
* NO.2 -ROOT- region correctness . If the .META. server killed in the NO.1
* is also carrying -ROOT- region, it is not needed
*/
if (rootServerNum != metaServerNum) {
// First abort master
abortMaster(cluster);
master = startMasterAndWaitUntilLogSplit(cluster);
// Second kill meta server
HRegionServer rootRS = cluster.getRegionServer(rootServerNum);
LOG.debug("Killing rootRS");
rootRS.kill();
rootRS.join();
/*
* Sleep double time of TestingMaster.sleep.duration, so we can ensure
* that master has already assigned ROOTandMETA or is blocking on
* assigning ROOTandMETA
*/
Thread.sleep(10000 * 2);
waitUntilMasterIsInitialized(master);
// Third check whether data is correct in meta region
assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
}
/* NO.3 data region correctness */
ServerManager serverManager = cluster.getMaster().getServerManager();
while (serverManager.areDeadServersInProgress()) {
Thread.sleep(100);
}
// Create a ZKW to use in the test
ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTUTIL);