* More of the same idea as in basicFail() but working with more instances and situations
*/
@Test(groups="complicatedfail", dependsOnGroups="basicfail1")
public void complicatedFail() throws Exception {
logger.debug("complicatedFail");
final Manager rm = this.locator.getManager();
VM[] allvms = rm.getGlobalAll();
assertEquals(0, allvms.length);
MockShutdownTrash.resetFailCount();
final int durationSecs = 240;
final int memory = 64;
final int numNodes = 5;
final CreateRequest creq =
this.populator().getCreateRequest("suite:failure:complicatedFail",
durationSecs, memory, numNodes);
final Caller caller = this.populator().getCaller();
final CreateResult result = rm.create(creq, caller);
final VM[] vms = result.getVMs();
assertEquals(5, vms.length);
for (int i = 0; i < vms.length; i++) {
assertNotNull(vms[i]);
logger.info("Leased vm '" + vms[i].getID() + '\'');
assertTrue(rm.exists(vms[i].getID(), Manager.INSTANCE));
}
Thread.sleep(2000L);
assertEquals(0, MockShutdownTrash.getFailCount());
// Fail at killing the instances:
MockShutdownTrash.setFail(true);
logger.warn("Set to fail.");
rm.trash(vms[0].getID(), Manager.INSTANCE, caller);
rm.trash(vms[1].getID(), Manager.INSTANCE, caller);
Thread.sleep(4000L);
int lastFailCount = MockShutdownTrash.getFailCount();
assertFalse(0 == lastFailCount);
assertTrue(rm.exists(vms[0].getID(), Manager.INSTANCE));
assertTrue(rm.exists(vms[1].getID(), Manager.INSTANCE));
allvms = rm.getGlobalAll();
assertEquals(5, allvms.length);
Thread.sleep(4000L);
assertTrue(rm.exists(vms[0].getID(), Manager.INSTANCE));
assertTrue(rm.exists(vms[1].getID(), Manager.INSTANCE));
allvms = rm.getGlobalAll();
assertEquals(5, allvms.length);
// Start succeeding to kill the instances:
MockShutdownTrash.setFail(false);
logger.warn("Set to succeed.");
Thread.sleep(15000L);
// Should now be gone:
assertFalse(rm.exists(vms[0].getID(), Manager.INSTANCE));
assertFalse(rm.exists(vms[1].getID(), Manager.INSTANCE));
allvms = rm.getGlobalAll();
assertEquals(3, allvms.length);
// Start counting from scratch
MockShutdownTrash.resetFailCount();
// Add two in a new group, with very short duration
final int durationSecs2 = 5;
final int memory2 = 64;
final int numNodes2 = 2;
final CreateRequest creq2 =
this.populator().getCreateRequest("suite:failure:complicatedFail2",
durationSecs2, memory2, numNodes2);
final CreateResult result2 = rm.create(creq2, caller);
VM[] vms2 = result2.getVMs();
assertEquals(2, vms2.length);
for (int i = 0; i < vms2.length; i++) {
assertNotNull(vms2[i]);
logger.info("Leased vm '" + vms2[i].getID() + '\'');
assertTrue(rm.exists(vms2[i].getID(), Manager.INSTANCE));
}
allvms = rm.getGlobalAll();
assertEquals(5, allvms.length);
// Wait for them to terminate successfully
Thread.sleep(8000L);
assertEquals(0, MockShutdownTrash.getFailCount());
// Should now be gone:
assertFalse(rm.exists(vms2[0].getID(), Manager.INSTANCE));
assertFalse(rm.exists(vms2[1].getID(), Manager.INSTANCE));
allvms = rm.getGlobalAll();
assertEquals(3, allvms.length);
// Old ones should remain:
assertTrue(rm.exists(vms[2].getID(), Manager.INSTANCE));
assertTrue(rm.exists(vms[3].getID(), Manager.INSTANCE));
assertTrue(rm.exists(vms[4].getID(), Manager.INSTANCE));
// Fail at killing the instances:
MockShutdownTrash.setFail(true);
logger.warn("Set to fail.");
// Try to kill all but one
rm.trash(vms[2].getID(), Manager.INSTANCE, caller);
rm.trash(vms[3].getID(), Manager.INSTANCE, caller);
Thread.sleep(4000L);
// Shouldn't work
allvms = rm.getGlobalAll();
assertEquals(3, allvms.length);
assertTrue(rm.exists(vms[2].getID(), Manager.INSTANCE));
assertTrue(rm.exists(vms[3].getID(), Manager.INSTANCE));
assertTrue(rm.exists(vms[4].getID(), Manager.INSTANCE));
// Start succeeding to kill the instances:
MockShutdownTrash.setFail(false);
logger.warn("Set to succeed.");
Thread.sleep(8000L);
// Should now be gone:
assertFalse(rm.exists(vms[2].getID(), Manager.INSTANCE));
assertFalse(rm.exists(vms[3].getID(), Manager.INSTANCE));
allvms = rm.getGlobalAll();
assertEquals(1, allvms.length);
// And this one should now die right away:
rm.trash(vms[4].getID(), Manager.INSTANCE, caller);
assertFalse(rm.exists(vms[4].getID(), Manager.INSTANCE));
allvms = rm.getGlobalAll();
assertEquals(0, allvms.length);
}