// Wait for connections to be finished
// DS must see expected numbers of DSs/RSs
waitForStableTopo(fakeRd1, 1, 1);
List<DSInfo> dsInfos = fakeRd1.getReplicasList();
DSInfo dsInfo = dsInfos.get(0);
assertEquals(dsInfo.getDsId(), FDS2_ID);
assertEquals(dsInfo.getStatus(), ServerStatus.NORMAL_STATUS);
/*
* Put DS2 in degraded status sending 4 safe read assured updates from DS1
* - 3 for window being full
* - 1 that is enqueued and makes the threshold value (1) reached and thus
* DS2 go into degraded status
*/
for (int i=1 ; i<=4 ; i++)
{
long startTime = System.currentTimeMillis();
try
{
fakeRd1.sendNewFakeUpdate();
} catch (TimeoutException e)
{
fail("No timeout is expected here");
}
long sendUpdateTime = System.currentTimeMillis() - startTime;
// RS should timeout as no listener in DS2
assertTrue((SMALL_TIMEOUT <= sendUpdateTime) && (sendUpdateTime <=
LONG_TIMEOUT));
}
// Wait for DS2 being degraded
boolean error = true;
for (int count = 0; count < 12; count++)
{
dsInfos = fakeRd1.getReplicasList();
if (dsInfos == null)
continue;
if (dsInfos.size() == 0)
continue;
dsInfo = dsInfos.get(0);
if ( (dsInfo.getDsId() == FDS2_ID) &&
(dsInfo.getStatus() == ServerStatus.DEGRADED_STATUS) )
{
error = false;
break;
}
else
{
sleep(1000);
}
}
if (error)
fail("DS2 not in degraded status");
sleep(500); // Sleep a while as counters are updated just after sending thread is unblocked
assertEquals(fakeRd1.getAssuredSrSentUpdates(), 4);
assertEquals(fakeRd1.getAssuredSrAcknowledgedUpdates(), 0);
assertEquals(fakeRd1.getAssuredSrNotAcknowledgedUpdates(), 4);
assertEquals(fakeRd1.getAssuredSrTimeoutUpdates(), 4);
assertEquals(fakeRd1.getAssuredSrWrongStatusUpdates(), 0);
assertEquals(fakeRd1.getAssuredSrReplayErrorUpdates(), 0);
Map<Integer, Integer> failedServer = fakeRd1.getAssuredSrServerNotAcknowledgedUpdates();
assertEquals(failedServer.size(), 1);
Integer nError = failedServer.get(FDS2_ID);
assertNotNull(nError);
assertEquals(nError.intValue(), 4);
assertEquals(fakeRd1.getAssuredSrReceivedUpdates(), 0);
assertEquals(fakeRd1.getAssuredSrReceivedUpdatesAcked(), 0);
assertEquals(fakeRd1.getAssuredSrReceivedUpdatesNotAcked(), 0);
assertEquals(fakeRd2.getAssuredSrSentUpdates(), 0);
assertEquals(fakeRd2.getAssuredSrAcknowledgedUpdates(), 0);
assertEquals(fakeRd2.getAssuredSrNotAcknowledgedUpdates(), 0);
assertEquals(fakeRd2.getAssuredSrTimeoutUpdates(), 0);
assertEquals(fakeRd2.getAssuredSrWrongStatusUpdates(), 0);
assertEquals(fakeRd2.getAssuredSrReplayErrorUpdates(), 0);
assertEquals(fakeRd2.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
assertEquals(fakeRd2.getAssuredSrReceivedUpdates(), 0);
assertEquals(fakeRd2.getAssuredSrReceivedUpdatesAcked(), 0);
assertEquals(fakeRd2.getAssuredSrReceivedUpdatesNotAcked(), 0);
assertEquals(fakeRd1.getReceivedUpdates(), 0);
assertEquals(fakeRd1.getWrongReceivedUpdates(), 0);
assertEquals(fakeRd2.getReceivedUpdates(), 0);
assertEquals(fakeRd2.getWrongReceivedUpdates(), 0);
assertTrue(fakeRd2.receivedUpdatesOk());
/*
* Send an assured update from DS 1 : should be acked as DS2 is degraded
* and RS should not consider it as eligible for assured
*/
long startTime = System.currentTimeMillis();
try
{
fakeRd1.sendNewFakeUpdate();
} catch (TimeoutException e)
{
fail("No timeout is expected here");
}
long sendUpdateTime = System.currentTimeMillis() - startTime;
// RS should ack quickly as DS2 degraded and not eligible for assured
assertTrue(sendUpdateTime < MAX_SEND_UPDATE_TIME);
sleep(500); // Sleep a while as counters are updated just after sending thread is unblocked
assertEquals(fakeRd1.getAssuredSrSentUpdates(), 5);
assertEquals(fakeRd1.getAssuredSrAcknowledgedUpdates(), 1);
assertEquals(fakeRd1.getAssuredSrNotAcknowledgedUpdates(), 4);
assertEquals(fakeRd1.getAssuredSrTimeoutUpdates(), 4);
assertEquals(fakeRd1.getAssuredSrWrongStatusUpdates(), 0);
assertEquals(fakeRd1.getAssuredSrReplayErrorUpdates(), 0);
failedServer = fakeRd1.getAssuredSrServerNotAcknowledgedUpdates();
assertEquals(failedServer.size(), 1);
nError = failedServer.get(FDS2_ID);
assertNotNull(nError);
assertEquals(nError.intValue(), 4);
assertEquals(fakeRd1.getAssuredSrReceivedUpdates(), 0);
assertEquals(fakeRd1.getAssuredSrReceivedUpdatesAcked(), 0);
assertEquals(fakeRd1.getAssuredSrReceivedUpdatesNotAcked(), 0);
assertEquals(fakeRd2.getAssuredSrSentUpdates(), 0);
assertEquals(fakeRd2.getAssuredSrAcknowledgedUpdates(), 0);
assertEquals(fakeRd2.getAssuredSrNotAcknowledgedUpdates(), 0);
assertEquals(fakeRd2.getAssuredSrTimeoutUpdates(), 0);
assertEquals(fakeRd2.getAssuredSrWrongStatusUpdates(), 0);
assertEquals(fakeRd2.getAssuredSrReplayErrorUpdates(), 0);
assertEquals(fakeRd2.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
assertEquals(fakeRd2.getAssuredSrReceivedUpdates(), 0);
assertEquals(fakeRd2.getAssuredSrReceivedUpdatesAcked(), 0);
assertEquals(fakeRd2.getAssuredSrReceivedUpdatesNotAcked(), 0);
assertEquals(fakeRd1.getReceivedUpdates(), 0);
assertEquals(fakeRd1.getWrongReceivedUpdates(), 0);
assertEquals(fakeRd2.getReceivedUpdates(), 0);
assertEquals(fakeRd2.getWrongReceivedUpdates(), 0);
assertTrue(fakeRd2.receivedUpdatesOk());
/*
* Put DS2 in normal status again (start listen service)
*/
fakeRd2.startListenService();
// Wait for DS2 being back to normal
error = true;
for (int count = 0; count < 12; count++)
{
dsInfos = fakeRd1.getReplicasList();
if (dsInfos == null)
continue;
if (dsInfos.size() == 0)
continue;
dsInfo = dsInfos.get(0);
if ( (dsInfo.getDsId() == FDS2_ID) &&
(dsInfo.getStatus() == ServerStatus.NORMAL_STATUS) )
{
error = false;
break;
}
else