memcachedsByNodeId.put(NODE_ID_3, _daemon3);
final String sessionId1 = post( _httpClient, TC_PORT_1, null, "key", "v1" ).getSessionId();
assertNotNull( sessionId1 );
final SessionIdFormat fmt = new SessionIdFormat();
final String nodeId = fmt.extractMemcachedId( sessionId1 );
final MemCacheDaemon<?> first = memcachedsByNodeId.get(nodeId);
// the memcached client writes async, so it's ok to wait a little bit (especially on windows)
assertNotNullElementWaitingWithProxy(0, 100, first.getCache()).get( key( sessionId1 ) );
assertNotNullElementWaitingWithProxy(0, 100, first.getCache()).get( key( fmt.createValidityInfoKeyName( sessionId1 ) ) );
// The executor needs some time to finish the backup...
final MemCacheDaemon<?> second = memcachedsByNodeId.get(nodeIdList.getNextNodeId(nodeId));
assertNotNullElementWaitingWithProxy(0, 4000, second.getCache()).get( key( fmt.createBackupKey( sessionId1 ) ) );
assertNotNullElementWaitingWithProxy(0, 200, second.getCache()).get( key( fmt.createBackupKey( fmt.createValidityInfoKeyName( sessionId1 ) ) ) );
// Shutdown the secondary memcached, so that the next backup should got to the next node
second.stop();
// Wait for update of nodeAvailabilityNodeCache
Thread.sleep(100l);
// Request / Update
final String sessionId2 = post( _httpClient, TC_PORT_1, sessionId1, "key", "v2" ).getSessionId();
assertEquals( sessionId2, sessionId1 );
final MemCacheDaemon<?> third = memcachedsByNodeId.get(nodeIdList.getNextNodeId(nodeIdList.getNextNodeId(nodeId)));
assertNotNullElementWaitingWithProxy(0, 4000, third.getCache()).get( key( fmt.createBackupKey( sessionId1 ) ) );
assertNotNullElementWaitingWithProxy(0, 200, third.getCache()).get( key( fmt.createBackupKey( fmt.createValidityInfoKeyName( sessionId1 ) ) ) );
// Shutdown the first node, so it should be loaded from the 3rd memcached
first.stop();
// Wait for update of nodeAvailabilityNodeCache
Thread.sleep(100l);
final Response response3 = get(_httpClient, TC_PORT_1, sessionId1);
final String sessionId3 = response3.getResponseSessionId();
assertNotNull(sessionId3);
assertFalse(sessionId3.equals(sessionId1));
assertEquals(sessionId3, fmt.createNewSessionId(sessionId1, fmt.extractMemcachedId(sessionId3)));
assertEquals(response3.get("key"), "v2");
}