}
@Test(groups={"small", "functional"})
public void testRelayFailOver() throws Exception
{
final Logger log = Logger.getLogger("TestRelayPullThread.testRelayFailOver");
log.setLevel(Level.INFO);
log.info("-------------- start -------------------");
List<String> sources = Arrays.asList("source1");
Properties clientProps = new Properties();
clientProps.setProperty("client.runtime.bootstrap.enabled", "false");
clientProps.setProperty("client.runtime.relay(1).name", "relay1");
clientProps.setProperty("client.runtime.relay(1).port", "10001");
clientProps.setProperty("client.runtime.relay(1).sources", "source1");
clientProps.setProperty("client.runtime.relay(2).name", "relay2");
clientProps.setProperty("client.runtime.relay(2).port", "10002");
clientProps.setProperty("client.runtime.relay(2).sources", "source1");
clientProps.setProperty("client.runtime.relay(3).name", "relay3");
clientProps.setProperty("client.runtime.relay(3).port", "10003");
clientProps.setProperty("client.runtime.relay(3).sources", "source1");
clientProps.setProperty("client.connectionDefaults.eventBuffer.maxSize", "100000");
clientProps.setProperty("client.connectionDefaults.pullerRetries.maxRetryNum", "9");
clientProps.setProperty("client.connectionDefaults.pullerRetries.sleepIncFactor", "1.0");
clientProps.setProperty("client.connectionDefaults.pullerRetries.sleepIncDelta", "1");
clientProps.setProperty("client.connectionDefaults.pullerRetries.initSleep", "1");
DatabusHttpClientImpl client = new DatabusHttpClientImpl("client.", clientProps);
Assert.assertNotNull(client, "client instantiation ok");
final DatabusHttpClientImpl.StaticConfig clientConf = client.getClientStaticConfig();
final DatabusSourcesConnection.StaticConfig srcConnConf = clientConf.getConnectionDefaults();
DatabusHttpClientImpl.RuntimeConfig clientRtConf = clientConf.getRuntime().build();
DbusEventBuffer.StaticConfig bufferConf = clientConf.getConnectionDefaults().getEventBuffer();
DbusEventBuffer relayBuffer = new DbusEventBuffer(bufferConf);
DbusEventBuffer bootstrapBuffer = new DbusEventBuffer(bufferConf);
//we keep the index of the next server we expect to see
AtomicInteger serverIdx = new AtomicInteger(-1);
Set<ServerInfo> relays = clientRtConf.getRelaysSet();
//generate the order in which we should see the servers
List<ServerInfo> relayOrder = new ArrayList<ServerInfo>(relays);
if (log.isInfoEnabled())
{
StringBuilder sb = new StringBuilder();
for (ServerInfo serverInfo: relayOrder)
{
sb.append(serverInfo.getName());
sb.append(" ");
}
log.info("Relay order:" + sb.toString());
}
//This guy always fails on /sources
final MockRelayConnection mockFailConn = new MockRelayConnection(null, null, null, serverIdx);
List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
sourcesResponse.add(new IdNamePair(1L, "source1"));
//This guy succeeds on /sources but fails on /register
final MockRelayConnection mockSuccessConn =
new MockRelayConnection(sourcesResponse, null, null, serverIdx);
DatabusRelayConnectionFactory mockConnFactory =
EasyMock.createMock("mockRelayFactory", DatabusRelayConnectionFactory.class);
//each server should be tried MAX_RETRIES time until all retries are exhausted
for (int i = 0; i < clientConf.getConnectionDefaults().getPullerRetries().getMaxRetryNum() / 3; ++i)
{
EasyMock.expect(mockConnFactory.createRelayConnection(
serverNameMatcher(serverIdx, relayOrder),
EasyMock.<ActorMessageQueue>notNull(),
EasyMock.<RemoteExceptionHandler>notNull())).andReturn(mockFailConn);
EasyMock.expect(mockConnFactory.createRelayConnection(
serverNameMatcher(serverIdx, relayOrder),
EasyMock.<ActorMessageQueue>notNull(),
EasyMock.<RemoteExceptionHandler>notNull())).andReturn(mockFailConn);
EasyMock.expect(mockConnFactory.createRelayConnection(
serverNameMatcher(serverIdx, relayOrder),
EasyMock.<ActorMessageQueue>notNull(),
EasyMock.<RemoteExceptionHandler>notNull())).andReturn(mockSuccessConn);
}
EasyMock.replay(mockConnFactory);
List<DatabusSubscription> sourcesSubList = DatabusSubscription.createSubscriptionList(sources);
//Dummy connection object as expected by the puller thread
// Note that in this case, it is ok to pass Set<relays> as all the relays serve the same source
//"source1"
ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sources);
DatabusSourcesConnection sourcesConn = new DatabusSourcesConnection(
srcConnConf, sourcesSubList, relays, null, null, null, relayBuffer, bootstrapBuffer,
Executors.newCachedThreadPool(), null, null, null, null, null, null, null, mockConnFactory, null,
null, null, null, new DbusEventV1Factory(), connStateFactory);
final RelayPullThread relayPuller =
new RelayPullThread("RelayPuller", sourcesConn, relayBuffer, connStateFactory, relays,
new ArrayList<DbusKeyCompositeFilterConfig>(),
!clientConf.getRuntime().getBootstrap().isEnabled(),
clientConf.isReadLatestScnOnErrorEnabled(),
clientConf.getPullerBufferUtilizationPct(),
Integer.MAX_VALUE,
ManagementFactory.getPlatformMBeanServer(),
new DbusEventV1Factory(),
null);
RemoteExceptionHandler mockRemoteExceptionHandler =
new MockRemoteExceptionHandler(sourcesConn, relayBuffer, relayPuller);
Field field = relayPuller.getClass().getDeclaredField("_remoteExceptionHandler");
field.setAccessible(true);
field.set(relayPuller, mockRemoteExceptionHandler);
mockFailConn.setCallback(relayPuller);
mockSuccessConn.setCallback(relayPuller);
//Let the show begin
Thread relayPullerThread = new Thread(relayPuller);
relayPullerThread.setDaemon(true);
relayPullerThread.start();
relayPuller.enqueueMessage(LifecycleMessage.createStartMessage());
TestUtil.assertWithBackoff(new ConditionCheck()
{
@Override
public boolean check()
{
return mockFailConn.getSourcesCallCounter() == 6;
}
}, "failConn: correct number of /sources", 500, log);
Assert.assertEquals(mockFailConn.getRegisterCallCounter(),
0,
"failConn: correct number of /register");
Assert.assertEquals(mockFailConn.getStreamCallCounter(),
0,
"failConn: correct number of /stream");
TestUtil.assertWithBackoff(new ConditionCheck()
{
@Override
public boolean check()
{
return mockSuccessConn.getSourcesCallCounter() == 3;
}
}, "successConn: correct number of /sources", 500, log);
Assert.assertEquals(mockSuccessConn.getRegisterCallCounter(),
3,
"successConn: correct number of /register");
Assert.assertEquals(mockSuccessConn.getStreamCallCounter(),
0,
"successConn: correct number of /stream");
TestUtil.assertWithBackoff(new ConditionCheck()
{
@Override
public boolean check()
{
return relayPuller.getComponentStatus().getStatus() ==
DatabusComponentStatus.Status.SUSPENDED_ON_ERROR;
}
}, "puller suspended because of out of retries", 500, log);
EasyMock.verify(mockConnFactory);
relayPuller.enqueueMessage(LifecycleMessage.createShutdownMessage());
relayPuller.awaitShutdown();
log.info("------------ done --------------");
}