/*
Copyright (c) 2012 LinkedIn Corp.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package com.linkedin.d2.discovery;
import com.linkedin.d2.D2BaseTest;
import com.linkedin.d2.balancer.clients.DynamicClient;
import com.linkedin.d2.balancer.util.LoadBalancerClientCli;
import com.linkedin.d2.balancer.util.LoadBalancerEchoServer;
import com.linkedin.d2.balancer.util.LoadBalancerUtil;
import com.linkedin.d2.discovery.stores.zk.ZKServer;
import com.linkedin.d2.discovery.stores.zk.ZKTestUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
@Test (groups = {"d2integration"})
public class TestD2ConfigWithSingleZKFailover extends D2BaseTest
{
private static final Logger _log = LoggerFactory.getLogger(TestD2ConfigWithSingleZKFailover.class);
private ArrayList<LoadBalancerEchoServer> _echoServers;
private static final String ZK_HOST = "127.0.0.1";
private int _zkPort = -999;
private static final String ECHO_SERVER_HOST = "127.0.0.1";
private static final int ECHO_SERVER_PORT1 = 2351;
private static final int ECHO_SERVER_PORT2 = 2352;
private static final int ECHO_SERVER_PORT3 = 2353;
private String _zkHosts;
private String _zkUriString;
private ZKServer _zkServer;
private LoadBalancerClientCli _cli;
private DynamicClient _client;
@BeforeTest
public void setup() throws IOException, Exception
{
// zkServer
_zkServer = ZKTestUtil.startZKServer();
_zkPort = _zkServer.getPort();
_zkHosts = ZK_HOST+":" + _zkPort;
_zkUriString = "zk://"+_zkHosts;
// Register clusters/services (two services per cluster)
LoadBalancerClientCli.runDiscovery(_zkHosts, "/d2", D2_CONFIG_DATA);
// Get LoadBalancer Client
_cli = new LoadBalancerClientCli(_zkHosts, "/d2");
// Echo servers startup
startAllEchoServers();
assertAllEchoServersRunning(_echoServers);
_client = _cli.createClient(_cli.getZKClient(), _zkUriString, "/d2", "service-1_1");
_log.info(LoadBalancerClientCli.printStores(_cli.getZKClient(), _zkUriString, "/d2"));
assertAllEchoServersRegistered(_cli.getZKClient(), _zkUriString, _echoServers);
}
@BeforeMethod
public void setupMethod() throws IOException, InterruptedException
{
try
{
_zkServer.startup();
}
catch (Exception e)
{
}
try
{
_echoServers.get(1).startServer();
_echoServers.get(2).startServer();
}
catch (Exception e)
{
}
try
{
_echoServers.get(1).markUp();
_echoServers.get(2).markUp();
}
catch (Exception e)
{
}
}
@AfterTest
public void teardown() throws IOException, InterruptedException, Exception
{
try
{
_cli.shutdown();
}
catch (Exception e)
{
_log.info("LoadBalancerClientCli shutdown failed.");
e.printStackTrace();
}
try
{
LoadBalancerUtil.syncShutdownClient(_client, _log);
}
catch (Exception e)
{
}
try
{
_zkServer.shutdown();
}
catch (Exception e)
{
_log.info("zk server shutdown failed.");
}
stopAllEchoServers(_echoServers);
}
@Test
public void testZkServerShutdown() throws IOException, InterruptedException, URISyntaxException, Exception
{
String [] expectedResponses;
String msg = generateMessage(_zkUriString);
expectedResponses = getExpectedResponses(0, msg);
assertMatch(_cli.sendRequest(_client, "cluster-1","service-1_1", msg), expectedResponses);
assertMatch(_cli.sendRequest(_client, "cluster-1","service-1_2", msg), expectedResponses);
// Shutdown zookeeper
_zkServer.shutdown(false);
msg = generateMessage(_zkUriString);
expectedResponses = getExpectedResponses(0, msg);
// Verify echo servers are still processing responses when zookeeper is down
assertMatch(_cli.sendRequest(_client, "cluster-1","service-1_1", msg), expectedResponses);
assertMatch(_cli.sendRequest(_client, "cluster-1","service-1_2", msg), expectedResponses);
}
@Test
public void testZkServerMultipleShutdownRestarts() throws IOException, InterruptedException, URISyntaxException, Exception
{
String [] expectedResponses;
for (int i=0; i < 10; i++)
{
String msg = generateMessage(_zkUriString);
expectedResponses = getExpectedResponses(0, msg);
assertMatch(_cli.sendRequest(_client, "cluster-1","service-1_1", msg), expectedResponses);
assertMatch(_cli.sendRequest(_client, "cluster-1","service-1_2", msg), expectedResponses);
assertEquals(_cli.sendRequest(_client, "cluster-2","service-2_1", msg), getExpectedResponse(0, msg,_echoServers.get(2).getResponsePostfixStringWithPort()));
// Shutdown zookeeper
_zkServer.shutdown(false);
msg = generateMessage(_zkUriString);
expectedResponses = getExpectedResponses(0, msg);
// Verify echo servers are still processing responses when zookeeper is down
assertMatch(_cli.sendRequest(_client, "cluster-1","service-1_1", msg), expectedResponses);
assertMatch(_cli.sendRequest(_client, "cluster-1","service-1_2", msg), expectedResponses);
assertEquals(_cli.sendRequest(_client, "cluster-2","service-2_1", msg), getExpectedResponse(0, msg,_echoServers.get(2).getResponsePostfixStringWithPort()));
// Restart zookeeper
_zkServer.startup();
}
_zkServer.shutdown(false);
}
@Test
public void testEchoServerMarkDownUp() throws IOException, InterruptedException, URISyntaxException, Exception
{
String [] expectedResponses;
String msg = generateMessage(_zkUriString);
expectedResponses = getExpectedResponses(0, msg);
assertMatch(_cli.sendRequest(_client, "cluster-1","service-1_1", msg), expectedResponses);
assertMatch(_cli.sendRequest(_client, "cluster-1","service-1_2", msg), expectedResponses);
assertEquals(_cli.sendRequest(_client, "cluster-2","service-2_1", msg), getExpectedResponse(0, msg,_echoServers.get(2).getResponsePostfixStringWithPort()));
// Echo Server mark down
_echoServers.get(1).markDown(); // mark down echo server with ECHO_SERVER_PORT2
_echoServers.get(2).markDown(); // mark down echo server with ECHO_SERVER_PORT3
String store = LoadBalancerClientCli.printStore(_cli.getZKClient(), _zkUriString, "/d2", "cluster-1", "service-1_1");
assertTrue(! store.contains(ECHO_SERVER_HOST+":"+ECHO_SERVER_PORT2),"Echo server with port "+ECHO_SERVER_PORT2+" was marked down but is still registered with ZK.");
msg = generateMessage(_zkUriString);
// Verify requests are not routed to echo servers with ECHO_SERVER_PORT2 and ECHO_SERVER_PORT3
assertEquals(_cli.sendRequest(_client, "cluster-1","service-1_1", msg), getExpectedResponse(0, msg,_echoServers.get(0).getResponsePostfixStringWithPort()));
assertEquals(_cli.sendRequest(_client, "cluster-1","service-1_2", msg), getExpectedResponse(0, msg,_echoServers.get(0).getResponsePostfixStringWithPort()));
try
{
String response = _cli.sendRequest(_client, "cluster-2","service-2_1", msg);
fail("Received response from marked down echo server with port=" + ECHO_SERVER_PORT3 + " "+response);
}
catch (Exception e)
{
// expected
}
// Echo Server mark up
_echoServers.get(1).markUp();
_echoServers.get(2).markUp();
msg = generateMessage(_zkUriString);
expectedResponses = getExpectedResponses(0, msg);
assertMatch(_cli.sendRequest(_client, "cluster-1","service-1_1", msg), expectedResponses);
assertMatch(_cli.sendRequest(_client, "cluster-1","service-1_2", msg), expectedResponses);
assertEquals(_cli.sendRequest(_client, "cluster-2","service-2_1", msg), getExpectedResponse(0, msg,_echoServers.get(2).getResponsePostfixStringWithPort()));
}
@Test
public void testEchoServerRestart() throws IOException, InterruptedException, URISyntaxException, Exception
{
String [] expectedResponses;
String msg = generateMessage(_zkUriString);
expectedResponses = getExpectedResponses(0, msg);
assertMatch(_cli.sendRequest(_client, "cluster-1","service-1_1", msg), expectedResponses);
assertMatch(_cli.sendRequest(_client, "cluster-1","service-1_2", msg), expectedResponses);
assertEquals(_cli.sendRequest(_client, "cluster-2","service-2_2", msg), getExpectedResponse(0, msg,_echoServers.get(2).getResponsePostfixStringWithPort()));
// Stop Echo Server
_echoServers.get(1).markDown();
_echoServers.get(1).stopServer(); // since we are running echo server in the same VM, this is not a real shutdown
msg = generateMessage(_zkUriString);
assertEquals(_cli.sendRequest(_client, "cluster-1","service-1_1", msg), getExpectedResponse(0, msg,_echoServers.get(0).getResponsePostfixStringWithPort()));
assertEquals(_cli.sendRequest(_client, "cluster-1","service-1_2", msg), getExpectedResponse(0, msg,_echoServers.get(0).getResponsePostfixStringWithPort()));
assertEquals(_cli.sendRequest(_client, "cluster-2","service-2_2", msg), getExpectedResponse(0, msg,_echoServers.get(2).getResponsePostfixStringWithPort()));
// Start Echo Server
try
{
_echoServers.get(1).startServer();
_echoServers.get(1).markUp();
}
catch (Exception e)
{
}
msg = generateMessage(_zkUriString);
expectedResponses = getExpectedResponses(0, msg);
assertMatch(_cli.sendRequest(_client, "cluster-1","service-1_2", msg), expectedResponses);
assertEquals(_cli.sendRequest(_client, "cluster-2","service-2_1", msg), getExpectedResponse(0, msg,_echoServers.get(2).getResponsePostfixStringWithPort()));
}
private String[] getExpectedResponses(int partitionId, String msg)
{
String resp = msg + ";WEIGHT="+partitionId+"/1.0";
// Two servers ECHO_SERVER_PORT1 and ECHO_SERVER_PORT2 are registered to the same "clister-1" and either one can return response
return new String[] {resp + _echoServers.get(0).getResponsePostfixStringWithPort(), resp + _echoServers.get(1).getResponsePostfixStringWithPort()};
}
private String getExpectedResponse(int partitionId, String msg, String postfix)
{
return msg + ";WEIGHT="+partitionId+"/1.0" + postfix;
}
private void startAllEchoServers() throws Exception
{
_echoServers = new ArrayList<LoadBalancerEchoServer>();
_echoServers.add(startEchoServer(ZK_HOST, _zkPort, ECHO_SERVER_HOST, ECHO_SERVER_PORT1, "cluster-1", "service-1_1", "service-1_2", "service-1_3" ));
_echoServers.add(startEchoServer(ZK_HOST, _zkPort, ECHO_SERVER_HOST, ECHO_SERVER_PORT2, "cluster-1", "service-1_1", "service-1_2", "service-1_3" ));
_echoServers.add(startEchoServer(ZK_HOST, _zkPort, ECHO_SERVER_HOST, ECHO_SERVER_PORT3, "cluster-2", "service-2_1", "service-2_2", "service-2_3" ));
}
}