// start one server
logger.info("Starting server1");
startNode("server1", settings);
ClusterService clusterService1 = ((InternalNode) node("server1")).injector().getInstance(ClusterService.class);
logger.info("Creating index [test]");
CreateIndexResponse createIndexResponse = client("server1").admin().indices().create(createIndexRequest("test")).actionGet();
assertThat(createIndexResponse.acknowledged(), equalTo(true));
logger.info("Running Cluster Health");
ClusterHealthResponse clusterHealth = client("server1").admin().cluster().health(clusterHealthRequest().waitForGreenStatus()).actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.status());
assertThat(clusterHealth.timedOut(), equalTo(false));
assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.GREEN));
assertThat(clusterHealth.relocatingShards(), equalTo(0));
assertThat(clusterHealth.activeShards(), equalTo(11));
assertThat(clusterHealth.activePrimaryShards(), equalTo(11));
ClusterState clusterState1 = clusterService1.state();
RoutingNode routingNodeEntry1 = clusterState1.readOnlyRoutingNodes().nodesToShards().get(clusterState1.nodes().localNodeId());
assertThat(routingNodeEntry1.numberOfShardsWithState(STARTED), equalTo(11));
// start another server
logger.info("Starting server2");
startNode("server2", settings);
Thread.sleep(200);
logger.info("Running Cluster Health");
clusterHealth = client("server1").admin().cluster().health(clusterHealthRequest().waitForGreenStatus().waitForRelocatingShards(0).waitForNodes("2")).actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.status());
assertThat(clusterHealth.timedOut(), equalTo(false));
assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.GREEN));
assertThat(clusterHealth.relocatingShards(), equalTo(0));
assertThat(clusterHealth.activeShards(), equalTo(11));
assertThat(clusterHealth.activePrimaryShards(), equalTo(11));
// sleep till the cluster state gets published, since we check the master
Thread.sleep(200);
ClusterService clusterService2 = ((InternalNode) node("server2")).injector().getInstance(ClusterService.class);
clusterState1 = clusterService1.state();
routingNodeEntry1 = clusterState1.readOnlyRoutingNodes().nodesToShards().get(clusterState1.nodes().localNodeId());
assertThat(routingNodeEntry1.numberOfShardsWithState(STARTED), anyOf(equalTo(6), equalTo(5)));
ClusterState clusterState2 = clusterService2.state();
RoutingNode routingNodeEntry2 = clusterState2.readOnlyRoutingNodes().nodesToShards().get(clusterState2.nodes().localNodeId());
assertThat(routingNodeEntry2.numberOfShardsWithState(STARTED), anyOf(equalTo(5), equalTo(6)));
// start another server
logger.info("Starting server3");
startNode("server3");
Thread.sleep(200);
ClusterService clusterService3 = ((InternalNode) node("server3")).injector().getInstance(ClusterService.class);
logger.info("Running Cluster Health");
clusterHealth = client("server1").admin().cluster().health(clusterHealthRequest().waitForGreenStatus().waitForRelocatingShards(0).waitForNodes("3")).actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.status());
assertThat(clusterHealth.timedOut(), equalTo(false));
assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.GREEN));
assertThat(clusterHealth.relocatingShards(), equalTo(0));
assertThat(clusterHealth.activeShards(), equalTo(11));
assertThat(clusterHealth.activePrimaryShards(), equalTo(11));
// sleep till the cluster state gets published, since we check the master
Thread.sleep(200);
clusterState1 = clusterService1.state();
routingNodeEntry1 = clusterState1.readOnlyRoutingNodes().nodesToShards().get(clusterState1.nodes().localNodeId());
assertThat(routingNodeEntry1.numberOfShardsWithState(STARTED), anyOf(equalTo(5), equalTo(4), equalTo(3)));
clusterState2 = clusterService2.state();
routingNodeEntry2 = clusterState2.readOnlyRoutingNodes().nodesToShards().get(clusterState2.nodes().localNodeId());
assertThat(routingNodeEntry2.numberOfShardsWithState(STARTED), anyOf(equalTo(5), equalTo(4), equalTo(3)));
ClusterState clusterState3 = clusterService3.state();
RoutingNode routingNodeEntry3 = clusterState3.readOnlyRoutingNodes().nodesToShards().get(clusterState3.nodes().localNodeId());
assertThat(routingNodeEntry3.numberOfShardsWithState(STARTED), equalTo(3));
assertThat(routingNodeEntry1.numberOfShardsWithState(STARTED) + routingNodeEntry2.numberOfShardsWithState(STARTED) + routingNodeEntry3.numberOfShardsWithState(STARTED), equalTo(11));
logger.info("Closing server1");
// kill the first server
closeNode("server1");
logger.info("Running Cluster Health");
clusterHealth = client("server3").admin().cluster().health(clusterHealthRequest().waitForGreenStatus().waitForNodes("2").waitForRelocatingShards(0)).actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.status());
assertThat(clusterHealth.timedOut(), equalTo(false));
assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.GREEN));
assertThat(clusterHealth.relocatingShards(), equalTo(0));
assertThat(clusterHealth.activeShards(), equalTo(11));
assertThat(clusterHealth.activePrimaryShards(), equalTo(11));
// sleep till the cluster state gets published, since we check the master
Thread.sleep(200);
clusterState2 = clusterService2.state();
routingNodeEntry2 = clusterState2.readOnlyRoutingNodes().nodesToShards().get(clusterState2.nodes().localNodeId());
assertThat(routingNodeEntry2.numberOfShardsWithState(STARTED), anyOf(equalTo(5), equalTo(6)));
clusterState3 = clusterService3.state();
routingNodeEntry3 = clusterState3.readOnlyRoutingNodes().nodesToShards().get(clusterState3.nodes().localNodeId());
assertThat(routingNodeEntry3.numberOfShardsWithState(STARTED), anyOf(equalTo(5), equalTo(6)));
assertThat(routingNodeEntry2.numberOfShardsWithState(STARTED) + routingNodeEntry3.numberOfShardsWithState(STARTED), equalTo(11));
logger.info("Deleting index [test]");
// last, lets delete the index
DeleteIndexResponse deleteIndexResponse = client("server2").admin().indices().delete(deleteIndexRequest("test")).actionGet();
assertThat(deleteIndexResponse.acknowledged(), equalTo(true));
Thread.sleep(500); // wait till the cluster state gets published
clusterState2 = clusterService2.state();
routingNodeEntry2 = clusterState2.readOnlyRoutingNodes().nodesToShards().get(clusterState2.nodes().localNodeId());
assertThat(routingNodeEntry2, nullValue());
clusterState3 = clusterService3.state();
routingNodeEntry3 = clusterState3.readOnlyRoutingNodes().nodesToShards().get(clusterState3.nodes().localNodeId());
assertThat(routingNodeEntry3, nullValue());
}