/*
* Copyright 2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.xd.dirt.server;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.CuratorWatcher;
import org.apache.curator.framework.imps.CuratorFrameworkState;
import org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.CollectionUtils;
import org.springframework.xd.dirt.cluster.ContainerAttributes;
import org.springframework.xd.dirt.core.Job;
import org.springframework.xd.dirt.core.JobDeploymentsPath;
import org.springframework.xd.dirt.core.ModuleDeploymentsPath;
import org.springframework.xd.dirt.core.Stream;
import org.springframework.xd.dirt.core.StreamDeploymentsPath;
import org.springframework.xd.dirt.job.JobFactory;
import org.springframework.xd.dirt.module.ModuleDeployer;
import org.springframework.xd.dirt.stream.StreamFactory;
import org.springframework.xd.dirt.zookeeper.Paths;
import org.springframework.xd.dirt.zookeeper.ZooKeeperConnection;
import org.springframework.xd.dirt.zookeeper.ZooKeeperUtils;
import org.springframework.xd.module.ModuleDeploymentProperties;
import org.springframework.xd.module.ModuleDescriptor;
import org.springframework.xd.module.ModuleType;
import org.springframework.xd.module.RuntimeModuleDeploymentProperties;
import org.springframework.xd.module.core.Module;
/**
* @author Mark Fisher
* @author David Turanski
* @author Ilayaperumal Gopinathan Listener for deployment requests for a container instance under {@link
* org.springframework.xd.dirt.zookeeper.Paths#DEPLOYMENTS}.
*/
class DeploymentListener implements PathChildrenCacheListener {
private static final Logger logger = LoggerFactory.getLogger(DeploymentListener.class);
/**
* The ZooKeeperConnection.
*/
private final ZooKeeperConnection zkConnection;
/**
* Watcher for modules deployed to this container under the {@link Paths#STREAMS} location.
*/
private final StreamModuleWatcher streamModuleWatcher;
/**
* Watcher for modules deployed to this container under the {@link Paths#JOBS} location.
*/
private final JobModuleWatcher jobModuleWatcher;
/**
* The ModuleDeployer this container delegates to when deploying a Module.
*/
private final ModuleDeployer moduleDeployer;
/**
* Metadata for the current Container.
*/
private final ContainerAttributes containerAttributes;
/**
* Job factory.
*/
private final JobFactory jobFactory;
/**
* Stream factory.
*/
private final StreamFactory streamFactory;
/**
* Prefix for tap channels.
*/
private static final String TAP_CHANNEL_PREFIX = "tap:";
/**
* Map of deployed modules.
*/
private final Map<ModuleDescriptor.Key, ModuleDescriptor> mapDeployedModules =
new ConcurrentHashMap<ModuleDescriptor.Key, ModuleDescriptor>();
/**
* Create an instance that will register the provided {@link ContainerAttributes} whenever the underlying {@link
* ZooKeeperConnection} is established. If that connection is already established at the time this instance receives
* a {@link org.springframework.context.event.ContextRefreshedEvent}, the attributes will be registered then.
* Otherwise, registration occurs within a callback that is invoked for connected events as well as reconnected
* events.
*
* @param containerAttributes runtime and configured attributes for the container
* @param streamFactory factory to construct {@link Stream}
* @param jobFactory factory to construct {@link Job}
* @param moduleDeployer module deployer
* @param zkConnection ZooKeeper connection
*/
public DeploymentListener(ZooKeeperConnection zkConnection, ModuleDeployer moduleDeployer,
ContainerAttributes containerAttributes, JobFactory jobFactory, StreamFactory streamFactory) {
this.zkConnection = zkConnection;
this.jobModuleWatcher = new JobModuleWatcher();
this.streamModuleWatcher = new StreamModuleWatcher();
this.moduleDeployer = moduleDeployer;
this.containerAttributes = containerAttributes;
this.jobFactory = jobFactory;
this.streamFactory = streamFactory;
}
/**
* {@inheritDoc}
*/
@Override
public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
ZooKeeperUtils.logCacheEvent(logger, event);
switch (event.getType()) {
case INITIALIZED:
break;
case CHILD_ADDED:
onChildAdded(client, event.getData());
break;
case CHILD_REMOVED:
onChildRemoved(client, event.getData());
break;
default:
break;
}
}
/**
* Event handler for new module deployments.
*
* @param client curator client
* @param data module data
*/
private void onChildAdded(CuratorFramework client, ChildData data) throws Exception {
String path = data.getPath();
ModuleDeploymentsPath moduleDeploymentsPath = new ModuleDeploymentsPath(path);
String unitName = moduleDeploymentsPath.getDeploymentUnitName();
String moduleType = moduleDeploymentsPath.getModuleType();
String moduleLabel = moduleDeploymentsPath.getModuleLabel();
int moduleSequence = moduleDeploymentsPath.getModuleSequence();
ModuleDescriptor.Key key = new ModuleDescriptor.Key(unitName, ModuleType.valueOf(moduleType), moduleLabel);
String container = moduleDeploymentsPath.getContainer();
Module module = null;
ModuleDeploymentStatus status;
RuntimeModuleDeploymentProperties properties = new RuntimeModuleDeploymentProperties();
properties.putAll(ZooKeeperUtils.bytesToMap(data.getData()));
try {
module = (ModuleType.job.toString().equals(moduleType)) ?
deployJobModule(client, unitName, moduleLabel, properties) :
deployStreamModule(client, unitName, moduleType, moduleLabel, properties);
if (module == null) {
status = new ModuleDeploymentStatus(container, moduleSequence, key, ModuleDeploymentStatus.State.failed,
"Module deployment returned null");
}
else {
status = new ModuleDeploymentStatus(container, moduleSequence, key,
ModuleDeploymentStatus.State.deployed, null);
}
}
catch (Exception e) {
status = new ModuleDeploymentStatus(container, moduleSequence, key, ModuleDeploymentStatus.State.failed,
ZooKeeperUtils.getStackTrace(e));
logger.error("Exception deploying module", e);
}
try {
writeModuleMetadata(client, module, path);
client.setData().forPath(status.buildPath(), ZooKeeperUtils.mapToBytes(status.toMap()));
}
catch (KeeperException.NoNodeException e) {
logger.warn("During deployment of module {} of type {} for {} with sequence number {}," +
"an undeployment request was detected; this module will be undeployed.", moduleLabel,
moduleType, unitName, moduleSequence);
if (logger.isTraceEnabled()) {
logger.trace("Path " + path + " was removed", e);
}
}
}
/**
* Event handler for deployment removals.
*
* @param client curator client
* @param data module data
*/
private void onChildRemoved(CuratorFramework client, ChildData data) throws Exception {
ModuleDeploymentsPath moduleDeploymentsPath = new ModuleDeploymentsPath(data.getPath());
String streamName = moduleDeploymentsPath.getDeploymentUnitName();
String moduleType = moduleDeploymentsPath.getModuleType();
String moduleLabel = moduleDeploymentsPath.getModuleLabel();
String moduleSequence = moduleDeploymentsPath.getModuleSequenceAsString();
undeployModule(streamName, moduleType, moduleLabel);
String path;
if (ModuleType.job.toString().equals(moduleType)) {
path = new JobDeploymentsPath().setJobName(streamName).setModuleLabel(moduleLabel)
.setContainer(containerAttributes.getId()).build();
}
else {
path = new StreamDeploymentsPath().setStreamName(streamName).setModuleType(moduleType)
.setModuleLabel(moduleLabel).setModuleSequence(moduleSequence)
.setContainer(this.containerAttributes.getId()).build();
}
if (client.checkExists().forPath(path) != null) {
logger.trace("Deleting path: {}", path);
client.delete().forPath(path);
}
}
/**
* If the provided module is not null, write its properties to the provided ZooKeeper path. If the module is null,
* no action is taken.
*
* @param client curator client
* @param module module for which to write properties; may be null
* @param path path to write properties to
* @throws Exception
*/
private void writeModuleMetadata(CuratorFramework client, Module module, String path) throws Exception {
if (module != null) {
Map<String, String> mapMetadata = new HashMap<String, String>();
CollectionUtils.mergePropertiesIntoMap(module.getProperties(), mapMetadata);
try {
client.create().withMode(CreateMode.EPHEMERAL)
.forPath(Paths.build(path, Paths.METADATA), ZooKeeperUtils.mapToBytes(mapMetadata));
}
catch (KeeperException.NodeExistsException ne) {
// This is likely to happen when the container disconnects and reconnects but the admin leader
// was not available to handle the container leaving event.
ModuleDescriptor descriptor = module.getDescriptor();
logger.info("The module metadata path for the module {} of type {} for {}" + "already exists.",
descriptor.getModuleLabel(), descriptor.getType().toString(), descriptor.getGroup());
}
}
}
/**
* Deploy the requested job.
*
* @param client curator client
* @param jobName job name
* @param jobLabel job label
* @param properties module deployment properties
* @return Module deployed job module
*/
private Module deployJobModule(CuratorFramework client, String jobName, String jobLabel,
RuntimeModuleDeploymentProperties properties) throws Exception {
logger.info("Deploying job '{}'", jobName);
String jobDeploymentPath = new JobDeploymentsPath().setJobName(jobName).setModuleLabel(jobLabel)
.setModuleSequence(properties.getSequenceAsString()).setContainer(containerAttributes.getId()).build();
Module module = null;
Job job = DeploymentLoader.loadJob(client, jobName, jobFactory);
if (job != null) {
ModuleDescriptor moduleDescriptor = job.getJobModuleDescriptor();
module = deployModule(moduleDescriptor, properties);
try {
// this indicates that the container has deployed the module
client.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(jobDeploymentPath);
// set a watch on this module in the job path;
// if the node is deleted this indicates an undeployment
client.getData().usingWatcher(jobModuleWatcher).forPath(jobDeploymentPath);
}
catch (KeeperException.NodeExistsException e) {
// This could possibly happen when the container disconnects and reconnects but
// admin leader was not available to process container removed event.
logger.info("Module for job {} already deployed", jobName);
}
}
// todo: this will return null if the job doesn't exist...needs review
return module;
}
/**
* Deploy the requested module for a stream.
*
* @param client curator client
* @param streamName name of the stream for the module
* @param moduleType module type
* @param moduleLabel module label
* @param properties module deployment properties
* @return Module deployed stream module
*/
private Module deployStreamModule(CuratorFramework client, String streamName, String moduleType, String moduleLabel,
RuntimeModuleDeploymentProperties properties) throws Exception {
logger.info("Deploying module '{}' for stream '{}'", moduleLabel, streamName);
String streamDeploymentPath = new StreamDeploymentsPath().setStreamName(streamName).setModuleType(moduleType)
.setModuleLabel(moduleLabel).setModuleSequence(properties.getSequenceAsString())
.setContainer(this.containerAttributes.getId()).build();
Module module = null;
Stream stream = DeploymentLoader.loadStream(client, streamName, streamFactory);
if (stream != null) {
ModuleDescriptor descriptor = stream.getModuleDescriptor(moduleLabel);
module = deployModule(descriptor, properties);
try {
// this indicates that the container has deployed the module
client.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(streamDeploymentPath);
// set a watch on this module in the stream path;
// if the node is deleted this indicates an undeployment
client.getData().usingWatcher(streamModuleWatcher).forPath(streamDeploymentPath);
}
catch (KeeperException.NodeExistsException e) {
// This could possibly happen when the container disconnects and reconnects but
// admin leader was not available to process container removed event.
logger.info("Module {} for stream {} already deployed", moduleLabel, streamName);
}
}
return module;
}
/**
* Deploy the requested module.
*
* @param moduleDescriptor descriptor for the module to be deployed
*/
private Module deployModule(ModuleDescriptor moduleDescriptor, ModuleDeploymentProperties deploymentProperties) {
logger.info("Deploying module {}", moduleDescriptor);
ModuleDescriptor.Key key = new ModuleDescriptor.Key(moduleDescriptor.getGroup(), moduleDescriptor.getType(),
moduleDescriptor.getModuleLabel());
mapDeployedModules.put(key, moduleDescriptor);
Module module = moduleDeployer.createModule(moduleDescriptor, deploymentProperties);
registerTap(moduleDescriptor);
this.moduleDeployer.deploy(module, moduleDescriptor);
return module;
}
/**
* Undeploy the requested module.
*
* @param streamName name of the stream for the module
* @param moduleType module type
* @param moduleLabel module label
*/
protected void undeployModule(String streamName, String moduleType, String moduleLabel) {
ModuleDescriptor.Key key = new ModuleDescriptor.Key(streamName, ModuleType.valueOf(moduleType), moduleLabel);
ModuleDescriptor descriptor = mapDeployedModules.get(key);
if (descriptor == null) {
// This is logged at trace level because every module undeployment
// will cause this to be logged. This is because there is a listener
// on both streams and modules, and the listener implementation for
// each will remove the module from the other, thus causing
// this method to be invoked twice per module undeployment.
logger.trace("Module {} already undeployed", moduleLabel);
}
else {
logger.info("Undeploying module {}", descriptor);
mapDeployedModules.remove(key);
this.moduleDeployer.undeploy(descriptor);
unregisterTap(descriptor);
}
}
/**
* Register the existence of a tap subscriber if the provided module has a tap channel as its source channel.
*
* @param descriptor {@link ModuleDescriptor} for the module
*/
private void registerTap(ModuleDescriptor descriptor) {
String tapChannelName = determineTapChannel(descriptor);
if (tapChannelName != null) {
try {
zkConnection.getClient().create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(
Paths.build(Paths.TAPS, tapChannelName, containerAttributes.getId(), descriptor.getGroup()));
}
catch (Exception e) {
// if it already exists, ignore
ZooKeeperUtils.wrapAndThrowIgnoring(e, KeeperException.NodeExistsException.class);
}
}
}
/**
* Unregister the tap subscriber if the provided module has a tap channel as its source channel.
*
* @param descriptor {@link ModuleDescriptor} for the module
*/
private void unregisterTap(ModuleDescriptor descriptor) {
CuratorFramework client = zkConnection.getClient();
String tapChannelName = determineTapChannel(descriptor);
if (tapChannelName != null) {
try {
try {
client.delete().forPath(Paths.build(Paths.TAPS, tapChannelName, this.containerAttributes.getId(),
descriptor.getGroup()));
}
catch (KeeperException.NoNodeException e) {
// already deleted, ignore
}
try {
// now try to delete the container node and then if successful, the tap node itself
client.delete().forPath(Paths.build(Paths.TAPS, tapChannelName, this.containerAttributes.getId()));
client.delete().forPath(Paths.build(Paths.TAPS, tapChannelName));
}
catch (KeeperException.NoNodeException e) {
// already deleted, ignore
}
catch (KeeperException.NotEmptyException e) {
// attempted to delete a node that still has other children, let it be
}
}
catch (Exception e) {
if (client.getState() == CuratorFrameworkState.STARTED) {
throw ZooKeeperUtils.wrapThrowable(e);
}
else {
logger.debug("Ignoring exception for tap un-registration due to closed ZooKeeper connection", e);
}
}
}
}
/**
* Determine whether the provided descriptor has a tap channel as its "source", and if so return the unqualified tap
* channel name. If not, return {@code null}.
*
* @param descriptor ModuleDescriptor whose source channel is checked
* @return unqualified tap channel name or {@code null}
*/
private String determineTapChannel(ModuleDescriptor descriptor) {
String sourceChannelName = descriptor.getSourceChannelName();
return (sourceChannelName != null && sourceChannelName.startsWith(TAP_CHANNEL_PREFIX)) ?
sourceChannelName.substring(TAP_CHANNEL_PREFIX.length()) : null;
}
void undeployAllModules() {
for (Iterator<ModuleDescriptor.Key> iterator = mapDeployedModules.keySet().iterator(); iterator.hasNext(); ) {
ModuleDescriptor.Key key = iterator.next();
try {
undeployModule(key.getGroup(), key.getType().name(), key.getLabel());
}
catch (Exception e) {
logger.warn("Exception while undeploying " + key, e);
}
iterator.remove();
}
}
/**
* Watcher for the modules deployed to this container under the {@link Paths#JOB_DEPLOYMENTS} location. If the node
* is deleted, this container will undeploy the module.
*/
class JobModuleWatcher implements CuratorWatcher {
/**
* {@inheritDoc}
*/
@Override
public void process(WatchedEvent event) throws Exception {
CuratorFramework client = zkConnection.getClient();
if (event.getType() == Watcher.Event.EventType.NodeDeleted) {
JobDeploymentsPath jobDeploymentsPath = new JobDeploymentsPath(event.getPath());
String jobName = jobDeploymentsPath.getJobName();
String moduleLabel = jobDeploymentsPath.getModuleLabel();
String moduleSequence = jobDeploymentsPath.getModuleSequenceAsString();
undeployModule(jobName, ModuleType.job.toString(), moduleLabel);
String deploymentPath = new ModuleDeploymentsPath().setContainer(containerAttributes.getId())
.setDeploymentUnitName(jobName).setModuleType(ModuleType.job.toString())
.setModuleLabel(moduleLabel).setModuleSequence(moduleSequence).build();
try {
if (client.checkExists().forPath(deploymentPath) != null) {
logger.trace("Deleting path: {}", deploymentPath);
client.delete().deletingChildrenIfNeeded().forPath(deploymentPath);
}
}
catch (Exception e) {
// it is common for a process shutdown to trigger this
// event; therefore any exception thrown while attempting
// to delete a deployment path will only be rethrown
// if the client is in a connected/started state
if (client.getState() == CuratorFrameworkState.STARTED) {
throw ZooKeeperUtils.wrapThrowable(e);
}
}
}
else {
logger.debug("Unexpected event {}, ZooKeeper state: {}", event.getType(), event.getState());
if (EnumSet.of(Watcher.Event.KeeperState.SyncConnected, Watcher.Event.KeeperState.SaslAuthenticated,
Watcher.Event.KeeperState.ConnectedReadOnly).contains(event.getState())) {
// this watcher is only interested in deletes for the purposes of
// undeploying modules;if any other change occurs the watch needs to be
// reestablished
try {
client.getData().usingWatcher(this).forPath(event.getPath());
}
catch (Exception e) {
logger.error("Exception setting up watch for path '{}': {}; ZooKeeper state: {}",
event.getPath(), e,
zkConnection.getClient().getZookeeperClient().getZooKeeper().getState());
if (logger.isDebugEnabled()) {
logger.debug("Full stack trace", e);
}
if (client.getState() == CuratorFrameworkState.STARTED) {
throw ZooKeeperUtils.wrapThrowable(e);
}
}
}
}
}
}
/**
* Watcher for the modules deployed to this container under the {@link Paths#STREAM_DEPLOYMENTS} location. If the
* node is deleted, this container will undeploy the module.
*/
class StreamModuleWatcher implements CuratorWatcher {
/**
* {@inheritDoc}
*/
@Override
public void process(WatchedEvent event) throws Exception {
CuratorFramework client = zkConnection.getClient();
if (event.getType() == Watcher.Event.EventType.NodeDeleted) {
StreamDeploymentsPath streamDeploymentsPath = new StreamDeploymentsPath(event.getPath());
String streamName = streamDeploymentsPath.getStreamName();
String moduleType = streamDeploymentsPath.getModuleType();
String moduleLabel = streamDeploymentsPath.getModuleLabel();
String moduleSequence = streamDeploymentsPath.getModuleSequenceAsString();
undeployModule(streamName, moduleType, moduleLabel);
String deploymentPath = new ModuleDeploymentsPath().setContainer(containerAttributes.getId())
.setDeploymentUnitName(streamName).setModuleType(moduleType).setModuleLabel(moduleLabel)
.setModuleSequence(moduleSequence).build();
try {
if (client.checkExists().forPath(deploymentPath) != null) {
logger.trace("Deleting path: {}", deploymentPath);
client.delete().deletingChildrenIfNeeded().forPath(deploymentPath);
}
}
catch (Exception e) {
// it is common for a process shutdown to trigger this
// event; therefore any exception thrown while attempting
// to delete a deployment path will only be rethrown
// if the client is in a connected/started state
if (client.getState() == CuratorFrameworkState.STARTED) {
throw ZooKeeperUtils.wrapThrowable(e);
}
}
}
else {
logger.debug("Unexpected event {}, ZooKeeper state: {}", event.getType(), event.getState());
if (EnumSet.of(Watcher.Event.KeeperState.SyncConnected, Watcher.Event.KeeperState.SaslAuthenticated,
Watcher.Event.KeeperState.ConnectedReadOnly).contains(event.getState())) {
// this watcher is only interested in deletes for the purposes of undeploying modules;
// if any other change occurs the watch needs to be reestablished
try {
client.getData().usingWatcher(this).forPath(event.getPath());
}
catch (Exception e) {
logger.error("Exception setting up watch for path '{}': {}; ZooKeeper state: {}",
event.getPath(), e,
zkConnection.getClient().getZookeeperClient().getZooKeeper().getState());
if (logger.isDebugEnabled()) {
logger.debug("Full stack trace", e);
}
if (client.getState() == CuratorFrameworkState.STARTED) {
throw ZooKeeperUtils.wrapThrowable(e);
}
}
}
}
}
}
}