Package com.fasterxml.clustermate.dw

Source Code of com.fasterxml.clustermate.dw.DWBasedService

package com.fasterxml.clustermate.dw;

import java.io.*;
import java.util.*;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.yammer.dropwizard.Service;
import com.yammer.dropwizard.assets.AssetsBundle;
import com.yammer.dropwizard.config.Bootstrap;
import com.yammer.dropwizard.config.Environment;
import com.yammer.dropwizard.lifecycle.Managed;

import com.fasterxml.storemate.shared.*;
import com.fasterxml.storemate.store.StorableStore;
import com.fasterxml.storemate.store.StoreOperationThrottler;
import com.fasterxml.storemate.store.backend.StoreBackend;
import com.fasterxml.storemate.store.backend.StoreBackendBuilder;
import com.fasterxml.storemate.store.backend.StoreBackendConfig;
import com.fasterxml.storemate.store.file.FileManager;
import com.fasterxml.storemate.store.impl.StorableStoreImpl;
import com.fasterxml.storemate.store.util.PartitionedWriteMutex;

import com.fasterxml.clustermate.api.EntryKey;
import com.fasterxml.clustermate.api.PathType;
import com.fasterxml.clustermate.api.RequestPathBuilder;
import com.fasterxml.clustermate.api.msg.ListItem;
import com.fasterxml.clustermate.jaxrs.IndexResource;
import com.fasterxml.clustermate.service.SharedServiceStuff;
import com.fasterxml.clustermate.service.Stores;
import com.fasterxml.clustermate.service.StartAndStoppable;
import com.fasterxml.clustermate.service.cfg.ServiceConfig;
import com.fasterxml.clustermate.service.cleanup.CleanerUpper;
import com.fasterxml.clustermate.service.cleanup.CleanupTask;
import com.fasterxml.clustermate.service.cluster.*;
import com.fasterxml.clustermate.service.metrics.AllOperationMetrics;
import com.fasterxml.clustermate.service.servlet.*;
import com.fasterxml.clustermate.service.store.*;
import com.fasterxml.clustermate.service.sync.SyncHandler;
import com.fasterxml.clustermate.std.JdkHttpClientPathBuilder;

public abstract class DWBasedService<
    K extends EntryKey,
    E extends StoredEntry<K>,
    L extends ListItem,
    SCONFIG extends ServiceConfig,
    CONF extends DWConfigBase<SCONFIG, CONF>
>
    extends Service<CONF>
{
    private final Logger LOG = LoggerFactory.getLogger(getClass());

    /**
     * List of {@link StartAndStoppable} objects we will dispatch start/stop calls to.
     */
    protected List<StartAndStoppable> _managed = null;
   
    /**
     * Marker flag used to indicate cases when service is run in test
     * mode; propagated through configuration.
     */
    protected final boolean _testMode;

    /**
     * We need to keep track of things created, to let tests access
     * the information
     */
    protected SharedServiceStuff _serviceStuff;
   
    /**
     * Container for various stores we use for data, metadata.
     */
    protected StoresImpl<K,E> _stores;
   
    /**
     * This object is needed to allow test code to work around usual
     * waiting time restrictions.
     */
    protected final TimeMaster _timeMaster;

    /**
     * And we better hang on to cluster view as well
     */
    protected ClusterViewByServerUpdatable _cluster;
   
    /**
     * Manager object that deals with data expiration and related
     * clean up tasks.
     */
    protected CleanerUpper<K,E> _cleanerUpper;

    /*
    /**********************************************************************
    /* Handlers
    /**********************************************************************
     */
   
    protected StoreHandler<K,E,L> _storeHandler;
   
    /*
    /**********************************************************************
    /* Construction
    /**********************************************************************
     */

    protected DWBasedService(TimeMaster timings)
    {
        this(timings, false);
    }

    protected DWBasedService(TimeMaster timings, boolean testMode)
    {
        super();
        _timeMaster = timings;
        _testMode = testMode;
    }

    /*
    /**********************************************************************
    /* Life-cycle
    /**********************************************************************
     */

    @Override
    public void initialize(Bootstrap<CONF> bootstrap) {
        // Static stuff from under /html (except for root  level things
        // like /index.html that need special handling)
        bootstrap.addBundle(new AssetsBundle("/html"));
    }

    public void _start() throws Exception
    {
        LOG.info("Starting up {} VManaged objects", _managed.size());
        for (StartAndStoppable managed : _managed) {
            LOG.info("Starting up: {}", managed.getClass().getName());
            managed.start();
        }
        LOG.info("VManaged object startup complete");

        /* 27-Mar-2013, tatu: Also: need to register shutdown hook to be
         *    able to do 'prepareForStop()'
         */
        Runtime.getRuntime().addShutdownHook(new Thread() {
            @Override
            public void run() {
                _prepareForStop();
            }
        });
    }

    public void _prepareForStop()
    {
        LOG.info("Calling prepareForStop on {} VManaged objects", _managed.size());
        for (StartAndStoppable managed : _managed) {
            try {
                managed.prepareForStop();
            } catch (Exception e) {
                LOG.warn("Problems with prepareForStop on {}: {}", managed.getClass().getName(),
                        e.getMessage());
            }
        }
        LOG.info("prepareForStop() for managed objects complete");
    }
   
    public void _stop() throws Exception
    {
        int count = _managed.size();
        LOG.info("Stopping {} VManaged objects", count);
        while (--count >= 0) {
            StartAndStoppable managed = _managed.remove(count);
            String desc = managed.getClass().getName();
            try {
                LOG.info("Stopping: {}", desc);
                managed.stop();
            } catch (Exception e) {
                LOG.warn(String.format("Problems trying to stop VManaged of type %s: (%s) %s",
                        desc, e.getClass().getName(), e.getMessage()),
                        e);
            }
        }
        LOG.info("VManaged object shutdown complete");
    }

    @Override
    public void run(CONF dwConfig, Environment environment) throws IOException
    {
        // first things first: we need to get start()/stop() calls, so:
        environment.manage(new Managed() {
            @Override
            public void start() throws Exception {
                _start();
            }

            @Override
            public void stop() throws Exception {
                _stop();
            }
        });

        /* 04-Jun-2013, tatu: Goddammit, disabling gzip filter is tricky due to
         *   data-binding... Object-values get re-created. So, need to patch after
         *   the fact. And hope it works...
         *  
         * NOTE: looks like this is too late, and won't have effect. If so, modifying
         * YAML/JSON config is the only way.
         */
        dwConfig.overrideGZIPEnabled(false);

        final SCONFIG config = dwConfig.getServiceConfig();
       
        _managed = new ArrayList<StartAndStoppable>();

        StoredEntryConverter<K,E,L> entryConverter = constructEntryConverter(config, environment);
        FileManager files = constructFileManager(config);

        _serviceStuff = constructServiceStuff(config, _timeMaster,
               entryConverter, files);
        if (_testMode) {
            _serviceStuff.markAsTest();
        }
       
        /* Let's try opening up StorableStore: must have been created,
         * and have tables we expect; otherwise we'll fail right away.
         */
        LOG.info("Trying to open Stores (StorableStore, node store, last-access store)");
         _stores = _constructStores(_serviceStuff);
         _managed.add(_stores);
        LOG.info("Opened StorableStore successfully");
        _stores.initAndOpen(false);

        // Then: read in cluster information (config file, backend store settings):
        final int port = dwConfig.getHttpConfiguration().getPort();
        LOG.info("Initializing cluster configuration (port {})...", port);
        final long startTime = _timeMaster.currentTimeMillis();
        ClusterViewByServerImpl<K,E> cl = new ClusterBootstrapper<K,E>(startTime, _serviceStuff, _stores)
                .bootstrap(port);
        _cluster = cl;
        _managed.add(_cluster);
    
        LOG.info("Cluster configuration setup complete, with {} nodes", _cluster.size());
       
        // Index page must be done via resource, otherwise will conflict with DW/JAX-RS Servlet:
        environment.addResource(new IndexResource(loadResource("/index.html"), loadResource("/favicon.jpg")));

        // Let's first construct handlers we use:
        LOG.info("Creating handlers for service endpoints");
        ClusterInfoHandler nodeH = constructClusterInfoHandler(_serviceStuff, _cluster);
        SyncHandler<K,E> syncH = constructSyncHandler(_serviceStuff, _stores, _cluster);
        _storeHandler = constructStoreHandler(_serviceStuff, _stores, _cluster);
        _managed.add(_storeHandler);

        LOG.info("Adding service end points");
        addServiceEndpoints(_serviceStuff, environment,
                nodeH, syncH, _storeHandler);

        LOG.info("Adding health checks");
        addHealthChecks(_serviceStuff, environment);

        LOG.info("Initializing background cleaner tasks");
        _cleanerUpper = constructCleanerUpper(_serviceStuff, _stores, _cluster);
        if (_cleanerUpper != null) {
            _managed.add(_cleanerUpper);
        }
        LOG.info("Initialization complete: HTTP service now running on port {}",
                dwConfig.getHttpConfiguration().getPort());
    }

    /*
    /**********************************************************************
    /* Factory methods: basic config objects
    /**********************************************************************
     */
   
    /**
     * Overridable method that is used for getting helper object used for
     * constructing {@link StoredEntry} instances to store in the
     * entry metadata store.
     */
    protected abstract StoredEntryConverter<K,E,L> constructEntryConverter(SCONFIG config,
            Environment environment);

    protected abstract FileManager constructFileManager(SCONFIG serviceConfig);

    protected abstract StoresImpl<K,E> constructStores(SharedServiceStuff stuff,
            SCONFIG serviceConfig, StorableStore store);   

    protected abstract SharedServiceStuff constructServiceStuff(SCONFIG serviceConfig,
            TimeMaster timeMaster, StoredEntryConverter<K,E,L> entryConverter,
            FileManager files);
   
    /*
    /**********************************************************************
    /* Factory methods for constructing handlers
    /**********************************************************************
     */

    protected abstract StoreHandler<K,E,L> constructStoreHandler(SharedServiceStuff serviceStuff,
            Stores<K,E> stores, ClusterViewByServer cluster);

    protected SyncHandler<K,E> constructSyncHandler(SharedServiceStuff stuff,
            StoresImpl<K,E> stores, ClusterViewByServerUpdatable cluster)
    {
        return new SyncHandler<K,E>(stuff, stores, cluster);
    }

    protected ClusterInfoHandler constructClusterInfoHandler(SharedServiceStuff stuff,
            ClusterViewByServerUpdatable cluster)
    {
        return new ClusterInfoHandler(stuff, cluster);
    }

    // since 0.9.6
    protected CleanerUpper<K,E> constructCleanerUpper(SharedServiceStuff stuff,
            Stores<K,E> stores, ClusterViewByServer cluster)
    {
        return new CleanerUpper<K,E>(_serviceStuff, _stores, _cluster,
                constructCleanupTasks());
    }

    // since 0.9.6
    protected abstract List<CleanupTask<?>> constructCleanupTasks();
   
    /*
    /**********************************************************************
    /* Factory methods: servlets
    /**********************************************************************
     */

    protected abstract StoreEntryServlet<K,E> constructStoreEntryServlet(SharedServiceStuff stuff,
            ClusterViewByServer cluster, StoreHandler<K,E,L> storeHandler);

    protected ServletBase constructNodeStatusServlet(SharedServiceStuff stuff,
            ClusterInfoHandler nodeHandler) {
        return new NodeStatusServlet(stuff, nodeHandler);
    }

    protected ServletBase constructNodeMetricsServlet(SharedServiceStuff stuff,
            ClusterViewByServer cluster, Stores<K,E> stores,
            AllOperationMetrics.Provider[] metrics) {
        return new NodeMetricsServlet(stuff, stores, metrics);
    }
       
    protected SyncListServlet<K,E> constructSyncListServlet(SharedServiceStuff stuff,
            ClusterViewByServer cluster, SyncHandler<K,E> syncHandler) {
        return new SyncListServlet<K,E>(stuff, cluster, syncHandler);
    }

    protected SyncPullServlet<K,E> constructSyncPullServlet(SharedServiceStuff stuff,
            ClusterViewByServer cluster, SyncHandler<K,E> syncHandler) {
        return new SyncPullServlet<K,E>(stuff, cluster, syncHandler);
    }

    protected StoreListServlet<K,E> constructStoreListServlet(SharedServiceStuff stuff,
            ClusterViewByServer cluster, StoreHandler<K,E,L> storeHandler) {
        return new StoreListServlet<K,E>(stuff, cluster, storeHandler);
    }
   
    /*
    /**********************************************************************
    /* Methods for service end point additions
    /**********************************************************************
     */

    /**
     * Method called to create service endpoints, given set of
     * handlers.
     */
    protected void addServiceEndpoints(SharedServiceStuff stuff,
            Environment environment,
            ClusterInfoHandler nodeHandler, SyncHandler<K,E> syncHandler,
            StoreHandler<K,E,L> storeHandler)
    {
        final ClusterViewByServer cluster = syncHandler.getCluster();

        EnumMap<PathType, ServletBase> servlets = new EnumMap<PathType, ServletBase>(PathType.class);
        servlets.put(PathType.NODE_STATUS, constructNodeStatusServlet(stuff, nodeHandler));

        ServletWithMetricsBase syncListServlet = constructSyncListServlet(stuff, cluster, syncHandler);
        servlets.put(PathType.SYNC_LIST, syncListServlet);
        ServletBase syncPullServlet = constructSyncPullServlet(stuff, cluster, syncHandler);
        servlets.put(PathType.SYNC_PULL, syncPullServlet);
        StoreEntryServlet<K,E> storeEntryServlet = constructStoreEntryServlet(stuff,
                cluster, storeHandler);
        servlets.put(PathType.STORE_ENTRY, storeEntryServlet);
        ServletWithMetricsBase storeListServlet = constructStoreListServlet(stuff,
                cluster, storeHandler);
        servlets.put(PathType.STORE_LIST, storeListServlet);

        servlets.put(PathType.NODE_METRICS, constructNodeMetricsServlet(stuff, cluster,
                storeHandler.getStores(),
                new AllOperationMetrics.Provider[] {
                    storeEntryServlet, storeListServlet, syncListServlet
                }
        ));


//        servlets.put(PathType.STORE_FIND_ENTRY, );
//        servlets.put(PathType.STORE_FIND_LIST, );
       
        ServiceDispatchServlet<K,E> dispatcher = new ServiceDispatchServlet<K,E>(cluster,
                null, stuff, servlets);

        RequestPathBuilder rootBuilder = rootPath(stuff.getServiceConfig());
        String rootPath = servletPath(rootBuilder);
        LOG.info("Registering main Dispatcher servlet at: {}", rootPath);
        environment.addServlet(dispatcher, rootPath);

        // // And finally servlet for for entry access
       
        addStoreEntryServlet(stuff, environment, cluster, _storeHandler);
    }

    /*
     * Old single-Servlet implementation:
     */
    /*
    {
        final ServiceConfig config = stuff.getServiceConfig();
        final ClusterViewByServer cluster = syncHandler.getCluster();

        // All paths are dynamic, so we need a mapper:
        RequestPathStrategy pathStrategy = stuff.getPathStrategy();
        RequestPathBuilder pathBuilder;

        pathBuilder = pathStrategy.appendNodeStatusPath(rootPath(config));

        // // And then start by adding cluster-info (aka node status)
        // // and sync end points as plain servlets
       
        NodeStatusServlet nsServlet = constructNodeStatusServlet(nodeHandler);
        if (nsServlet != null) {
            environment.addServlet(nsServlet, servletPath(pathBuilder));
        }
   
        pathBuilder = pathStrategy.appendSyncListPath(rootPath(config));
        environment.addServlet(new SyncListServlet<K,E>(stuff, cluster, syncHandler),
                servletPath(pathBuilder));
        pathBuilder = pathStrategy.appendSyncPullPath(rootPath(config));
        environment.addServlet(new SyncPullServlet<K,E>(stuff, cluster, syncHandler),
                servletPath(pathBuilder));

        // // And finally servlet for for entry access
       
        addStoreEntryServlet(stuff, environment, pathStrategy, cluster, _storeHandler);
    }
    */
   
    /**
     * Overridable method used for hooking standard entry access endpoint into
     * alternate location. Usually used for backwards compatibility purposes.
     */
    protected void addStoreEntryServlet(SharedServiceStuff stuff,
            Environment environment,
            ClusterViewByServer cluster,
            StoreHandler<K,E,?> storeHandler)
    {
    }
   
    protected void addHealthChecks(SharedServiceStuff stuff,
            Environment environment) { }
   
    /*
    /**********************************************************************
    /* Internal methods
    /**********************************************************************
     */
   
    protected StoresImpl<K,E> _constructStores(SharedServiceStuff stuff)
        throws IOException
    {
        final SCONFIG v = stuff.getServiceConfig();
        StoreBackendBuilder<?> b = v.instantiateBackendBuilder();
        StoreBackendConfig backendConfig = v._storeBackendConfigOverride;
        if (backendConfig == null) { // no overrides, use databinding
            Class<? extends StoreBackendConfig> cfgType = b.getConfigClass();
            if (v.storeBackendConfig == null) {
                throw new IllegalStateException("Missing 'v.storeBackendConfig");
            }
            backendConfig = stuff.convertValue(v.storeBackendConfig, cfgType);
        }
        StoreBackend backend = b.with(v.storeConfig)
                .with(backendConfig)
                .build();
        StorableStore store = new StorableStoreImpl(v.storeConfig, backend, _timeMaster,
               stuff.getFileManager(),
               _constructThrottler(stuff), _constructWriteMutex(stuff));
        return constructStores(stuff, v, store);
    }

    /**
     * Factory method called to instantiate {@link StoreOperationThrottler}
     * to use for throttling underlying local database operations.
     * If null is returned, store is free to use whatever default throttling
     * mechanism it needs to for ensuring consistency, but nothing more.
     *<p>
     * Default implementation simply returns null to let the default throttler
     * be used.
     *
     * @since 0.9.9
     */
    protected StoreOperationThrottler _constructThrottler(SharedServiceStuff stuff)
    {
        // null -> use the default implementation
        return null;
    }

    /**
     * Factory method called to instantiate {@link PartitionedWriteMutex}
     * to use for ensuring atomicity of read-modify-write operations.
     * If null is returned, store is free to use its default implementation.
     *<p>
     * Default implementation simply returns null to let the default implementation
     * be used.
     *
     * @since 0.9.10
     */
    protected PartitionedWriteMutex _constructWriteMutex(SharedServiceStuff stuff)
    {
        // null -> use the default implementation
        return null;
    }
   
    /*
    /**********************************************************************
    /* Accessors for tests
    /**********************************************************************
     */

    public TimeMaster getTimeMaster() {
        return _timeMaster;
    }
   
    /*
    /**********************************************************************
    /* Helper methods
    /**********************************************************************
     */

    protected RequestPathBuilder rootPath(ServiceConfig config)
    {
        return new JdkHttpClientPathBuilder("localhost")
            .addPathSegments(config.servicePathRoot);
    }

    /**
     * Helper method for constructing Servlet registration path, given
     * a basic end point path definition. Currently just verifies prefix
     * and suffix slashes and adds '*' as necessary.
     */
    protected String servletPath(RequestPathBuilder pathBuilder)
    {
        String base = pathBuilder.getPath();
        if (!base.endsWith("*")) {
            if (base.endsWith("/")) {
                base += "*";
            } else {
                base += "/*";
            }
        }
        if (!base.startsWith("/")) {
            base = "/"+base;
        }
        return base;
    }

    protected byte[] loadResource(String ref) throws IOException
    {
        ByteArrayOutputStream bytes = new ByteArrayOutputStream(4000);
        InputStream in = getClass().getResourceAsStream(ref);
        byte[] buffer = new byte[4000];
        int count;
    
        while ((count = in.read(buffer)) > 0) {
            bytes.write(buffer, 0, count);
        }
        in.close();
        byte[] data = bytes.toByteArray();
        if (data.length == 0) {
            String msg = "Could not find resource '"+ref+"'";
            LOG.error(msg);
            throw new IllegalArgumentException(msg);
        }
        return bytes.toByteArray();
    }
}
TOP

Related Classes of com.fasterxml.clustermate.dw.DWBasedService

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.