Package com.marklogic.developer.corb

Source Code of com.marklogic.developer.corb.Manager$CallerBlocksPolicy

/*
* Copyright (c)2005-2010 Mark Logic Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* The use of the Apache License does not indicate that this project is
* affiliated with the Apache Software Foundation.
*/
package com.marklogic.developer.corb;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;

import com.marklogic.developer.SimpleLogger;
import com.marklogic.xcc.AdhocQuery;
import com.marklogic.xcc.Content;
import com.marklogic.xcc.ContentCreateOptions;
import com.marklogic.xcc.ContentFactory;
import com.marklogic.xcc.ContentSource;
import com.marklogic.xcc.ContentSourceFactory;
import com.marklogic.xcc.Request;
import com.marklogic.xcc.RequestOptions;
import com.marklogic.xcc.ResultItem;
import com.marklogic.xcc.ResultSequence;
import com.marklogic.xcc.SecurityOptions;
import com.marklogic.xcc.Session;
import com.marklogic.xcc.exceptions.RequestException;
import com.marklogic.xcc.exceptions.XccConfigException;
import com.marklogic.xcc.exceptions.XccException;
import com.marklogic.xcc.types.XSInteger;
import com.marklogic.xcc.types.XdmItem;

/**
* @author Michael Blakeley, MarkLogic Corporation
* @author Colleen Whitney, MarkLogic Corporation
*/
public class Manager implements Runnable {

    public static String VERSION = "2010-08-24.1";

    public class CallerBlocksPolicy implements RejectedExecutionHandler {

        private BlockingQueue<Runnable> queue;

        private boolean warning = false;

        /*
         * (non-Javadoc)
         *
         * @see
         * java.util.concurrent.RejectedExecutionHandler#rejectedExecution(java
         * .lang.Runnable, java.util.concurrent.ThreadPoolExecutor)
         */
        public void rejectedExecution(Runnable r,
                ThreadPoolExecutor executor) {
            if (null == queue) {
                queue = executor.getQueue();
            }
            try {
                // block until space becomes available
                if (!warning) {
                    logger.fine("queue is full: size = " + queue.size()
                            + " (will only appear once!)");
                    warning = true;
                }
                queue.put(r);
            } catch (InterruptedException e) {
                // reset interrupt status and exit
                Thread.interrupted();
                // someone is trying to interrupt us
                throw new RejectedExecutionException(e);
            }
        }

    }

    private static String versionMessage = "version " + VERSION + " on "
            + System.getProperty("java.version") + " ("
            + System.getProperty("java.runtime.name") + ")";

    /**
     *
     */
    private static final String DECLARE_NAMESPACE_MLSS_XDMP_STATUS_SERVER = "declare namespace mlss = 'http://marklogic.com/xdmp/status/server'\n";

    /**
     *
     */
    private static final String XQUERY_VERSION_0_9_ML = "xquery version \"0.9-ml\"\n";

    /**
     *
     */
    private static final String NAME = Manager.class.getName();

    private URI connectionUri;

    private String collection;

    private TransformOptions options = new TransformOptions();

    private ThreadPoolExecutor pool = null;

    private ContentSource contentSource;

    private Monitor monitor;

    private SimpleLogger logger;

    private String moduleUri;

    private Thread monitorThread;

    private UriQueue uriQueue;

    private ExecutorCompletionService<String> completionService;

    /**
     * @param connectionUri
     * @param collection
     * @param modulePath
     * @param uriListPath
     */
    public Manager(URI connectionUri, String collection) {
        this.connectionUri = connectionUri;
        this.collection = collection;
    }

    /**
     * @param args
     * @throws URISyntaxException
     */
    public static void main(String[] args) throws URISyntaxException {
        if (args.length < 3) {
            usage();
            return;
        }

        // gather inputs
        URI connectionUri = new URI(args[0]);
        String collection = args[1];

        Manager tm = new Manager(connectionUri, collection);
        // options
        TransformOptions options = tm.getOptions();

        options.setProcessModule(args[2]);

        if (args.length > 3 && !args[3].equals("")) {
            options.setThreadCount(Integer.parseInt(args[3]));
        }
        if (args.length > 4 && !args[4].equals("")) {
            options.setUrisModule(args[4]);
        }
        if (args.length > 5 && !args[5].equals("")) {
            options.setModuleRoot(args[5]);
        }
        if (args.length > 6 && !args[6].equals("")) {
            options.setModulesDatabase(args[6]);
        }
        if (args.length > 7 && !args[7].equals("")) {
            if (args[7].equals("false") || args[7].equals("0"))
                options.setDoInstall(false);
        }
        tm.run();
    }

    /**
     * @return
     */
    private TransformOptions getOptions() {
        return options;
    }

    /**
     *
     */
    private static void usage() {
        PrintStream err = System.err;
        err.println("\nusage:");
        err.println("\t" + NAME
                + " xcc://user:password@host:port/[ database ]"
                + " input-selector module-name.xqy"
                + " [ thread-count [ uris-module [ module-root"
                + " [ modules-database [ install ] ] ] ] ]");
    }

    /*
     * (non-Javadoc)
     *
     * @see java.lang.Runnable#run()
     */
    public void run() {
        configureLogger();
        logger.info(NAME + " starting: " + versionMessage);
        long maxMemory = Runtime.getRuntime().maxMemory() / (1024 * 1024);
        logger.info("maximum heap size = " + maxMemory + " MiB");

        prepareContentSource();
        registerStatusInfo();
        prepareModules();
        monitorThread = preparePool();

        try {
            populateQueue();

            while (monitorThread.isAlive()) {
                try {
                    monitorThread.join();
                } catch (InterruptedException e) {
                    // reset interrupt status and continue
                    Thread.interrupted();
                    logger.logException(
                            "interrupted while waiting for monitor", e);
                }
            }
        } catch (XccException e) {
            logger.logException(connectionUri.toString(), e);
            stop();
            // fatal
            throw new RuntimeException(e);
        }
    }

    /**
     * @return
     */
    private Thread preparePool() {
        RejectedExecutionHandler policy = new CallerBlocksPolicy();
        int threads = options.getThreadCount();
        // an array queue should be somewhat lighter-weight
        BlockingQueue<Runnable> workQueue = new ArrayBlockingQueue<Runnable>(
                options.getQueueSize());
        pool = new ThreadPoolExecutor(threads, threads, 16,
                TimeUnit.SECONDS, workQueue, policy);
        pool.prestartAllCoreThreads();
        completionService = new ExecutorCompletionService<String>(pool);
        monitor = new Monitor(pool, completionService, this, logger);
        Thread monitorThread = new Thread(monitor);
        return monitorThread;
    }

    /**
     * @throws IOException
     * @throws RequestException
     *
     */
    private void prepareModules() {
        String[] resourceModules = new String[] {
                options.getUrisModule(), options.getProcessModule() };
        String modulesDatabase = options.getModulesDatabase();
        logger.info("checking modules, database: " + modulesDatabase);
        Session session = contentSource.newSession(modulesDatabase);
        InputStream is = null;
        Content c = null;
        ContentCreateOptions opts = ContentCreateOptions
                .newTextInstance();
        try {
            for (int i = 0; i < resourceModules.length; i++) {
                // Start by checking install flag.
                if (!options.isDoInstall()) {
                    logger.info("Skipping module installation: "
                            + resourceModules[i]);
                    continue;
                }
                // Next check: if XCC is configured for the filesystem, warn
                // user
                else if (options.getModulesDatabase().equals("")) {
                    logger
                            .warning("XCC configured for the filesystem: please install modules manually");
                    return;
                }
                // Finally, if it's configured for a database, install.
                else {
                    File f = new File(resourceModules[i]);
                    // If not installed, are the specified files on the
                    // filesystem?
                    if (f.exists()) {
                        moduleUri = options.getModuleRoot() + f.getName();
                        c = ContentFactory.newContent(moduleUri, f, opts);
                    }
                    // finally, check package
                    else {
                        logger.warning("looking for "
                                + resourceModules[i] + " as resource");
                        moduleUri = options.getModuleRoot()
                                + resourceModules[i];
                        is = this.getClass().getResourceAsStream(
                                resourceModules[i]);
                        if (null == is) {
                            throw new NullPointerException(
                                    resourceModules[i]
                                            + " could not be found on the filesystem,"
                                            + " or in package resources");
                        }
                        c = ContentFactory
                                .newContent(moduleUri, is, opts);
                    }
                    session.insertContent(c);
                }
            }
        } catch (IOException e) {
            logger.logException("fatal error", e);
            throw new RuntimeException(e);
        } catch (RequestException e) {
            logger.logException("fatal error", e);
            throw new RuntimeException(e);
        } finally {
            session.close();
        }
    }

    /**
     *
     */
    private void prepareContentSource() {
        logger.info("using content source " + connectionUri);
        try {
            // support SSL
            boolean ssl = connectionUri.getScheme().equals("xccs");
            contentSource = ssl ? ContentSourceFactory.newContentSource(
                    connectionUri, newTrustAnyoneOptions())
                    : ContentSourceFactory
                            .newContentSource(connectionUri);
        } catch (XccConfigException e) {
            logger.logException(connectionUri.toString(), e);
            throw new RuntimeException(e);
        } catch (KeyManagementException e) {
            logger.logException(connectionUri.toString(), e);
            throw new RuntimeException(e);
        } catch (NoSuchAlgorithmException e) {
            logger.logException(connectionUri.toString(), e);
            throw new RuntimeException(e);
        }
    }

    private void registerStatusInfo() {
        Session session = contentSource.newSession();
        AdhocQuery q = session.newAdhocQuery(XQUERY_VERSION_0_9_ML
                + DECLARE_NAMESPACE_MLSS_XDMP_STATUS_SERVER
                + "let $status := \n"
                + " xdmp:server-status(xdmp:host(), xdmp:server())\n"
                + "let $modules := $status/mlss:modules\n"
                + "let $root := $status/mlss:root\n"
                + "return (data($modules), data($root))");
        ResultSequence rs = null;
        try {
            rs = session.submitRequest(q);
        } catch (RequestException e) {
            e.printStackTrace();
        } finally {
            session.close();
        }
        while (rs.hasNext()) {
            ResultItem rsItem = rs.next();
            XdmItem item = rsItem.getItem();
            if (rsItem.getIndex() == 0 && item.asString().equals("0")) {
                options.setModulesDatabase("");
            }
            if (rsItem.getIndex() == 1) {
                options.setXDBC_ROOT(item.asString());
            }
        }
        logger.info("Configured modules db: "
                + options.getModulesDatabase());
        logger.info("Configured modules root: " + options.getXDBC_ROOT());
        logger.info("Configured uri module: " + options.getUrisModule());
        logger.info("Configured process module: "
                + options.getProcessModule());
    }

    /**
     * @throws XccException
     */
    private void populateQueue() throws XccException {
        logger.info("populating queue");

        TaskFactory tf = new TaskFactory(contentSource, options
                .getModuleRoot()
                + options.getProcessModule());
        uriQueue = new UriQueue(completionService, pool, tf, monitor,
                new LinkedBlockingQueue<String>(), logger);

        // must not cache the results, or we quickly run out of memory
        RequestOptions requestOptions = new RequestOptions();
        requestOptions.setCacheResult(false);

        Session session = null;
        int count = 0;
        int total = -1;

        try {
            session = contentSource.newSession();
            String urisModule = options.getModuleRoot()
                    + options.getUrisModule();
            logger.info("invoking module " + urisModule);
            Request req = session.newModuleInvoke(urisModule);
            // NOTE: collection will be treated as a CWSV
            req.setNewStringVariable("URIS", collection);
            // TODO support DIRECTORY as type
            req.setNewStringVariable("TYPE",
                    TransformOptions.COLLECTION_TYPE);
            req.setNewStringVariable("PATTERN", "[,\\s]+");
            req.setOptions(requestOptions);

            ResultSequence res = session.submitRequest(req);

            // like a Pascal string, the first item will be the count
            total = ((XSInteger) res.next().getItem()).asPrimitiveInt();
            logger.info("expecting total " + total);
            if (0 == total) {
                logger.info("nothing to process");
                stop();
                return;
            }

            uriQueue.setExpected(total);
            uriQueue.start();

            monitor.setTaskCount(total);
            monitorThread.start();

            // this may return millions of items:
            // try to be memory-efficient
            count = 0;
            String uri;
            // check pool occasionally, for fast-fail
            while (res.hasNext() && null != pool) {
                uri = res.next().asString();
                uriQueue.add(uri);
                if (null == pool) {
                    break;
                }
                count++;
                String msg = "queued " + count + "/" + total + ": " + uri;
                if (0 == count % 10000) {
                    logger.info(msg);
                } else {
                    logger.finest(msg);
                }
                if (count > total) {
                    logger
                            .warning("expected " + total + ", got "
                                    + count);
                    logger.warning("check your uri module!");
                }
            }
            logger.info("queued " + count + "/" + total);
        } catch (XccException e) {
            stop();
            throw e;
        } finally {
            if (null != session) {
                session.close();
            }
            // there won't be any more tasks
            if (null != uriQueue) {
                uriQueue.shutdown();
            }
        }
        // if the pool went away, the monitor stopped it: bail out.
        if (null == pool) {
            return;
        }

        assert total == count;
        logger.fine("queue is populated with " + total + " tasks");
    }

    private void configureLogger() {
        if (logger == null) {
            logger = SimpleLogger.getSimpleLogger();
        }
        Properties props = new Properties();
        props.setProperty("LOG_LEVEL", options.getLogLevel());
        props.setProperty("LOG_HANDLER", options.getLogHandler());
        logger.configureLogger(props);
    }

    /**
     * @param e
     */
    public void stop() {
        logger.info("cleaning up");
        if (null != pool) {
            List<Runnable> remaining = pool.shutdownNow();
            if (remaining.size() > 0) {
                logger.warning("thread pool was shut down with "
                        + remaining.size() + " pending tasks");
            }
            pool = null;
        }
        if (null != monitor) {
            monitor.shutdownNow();
        }
        if (null != monitorThread) {
            monitorThread.interrupt();
        }
        if (null != uriQueue) {
            uriQueue.halt();
        }
    }

    /**
     * @param e
     */
    public void stop(ExecutionException e) {
        logger.logException("fatal error", e.getCause());
        logger.warning("exiting due to fatal error");
        stop();
    }

    protected static SecurityOptions newTrustAnyoneOptions()
            throws KeyManagementException, NoSuchAlgorithmException {
        TrustManager[] trust = new TrustManager[] { new X509TrustManager() {
            public X509Certificate[] getAcceptedIssuers() {
                return new X509Certificate[0];
            }

            /**
             * @throws CertificateException
             */
            public void checkClientTrusted(X509Certificate[] certs,
                    String authType) throws CertificateException {
                // no exception means it's okay
            }

            /**
             * @throws CertificateException
             */
            public void checkServerTrusted(X509Certificate[] certs,
                    String authType) throws CertificateException {
                // no exception means it's okay
            }
        } };

        SSLContext sslContext = SSLContext.getInstance("SSLv3");
        sslContext.init(null, trust, null);
        return new SecurityOptions(sslContext);
    }
}
TOP

Related Classes of com.marklogic.developer.corb.Manager$CallerBlocksPolicy

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.