Package org.exist.storage

Source Code of org.exist.storage.BrokerPool$StatusReporter

/*
* eXist Open Source Native XML Database
* Copyright (C) 2003-2014 The eXist-db Project
* http://exist-db.org
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*  $Id$
*/
package org.exist.storage;

import org.apache.log4j.Logger;
import org.exist.Database;
import org.exist.EXistException;
import org.exist.collections.Collection;
import org.exist.collections.CollectionCache;
import org.exist.collections.CollectionConfiguration;
import org.exist.collections.CollectionConfigurationManager;
import org.exist.collections.triggers.*;
import org.exist.config.ConfigurationDocumentTrigger;
import org.exist.config.Configurator;
import org.exist.config.annotation.ConfigurationClass;
import org.exist.config.annotation.ConfigurationFieldAsAttribute;
import org.exist.debuggee.Debuggee;
import org.exist.debuggee.DebuggeeFactory;
import org.exist.dom.SymbolTable;
import org.exist.indexing.IndexManager;
import org.exist.management.AgentFactory;
import org.exist.numbering.DLNFactory;
import org.exist.numbering.NodeIdFactory;
import org.exist.plugin.PluginsManager;
import org.exist.plugin.PluginsManagerImpl;
import org.exist.repo.ClasspathHelper;
import org.exist.repo.ExistRepository;
import org.exist.scheduler.Scheduler;
import org.exist.scheduler.impl.QuartzSchedulerImpl;
import org.exist.scheduler.impl.SystemTaskJobImpl;
import org.exist.security.*;
import org.exist.security.SecurityManager;
import org.exist.security.internal.SecurityManagerImpl;
import org.exist.storage.btree.DBException;
import org.exist.storage.lock.DeadlockDetection;
import org.exist.storage.lock.FileLock;
import org.exist.storage.lock.Lock;
import org.exist.storage.lock.ReentrantReadWriteLock;
import org.exist.storage.sync.Sync;
import org.exist.storage.sync.SyncTask;
import org.exist.storage.txn.TransactionException;
import org.exist.storage.txn.TransactionManager;
import org.exist.storage.txn.Txn;
import org.exist.util.*;
import org.exist.util.Configuration.StartupTriggerConfig;
import org.exist.util.hashtable.MapRWLock;
import org.exist.util.hashtable.MapRWLock.LongOperation;
import org.exist.xmldb.ShutdownListener;
import org.exist.xmldb.XmldbURI;
import org.exist.xquery.PerformanceStats;
import org.expath.pkg.repo.PackageException;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.text.NumberFormat;
import java.util.*;
import java.util.Map.Entry;

/**
* This class controls all available instances of the database.
* Use it to configure, start and stop database instances.
* You may have multiple instances defined, each using its own configuration.
* To define multiple instances, pass an identification string to {@link #configure(String, int, int, Configuration)}
* and use {@link #getInstance(String)} to retrieve an instance.
*
* @author Wolfgang Meier <wolfgang@exist-db.org>
* @author Pierrick Brihaye <pierrick.brihaye@free.fr>
*/
//TODO : in the future, separate the design between the Map of DBInstances and their non static implementation
@ConfigurationClass("pool")
public class BrokerPool implements Database {

    private final static Logger LOG = Logger.getLogger(BrokerPool.class);

    private final static TreeMap<String, BrokerPool> instances = new TreeMap<>();
    private final static Map<String, Throwable> instancesInitializtionException = new TreeMap<>();

    //on-start, ready, go
    /*** initializing sub-components */
    public final static String SIGNAL_STARTUP = "startup";
    /*** ready for recovery & read-only operations */
    public final static String SIGNAL_READINESS = "ready";
    /*** ready for writable operations */
    public final static String SIGNAL_WRITABLE = "writable";
    /*** ready for writable operations */
    public final static String SIGNAL_STARTED = "started";
    /*** running shutdown sequence */
    public final static String SIGNAL_SHUTDOWN = "shutdown";
    /*** recovery aborted, db stopped */
    public final static String SIGNAL_ABORTED = "aborted";

    /**
     * The name of a default database instance for those who are too lazy to provide parameters ;-).
     */
    public final static String DEFAULT_INSTANCE_NAME = "exist";
    public static final String CONFIGURATION_CONNECTION_ELEMENT_NAME = "db-connection";
    public static final String CONFIGURATION_STARTUP_ELEMENT_NAME = "startup";
    public static final String CONFIGURATION_POOL_ELEMENT_NAME = "pool";
    public static final String CONFIGURATION_SECURITY_ELEMENT_NAME = "security";
    public static final String CONFIGURATION_RECOVERY_ELEMENT_NAME = "recovery";
    public static final String DISK_SPACE_MIN_ATTRIBUTE = "minDiskSpace";

    public static final String DATA_DIR_ATTRIBUTE = "files";
    //TODO : move elsewhere ?
    public final static String RECOVERY_ENABLED_ATTRIBUTE = "enabled";
    public final static String RECOVERY_POST_RECOVERY_CHECK = "consistency-check";
    //TODO : move elsewhere ?
    public final static String COLLECTION_CACHE_SIZE_ATTRIBUTE = "collectionCacheSize";
    public final static String MIN_CONNECTIONS_ATTRIBUTE = "min";
    public final static String MAX_CONNECTIONS_ATTRIBUTE = "max";
    public final static String SYNC_PERIOD_ATTRIBUTE = "sync-period";
    public final static String SHUTDOWN_DELAY_ATTRIBUTE = "wait-before-shutdown";
    public final static String NODES_BUFFER_ATTRIBUTE = "nodesBuffer";

    //Various configuration property keys (set by the configuration manager)
    public final static String PROPERTY_STARTUP_TRIGGERS = "startup.triggers";
    public final static String PROPERTY_DATA_DIR = "db-connection.data-dir";
    public final static String PROPERTY_MIN_CONNECTIONS = "db-connection.pool.min";
    public final static String PROPERTY_MAX_CONNECTIONS = "db-connection.pool.max";
    public final static String PROPERTY_SYNC_PERIOD = "db-connection.pool.sync-period";
    public final static String PROPERTY_SHUTDOWN_DELAY = "wait-before-shutdown";
    public static final String DISK_SPACE_MIN_PROPERTY = "db-connection.diskSpaceMin";

    //TODO : move elsewhere ?
    public final static String PROPERTY_COLLECTION_CACHE_SIZE = "db-connection.collection-cache-size";
    //TODO : move elsewhere ? Get fully qualified class name ?
    public final static String DEFAULT_SECURITY_CLASS = "org.exist.security.internal.SecurityManagerImpl";
    public final static String PROPERTY_SECURITY_CLASS = "db-connection.security.class";
    public final static String PROPERTY_RECOVERY_ENABLED = "db-connection.recovery.enabled";
    public final static String PROPERTY_RECOVERY_CHECK = "db-connection.recovery.consistency-check";
    public final static String PROPERTY_SYSTEM_TASK_CONFIG = "db-connection.system-task-config";
    public static final String PROPERTY_NODES_BUFFER = "db-connection.nodes-buffer";
    public static final String PROPERTY_EXPORT_ONLY = "db-connection.emergency";

    public static final String DOC_ID_MODE_ATTRIBUTE = "doc-ids";

    public static final String DOC_ID_MODE_PROPERTY = "db-connection.doc-ids.mode";

    //TODO : inline the class ? or... make it configurable ?
    // WM: inline. I don't think users need to be able to overwrite this.
    // They can register their own shutdown hooks any time.
    private final static Thread shutdownHook = new Thread() {
        /**
         * Make sure that all instances are cleanly shut down.
         */
        @Override
        public void run() {
            LOG.info("Executing shutdown thread");
            BrokerPool.stopAll(true);
        }
    };

    //TODO : make this default value configurable ? useless if we have a registerShutdownHook(Thread aThread) method (null = deregister)
    private static boolean registerShutdownHook = true;

    private static Observer statusObserver = null;
    private StatusReporter statusReporter = null;

    /**
     * Whether of not the JVM should run the shutdown thread.
     *
     * @param register <code>true</code> if the JVM should run the thread
     */
    //TODO : rename as activateShutdownHook ? or registerShutdownHook(Thread aThread)
    // WM: it is probably not necessary to allow users to register their own hook. This method
    // is only used once, by class org.exist.jetty.JettyStart, which registers its own hook.
    public final static void setRegisterShutdownHook(boolean register) {
        /*
         * TODO : call Runtime.getRuntime().removeShutdownHook or Runtime.getRuntime().registerShutdownHook
         * depending of the value of register
         * Since Java doesn't provide a convenient way to know if a shutdown hook has been registrered,
         * we may have to catch IllegalArgumentException
         */
        //TODO : check that the JVM is not shutting down
        registerShutdownHook = register;
    }

    //TODO : make it non-static since every database instance may have its own policy.
    //TODO : make a default value that could be overwritten by the configuration
    // WM: this is only used by junit tests to test the recovery process.
    /**
     * For testing only: triggers a database corruption by disabling the page caches. The effect is
     * similar to a sudden power loss or the jvm being killed. The flag is used by some
     * junit tests to test the recovery process.
     */
    public static boolean FORCE_CORRUPTION = false;

    /**
     * Creates and configures a default database instance and adds it to the pool.
     * Call this before calling {link #getInstance()}.
     * If a default database instance already exists, the new configuration is ignored.
     *
     * @param minBrokers The minimum number of concurrent brokers for handling requests on the database instance.
     * @param maxBrokers The maximum number of concurrent brokers for handling requests on the database instance.
     * @param config     The configuration object for the database instance
     *
     * @throws EXistException If the initialization fails.
     */
    //TODO : in the future, we should implement a Configurable interface 
    public final static void configure(final int minBrokers, final int maxBrokers, final Configuration config)
        throws EXistException, DatabaseConfigurationException {
        configure(DEFAULT_INSTANCE_NAME, minBrokers, maxBrokers, config);
    }

    /**
     * Creates and configures a database instance and adds it to the pool.
     * Call this before calling {link #getInstance()}.
     * If a database instance with the same name already exists, the new configuration is ignored.
     *
     * @param instanceName A <strong>unique</strong> name for the database instance.
     *                     It is possible to have more than one database instance (with different configurations for example).
     * @param minBrokers   The minimum number of concurrent brokers for handling requests on the database instance.
     * @param maxBrokers   The maximum number of concurrent brokers for handling requests on the database instance.
     * @param config       The configuration object for the database instance
     *
     * @throws EXistException If the initialization fails.
     */
    //TODO : in the future, we should implement a Configurable interface
    public final static void configure(
        final String instanceName, final int minBrokers, final int maxBrokers,
        final Configuration config) throws EXistException {
        //Check if there is a database instance in the pool with the same id
        BrokerPool instance = instances.get(instanceName);
        if(instance == null) {
            LOG.debug("configuring database instance '" + instanceName + "'...");
            try {
                //Create the instance
                instance = new BrokerPool(instanceName, minBrokers, maxBrokers, config);
                //Add it to the pool
                instances.put(instanceName, instance);
                //We now have at least an instance...
                if(instances.size() == 1) {
                    //... so a ShutdownHook may be interesting
                    if(registerShutdownHook) {
                        try {
                            //... currently an eXist specific one. TODO : make it configurable ?
                            Runtime.getRuntime().addShutdownHook(shutdownHook);
                            LOG.debug("shutdown hook registered");
                        } catch(final IllegalArgumentException e) {
                            LOG.warn("shutdown hook already registered");
                        }
                    }
                }
            } catch(final Throwable ex) {
                // Catch all possible issues and report.
                LOG.error("Unable to initialize database instance '" + instanceName
                    + "': " + ex.getMessage(), ex);
                instancesInitializtionException.put(instanceName, ex);
                // TODO: Add throw of exception? DW
            }
            //TODO : throw an exception here rather than silently ignore an *explicit* parameter ?
            // WM: maybe throw an exception. Users can check if a db is already configured.
        } else {
            LOG.warn("database instance '" + instanceName + "' is already configured");
        }
    }

    /**
     * Returns whether or not the default database instance is configured.
     *
     * @return <code>true</code> if it is configured
     */
    //TODO : in the future, we should implement a Configurable interface
    public final static boolean isConfigured() {
        return isConfigured(DEFAULT_INSTANCE_NAME);
    }

    /**
     * Returns whether or not a database instance is configured.
     *
     * @param id The name of the database instance
     * @return <code>true</code> if it is configured
     */
    //TODO : in the future, we should implement a Configurable interface 
    public final static boolean isConfigured(final String id) {
        //Check if there is a database instance in the pool with the same id
        final BrokerPool instance = instances.get(id);
        //No : it *can't* be configured
        if(instance == null) {
            return false;
        }
        //Yes : it *may* be configured
        return instance.isInstanceConfigured();
    }

    /**
     * Returns a broker pool for the default database instance.
     *
     * @return The broker pool
     * @throws EXistException If the database instance is not available (not created, stopped or not configured)
     */
    public final static BrokerPool getInstance() throws EXistException {
        return getInstance(DEFAULT_INSTANCE_NAME);
    }

    /**
     * Returns a broker pool for a database instance.
     *
     * @param instanceName The name of the database instance
     * @return The broker pool
     *
     * @throws EXistException If the instance is not available (not created, stopped or not configured)
     */
    public final static BrokerPool getInstance(final String instanceName) throws EXistException {
        //Check if there is a database instance in the pool with the same id
        final BrokerPool instance = instances.get(instanceName);
        if(instance != null)
        //TODO : call isConfigured(id) and throw an EXistException if relevant ?
        {
            return instance;
        }

        final Throwable exception = instancesInitializtionException.get(instanceName);
        if(exception != null) {
            if(exception instanceof EXistException) {
                throw (EXistException) exception;
            }
            throw new EXistException(exception);
        }

        throw new EXistException("database instance '" + instanceName + "' is not available");
    }

    /**
     * Returns an iterator over the database instances.
     *
     * @return The iterator
     */
    public final static Iterator<BrokerPool> getInstances() {
        return instances.values().iterator();
    }

    public final static boolean isInstancesEmpty() {
        return instances.values().isEmpty();
    }

    /**
     * Stops the default database instance. After calling this method, it is
     * no longer configured.
     *
     * @throws EXistException If the default database instance is not available (not created, stopped or not configured)
     */
    public final static void stop() throws EXistException {
        stop(DEFAULT_INSTANCE_NAME);
    }

    /**
     * Stops the given database instance. After calling this method, it is
     * no longer configured.
     *
     * @param id The name of the database instance
     *
     * @throws EXistException If the database instance is not available (not created, stopped or not configured)
     */
    public final static void stop(final String id) throws EXistException {
        final BrokerPool instance = getInstance(id);
        instance.shutdown();
    }

    /**
     * Stops all the database instances. After calling this method, the database instances are
     * no longer configured.
     *
     * @param killed <code>true</code> when invoked by an exiting JVM
     */
    public final static void stopAll(final boolean killed) {
        for(final BrokerPool instance : instances.values()) {
            if(instance.conf != null) {
                //Shut it down
                instance.shutdown(killed);
            }
        }
        //Clear the living instances container : they are all sentenced to death...
        instances.clear();
    }

    public final static void systemInfo() {
        for(final BrokerPool instance : instances.values()) {
            instance.printSystemInfo();
        }
    }

    public static void registerStatusObserver(final Observer observer) {
        statusObserver = observer;
        LOG.debug("registering observer: " + observer.getClass().getName());
    }

  /* END OF STATIC IMPLEMENTATION */

    /**
     * Default values
     */
    //TODO : make them static when we have 2 classes
    private final int DEFAULT_MIN_BROKERS = 1;
    private final int DEFAULT_MAX_BROKERS = 15;
    public final long DEFAULT_SYNCH_PERIOD = 120000;
    public final long DEFAULT_MAX_SHUTDOWN_WAIT = 45000;
    //TODO : move this default setting to org.exist.collections.CollectionCache ?
    public final int DEFAULT_COLLECTION_BUFFER_SIZE = 64;

    public static final String PROPERTY_PAGE_SIZE = "db-connection.page-size";
    public static final int DEFAULT_PAGE_SIZE = 4096;

    /**
     * <code>true</code> if the database instance is able to handle transactions.
     */
    private boolean transactionsEnabled;

    /**
     * The name of the database instance
     */
    private String instanceName;

    //TODO: change 0 = initializing, 1 = operating, -1 = shutdown  (shabanovd)
    private final static int SHUTDOWN = -1;
    private final static int INITIALIZING = 0;
    private final static int OPERATING = 1;

    // volatile so this doesn't get optimized away or into a CPU register in some thread
    private volatile int status = INITIALIZING;

    /**
     * The number of brokers for the database instance
     */
    private int brokersCount = 0;

    /**
     * The minimal number of brokers for the database instance
     */
    @ConfigurationFieldAsAttribute("min")
    private int minBrokers;

    /**
     * The maximal number of brokers for the database instance
     */
    @ConfigurationFieldAsAttribute("max")
    private int maxBrokers;

    /**
     * The number of inactive brokers for the database instance
     */
    private Stack<DBBroker> inactiveBrokers = new Stack<>();

    /**
     * The number of active brokers for the database instance
     */
    private MapRWLock<Thread, DBBroker> activeBrokers = new MapRWLock<>(new IdentityHashMap<Thread, DBBroker>());

    /**
     * The configuration object for the database instance
     */
    protected Configuration conf = null;

    /**
     * <code>true</code> if a cache synchronization event is scheduled
     */
    //TODO : rename as syncScheduled ?
    //TODO : alternatively, delete this member and create a Sync.NOSYNC event
    private boolean syncRequired = false;

    /**
     * The kind of scheduled cache synchronization event.
     * One of {@link org.exist.storage.sync.Sync#MAJOR_SYNC} or {@link org.exist.storage.sync.Sync#MINOR_SYNC}
     */
    private int syncEvent = 0;

    private boolean checkpoint = false;

    /**
     * <code>true</code> if the database instance is running in read-only mode.
     */
    //TODO : this should be computed by the DBrokers depending of their configuration/capabilities
    //TODO : for now, this member is used for recovery management
    private boolean isReadOnly;

    @ConfigurationFieldAsAttribute("pageSize")
    private int pageSize;

    private FileLock dataLock;

    /**
     * The transaction manager of the database instance.
     */
    private TransactionManager transactionManager = null;

    /**
     * Delay (in ms) for running jobs to return when the database instance shuts down.
     */
    @ConfigurationFieldAsAttribute("wait-before-shutdown")
    private long maxShutdownWait;

    /**
     * The scheduler for the database instance.
     */
    @ConfigurationFieldAsAttribute("scheduler")
    private Scheduler scheduler;

    /**
     * Manages pluggable index structures.
     */
    private IndexManager indexManager;

    /**
     * Global symbol table used to encode element and attribute qnames.
     */
    private SymbolTable symbols;

    /**
     * Cache synchronization on the database instance.
     */
    @ConfigurationFieldAsAttribute("sync-period")
    private long majorSyncPeriod = DEFAULT_SYNCH_PERIOD;        //the period after which a major sync should occur
    private long lastMajorSync = System.currentTimeMillis();    //time the last major sync occurred

    private long diskSpaceMin = 64 * 1024L * 1024L;

    /**
     * The listener that is notified when the database instance shuts down.
     */
    private ShutdownListener shutdownListener = null;

    /**
     * The security manager of the database instance.
     */
    private SecurityManager securityManager = null;

    /**
     * The plugin manager.
     */
    private PluginsManagerImpl pluginManager = null;


    /**
     * The global notification service used to subscribe
     * to document updates.
     */
    private NotificationService notificationService = null;

    /**
     * The cache in which the database instance may store items.
     */

    private DefaultCacheManager cacheManager;

    private CollectionCacheManager collectionCacheMgr;

    private long reservedMem;

    /**
     * The pool in which the database instance's <strong>compiled</strong> XQueries are stored.
     */
    private XQueryPool xQueryPool;

    /**
     * The monitor in which the database instance's strong>running</strong> XQueries are managed.
     */
    private ProcessMonitor processMonitor;

    /**
     * Global performance stats to gather function execution statistics
     * from all queries running on this database instance.
     */
    private PerformanceStats xqueryStats;

    /**
     * The global manager for accessing collection configuration files from the database instance.
     */
    private CollectionConfigurationManager collectionConfigurationManager = null;

    /**
     * The cache in which the database instance's collections are stored.
     */
    //TODO : rename as collectionsCache ?
    protected CollectionCache collectionCache;

    /**
     * The pool in which the database instance's readers are stored.
     */
    protected XMLReaderPool xmlReaderPool;

    private NodeIdFactory nodeFactory = new DLNFactory();

    //TODO : is another value possible ? If no, make it static
    // WM: no, we need one lock per database instance. Otherwise we would lock another database.
    private Lock globalXUpdateLock = new ReentrantReadWriteLock("xupdate");

    private Subject serviceModeUser = null;
    private boolean inServiceMode = false;

    //the time that the database was started
    private final Calendar startupTime = Calendar.getInstance();

    private BrokerWatchdog watchdog = null;

    private ClassLoader classLoader;

    private ExistRepository expathRepo = null;

    /**
     * Creates and configures the database instance.
     *
     * @param instanceName A name for the database instance.
     * @param minBrokers   The minimum number of concurrent brokers for handling requests on the database instance.
     * @param maxBrokers   The maximum number of concurrent brokers for handling requests on the database instance.
     * @param conf         The configuration object for the database instance
     *
     * @throws EXistException If the initialization fails.
     */
    //TODO : Then write a configure(int minBrokers, int maxBrokers, Configuration conf) method
    private BrokerPool(final String instanceName, final int minBrokers, final int maxBrokers, final Configuration conf)
        throws EXistException, DatabaseConfigurationException {

        Integer anInteger;
        Long aLong;
        Boolean aBoolean;
        final NumberFormat nf = NumberFormat.getNumberInstance();

        this.classLoader = Thread.currentThread().getContextClassLoader();

        //TODO : ensure that the instance name is unique ?
        //WM: needs to be done in the configure method.
        this.instanceName = instanceName;

        //TODO : find a nice way to (re)set the default values
        //TODO : create static final members for configuration keys
        this.minBrokers = DEFAULT_MIN_BROKERS;
        this.maxBrokers = DEFAULT_MAX_BROKERS;
        this.maxShutdownWait = DEFAULT_MAX_SHUTDOWN_WAIT;
        //TODO : read from configuration
        this.transactionsEnabled = true;

        this.minBrokers = minBrokers;
        this.maxBrokers = maxBrokers;
        /*
     * strange enough, the settings provided by the constructor may be overridden
     * by the ones *explicitly* provided by the constructor
     * TODO : consider a private constructor BrokerPool(String instanceName) then configure(int minBrokers, int maxBrokers, Configuration config)
     */
        anInteger = (Integer) conf.getProperty(PROPERTY_MIN_CONNECTIONS);
        if(anInteger != null) {
            this.minBrokers = anInteger;
        }
        anInteger = (Integer) conf.getProperty(PROPERTY_MAX_CONNECTIONS);
        if(anInteger != null) {
            this.maxBrokers = anInteger;
        }
        //TODO : sanity check : minBrokers shall be lesser than or equal to maxBrokers
        //TODO : sanity check : minBrokers shall be positive
        LOG.info("database instance '" + instanceName + "' will have between " + nf.format(this.minBrokers) + " and " + nf.format(this.maxBrokers) + " brokers");

        //TODO : use the periodicity of a SystemTask (see below)
        aLong = (Long) conf.getProperty(PROPERTY_SYNC_PERIOD);
        if(aLong != null)
      /*this.*/ {
            majorSyncPeriod = aLong;
        }
        //TODO : sanity check : the synch period should be reasonable
        LOG.info("database instance '" + instanceName + "' will be synchronized every " + nf.format(/*this.*/majorSyncPeriod) + " ms");

        aLong = (Long) conf.getProperty(BrokerPool.PROPERTY_SHUTDOWN_DELAY);
        if(aLong != null) {
            this.maxShutdownWait = aLong;
        }
        //TODO : sanity check : the shutdown period should be reasonable
        LOG.info("database instance '" + instanceName + "' will wait  " + nf.format(this.maxShutdownWait) + " ms during shutdown");

        aBoolean = (Boolean) conf.getProperty(PROPERTY_RECOVERY_ENABLED);
        if(aBoolean != null) {
            this.transactionsEnabled = aBoolean;
        }
        LOG.info("database instance '" + instanceName + "' is enabled for transactions : " + this.transactionsEnabled);

        final Integer min = (Integer) conf.getProperty(BrokerPool.DISK_SPACE_MIN_PROPERTY);
        if(min != null) {
            diskSpaceMin = min * 1024L * 1024L;
        }

        pageSize = conf.getInteger(PROPERTY_PAGE_SIZE);
        if(pageSize < 0) {
            pageSize = DEFAULT_PAGE_SIZE;
        }

        //TODO : move this to initialize ? (cant as we need it for FileLockHeartBeat)
        scheduler = new QuartzSchedulerImpl(this, conf);

        //TODO : since we need one :-( (see above)
        this.isReadOnly = !canReadDataDir(conf);
        LOG.debug("isReadOnly: " + isReadOnly);
        //Configuration is valid, save it
        this.conf = conf;

        //TODO : in the future, we should implement an Initializable interface
        try {
            initialize();
        } catch(final Throwable e) {
            // remove that file lock we may have acquired in canReadDataDir
            if(dataLock != null && !isReadOnly)
                dataLock.release();

            if(!instances.containsKey(instanceName)) {
                instancesInitializtionException.put(instanceName, e);
            }

            if(e instanceof EXistException) {
                throw (EXistException) e;
            }

            if(e instanceof DatabaseConfigurationException) {
                throw (DatabaseConfigurationException) e;
            }

            throw new EXistException(e);
        }

        //TODO : move this to initialize ?
        //setup database synchronization job
        if(majorSyncPeriod > 0) {
            //TODO : why not automatically register Sync in system tasks ?
//            scheduler.createPeriodicJob(2500, new Sync(), 2500);
            final SyncTask syncTask = new SyncTask();
            syncTask.configure(conf, null);
            scheduler.createPeriodicJob(2500, new SystemTaskJobImpl(SyncTask.getJobName(), syncTask), 2500);
        }

        if("yes".equals(System.getProperty("trace.brokers", "no"))) {
            watchdog = new BrokerWatchdog();
        }
    }

    //TODO : create a canReadJournalDir() method in the *relevant* class. The two directories may be different.
    protected boolean canReadDataDir(final Configuration conf) throws EXistException {
        String dataDir = (String) conf.getProperty(PROPERTY_DATA_DIR);
        if(dataDir == null) {
            dataDir = "data";
        } //TODO : DEFAULT_DATA_DIR

        final File dir = new File(dataDir);
        if(!dir.exists()) {
            try {
                //TODO : shall we force the creation ? use a parameter to decide ?
                LOG.info("Data directory '" + dir.getAbsolutePath() + "' does not exist. Creating one ...");
                dir.mkdirs();
            } catch(final SecurityException e) {
                LOG.info("Cannot create data directory '" + dir.getAbsolutePath() + "'. Switching to read-only mode.");
                return false;
            }
        }

        //Save it for further use.
        //TODO : "data-dir" has sense for *native* brokers
        conf.setProperty(PROPERTY_DATA_DIR, dataDir);
        if(!dir.canWrite()) {
            LOG.info("Cannot write to data directory: " + dir.getAbsolutePath() + ". Switching to read-only mode.");
            return false;
        }

        // try to acquire lock on the data dir
        dataLock = new FileLock(this, dir, "dbx_dir.lck");

        try {
            final boolean locked = dataLock.tryLock();
            if(!locked) {
                throw new EXistException("The database directory seems to be locked by another " +
                    "database instance. Found a valid lock file: " + dataLock.getFile());
            }
        } catch(final ReadOnlyException e) {
            LOG.info(e.getMessage() + ". Switching to read-only mode!!!");
            return false;
        }
        return true;
    }

    /**
     * Initializes the database instance.
     *
     * @throws EXistException
     */
    protected void initialize() throws EXistException, DatabaseConfigurationException {
        if(LOG.isDebugEnabled()) {
            LOG.debug("initializing database instance '" + instanceName + "'...");
        }

        //Flag to indicate that we are initializing
        status = INITIALIZING;

        // Don't allow two threads to do a race on this. May be irrelevant as this is only called
        // from the constructor right now.
        synchronized(this) {
            try {
                statusReporter = new StatusReporter(SIGNAL_STARTUP);
                if(statusObserver != null) {
                    statusReporter.addObserver(statusObserver);
                }
                final Thread statusThread = new Thread(statusReporter);
                statusThread.start();

                // statusReporter may have to be terminated or the thread can/will hang.
                try {
                    final boolean exportOnly = (Boolean) conf.getProperty(PROPERTY_EXPORT_ONLY, false);

                    //create the security manager
                    securityManager = new SecurityManagerImpl(this);

                    //REFACTOR : construct then configure
                    cacheManager = new DefaultCacheManager(this);

                    //REFACTOR : construct then configure
                    xQueryPool = new XQueryPool(conf);
                    //REFACTOR : construct then... configure
                    processMonitor = new ProcessMonitor(maxShutdownWait);
                    xqueryStats = new PerformanceStats(this);

                    //REFACTOR : construct then... configure
                    xmlReaderPool = new XMLReaderPool(conf, new XMLReaderObjectFactory(this), 5, 0);

                    //REFACTOR : construct then... configure
                    int bufferSize = conf.getInteger(PROPERTY_COLLECTION_CACHE_SIZE);
                    if(bufferSize == -1) {
                        bufferSize = DEFAULT_COLLECTION_BUFFER_SIZE;
                    }
                    collectionCache = new CollectionCache(this, bufferSize, 0.0001);
                    collectionCacheMgr = new CollectionCacheManager(this, collectionCache);

                    // compute how much memory should be reserved for caches to grow
                    final Runtime rt = Runtime.getRuntime();
                    final long maxMem = rt.maxMemory();
                    final long minFree = maxMem / 5;
                    reservedMem = cacheManager.getTotalMem() + collectionCacheMgr.getMaxTotal() + minFree;
                    LOG.debug("Reserved memory: " + reservedMem + "; max: " + maxMem + "; min: " + minFree);

                    notificationService = new NotificationService();

                    //REFACTOR : construct then... configure
                    //TODO : journal directory *may* be different from BrokerPool.PROPERTY_DATA_DIR
                    transactionManager = new TransactionManager(this, new File((String) conf.getProperty(BrokerPool.PROPERTY_DATA_DIR)), isTransactional());
                    try {
                        transactionManager.initialize();
                    } catch(final ReadOnlyException e) {
                        LOG.warn(e.getMessage() + ". Switching to read-only mode!!!");
                        isReadOnly = true;
                    }
                    // If the initialization fails after transactionManager has been created this method better cleans up
                    // or the FileSyncThread for the journal can/will hang.
                    try {
                        symbols = new SymbolTable(this, conf);
                        isReadOnly = isReadOnly || !symbols.getFile().canWrite();

                        indexManager = new IndexManager(this, conf);

                        //TODO : replace the following code by get()/release() statements ?
                        // WM: I would rather tend to keep this broker reserved as a system broker.
                        // create a first broker to initialize the security manager
                        //createBroker();
                        //TODO : this broker is *not* marked as active and *might* be reused by another process ! Is it intended ?
                        // at this stage, the database is still single-threaded, so reusing the broker later is not a problem.
                        //DBBroker broker = inactiveBrokers.peek();
                        // dmitriy: Security issue: better to use proper get()/release() way, because of sub-processes (SecurityManager as example)
                        final DBBroker broker = get(securityManager.getSystemSubject());
                        try {

                            if(isReadOnly()) {
                                transactionManager.setEnabled(false);
                            }

                            //Run the recovery process
                            //TODO : assume
                            boolean recovered = false;
                            if(isTransactional()) {
                                recovered = transactionManager.runRecovery(broker);
                                //TODO : extract the following from this block ? What if we ware not transactional ? -pb
                                if(!recovered) {
                                    try {
                                        if(broker.getCollection(XmldbURI.ROOT_COLLECTION_URI) == null) {
                                            final Txn txn = transactionManager.beginTransaction();
                                            try {
                                                //TODO : use a root collection final member
                                                broker.getOrCreateCollection(txn, XmldbURI.ROOT_COLLECTION_URI);
                                                transactionManager.commit(txn);
                                            } catch(final IOException | TriggerException | PermissionDeniedException e) {
                                                transactionManager.abort(txn);
                                            } finally {
                                                transactionManager.close(txn);
                                            }
                                        }
                                    } catch(final PermissionDeniedException pde) {
                                        LOG.fatal(pde.getMessage(), pde);
                                    }
                                }
                            }

                  /* initialise required collections if they don't exist yet */
                            if(!exportOnly) {
                                try {
                                    initialiseSystemCollections(broker);
                                } catch(final PermissionDeniedException pde) {
                                    LOG.error(pde.getMessage(), pde);
                                    throw new EXistException(pde.getMessage(), pde);
                                }
                            }

                            //create the plugin manager
                            pluginManager = new PluginsManagerImpl(this, broker);

                            //TODO : from there, rethink the sequence of calls.
                            // WM: attention: a small change in the sequence of calls can break
                            // either normal startup or recovery.

                            status = OPERATING;

                            statusReporter.setStatus(SIGNAL_READINESS);

                            //Get a manager to handle further collections configuration
                            initCollectionConfigurationManager(broker);

                            //wake-up the plugins manager
                            pluginManager.start(broker);

                            //wake-up the security manager
                            securityManager.attach(this, broker);

                            //have to do this after initializing = false
                            // so that the policies collection is saved
                            if(securityManager.isXACMLEnabled()) {
                                securityManager.getPDP().initializePolicyCollection();
                            }

                            //If necessary, launch a task to repair the DB
                            //TODO : merge this with the recovery process ?
                            //XXX: don't do if READONLY mode
                            if(recovered) {
                                if(!exportOnly) {
                                    reportStatus("Reindexing database files...");
                                    try {
                                        broker.repair();
                                    } catch(final PermissionDeniedException e) {
                                        LOG.warn("Error during recovery: " + e.getMessage(), e);
                                    }
                                }

                                if(((Boolean) conf.getProperty(PROPERTY_RECOVERY_CHECK)).booleanValue()) {
                                    final ConsistencyCheckTask task = new ConsistencyCheckTask();
                                    final Properties props = new Properties();
                                    props.setProperty("backup", "no");
                                    props.setProperty("output", "sanity");
                                    task.configure(conf, props);
                                    task.execute(broker);
                                }
                            }

                            //OK : the DB is repaired; let's make a few RW operations
                            statusReporter.setStatus(SIGNAL_WRITABLE);

                            //initialize configurations watcher trigger
                            if(!exportOnly) {
                                try {
                                    initialiseTriggersForCollections(broker, XmldbURI.SYSTEM_COLLECTION_URI);
                                } catch(final PermissionDeniedException pde) {
                                    //XXX: do not catch exception!
                                    LOG.error(pde.getMessage(), pde);
                                }
                            }

                            // remove temporary docs
                            try {
                                broker.cleanUpTempResources(true);
                            } catch(final PermissionDeniedException pde) {
                                LOG.error(pde.getMessage(), pde);
                            }

                            sync(broker, Sync.MAJOR_SYNC);

                            //require to allow access by BrokerPool.getInstance();
                            instances.put(instanceName, this);

                            try {
                                // initialize EXPath repository so startup triggers can access it
                                expathRepo = ExistRepository.getRepository(this.conf);
                            } catch(final PackageException e) {
                                LOG.warn("Failed to initialize expath repository: " + e.getMessage() + " - this is not fatal, but " +
                                    "the package manager may not work.");
                            }

                            callStartupTriggers((List<StartupTriggerConfig>) conf.getProperty(BrokerPool.PROPERTY_STARTUP_TRIGGERS), broker);
                        } finally {
                            release(broker);
                        }

                        //Create a default configuration file for the root collection
                        //TODO : why can't we call this from within CollectionConfigurationManager ?
                        //TODO : understand why we get a test suite failure
                        //collectionConfigurationManager.checkRootCollectionConfigCollection(broker);
                        //collectionConfigurationManager.checkRootCollectionConfig(broker);


                /* TODO: start adam */

                        //Schedule the system tasks
                /*for (int i = 0; i < systemTasks.size(); i++) {
                            //TODO : remove first argument when SystemTask has a getPeriodicity() method
                            initSystemTask((SingleInstanceConfiguration.SystemTaskConfig) systemTasksPeriods.get(i), (SystemTask)systemTasks.get(i));
                        }
                    systemTasksPeriods = null;*/

                /* TODO: end adam */

                        //Create the minimal number of brokers required by the configuration
                        for(int i = 1; i < minBrokers; i++) {
                            createBroker();
                        }

                        // register some MBeans to provide access to this instance
                        AgentFactory.getInstance().initDBInstance(this);

                        if(LOG.isDebugEnabled()) {
                            LOG.debug("database instance '" + instanceName + "' initialized");
                        }

                        //setup any configured jobs
                        //scheduler.setupConfiguredJobs();

                        //execute any startup jobs
                        //scheduler.executeStartupJobs();

                        scheduler.run();

                        ClasspathHelper.updateClasspath(this);

                        statusReporter.setStatus(SIGNAL_STARTED);
                    } catch(final Throwable t) {
                        if(isTransactional() && transactionManager != null) {
                            transactionManager.shutdown();
                        }
                        throw t;
                    }
                } catch(final EXistException | DatabaseConfigurationException e) {
                    throw e;
                } catch(final Throwable t) {
                    throw new EXistException(t.getMessage(), t);
                }
            } finally {
                if(statusReporter != null) {
                    statusReporter.terminate();
                    statusReporter = null;
                }
            }
        }
    }

    //TODO : remove the period argument when SystemTask has a getPeriodicity() method
    //TODO : make it protected ?
    /*private void initSystemTask(SingleInstanceConfiguration.SystemTaskConfig config, SystemTask task) throws EXistException {
        try {
            if (config.getCronExpr() == null) {
                LOG.debug("Scheduling system maintenance task " + task.getClass().getName() + " every " +
                        config.getPeriod() + " ms");
                scheduler.createPeriodicJob(config.getPeriod(), new SystemTaskJobImpl(task), config.getPeriod());
            } else {
                LOG.debug("Scheduling system maintenance task " + task.getClass().getName() +
                        " with cron expression: " + config.getCronExpr());
                scheduler.createCronJob(config.getCronExpr(), new SystemTaskJobImpl(task));
            }
        } catch (Exception e) {
      LOG.warn(e.getMessage(), e);
            throw new EXistException("Failed to initialize system maintenance task: " + e.getMessage());
        }
    }*/

    private void callStartupTriggers(final List<StartupTriggerConfig> startupTriggerConfigs, final DBBroker sysBroker) {
        if(startupTriggerConfigs == null) {
            return;
        }

        for(final StartupTriggerConfig startupTriggerConfig : startupTriggerConfigs) {
            try {
                final Class<StartupTrigger> clazz = (Class<StartupTrigger>) Class.forName(startupTriggerConfig.getClazz());
                final StartupTrigger startupTrigger = clazz.newInstance();
                startupTrigger.execute(sysBroker, startupTriggerConfig.getParams());
            } catch(final ClassNotFoundException | IllegalAccessException | InstantiationException e) {
                LOG.error("Could not call StartupTrigger class: " + startupTriggerConfig + ". SKIPPING! " + e.getMessage(), e);
            } catch(final RuntimeException re) {
                LOG.warn("StartupTrigger threw RuntimeException: " + re.getMessage() + ". IGNORING!", re);
            }
        }
        // trigger a checkpoint after processing all startup triggers
        checkpoint = true;
        triggerSync(Sync.MAJOR_SYNC);
    }

    /**
     * Initialise system collections, if it doesn't exist yet
     *
     * @param sysBroker        The system broker from before the brokerpool is populated
     * @param sysCollectionUri XmldbURI of the collection to create
     * @param permissions      The permissions to set on the created collection
     */
    private void initialiseSystemCollection(final DBBroker sysBroker, final XmldbURI sysCollectionUri, final int permissions) throws EXistException, PermissionDeniedException {
        Collection collection = sysBroker.getCollection(sysCollectionUri);
        if(collection == null) {
            final TransactionManager transact = getTransactionManager();
            final Txn txn = transact.beginTransaction();
            try {
                collection = sysBroker.getOrCreateCollection(txn, sysCollectionUri);
                if(collection == null) {
                    throw new IOException("Could not create system collection: " + sysCollectionUri);
                }
                collection.setPermissions(permissions);
                sysBroker.saveCollection(txn, collection);

                transact.commit(txn);
            } catch(final Exception e) {
                transact.abort(txn);
                e.printStackTrace();
                final String msg = "Initialisation of system collections failed: " + e.getMessage();
                LOG.error(msg, e);
                throw new EXistException(msg, e);
            } finally {
                transact.close(txn);
            }
        }
    }

    /**
     * Initialize required system collections, if they don't exist yet
     *
     * @param broker - The system broker from before the brokerpool is populated
     * @throws EXistException If a system collection cannot be created
     */
    private void initialiseSystemCollections(final DBBroker broker) throws EXistException, PermissionDeniedException {
        //create /db/system
        initialiseSystemCollection(broker, XmldbURI.SYSTEM_COLLECTION_URI, Permission.DEFAULT_SYSTEM_COLLECTION_PERM);
    }

    private void initialiseTriggersForCollections(final DBBroker broker, final XmldbURI uri) throws EXistException, PermissionDeniedException {
        final Collection collection = broker.getCollection(uri);

        //initialize configurations watcher trigger
        if(collection != null) {
            final CollectionConfigurationManager manager = getConfigurationManager();
            final CollectionConfiguration collConf = manager.getOrCreateCollectionConfiguration(broker, collection);

            final DocumentTriggerProxy triggerProxy = new DocumentTriggerProxy(ConfigurationDocumentTrigger.class); //, collection.getURI());
            collConf.documentTriggers().add(triggerProxy);
        }
    }

    public long getReservedMem() {
        return reservedMem - cacheManager.getSizeInBytes();
    }

    public int getPageSize() {
        return pageSize;
    }

    /**
     * Returns the class loader used when this BrokerPool was configured.
     */
    public ClassLoader getClassLoader() {
        return this.classLoader;
    }

    /**
     * Whether or not the database instance is being initialized.
     *
     * @return <code>true</code> is the database instance is being initialized
     */
    //TODO : let's be positive and rename it as isInitialized ?
    public boolean isInitializing() {
        return status == INITIALIZING;
    }

    /**
     * Returns the database instance's name.
     *
     * @return The id
     */
    //TODO : rename getInstanceName
    public String getId() {
        return instanceName;
    }

    /**
     * Returns the number of brokers currently serving requests for the database instance.
     *
     * @return The brokers count
     * @deprecated use countActiveBrokers
     */
    //TODO : rename as getActiveBrokers ?
    public int active() {
        return activeBrokers.size();
    }

    /**
     * Returns the number of brokers currently serving requests for the database instance.
     *
     * @return The active brokers count.
     */
    @Override
    public int countActiveBrokers() {
        return activeBrokers.size();
    }

    public Map<Thread, DBBroker> getActiveBrokers() {
        final Map<Thread, DBBroker> res = new HashMap<>(activeBrokers.size());

        activeBrokers.readOperation(new LongOperation<Thread, DBBroker>() {
            @Override
            public void execute(final Map<Thread, DBBroker> map) {
                res.putAll(map);
            }
        });

        return res;
    }

    /**
     * Returns the number of inactive brokers for the database instance.
     *
     * @return The brokers count
     */
    //TODO : rename as getInactiveBrokers ?
    public int available() {
        return inactiveBrokers.size();
    }

    //TODO : getMin() method ?

    /**
     * Returns the maximal number of brokers for the database instance.
     *
     * @return The brokers count
     */
    //TODO : rename as getMaxBrokers ?
    public int getMax() {
        return maxBrokers;
    }

    public int total() {
        return brokersCount;
    }

    /**
     * Returns whether the database instance has been configured.
     *
     * @return <code>true</code> if the datbase instance is configured
     */
    public final boolean isInstanceConfigured() {
        return conf != null;
    }

    /**
     * Returns the configuration object for the database instance.
     *
     * @return The configuration
     */
    public Configuration getConfiguration() {
        return conf;
    }

    public ExistRepository getExpathRepo() {
        return expathRepo;
    }

    //TODO : rename as setShutdwonListener ?
    public void registerShutdownListener(final ShutdownListener listener) {
        //TODO : check that we are not shutting down
        shutdownListener = listener;
    }

    public NodeIdFactory getNodeFactory() {
        return nodeFactory;
    }

    /**
     * Returns the database instance's security manager
     *
     * @return The security manager
     */
    public SecurityManager getSecurityManager() {
        return securityManager;
    }


    /**
     * Returns the Scheduler
     *
     * @return The scheduler
     */
    public Scheduler getScheduler() {
        return scheduler;
    }

    public SymbolTable getSymbols() {
        return symbols;
    }

    public NotificationService getNotificationService() {
        return notificationService;
    }

    /**
     * Returns whether transactions can be handled by the database instance.
     *
     * @return <code>true</code> if transactions can be handled
     */
    public boolean isTransactional() {
        //TODO : confusion between dataDir and a so-called "journalDir" !
        return !isReadOnly && transactionsEnabled;
    }

    public boolean isReadOnly() {
        final long freeSpace = dataLock.getFile().getUsableSpace();
        if(freeSpace < diskSpaceMin) {
            LOG.fatal("Partition containing DATA_DIR: " + dataLock.getFile().getAbsolutePath() + " is running out of disk space. " +
                "Switching eXist-db to read only to prevent data loss!");
            setReadOnly();
        }

        return isReadOnly;
    }

    public void setReadOnly() {
        LOG.info("Switching to read-only mode!!!");
        isReadOnly = true;
    }

    public boolean isInServiceMode() {
        return inServiceMode;
    }

    public TransactionManager getTransactionManager() {
        return this.transactionManager;
    }

    /**
     * Returns a manager for accessing the database instance's collection configuration files.
     *
     * @return The manager
     */
    @Override
    public CollectionConfigurationManager getConfigurationManager() {
        return collectionConfigurationManager;
    }

    public void initCollectionConfigurationManager(final DBBroker broker) {
        if(collectionConfigurationManager == null) {
            try {
                collectionConfigurationManager = new CollectionConfigurationManager(broker);
            } catch(final Exception e) {
                LOG.error("Found an error while initializing database: " + e.getMessage(), e);
            }
        }
    }


    /**
     * Returns a cache in which the database instance's collections are stored.
     *
     * @return The cache
     */
    public CollectionCache getCollectionsCache() {
        return collectionCache;
    }

    /**
     * Returns a cache in which the database instance's may store items.
     *
     * @return The cache
     */
    @Override
    public DefaultCacheManager getCacheManager() {
        return cacheManager;
    }

    public CollectionCacheManager getCollectionCacheMgr() {
        return collectionCacheMgr;
    }

    /**
     * Returns the index manager which handles all additional indexes not
     * being part of the database core.
     *
     * @return The IndexManager
     */
    public IndexManager getIndexManager() {
        return indexManager;
    }

    /**
     * Returns a pool in which the database instance's <strong>compiled</strong> XQueries are stored.
     *
     * @return The pool
     */
    public XQueryPool getXQueryPool() {
        return xQueryPool;
    }

    /**
     * Returns a monitor in which the database instance's <strong>running</strong> XQueries are managed.
     *
     * @return The monitor
     */
    public ProcessMonitor getProcessMonitor() {
        return processMonitor;
    }

    /**
     * Returns the global profiler used to gather execution statistics
     * from all XQueries running on this db instance.
     *
     * @return the profiler
     */
    public PerformanceStats getPerformanceStats() {
        return xqueryStats;
    }

    /**
     * Returns a pool in which the database instance's readers are stored.
     *
     * @return The pool
     */
    public XMLReaderPool getParserPool() {
        return xmlReaderPool;
    }

    /**
     * Returns the global update lock for the database instance.
     * This lock is used by XUpdate operations to avoid that
     * concurrent XUpdate requests modify the database until all
     * document locks have been correctly set.
     *
     * @return The global lock
     */
    //TODO : rename as getUpdateLock ?
    public Lock getGlobalUpdateLock() {
        return globalXUpdateLock;
    }

    /**
     * Creates an inactive broker for the database instance.
     *
     * @return The broker
     * @throws EXistException
     */
    protected DBBroker createBroker() throws EXistException {
        //TODO : in the future, don't pass the whole configuration, just the part relevant to brokers
        final DBBroker broker = BrokerFactory.getInstance(this, this.getConfiguration());
        inactiveBrokers.push(broker);
        brokersCount++;
        broker.setId(broker.getClass().getName() + '_' + instanceName + "_" + brokersCount);
        LOG.debug(
            "created broker '" + broker.getId() + " for database instance '" + instanceName + "'");
        return broker;
    }

    //Seems dangerous and redundant as you myst acquire a broker yourself first, just use broker.setUser()
    public boolean setSubject(final Subject subject) {
        //synchronized(this) {
        //Try to get an active broker
        final DBBroker broker = activeBrokers.get(Thread.currentThread());
        if(broker != null) {
            broker.setSubject(subject);
            return true;
        }
        //}

        return false;
    }

    /*
     * Seems dangerous and redundant as you must acquire a broker yourself first, just use broker.getUser()
     *
     * yes, you have to authenticate before any action
     * try {
     *   broker = db.authenticate(...);
     *
     *   ...actions...
     *
     *  broker = db.getBroker();
     * } finally {
     *  db.release();
     * }
     */
    public Subject getSubject() {
        //synchronized(this) {
        //Try to get an active broker
        final DBBroker broker = activeBrokers.get(Thread.currentThread());
        if(broker != null) {
            return broker.getSubject();
        }
        //}

        return securityManager.getGuestSubject();
    }

    /**
     * Get active broker for current thread
     *
     * @return Database broker
     * @throws RuntimeException NO broker available for current thread.
     */
    public DBBroker getActiveBroker() { //throws EXistException {
        //synchronized(this) {
        //Try to get an active broker
        final DBBroker broker = activeBrokers.get(Thread.currentThread());
        if(broker == null) {
            final StringBuilder sb = new StringBuilder();
            sb.append("Broker was not obtained for thread '");
            sb.append(Thread.currentThread());
            sb.append("'.");
            sb.append(System.getProperty("line.separator"));
            activeBrokers.readOperation(new LongOperation<Thread, DBBroker>() {
                @Override
                public void execute(Map<Thread, DBBroker> map) {
                    for(final Entry<Thread, DBBroker> entry : map.entrySet()) {

//              if (entry.getKey().equals(Thread.currentThread()))
//                return entry.getValue();

                        sb.append(entry.getKey());
                        sb.append(" = ");
                        sb.append(entry.getValue());
                        sb.append(System.getProperty("line.separator"));
                    }
                }
            });
            LOG.debug(sb.toString());
            throw new RuntimeException(sb.toString());
        }
        return broker;
        //}
    }

    public DBBroker authenticate(final String username, final Object credentials) throws AuthenticationException {
        final Subject subject = getSecurityManager().authenticate(username, credentials);

        try {
            return get(subject);
        } catch(final Exception e) {
            throw new AuthenticationException(AuthenticationException.UNNOWN_EXCEPTION, e);
        }
    }

    public DBBroker getBroker() throws EXistException {
        return get(null);
    }

    /**
     * Returns an active broker for the database instance.
     *
     * @return The broker
     * @throws EXistException If the instance is not available (stopped or not configured)
     */
    //TODO : rename as getBroker ? getInstance (when refactored) ?
    public DBBroker get(final Subject user) throws EXistException {
        if(!isInstanceConfigured()) {
            throw new EXistException("database instance '" + instanceName + "' is not available");
        }

        //Try to get an active broker
        DBBroker broker = activeBrokers.get(Thread.currentThread());
        //Use it...
        //TOUNDERSTAND (pb) : why not pop a broker from the inactive ones rather than maintaining reference counters ?
        // WM: a thread may call this more than once in the sequence of operations, i.e. calls to get/release can
        // be nested. Returning a new broker every time would lead to a deadlock condition if two threads have
        // to wait for a broker to become available. We thus use reference counts and return
        // the same broker instance for each thread.
        if(broker != null) {
            //increase its number of uses
            broker.incReferenceCount();
            if(user != null) {
                broker.setSubject(user);
            }
            return broker;
            //TODO : share the code with what is below (including notifyAll) ?
            // WM: notifyAll is not necessary if we don't have to wait for a broker.
        }

        //No active broker : get one ASAP

        while(serviceModeUser != null && user != null && !user.equals(serviceModeUser)) {
            try {
                LOG.debug("Db instance is in service mode. Waiting for db to become available again ...");
                wait();
            } catch(final InterruptedException e) {
            }
        }

        synchronized(this) {
            //Are there any available brokers ?
            if(inactiveBrokers.isEmpty()) {
                //There are no available brokers. If allowed...
                if(brokersCount < maxBrokers)
                //... create one
                {
                    createBroker();
                } else
                    //... or wait until there is one available
                    while(inactiveBrokers.isEmpty()) {
                        LOG.debug("waiting for a broker to become available");
                        try {
                            this.wait();
                        } catch(final InterruptedException e) {
                            //nothing to be done!
                        }
                    }
            }
            broker = inactiveBrokers.pop();
            //activate the broker
            activeBrokers.put(Thread.currentThread(), broker);

            if(watchdog != null) {
                watchdog.add(broker);
            }

            broker.incReferenceCount();
            if(user != null) {
                broker.setSubject(user);
            } else {
                broker.setSubject(securityManager.getGuestSubject());
            }
            //Inform the other threads that we have a new-comer
            // TODO: do they really need to be informed here???????
            this.notifyAll();
            return broker;
        }
    }

    /**
     * Releases a broker for the database instance. If it is no more used, make if invactive.
     * If there are pending system maintenance tasks,
     * the method will block until these tasks have finished.
     *
     * @param broker The broker to be released
     */
    //TODO : rename as releaseBroker ? releaseInstance (when refactored) ?
    public void release(final DBBroker broker) {

        // might be null as release() is often called within a finally block
        if(broker == null) {
            return;
        }

        //first check that the broker is active ! If not, return immediately.
        broker.decReferenceCount();
        if(broker.getReferenceCount() > 0) {
            //it is still in use and thus can't be marked as inactive
            return;
        }

        synchronized(this) {
            //Broker is no more used : inactivate it
            for(final DBBroker inactiveBroker : inactiveBrokers) {
                if(broker == inactiveBroker) {
                    LOG.error("Broker is already in the inactive list!!!");
                    return;
                }
            }

            if(activeBrokers.remove(Thread.currentThread()) == null) {
                LOG.error("release() has been called from the wrong thread for broker " + broker.getId());
                // Cleanup the state of activeBrokers

                activeBrokers.writeOperation(new LongOperation<Thread, DBBroker>() {
                    @Override
                    public void execute(Map<Thread, DBBroker> map) {
                        for(final Object t : map.keySet()) {
                            if(map.get(t) == broker) {
                                final EXistException ex = new EXistException();
                                LOG.error("release() has been called from '" + Thread.currentThread() + "', but occupied at '" + t + "'.", ex);

                                map.remove(t);
                                break;
                            }
                        }
                    }
                });
            }
            final Subject lastUser = broker.getSubject();
            broker.setSubject(securityManager.getGuestSubject());
            inactiveBrokers.push(broker);
            if(watchdog != null) {
                watchdog.remove(broker);
            }

            //If the database is now idle, do some useful stuff
            if(activeBrokers.size() == 0) {
                //TODO : use a "clean" dedicated method (we have some below) ?
                if(syncRequired) {
                    //Note that the broker is not yet really inactive ;-)
                    sync(broker, syncEvent);
                    this.syncRequired = false;
                    this.checkpoint = false;
                }
                if(serviceModeUser != null && !lastUser.equals(serviceModeUser)) {
                    inServiceMode = true;
                }
            }
            //Inform the other threads that someone is gone
            this.notifyAll();
        }
    }

    public DBBroker enterServiceMode(final Subject user) throws PermissionDeniedException {
        if(!user.hasDbaRole()) {
            throw new PermissionDeniedException("Only users of group dba can switch the db to service mode");
        }

        serviceModeUser = user;
        synchronized(this) {
            if(activeBrokers.size() != 0) {
                while(!inServiceMode) {
                    try {
                        wait();
                    } catch(final InterruptedException e) {
                        //nothing to be done
                    }
                }
            }
        }

        inServiceMode = true;
        final DBBroker broker = inactiveBrokers.peek();
        checkpoint = true;
        sync(broker, Sync.MAJOR_SYNC);
        checkpoint = false;
        // Return a broker that can be used to perform system tasks
        return broker;
    }

    public void exitServiceMode(final Subject user) throws PermissionDeniedException {
        if(!user.equals(serviceModeUser)) {
            throw new PermissionDeniedException("The db has been locked by a different user");
        }
        serviceModeUser = null;
        inServiceMode = false;
        synchronized(this) {
            this.notifyAll();
        }
    }

    public void reportStatus(final String message) {
        if(statusReporter != null) {
            statusReporter.setStatus(message);
        }
    }

    public long getMajorSyncPeriod() {
        return majorSyncPeriod;
    }

    public long getLastMajorSync() {
        return lastMajorSync;
    }

    /**
     * Executes a waiting cache synchronization for the database instance.
     *
     * @param broker    A broker responsible for executing the job
     * @param syncEvent One of {@link org.exist.storage.sync.Sync#MINOR_SYNC} or {@link org.exist.storage.sync.Sync#MINOR_SYNC}
     */
    //TODO : rename as runSync ? executeSync ?
    //TOUNDERSTAND (pb) : *not* synchronized, so... "executes" or, rather, "schedules" ? "executes" (WM)
    //TOUNDERSTAND (pb) : why do we need a broker here ? Why not get and release one when we're done ?
    // WM: the method will always be under control of the BrokerPool. It is guaranteed that no
    // other brokers are active when it is called. That's why we don't need to synchronize here.
    //TODO : make it protected ?
    public void sync(final DBBroker broker, final int syncEvent) {
        broker.sync(syncEvent);
        final Subject user = broker.getSubject();
        //TODO : strange that it is set *after* the sunc method has been called.
        broker.setSubject(securityManager.getSystemSubject());
        if(syncEvent == Sync.MAJOR_SYNC) {
            LOG.debug("Major sync");
            try {
                if(!FORCE_CORRUPTION) {
                    transactionManager.checkpoint(checkpoint);
                }
            } catch(final TransactionException e) {
                LOG.warn(e.getMessage(), e);
            }
            cacheManager.checkCaches();

            if(pluginManager != null) {
                pluginManager.sync(broker);
            }

            lastMajorSync = System.currentTimeMillis();
            if(LOG.isDebugEnabled()) {
                notificationService.debug();
            }
        } else {
            cacheManager.checkDistribution();
//            LOG.debug("Minor sync");
        }
        //TODO : touch this.syncEvent and syncRequired ?

        //After setting the SYSTEM_USER above we must change back to the DEFAULT User to prevent a security problem
        //broker.setUser(User.DEFAULT);
        broker.setSubject(user);
    }

    /**
     * Schedules a cache synchronization for the database instance. If the database instance is idle,
     * the cache synchronization will be run immediately. Otherwise, the task will be deferred
     * until all running threads have returned.
     *
     * @param syncEvent One of {@link org.exist.storage.sync.Sync#MINOR_SYNC} or
     *                  {@link org.exist.storage.sync.Sync#MINOR_SYNC}
     */
    public void triggerSync(final int syncEvent) {
        //TOUNDERSTAND (pb) : synchronized, so... "schedules" or, rather, "executes" ? "schedules" (WM)
        if(status == SHUTDOWN) {
            return;
        }
        LOG.debug("Triggering sync: " + syncEvent);
        synchronized(this) {
            //Are there available brokers ?
            // TOUNDERSTAND (pb) : the trigger is ignored !
            // WM: yes, it seems wrong!!
//      if(inactiveBrokers.size() == 0)
//        return;
            //TODO : switch on syncEvent and throw an exception if it is inaccurate ?
            //Is the database instance idle ?
            if(inactiveBrokers.size() == brokersCount) {
                //Borrow a broker
                //TODO : this broker is *not* marked as active and may be reused by another process !
                // No other brokers are running at this time, so there's no risk.
                //TODO : use get() then release the broker ?
                // No, might lead to a deadlock.
                final DBBroker broker = inactiveBrokers.pop();
                //Do the synchronization job
                sync(broker, syncEvent);
                inactiveBrokers.push(broker);
                syncRequired = false;
            } else {
                //Put the synchronization job into the queue
                //TODO : check that we don't replace high priority Sync.MAJOR_SYNC by a lesser priority sync !
                this.syncEvent = syncEvent;
                syncRequired = true;
            }
        }
    }

    /**
     * Schedules a system maintenance task for the database instance. If the database is idle,
     * the task will be run immediately. Otherwise, the task will be deffered
     * until all running threads have returned.
     *
     * @param task The task
     */
    //TOUNDERSTAND (pb) : synchronized, so... "schedules" or, rather, "executes" ?
    public void triggerSystemTask(final SystemTask task) {
        transactionManager.triggerSystemTask(task);
    }

    /**
     * Shuts downs the database instance
     */
    public void shutdown() {
        shutdown(false);
    }

    public boolean isShuttingDown() {
        return status == SHUTDOWN;
    }

    /**
     * Shuts downs the database instance
     *
     * @param killed <code>true</code> when the JVM is (cleanly) exiting
     */
    public void shutdown(final boolean killed) {
        if(status == SHUTDOWN) {
            // we are already shut down
            return;
        }

        LOG.info("Database is shutting down ...");

        status = SHUTDOWN;

        processMonitor.stopRunningJobs();
        final java.util.concurrent.locks.Lock lock = transactionManager.getLock();
        try {
            // wait for currently running system tasks before we shutdown
            // they will have a lock on the transactionManager
            lock.lock();

            synchronized(this) {
                // these may be used and set by other threads for the same or some other purpose
                // (unlikely). Take no chances.
                statusReporter = new StatusReporter(SIGNAL_SHUTDOWN);
                if(statusObserver != null) {
                    statusReporter.addObserver(statusObserver);
                }
                final Thread statusThread = new Thread(statusReporter);
                statusThread.start();

                // release transaction log to allow remaining brokers to complete
                // their job
                lock.unlock();

                // DW: only in debug mode
                if(LOG.isDebugEnabled()) {
                    notificationService.debug();
                }

                //Notify all running tasks that we are shutting down

                //Shutdown the scheduler
                scheduler.shutdown(false);    //asynchronous

                while(!scheduler.isShutdown()) {
                    //wait for shutdown
                    try {
                        wait(250);
                    } catch(final InterruptedException e) {
                        //nothing to be done
                    }
                }

                //Notify all running XQueries that we are shutting down
                processMonitor.killAll(500);
                //TODO : close other objects using varying methods ? set them to null ?
                //cacheManager.something();
                //xQueryPool.something();
                //collectionConfigurationManager.something();
                //collectionCache.something();
                //xmlReaderPool.close();

                if(isTransactional()) {
                    transactionManager.getJournal().flushToLog(true, true);
                }

                final long waitStart = System.currentTimeMillis();
                //Are there active brokers ?
                if(activeBrokers.size() > 0) {
                    printSystemInfo();
                    LOG.info("Waiting " + maxShutdownWait + "ms for remaining threads to shut down...");
                    while(activeBrokers.size() > 0) {
                        try {
                            //Wait until they become inactive...
                            this.wait(1000);
                        } catch(final InterruptedException e) {
                            //nothing to be done
                        }

                        //...or force the shutdown
                        if(maxShutdownWait > -1 && System.currentTimeMillis() - waitStart > maxShutdownWait) {
                            LOG.warn("Not all threads returned. Forcing shutdown ...");
                            break;
                        }
                    }
                }
                LOG.debug("Calling shutdown ...");

                if(pluginManager != null)
                    try {
                        pluginManager.stop(null);
                    } catch(final EXistException e) {
                        LOG.warn("Error during plugin manager shutdown: " + e.getMessage(), e);
                    }

                // closing down external indexes
                try {
                    indexManager.shutdown();
                } catch(final DBException e) {
                    LOG.warn("Error during index shutdown: " + e.getMessage(), e);
                }

                //TODO : replace the following code by get()/release() statements ?
                // WM: deadlock risk if not all brokers returned properly.
                DBBroker broker = null;
                if(inactiveBrokers.isEmpty())
                    try {
                        broker = createBroker();
                    } catch(final EXistException e) {
                        LOG.warn("could not create instance for shutdown. Giving up.");
                    }
                else
                //TODO : this broker is *not* marked as active and may be reused by another process !
                //TODO : use get() then release the broker ?
                // WM: deadlock risk if not all brokers returned properly.
                //TODO: always createBroker? -dmitriy
                {
                    broker = inactiveBrokers.peek();
                }

                //TOUNDERSTAND (pb) : shutdown() is called on only *one* broker ?
                // WM: yes, the database files are shared, so only one broker is needed to close them for all
                if(broker != null) {
                    broker.setSubject(securityManager.getSystemSubject());
                    broker.shutdown();
                }
                collectionCacheMgr.deregisterCache(collectionCache);

                // do not write a checkpoint if some threads did not return before shutdown
                // there might be dirty transactions
                transactionManager.shutdown();

                // deregister JMX MBeans
                AgentFactory.getInstance().closeDBInstance(this);

                //Invalidate the configuration
                conf = null;
                //Clear the living instances container
                instances.remove(instanceName);

                if(!isReadOnly)
                // release the lock on the data directory
                {
                    dataLock.release();
                }

                LOG.info("shutdown complete !");

                //Last instance closes the house...
                //TOUNDERSTAND (pb) : !killed or, rather, killed ?
                // TODO: WM: check usage of killed!
                if(instances.size() == 0 && !killed) {
                    LOG.debug("removing shutdown hook");
                    try {
                        Runtime.getRuntime().removeShutdownHook(shutdownHook);
                    } catch(final IllegalStateException e) {
                        //ignore IllegalStateException("Shutdown in progress");
                    }
                }
                if(shutdownListener != null) {
                    shutdownListener.shutdown(instanceName, instances.size());
                }

                statusReporter.terminate();
                statusReporter = null;
            }
        } finally {
            // clear instance variables, just to be sure they will be garbage collected
            // the test suite restarts the db a few hundred times
            Configurator.clear(this);
            transactionManager = null;
            collectionCache = null;
            collectionCacheMgr = null;
            xQueryPool = null;
            processMonitor = null;
            collectionConfigurationManager = null;
            notificationService = null;
            indexManager = null;
            scheduler = null;
            xmlReaderPool = null;
            shutdownListener = null;
            securityManager = null;
            notificationService = null;
        }
    }

    public BrokerWatchdog getWatchdog() {
        return watchdog;
    }

    //TODO : move this elsewhere
    public void triggerCheckpoint() {
        if(syncRequired) {
            return;
        }
        synchronized(this) {
            syncEvent = Sync.MAJOR_SYNC;
            syncRequired = true;
            checkpoint = true;
        }
    }

    private Debuggee debuggee = null;

    public Debuggee getDebuggee() {
        synchronized(this) {
            if(debuggee == null) {
                debuggee = DebuggeeFactory.getInstance();
            }
        }

        return debuggee;
    }

    public Calendar getStartupTime() {
        return startupTime;
    }

    public void printSystemInfo() {
        final StringWriter sout = new StringWriter();
        final PrintWriter writer = new PrintWriter(sout);

        writer.println("SYSTEM INFO");
        writer.format("Database instance: %s\n", getId());
        writer.println("-------------------------------------------------------------------");
        if(watchdog != null) {
            watchdog.dump(writer);
        }
        DeadlockDetection.debug(writer);

        final String s = sout.toString();
        LOG.info(s);
        System.err.println(s);
    }

    private class StatusReporter extends Observable implements Runnable {

        private String status;
        private volatile boolean terminate = false;

        public StatusReporter(final String status) {
            this.status = status;
        }

        public synchronized void setStatus(final String status) {
            this.status = status;
            this.setChanged();
            this.notifyObservers(status);
        }

        public synchronized void terminate() {
            this.terminate = true;
            this.notifyAll();
        }

        public void run() {
            while(!terminate) {
                synchronized(this) {
                    try {
                        wait(500);
                    } catch(final InterruptedException e) {
                        // nothing to do
                    }
                }
                this.setChanged();
                this.notifyObservers(status);
            }
        }
    }

    @Override
    public File getStoragePlace() {
        return new File((String) conf.getProperty(BrokerPool.PROPERTY_DATA_DIR));
    }

    private final List<TriggerProxy<? extends DocumentTrigger>> documentTriggers = new ArrayList<>();
    private final List<TriggerProxy<? extends CollectionTrigger>> collectionTriggers = new ArrayList<>();

    @Override
    public List<TriggerProxy<? extends DocumentTrigger>> getDocumentTriggers() {
        return documentTriggers;
    }

    @Override
    public List<TriggerProxy<? extends CollectionTrigger>> getCollectionTriggers() {
        return collectionTriggers;
    }

    @Override
    public void registerDocumentTrigger(final Class<? extends DocumentTrigger> clazz) {
        documentTriggers.add(new DocumentTriggerProxy(clazz));
    }

    @Override
    public void registerCollectionTrigger(final Class<? extends CollectionTrigger> clazz) {
        collectionTriggers.add(new CollectionTriggerProxy(clazz));
    }

    public PluginsManager getPluginsManager() {
        return pluginManager;
    }

    protected MetaStorage metaStorage = null;

    public MetaStorage getMetaStorage() {
        return metaStorage;
    }
}
TOP

Related Classes of org.exist.storage.BrokerPool$StatusReporter

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.