*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*   http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing,

* software distributed under the License is distributed on an

* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

* KIND, either express or implied.  See the License for the

* specific language governing permissions and limitations

* under the License.

*

*/

* BDBMessageStore implements a persistent {@link MessageStore} using the BDB high performance log.

*

* <p/><table id="crc"><caption>CRC Card</caption> <tr><th> Responsibilities <th> Collaborations <tr><td> Accept

* transaction boundary demarcations: Begin, Commit, Abort. <tr><td> Store and remove queues. <tr><td> Store and remove

* exchanges. <tr><td> Store and remove messages. <tr><td> Bind and unbind queues to exchanges. <tr><td> Enqueue and

* dequeue messages to queues. <tr><td> Generate message identifiers. </table>

*/

    private static final Logger _log = Logger.getLogger(BDBMessageStore.class);

    private static final int LOCK_RETRY_ATTEMPTS = 5;

    static final int DATABASE_FORMAT_VERSION = 5;

    private static final String DATABASE_FORMAT_VERSION_PROPERTY = "version";

    public static final String ENVIRONMENT_PATH_PROPERTY = "environment-path";

    private Environment _environment;

    private String MESSAGEMETADATADB_NAME = "messageMetaDataDb";

    private String MESSAGECONTENTDB_NAME = "messageContentDb";

    private String QUEUEBINDINGSDB_NAME = "queueBindingsDb";

    private String DELIVERYDB_NAME = "deliveryDb";

    private String EXCHANGEDB_NAME = "exchangeDb";

    private String QUEUEDB_NAME = "queueDb";

    private String BRIDGEDB_NAME = "bridges";

    private String LINKDB_NAME = "links";

    private String XIDDB_NAME = "xids";

    private Database _messageMetaDataDb;

    private Database _messageContentDb;

    private Database _queueBindingsDb;

    private Database _deliveryDb;

    private Database _exchangeDb;

    private Database _queueDb;

    private Database _bridgeDb;

    private Database _linkDb;

    private Database _xidDb;

    /* =======

     * Schema:

     * =======

     *

     * Queue:

     * name(AMQShortString) - name(AMQShortString), owner(AMQShortString),

     *                        arguments(FieldTable encoded as binary), exclusive (boolean)

     *

     * Exchange:

     * name(AMQShortString) - name(AMQShortString), typeName(AMQShortString), autodelete (boolean)

     *

     * Binding:

     * exchangeName(AMQShortString), queueName(AMQShortString), routingKey(AMQShortString),

     *                                            arguments (FieldTable encoded as binary) - 0 (zero)

     *

     * QueueEntry:

     * queueName(AMQShortString), messageId (long) - 0 (zero)

     *

     * Message (MetaData):

     * messageId (long) - bodySize (integer), metaData (MessageMetaData encoded as binary)

     *

     * Message (Content):

     * messageId (long), byteOffset (integer) - dataLength(integer), data(binary)

     */

    private LogSubject _logSubject;

    private final AtomicLong _messageId = new AtomicLong(0);

    private final CommitThread _commitThread = new CommitThread("Commit-Thread");

    // Factory Classes to create the TupleBinding objects that reflect the version instance of this BDBStore

    private MessageMetaDataTupleBindingFactory _metaDataTupleBindingFactory;

    private QueueTupleBindingFactory _queueTupleBindingFactory;

    private BindingTupleBindingFactory _bindingTupleBindingFactory;

    /** The data version this store should run with */

    private int _version;

    private enum State

    {

        INITIAL,

        CONFIGURING,

        CONFIGURED,

        RECOVERING,

        STARTED,

        CLOSING,

        CLOSED

    }

    private State _state = State.INITIAL;

    private TransactionConfig _transactionConfig = new TransactionConfig();

    private boolean _readOnly = false;

    private boolean _configured;

    public BDBMessageStore()

    {

        this(DATABASE_FORMAT_VERSION);

    }

    public BDBMessageStore(int version)

    {

        _version = version;

    }

    private void setDatabaseNames(int version)

    {

        if (version > 1)

        {

            MESSAGEMETADATADB_NAME += "_v" + version;

            MESSAGECONTENTDB_NAME += "_v" + version;

            QUEUEDB_NAME += "_v" + version;

            DELIVERYDB_NAME += "_v" + version;

            EXCHANGEDB_NAME += "_v" + version;

            QUEUEBINDINGSDB_NAME += "_v" + version;

            LINKDB_NAME += "_v" + version;

            BRIDGEDB_NAME += "_v" + version;

           

            XIDDB_NAME += "_v" + version;

        }

    }

    public void configureConfigStore(String name,

                                     ConfigurationRecoveryHandler recoveryHandler,

                                     Configuration storeConfiguration,

                                     LogSubject logSubject) throws Exception

    {

        CurrentActor.get().message(logSubject, ConfigStoreMessages.CREATED(this.getClass().getName()));

        if(!_configured)

        {

            _logSubject = logSubject;

            configure(name,storeConfiguration);

            _configured = true;

            stateTransition(State.CONFIGURING, State.CONFIGURED);

        }

        recover(recoveryHandler);

        stateTransition(State.RECOVERING, State.STARTED);

    }

    public void configureMessageStore(String name,

                                      MessageStoreRecoveryHandler recoveryHandler,

                                      Configuration storeConfiguration,

                                      LogSubject logSubject) throws Exception

    {

        CurrentActor.get().message(logSubject, MessageStoreMessages.CREATED(this.getClass().getName()));

        if(!_configured)

        {

            _logSubject = logSubject;

            configure(name,storeConfiguration);

            _configured = true;

            stateTransition(State.CONFIGURING, State.CONFIGURED);

        }

        recoverMessages(recoveryHandler);

    }

    public void configureTransactionLog(String name, TransactionLogRecoveryHandler recoveryHandler,

            Configuration storeConfiguration, LogSubject logSubject) throws Exception

    {

        CurrentActor.get().message(logSubject, TransactionLogMessages.CREATED(this.getClass().getName()));

        if(!_configured)

        {

            _logSubject = logSubject;

            configure(name,storeConfiguration);

            _configured = true;

            stateTransition(State.CONFIGURING, State.CONFIGURED);

        }

        recoverQueueEntries(recoveryHandler);

       

    }

    public org.apache.qpid.server.store.MessageStore.Transaction newTransaction()

    {

        return new BDBTransaction();

    }

    /**

     * Called after instantiation in order to configure the message store.

     *

     * @param name The name of the virtual host using this store

     * @return whether a new store environment was created or not (to indicate whether recovery is necessary)

     *

     * @throws Exception If any error occurs that means the store is unable to configure itself.

     */

    public boolean configure(String name, Configuration storeConfig) throws Exception

    {

        File environmentPath = new File(storeConfig.getString(ENVIRONMENT_PATH_PROPERTY,

                                System.getProperty("QPID_WORK") + "/bdbstore/" + name));

        if (!environmentPath.exists())

        {

            if (!environmentPath.mkdirs())

            {

                throw new IllegalArgumentException("Environment path " + environmentPath + " could not be read or created. "

                                                   + "Ensure the path is correct and that the permissions are correct.");

            }

        }

        CurrentActor.get().message(_logSubject, MessageStoreMessages.STORE_LOCATION(environmentPath.getAbsolutePath()));

        _version = storeConfig.getInt(DATABASE_FORMAT_VERSION_PROPERTY, DATABASE_FORMAT_VERSION);

        return configure(environmentPath, false);

    }

    /**

     * @param environmentPath location for the store to be created in/recovered from

     * @param readonly if true then don't allow modifications to an existing store, and don't create a new store if none exists

     * @return whether or not a new store environment was created

     * @throws AMQStoreException

     * @throws DatabaseException

     */

    protected boolean configure(File environmentPath, boolean readonly) throws AMQStoreException, DatabaseException

    {

        _readOnly = readonly;

        stateTransition(State.INITIAL, State.CONFIGURING);

        _log.info("Configuring BDB message store");

        createTupleBindingFactories(_version);

        setDatabaseNames(_version);

        return setupStore(environmentPath, readonly);

    }

    private void createTupleBindingFactories(int version)

    {

            _bindingTupleBindingFactory = new BindingTupleBindingFactory(version);

            _queueTupleBindingFactory = new QueueTupleBindingFactory(version);

            _metaDataTupleBindingFactory = new MessageMetaDataTupleBindingFactory(version);

    }

    /**

     * Move the store state from CONFIGURING to STARTED.

     *

     * This is required if you do not want to perform recovery of the store data

     *

     * @throws AMQStoreException if the store is not in the correct state

     */

    public void start() throws AMQStoreException

    {

        stateTransition(State.CONFIGURING, State.STARTED);

    }

    private boolean setupStore(File storePath, boolean readonly) throws DatabaseException, AMQStoreException

    {

        checkState(State.CONFIGURING);

        boolean newEnvironment = createEnvironment(storePath, readonly);

        verifyVersionByTables();

        openDatabases(readonly);

        if (!readonly)

        {

            _commitThread.start();

        }

        return newEnvironment;

    }

    private void verifyVersionByTables() throws DatabaseException

    {

        for (String s : _environment.getDatabaseNames())

        {

            int versionIndex = s.indexOf("_v");

            // lack of _v index suggests DB is v1

            // so if _version is not v1 then error

            if (versionIndex == -1)

            {

                if (_version != 1)

                {

                    closeEnvironment();

                    throw new IllegalArgumentException("Error: Unable to load BDBStore as version " + _version

                                            + ". Store on disk contains version 1 data.");

                }

                else // DB is v1 and _version is v1

                {

                    continue;

                }

            }

            // Otherwise Check Versions

            int version = Integer.parseInt(s.substring(versionIndex + 2));

            if (version != _version)

            {

                closeEnvironment();

                throw new IllegalArgumentException("Error: Unable to load BDBStore as version " + _version

                                            + ". Store on disk contains version " + version + " data.");

            }

        }

    }

    private synchronized void stateTransition(State requiredState, State newState) throws AMQStoreException

    {

        if (_state != requiredState)

        {

            throw new AMQStoreException("Cannot transition to the state: " + newState + "; need to be in state: " + requiredState

                                   + "; currently in state: " + _state);

        }

        _state = newState;

    }

    private void checkState(State requiredState) throws AMQStoreException

    {

        if (_state != requiredState)

        {

            throw new AMQStoreException("Unexpected state: " + _state + "; required state: " + requiredState);

        }

    }

    private boolean createEnvironment(File environmentPath, boolean readonly) throws DatabaseException

    {

        _log.info("BDB message store using environment path " + environmentPath.getAbsolutePath());

        EnvironmentConfig envConfig = new EnvironmentConfig();

        // This is what allows the creation of the store if it does not already exist.

        envConfig.setAllowCreate(true);

        envConfig.setTransactional(true);

        envConfig.setConfigParam("je.lock.nLockTables", "7");

        // Added to help diagnosis of Deadlock issue

        // http://www.oracle.com/technology/products/berkeley-db/faq/je_faq.html#23

        if (Boolean.getBoolean("qpid.bdb.lock.debug"))

        {

            envConfig.setConfigParam("je.txn.deadlockStackTrace", "true");

            envConfig.setConfigParam("je.txn.dumpLocks", "true");

        }

        // Set transaction mode

        _transactionConfig.setReadCommitted(true);

        //This prevents background threads running which will potentially update the store.

        envConfig.setReadOnly(readonly);

        try

        {

            _environment = new Environment(environmentPath, envConfig);

            return false;

        }

        catch (DatabaseException de)

        {

            if (de.getMessage().contains("Environment.setAllowCreate is false"))

            {

                //Allow the creation this time

                envConfig.setAllowCreate(true);

                if (_environment != null )

                {

                    _environment.cleanLog();

                    _environment.close();

                }

                _environment = new Environment(environmentPath, envConfig);

                return true;

            }

            else

            {

                throw de;

            }

        }

    }

    private void openDatabases(boolean readonly) throws DatabaseException

    {

        DatabaseConfig dbConfig = new DatabaseConfig();

        dbConfig.setTransactional(true);

        dbConfig.setAllowCreate(true);

        //This is required if we are wanting read only access.

        dbConfig.setReadOnly(readonly);

        _messageMetaDataDb = openDatabase(MESSAGEMETADATADB_NAME, dbConfig);

        _queueDb = openDatabase(QUEUEDB_NAME, dbConfig);

        _exchangeDb = openDatabase(EXCHANGEDB_NAME, dbConfig);

        _queueBindingsDb = openDatabase(QUEUEBINDINGSDB_NAME, dbConfig);

        _messageContentDb = openDatabase(MESSAGECONTENTDB_NAME, dbConfig);

        _deliveryDb = openDatabase(DELIVERYDB_NAME, dbConfig);

        _linkDb = openDatabase(LINKDB_NAME, dbConfig);

        _bridgeDb = openDatabase(BRIDGEDB_NAME, dbConfig);

        _xidDb = openDatabase(XIDDB_NAME, dbConfig);

    }

    private Database openDatabase(final String dbName, final DatabaseConfig dbConfig)

    {

        // if opening read-only and the database doesn't exist, then you can't create it

        return dbConfig.getReadOnly() && !_environment.getDatabaseNames().contains(dbName)

               ? null

               : _environment.openDatabase(null, dbName, dbConfig);

    }

    /**

     * Called to close and cleanup any resources used by the message store.

     *

     * @throws Exception If the close fails.

     */

    public void close() throws Exception

    {

        if (_state != State.STARTED)

        {

            return;

        }

        _state = State.CLOSING;

        _commitThread.close();

        _commitThread.join();

        if (_messageMetaDataDb != null)

        {

            _log.info("Closing message metadata database");

            _messageMetaDataDb.close();

        }

        if (_messageContentDb != null)

        {

            _log.info("Closing message content database");

            _messageContentDb.close();

        }

        if (_exchangeDb != null)

        {

            _log.info("Closing exchange database");

            _exchangeDb.close();

        }

        if (_queueBindingsDb != null)

        {

            _log.info("Closing bindings database");

            _queueBindingsDb.close();

        }

        if (_queueDb != null)

        {

            _log.info("Closing queue database");

            _queueDb.close();

        }

        if (_deliveryDb != null)

        {

            _log.info("Close delivery database");

            _deliveryDb.close();

        }

        if (_bridgeDb != null)

        {

            _log.info("Close bridge database");

            _bridgeDb.close();

        }

        if (_linkDb != null)

        {

            _log.info("Close link database");

            _linkDb.close();

        }

        if (_xidDb != null)

        {

            _log.info("Close xid database");

            _xidDb.close();

        }

        closeEnvironment();

        _state = State.CLOSED;

        CurrentActor.get().message(_logSubject,MessageStoreMessages.CLOSED());

    }

    private void closeEnvironment() throws DatabaseException

    {

        if (_environment != null)

        {

            if(!_readOnly)

            {

                // Clean the log before closing. This makes sure it doesn't contain

                // redundant data. Closing without doing this means the cleaner may not

                // get a chance to finish.

                _environment.cleanLog();

            }

            _environment.close();

        }

    }

    public void recover(ConfigurationRecoveryHandler recoveryHandler) throws AMQStoreException

    {

        stateTransition(State.CONFIGURED, State.RECOVERING);

        CurrentActor.get().message(_logSubject,MessageStoreMessages.RECOVERY_START());

        try

        {

            QueueRecoveryHandler qrh = recoveryHandler.begin(this);

            loadQueues(qrh);

            ExchangeRecoveryHandler erh = qrh.completeQueueRecovery();

            loadExchanges(erh);

            BindingRecoveryHandler brh = erh.completeExchangeRecovery();

            recoverBindings(brh);

            ConfigurationRecoveryHandler.BrokerLinkRecoveryHandler lrh = brh.completeBindingRecovery();

            recoverBrokerLinks(lrh);

        }

        catch (DatabaseException e)

        {

            throw new AMQStoreException("Error recovering persistent state: " + e.getMessage(), e);

        }

    }

    private void loadQueues(QueueRecoveryHandler qrh) throws DatabaseException

    {

        Cursor cursor = null;

        try

        {

            cursor = _queueDb.openCursor(null, null);

            DatabaseEntry key = new DatabaseEntry();

            DatabaseEntry value = new DatabaseEntry();

            TupleBinding binding = _queueTupleBindingFactory.getInstance();

            while (cursor.getNext(key, value, LockMode.RMW) == OperationStatus.SUCCESS)

            {

                QueueRecord queueRecord = (QueueRecord) binding.entryToObject(value);

                String queueName = queueRecord.getNameShortString() == null ? null :

                                        queueRecord.getNameShortString().asString();

                String owner = queueRecord.getOwner() == null ? null :

                                        queueRecord.getOwner().asString();

                boolean exclusive = queueRecord.isExclusive();

                FieldTable arguments = queueRecord.getArguments();

                qrh.queue(queueName, owner, exclusive, arguments);

            }

        }

        finally

        {

            if (cursor != null)

            {

                cursor.close();

            }

        }

    }

    private void loadExchanges(ExchangeRecoveryHandler erh) throws DatabaseException

    {

        Cursor cursor = null;

        try

        {

            cursor = _exchangeDb.openCursor(null, null);

            DatabaseEntry key = new DatabaseEntry();

            DatabaseEntry value = new DatabaseEntry();

            TupleBinding binding = new ExchangeTB();

            while (cursor.getNext(key, value, LockMode.RMW) == OperationStatus.SUCCESS)

            {

                ExchangeRecord exchangeRec = (ExchangeRecord) binding.entryToObject(value);

                String exchangeName = exchangeRec.getNameShortString() == null ? null :

                                      exchangeRec.getNameShortString().asString();

                String type = exchangeRec.getType() == null ? null :

                              exchangeRec.getType().asString();

                boolean autoDelete = exchangeRec.isAutoDelete();

                erh.exchange(exchangeName, type, autoDelete);

            }

        }

        finally

        {

            if (cursor != null)

            {

                cursor.close();

            }

        }

    }

    private void recoverBindings(BindingRecoveryHandler brh) throws DatabaseException

    {

        Cursor cursor = null;

        try

        {

            cursor = _queueBindingsDb.openCursor(null, null);

            DatabaseEntry key = new DatabaseEntry();

            DatabaseEntry value = new DatabaseEntry();

            TupleBinding<BindingRecord> binding = _bindingTupleBindingFactory.getInstance();

            while (cursor.getNext(key, value, LockMode.RMW) == OperationStatus.SUCCESS)

            {

                //yes, this is retrieving all the useful information from the key only.

                //For table compatibility it shall currently be left as is

                BindingRecord bindingRecord = binding.entryToObject(key);

                String exchangeName = bindingRecord.getExchangeName() == null ? null :

                                      bindingRecord.getExchangeName().asString();

                String queueName = bindingRecord.getQueueName() == null ? null :

                                   bindingRecord.getQueueName().asString();

                String routingKey = bindingRecord.getRoutingKey() == null ? null :

                                    bindingRecord.getRoutingKey().asString();

                ByteBuffer argumentsBB = (bindingRecord.getArguments() == null ? null :

                    java.nio.ByteBuffer.wrap(bindingRecord.getArguments().getDataAsBytes()));

                brh.binding(exchangeName, queueName, routingKey, argumentsBB);

            }

        }

        finally

        {

            if (cursor != null)

            {

                cursor.close();

            }

        }

    }

    private void recoverBrokerLinks(final ConfigurationRecoveryHandler.BrokerLinkRecoveryHandler lrh)

    {

        Cursor cursor = null;

        try

        {

            cursor = _linkDb.openCursor(null, null);

            DatabaseEntry key = new DatabaseEntry();

            DatabaseEntry value = new DatabaseEntry();

            while (cursor.getNext(key, value, LockMode.RMW) == OperationStatus.SUCCESS)

            {

                UUID id = UUIDTupleBinding.getInstance().entryToObject(key);

                long createTime = LongBinding.entryToLong(value);

                Map<String,String> arguments = StringMapBinding.getInstance().entryToObject(value);

                ConfigurationRecoveryHandler.BridgeRecoveryHandler brh = lrh.brokerLink(id, createTime, arguments);

                recoverBridges(brh, id);

            }

        }

        finally

        {

            if (cursor != null)

            {

                cursor.close();

            }

        }

    }

    private void recoverBridges(final ConfigurationRecoveryHandler.BridgeRecoveryHandler brh, final UUID linkId)

    {

        Cursor cursor = null;

        try

        {

            cursor = _bridgeDb.openCursor(null, null);

            DatabaseEntry key = new DatabaseEntry();

            DatabaseEntry value = new DatabaseEntry();

            while (cursor.getNext(key, value, LockMode.RMW) == OperationStatus.SUCCESS)

            {

                UUID id = UUIDTupleBinding.getInstance().entryToObject(key);

                UUID parentId = UUIDTupleBinding.getInstance().entryToObject(value);

                if(parentId.equals(linkId))

                {

                    long createTime = LongBinding.entryToLong(value);

                    Map<String,String> arguments = StringMapBinding.getInstance().entryToObject(value);

                    brh.bridge(id,createTime,arguments);

                }

            }

            brh.completeBridgeRecoveryForLink();

        }

        finally

        {

            if (cursor != null)

            {

                cursor.close();

            }

        }

    }

    private void recoverMessages(MessageStoreRecoveryHandler msrh) throws DatabaseException

    {

        StoredMessageRecoveryHandler mrh = msrh.begin();

        Cursor cursor = null;

        try

        {

            cursor = _messageMetaDataDb.openCursor(null, null);

            DatabaseEntry key = new DatabaseEntry();

            DatabaseEntry value = new DatabaseEntry();

            EntryBinding valueBinding = _metaDataTupleBindingFactory.getInstance();

            long maxId = 0;

            while (cursor.getNext(key, value, LockMode.RMW) == OperationStatus.SUCCESS)

            {

                long messageId = LongBinding.entryToLong(key);

                StorableMessageMetaData metaData = (StorableMessageMetaData) valueBinding.entryToObject(value);

                StoredBDBMessage message = new StoredBDBMessage(messageId, metaData, false);

                mrh.message(message);

                maxId = Math.max(maxId, messageId);

            }

            _messageId.set(maxId);

        }

        catch (DatabaseException e)

        {

            _log.error("Database Error: " + e.getMessage(), e);

            throw e;

        }

        finally

        {

            if (cursor != null)

            {

                cursor.close();

            }

        }

    }

    private void recoverQueueEntries(TransactionLogRecoveryHandler recoveryHandler)

    throws DatabaseException

    {

        QueueEntryRecoveryHandler qerh = recoveryHandler.begin(this);

        ArrayList<QueueEntryKey> entries = new ArrayList<QueueEntryKey>();

        Cursor cursor = null;

        try

        {

            cursor = _deliveryDb.openCursor(null, null);

            DatabaseEntry key = new DatabaseEntry();

            EntryBinding keyBinding = new QueueEntryTB();

            DatabaseEntry value = new DatabaseEntry();

            while (cursor.getNext(key, value, LockMode.RMW) == OperationStatus.SUCCESS)

            {

                QueueEntryKey qek = (QueueEntryKey) keyBinding.entryToObject(key);