Package com.sleepycat.je.log

Examples of com.sleepycat.je.log.LogManager


        nCheckpoints++;

        boolean success = false;
        boolean traced = false;

        LogManager logManager = envImpl.getLogManager();

        /* dirtyMap keeps track of the INs to be written out by the ckpt. */
        DirtyINMap dirtyMap = new DirtyINMap(envImpl);
        try {

            /*
             * Eviction can run during checkpoint as long as it follows the
             * same rules for using provisional logging and for propagating
             * logging of the checkpoint dirty set up the tree. We have to lock
             * out the evictor after the logging of checkpoint start until
             * we've selected the dirty set and decided on the highest level to
             * be flushed. See SR 11163, 11349.
             */
            long checkpointStart = DbLsn.NULL_LSN;
            long firstActiveLsn = DbLsn.NULL_LSN;

            synchronized (envImpl.getEvictor()) {

                /* Log the checkpoint start. */
                SingleItemEntry startEntry =
                    new SingleItemEntry(LogEntryType.LOG_CKPT_START,
                                        new CheckpointStart(checkpointId,
                                                            invokingSource));
                checkpointStart =
                    logManager.log(startEntry,
                                   ReplicationContext.NO_REPLICATE);

                /*
                 * Note the first active LSN point. The definition of
                 * firstActiveLsn is that all log entries for active
                 * transactions are equal to or after that LSN.
                 */
                firstActiveLsn = envImpl.getTxnManager().getFirstActiveLsn();

                if (firstActiveLsn == DbLsn.NULL_LSN) {
                    firstActiveLsn = checkpointStart;
                } else {
                    if (DbLsn.compareTo(checkpointStart, firstActiveLsn) < 0) {
                        firstActiveLsn = checkpointStart;
                    }
                }

                /*
                 * Find the set of dirty INs that must be logged.  Update the
                 * highestFlushLevels volatile field so it will be seen by the
                 * evictor, before starting to flush dirty nodes.
                 */
                highestFlushLevels = dirtyMap.selectDirtyINsForCheckpoint
                    (flushAll, flushExtraLevel);
            }

            /*
             * Add the dirty map to the memory budget, outside the evictor
             * synchronization section.
             */
            dirtyMap.addCostToMemoryBudget();

            /* Call hook after dirty map creation and before flushing. */
            TestHookExecute.doHookIfSet(beforeFlushHook);

            /* Flush IN nodes. */
            boolean allowDeltas = !config.getMinimizeRecoveryTime();
            flushDirtyNodes(envImpl, dirtyMap, highestFlushLevels, allowDeltas,
                            checkpointStart, highPriority, flushStats);

            /*
             * Flush MapLNs if not already done by flushDirtyNodes.  Only flush
             * a database if it has not already been flushed since checkpoint
             * start.  Lastly, flush the DB mapping tree root.
             */
            dirtyMap.flushMapLNs(checkpointStart);
            dirtyMap.flushRoot(checkpointStart);

            /*
             * Flush replication information if necessary so that the VLSNIndex
             * cache is flushed and is recoverable.
             */
            envImpl.preCheckpointEndFlush();

            /*
             * Flush utilization info AFTER flushing IN nodes to reduce the
             * inaccuracies caused by the sequence FileSummaryLN-LN-BIN.
             */
            envImpl.getUtilizationProfile().flushFileUtilization
                (envImpl.getUtilizationTracker().getTrackedFiles());

            DbTree dbTree = envImpl.getDbTree();
            boolean willDeleteFiles = !cleanerState.isEmpty();
            CheckpointEnd ckptEnd = new CheckpointEnd
                (invokingSource, checkpointStart, envImpl.getRootLsn(),
                 firstActiveLsn,
                 envImpl.getNodeSequence().getLastLocalNodeId(),
                 envImpl.getNodeSequence().getLastReplicatedNodeId(),
                 dbTree.getLastLocalDbId(), dbTree.getLastReplicatedDbId(),
                 envImpl.getTxnManager().getLastLocalTxnId(),
                 envImpl.getTxnManager().getLastReplicatedTxnId(),
                 checkpointId,
                 willDeleteFiles);

            SingleItemEntry endEntry =
                new SingleItemEntry(LogEntryType.LOG_CKPT_END, ckptEnd);

            /*
             * Log checkpoint end and update state kept about the last
             * checkpoint location. Send a trace message *before* the
             * checkpoint end log entry. This is done so that the normal trace
             * message doesn't affect the time-based isRunnable() calculation,
             * which only issues a checkpoint if a log record has been written
             * since the last checkpoint.
             */
            trace(envImpl, invokingSource, true);
            traced = true;

            /*
             * Always flush to ensure that cleaned files are not referenced,
             * and to ensure that this checkpoint is not wasted if we crash.
             */
            lastCheckpointEnd =
                logManager.logForceFlush(endEntry,
                                         true /*fsyncRequired*/,
                                         ReplicationContext.NO_REPLICATE);

            lastCheckpointStart = checkpointStart;

View Full Code Here


                                        long checkpointStart,
                                        boolean highPriority,
                                        FlushStats fstats)
        throws DatabaseException {

        LogManager logManager = envImpl.getLogManager();
        DbTree dbTree = envImpl.getDbTree();

        /*
         * Use a tracker to count lazily compressed, deferred write, LNs as
         * obsolete.  A local tracker is used to accumulate tracked obsolete
         * info so it can be added in a single call under the log write latch.
         * [#15365]
         */
        LocalUtilizationTracker localTracker =
            new LocalUtilizationTracker(envImpl);

        while (dirtyMap.getNumLevels() > 0) {

            /* Work on one level's worth of nodes in ascending level order. */
            Integer currentLevel = dirtyMap.getLowestLevelSet();
            int currentLevelVal = currentLevel.intValue();

            /*
             * Flush MapLNs just prior to flushing the first level of the
             * mapping tree.  Only flush a database if it has not already been
             * flushed since checkpoint start.
             */
            if (currentLevelVal == IN.DBMAP_LEVEL) {
                dirtyMap.flushMapLNs(checkpointStart);
            }

            /* Flush the nodes at the current level. */
            while (true) {
                CheckpointReference targetRef =
                    dirtyMap.removeNextNode(currentLevel);
                if (targetRef == null) {
                    break;
                }

                /*
                 * Check to make sure the DB was not deleted after putting it
                 * in the dirty map, and prevent the DB from being deleted
                 * while we're working with it.
                 */
                DatabaseImpl db = null;
                try {
                    db = dbTree.getDb(targetRef.dbId);
                    if (db != null && !db.isDeleted()) {

                        /* Flush if we're below maxFlushLevel. */
                        int maxFlushLevel = getHighestFlushLevelInternal
                            (db, highestFlushLevels);
                        if (currentLevelVal <= maxFlushLevel) {

                            /* Evict before each operation. */
                            envImpl.criticalEviction(true /*backgroundIO*/);

                            flushIN
                                (envImpl, db, logManager, targetRef, dirtyMap,
                                 currentLevelVal, maxFlushLevel, allowDeltas,
                                 checkpointStart, highPriority,
                                 fstats, localTracker,
                                 true /*allowLogSubtree*/);

                            /*
                             * Sleep if background read/write limit was
                             * exceeded.
                             */
                            envImpl.sleepAfterBackgroundIO();
                        }
                    }
                } finally {
                    dbTree.releaseDb(db);
                }
            }

            /* We're done with this level. */
            dirtyMap.removeLevel(currentLevel);
        }

        /*
         * Count obsolete nodes tracked doing lazy compression.  All latches
         * must have been released. [#15365]
         *
         * Do not flush FileSummaryLNs/MapLNs (do not call
         * UtilizationProfile.flushLocalTracker) here because that flushing is
         * already done by the checkpoint.
         */
        logManager.transferToUtilizationTracker(localTracker);
    }
View Full Code Here

                                       Provisional provisional,
                                       FlushStats fstats,
                                       LocalUtilizationTracker localTracker)
        throws DatabaseException {

        LogManager logManager = envImpl.getLogManager();

        INLogContext context = new INLogContext();
        context.nodeDb = parent.getDatabase();
        context.backgroundIO = true;
        context.allowDeltas = allowDeltas;

        boolean mustLogParent = false;
        List<INLogItem> itemList = new ArrayList<INLogItem>();

        try {
            for (int index : indicesToLog) {
                IN child = (IN) parent.getTarget(index);

                /* Remove it from dirty map if it is present. */
                dirtyMap.removeNode(child.getLevel(), child.getNodeId());

                /*
                 * Latch and add item with valid parentIndex, so we will
                 * release the latch in the finally statement.
                 */
                child.latch(CacheMode.UNCHANGED);
                INLogItem item = new INLogItem();
                item.parentIndex = index;
                itemList.add(item);

                /*
                 * Compress this node if necessary. Note that this may dirty
                 * the node.
                 */
                envImpl.lazyCompress(child, localTracker);

                if (child.getDirty()) {

                    if (child.getDatabase().isDurableDeferredWrite()) {

                        /*
                         * Find dirty descendants to avoid logging nodes with
                         * never-logged children. See [#13936] and
                         * IN.logDirtyChildren for description of the case.
                         *
                         * Note that we must log both dirty and never-logged
                         * descendants to be sure to have a consistent view of
                         * the split. If we didn't, we could end up with the
                         * post-split version of a new sibling and the
                         * pre-split version of an split sibling in the log,
                         * which could result in a recovery where descendants
                         * are incorrectly duplicated, because they are in both
                         * the pre-split split sibling, and the post-split
                         * version of the new sibling.
                         */
                        child.logDirtyChildren();
                    }

                    /* Set default params. */
                    item.provisional = provisional;
                    item.repContext = ReplicationContext.NO_REPLICATE;
                    item.parent = parent;

                    /*
                     * Allow child to perform "before log" processing.  Note
                     * that child decides whether to log a delta. Only BINs
                     * that fall into the required percentages and have not
                     * been cleaned will be logged with a delta.
                     */
                    child.beforeLog(logManager, item, context);
                } else {
                    /* Do not process if not dirty.  Unlatch now. */
                    itemList.remove(itemList.size() - 1);
                    child.releaseLatch();

                    /* Log parent if child has already been flushed. */
                    mustLogParent = true;
                }
            }

            /*
             * Log all siblings at once.  Limitations of Java generics prevent
             * conversion from List<INLogItem> to List<LogItem> even by
             * casting, so we convert to an array instead.
             */
            LogItem[] itemArray = new LogItem[itemList.size()];
            logManager.multiLog(itemList.toArray(itemArray), context);

            for (INLogItem item : itemList) {
                IN child = (IN) parent.getTarget(item.parentIndex);

                /* Allow child to perform "after log" processing. */
 
View Full Code Here

             * be present in the log for recovery. It also ensures that all log
             * entries will be flushed to disk and the TxnChain will not have
             * to worry about entries that are in log buffers when constructing
             * the rollback information.
             */
            LogManager logManager = repImpl.getLogManager();
            LogEntry rollbackStart =
                new SingleItemEntry(LogEntryType.LOG_ROLLBACK_START,
                                    new RollbackStart(matchpointVLSN,
                                                      matchpointLsn,
                                                      activeTxns.keySet()));
            long rollbackStartLsn =
                logManager.logForceFlush(rollbackStart,
                                         true, // fsyncRequired,
                                         ReplicationContext.NO_REPLICATE);
            rollbackStatus = RBSTATUS_LOG_RBSTART;

            /*
             * 2. Do rollback in memory. Undo any operations that were logged
             * after the matchpointLsn, and save the LSNs for those log
             * entries.. There should be something to undo, because we checked
             * earlier that there were log entries after the matchpoint.
             */
            List<Long> rollbackLsns = new ArrayList<Long>();
            for (ReplayTxn replayTxn : activeTxns.values()) {
                Collection<Long> txnRollbackLsns =
                    replayTxn.rollback(matchpointLsn);

                /*
                 * Txns that were entirely rolled back should have been removed
                 * from the activeTxns map.
                 */
                assert checkRemoved(replayTxn) :
                    "Should have removed " + replayTxn;

                rollbackLsns.addAll(txnRollbackLsns);
            }
            rollbackStatus = RBSTATUS_MEM_ROLLBACK;
            assert rollbackLsns.size() != 0;

            /*
             * 3 & 4 - Mark the rolled back log entries as invisible. After all
             * are done, fsync the set of files. By waiting, some may have made
             * it out on their own.
             */
            RollbackTracker.makeInvisible(repImpl, rollbackLsns);
            rollbackStatus = RBSTATUS_INVISIBLE;

            /*
             * 5. Log RollbackEnd. Flush it so that we can use it to optimize
             * recoveries later on. If the RollbackEnd exists, we can skip the
             * step of re-making LNs invisible.
             */
            logManager.logForceFlush
                (new SingleItemEntry(LogEntryType.LOG_ROLLBACK_END,
                                     new RollbackEnd(matchpointLsn,
                                                     rollbackStartLsn)),
                 true, // fsyncRequired
                 ReplicationContext.NO_REPLICATE);
View Full Code Here

        /*
         * Update the MapLNs before deleting FileSummaryLNs in case there is an
         * error during this process.  If a FileSummaryLN exists, we will redo
         * this process during the next recovery (populateCache).
         */
        final LogManager logManager = env.getLogManager();
        final DbTree dbTree = env.getDbTree();
        /* Only call logMapTreeRoot once for ID and NAME DBs. */
        DatabaseImpl idDatabase = dbTree.getDb(DbTree.ID_DB_ID);
        DatabaseImpl nameDatabase = dbTree.getDb(DbTree.NAME_DB_ID);
        boolean logRoot = false;
        if (logManager.removeDbFileSummary(idDatabase, fileNum)) {
            logRoot = true;
        }
        if (logManager.removeDbFileSummary(nameDatabase, fileNum)) {
            logRoot = true;
        }
        if (logRoot) {
            env.logMapTreeRoot();
        }
        /* Use DB ID set if available to avoid full scan of ID DB. */
        if (databases != null) {
            for (DatabaseId dbId : databases) {
                if (!dbId.equals(DbTree.ID_DB_ID) &&
                    !dbId.equals(DbTree.NAME_DB_ID)) {
                    DatabaseImpl db = dbTree.getDb(dbId);
                    try {
                        if (db != null &&
                            logManager.removeDbFileSummary(db, fileNum)) {
                            dbTree.modifyDbRoot(db);
                        }
                    } finally {
                        dbTree.releaseDb(db);
                    }
                }
            }
        } else {

            /*
             * Use LockType.NONE for traversing the ID DB so that a lock is not
             * held when calling modifyDbRoot, which must release locks to
             * handle deadlocks.
             */
            CursorImpl.traverseDbWithCursor(idDatabase,
                                            LockType.NONE,
                                            true /*allowEviction*/,
                                            new CursorImpl.WithCursor() {
                public boolean withCursor(CursorImpl cursor,
                                          DatabaseEntry key,
                                          DatabaseEntry data)
                    throws DatabaseException {

                    MapLN mapLN = (MapLN) cursor.getCurrentLN(LockType.NONE);
                    if (mapLN != null) {
                        DatabaseImpl db = mapLN.getDatabase();
                        if (logManager.removeDbFileSummary(db, fileNum)) {

                            /*
                             * Because we're using dirty-read, silently do
                             * nothing if the DB does not exist
                             * (mustExist=false).
 
View Full Code Here

         * root, IN.deleteEntry will traverse downwards.
         */
        subtreeRoot = search
            (idKey, SearchType.DELETE, -1, null, true /*updateGeneration*/);

        LogManager logManager =
            database.getDbEnvironment().getLogManager();
        if (subtreeRoot == null) {

            /*
             * The root is the top of this subtree. If there are no more
             * entries left in the root, delete the whole tree.  There's a
             * window on the rootLatch between the time that search releases
             * the rootLatch and the acquire below.  Something could insert
             * into the tree.  Use validateSubtreeForDelete to ensure that it's
             * still empty.
             */
            rootLatch.acquire();
            try {
                IN rootIN = (IN) root.fetchTarget(database, null);
               
    DbConfigManager configManager =
        database.getDbEnvironment().getConfigManager();
    boolean purgeRoot = configManager.getBoolean
        (EnvironmentParams.COMPRESSOR_PURGE_ROOT);

    /**
     * We've encountered the last empty subtree of the tree.  In
     * general, there's no reason to delete this last
     * IN->...IN->BIN subtree since we're likely to to add more
     * nodes to this tree again.  Deleting the subtree also adds to
     * the space used by the log since a MapLN needs to be written
     * when the root is nulled, and a MapLN, IN (root), BIN needs
     * to be written when the root is recreated.
     *
     * Consider a queue application which frequently inserts and
     * deletes entries and often times leaves the tree empty, but
     * will insert new records again.
     *
     * An optimization might be to prune the multiple IN path to
     * the last BIN (if it even exists) to just a root IN pointing
     * to the single BIN, but this doesn't feel like it's worth the
     * trouble since the extra depth doesn't matter all that much.
     *
     * If je.compressor.purgeRoot is true, then we null the root.
     */
                if (purgeRoot &&
        (rootIN.getNEntries() <= 1) &&
                    (rootIN.validateSubtreeBeforeDelete(0))) {

                    /*
                     * The tree is empty, clear out the IN list.  Can't just
                     * call clear() because there are IN's from more than one
                     * Database on the list.
                     */
                    root = null;
                    treeEmpty = true;

                    /*
                     * Record the root deletion for recovery. Do this within
                     * the root latch. We need to put this log entry into the
                     * log before another thread comes in and creates a new
                     * rootIN for this database.
                     *
                     * For example,
                     * LSN 1000 IN delete info entry
                     * LSN 1010 new IN, for next set of inserts
                     * LSN 1020 new BIN, for next set of inserts.
                     *
                     * The entry at 1000 is needed so that LSN 1010 will
                     * properly supercede all previous IN entries in the tree.
                     */
                    logManager.log(new INDeleteInfo
                                   (rootIN.getNodeId(),
                                    rootIN.getIdentifierKey(),
                                    database.getId()));

                    /* Count obsolete nodes after logging the delete info. */
                    accountForSubtreeRemoval(inMemoryINs, rootIN, tracker);
                }
            } finally {
                rootLatch.release();
            }

        } else {
      try {
    int index = subtreeRoot.findEntry(idKey, false, false);
    IN subtreeRootIN = (IN) subtreeRoot.fetchTarget(index);
    boolean deleteOk = subtreeRoot.deleteEntry(index, true);
    assert deleteOk;

                /*
                 * Record in the log the nodeid of the highest node in the
                 * subtree that we're deleting. We'll use this later to
                 * navigate to the right place if we need to replay this
                 * delete.
                 */
                logManager.log(new INDeleteInfo
             (subtreeRootIN.getNodeId(),
        subtreeRootIN.getIdentifierKey(),
        database.getId()));

                /* Count obsolete nodes after logging the delete info. */
 
View Full Code Here

  DIN duplicateRoot = null;
  boolean dupCountLNLocked = false;
  DupCountLN dcl = null;
  BasicLocker locker = new BasicLocker(env);
        LogManager logManager =
            database.getDbEnvironment().getLogManager();

  try {
            int index = in.findEntry(dupKey, false, true);
      if (index >= 0) {
    duplicateRoot = (DIN) in.fetchTarget(index);
    duplicateRoot.latch();

    ChildReference dclRef = duplicateRoot.getDupCountLNRef();
    dcl = (DupCountLN)
        dclRef.fetchTarget(database, duplicateRoot);

    /* Read lock the dup count LN. */
    if (locker.nonBlockingReadLock(dcl.getNodeId(), database) ==
                    LockGrantType.DENIED) {
        return false;
    } else {
        dupCountLNLocked = true;
    }

    /*
     * We don't release the latch on 'in' before we search the
     * duplicate tree below because we might be deleting the whole
     * subtree from the IN and we want to keep it latched until we
     * know.
     */
    IN subtreeRoot;
    try {
        subtreeRoot = searchSubTree(duplicateRoot,
            idKey,
            SearchType.DELETE,
            -1,
                                                null,
                                                true /*updateGeneration*/);
    } catch (NodeNotEmptyException NNEE) {

        /*
         * We can't delete the subtree because there are still
         * cursors pointing to the lowest node on it.
         */
        in.releaseLatch();
        throw NNEE;
    }

    if (subtreeRoot == null) {
        /* We're deleting the duplicate root. */
        BIN bin = (BIN) in;
        if (bin.nCursors() == 0) {
      try {

          /*
           * duplicateRoot is not currently latched.  Relatch
           * it and recheck if it still is deletable.
           */
          duplicateRoot.latch();
          if (duplicateRoot.isValidForDelete()) {
        boolean deleteOk =
            bin.deleteEntry(index, true);
        assert deleteOk;

        logManager.log(new INDupDeleteInfo
                 (duplicateRoot.getNodeId(),
            duplicateRoot.getMainTreeKey(),
            duplicateRoot.getDupTreeKey(),
            database.getId()));

                                /*
                                 * Count obsolete nodes after logging the
                                 * delete info.
                                 */
                                accountForSubtreeRemoval
            (inMemoryINs, duplicateRoot, tracker);

        if (bin.getNEntries() == 0) {
            database.getDbEnvironment().
          addToCompressorQueue(bin, null, false);
        }
          }
      } finally {
          duplicateRoot.releaseLatch();
      }
        } else {

      /*
       * Don't delete anything off this IN if there are
       * cursors referring to it.
       */
      ret = false;
        }
        in.releaseLatch();
    } else {
        try {
      /* We're deleting a portion of the duplicate tree. */
      in.releaseLatch();
      int dupIndex =
          subtreeRoot.findEntry(idKey, false, false);
      IN rootIN = (IN) subtreeRoot.fetchTarget(dupIndex);
      boolean deleteOk =
          subtreeRoot.deleteEntry(dupIndex, true);
      assert deleteOk;

      /*
       * Record in the log the nodeid of the highest node in
       * the subtree that we're deleting. We'll use this
       * later to navigate to the right place if we need to
       * replay this delete.
       */
      logManager.log(new INDupDeleteInfo
               (rootIN.getNodeId(),
          rootIN.getMainTreeKey(),
          rootIN.getDupTreeKey(),
          database.getId()));

View Full Code Here

        /*
         * Create a new root IN, insert the current root IN into it, and then
         * call split.
         */
        EnvironmentImpl env = database.getDbEnvironment();
        LogManager logManager = env.getLogManager();
        INList inMemoryINs = env.getInMemoryINs();

        IN curRoot = null;
        curRoot = (IN) root.fetchTarget(database, null);
        curRoot.latch();
View Full Code Here

                /* Continue down a level */
                parent = child;
            } while (!(parent instanceof BIN));

            boolean startedSplits = false;
            LogManager logManager =
                database.getDbEnvironment().getLogManager();

            /*
             * Process the accumulated nodes from the bottom up. Split each
             * node if required. If the node should not split, we check if
View Full Code Here

        throws DatabaseException {

        validateInsertArgs(allowDuplicates);

        EnvironmentImpl env = database.getDbEnvironment();
        LogManager logManager = env.getLogManager();
        INList inMemoryINs = env.getInMemoryINs();

        /* Find and latch the relevant BIN. */
        BIN bin = null;
        try {
View Full Code Here

TOP

Related Classes of com.sleepycat.je.log.LogManager

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.