Package org.opensolaris.opengrok.history

Source Code of org.opensolaris.opengrok.history.HistoryGuru

/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* See LICENSE.txt included in this distribution for the specific
* language governing permissions and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at LICENSE.txt.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/

/*
* Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved.
*/
package org.opensolaris.opengrok.history;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.opensolaris.opengrok.OpenGrokLogger;
import org.opensolaris.opengrok.configuration.Configuration.RemoteSCM;
import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
import org.opensolaris.opengrok.index.IgnoredNames;
import org.opensolaris.opengrok.util.Statistics;

/**
* The HistoryGuru is used to implement an transparent layer to the various
* source control systems.
*
* @author Chandan
*/
public final class HistoryGuru {
    private static final Logger log = OpenGrokLogger.getLogger();

    /** The one and only instance of the HistoryGuru */
    private static HistoryGuru instance = new HistoryGuru();

    /** The history cache to use */
    private final HistoryCache historyCache;

    private Map<String, Repository> repositories =
        new HashMap<String, Repository>();
    private final int scanningDepth;

    /**
     * Creates a new instance of HistoryGuru, and try to set the default
     * source control system.
     */
    private HistoryGuru() {
        HistoryCache cache = null;
        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
        scanningDepth=env.getScanningDepth();
        if (env.useHistoryCache()) {
            if (env.storeHistoryCacheInDB()) {
                cache = new JDBCHistoryCache();
            } else {
                cache = new FileHistoryCache();
            }
            try {
                cache.initialize();
            } catch (HistoryException he) {
                log.log(Level.WARNING,
                        "Failed to initialize the history cache", he);
                // Failed to initialize, run without a history cache
                cache = null;
            }
        }
        historyCache = cache;
    }

    /**
     * Get the one and only instance of the HistoryGuru
     * @return the one and only HistoryGuru instance
     */
    public static HistoryGuru getInstance()  {
        return instance;
    }

    /**
     * Return whether or not a cache should be used for the history log.
     * @return {@code true} if the history cache has been enabled and
     * initialized, {@code false} otherwise
     */
    private boolean useCache() {
        return historyCache != null;
    }

    /**
     * Get a string with information about the history cache.
     *
     * @return a free form text string describing the history cache instance
     * @throws HistoryException if an error occurred while getting the info
     */
    public String getCacheInfo() throws HistoryException {
        return historyCache == null ? "No cache" : historyCache.getInfo();
    }

    /**
     * Annotate the specified revision of a file.
     *
     * @param file the file to annotate
     * @param rev the revision to annotate (<code>null</code> means BASE)
     * @return file annotation, or <code>null</code> if the
     * <code>HistoryParser</code> does not support annotation
     * @throws IOException
     */
    public Annotation annotate(File file, String rev) throws IOException {
        Annotation ret = null;

        Repository repo = getRepository(file);
        if (repo != null) {
            ret = repo.annotate(file, rev);
            History hist = null;
            try {
                hist = repo.getHistory(file);
            } catch (HistoryException ex) {
                Logger.getLogger(HistoryGuru.class.getName()).log(Level.FINEST,
                    "Cannot get messages for tooltip: ", ex);
            }
            if (hist != null && ret != null) {
             Set<String> revs = ret.getRevisions();
             // !!! cannot do this because of not matching rev ids (keys)
             // first is the most recent one, so we need the position of "rev"
             // until the end of the list
             //if (hent.indexOf(rev)>0) {
             //     hent = hent.subList(hent.indexOf(rev), hent.size());
             //}
             for (HistoryEntry he : hist.getHistoryEntries()) {
                String hist_rev = he.getRevision();
                String short_rev = repo.getRevisionForAnnotate(hist_rev);
                if (revs.contains(short_rev)) {
                    ret.addDesc(short_rev, "changeset: " + he.getRevision() +
                        "\nsummary: " + he.getMessage() + "\nuser: " +
                        he.getAuthor() + "\ndate: " + he.getDate());
                }
             }
            }
        }

        return ret;
    }

    /**
     * Get the appropriate history reader for given file.
     *
     * @param file The file to get the history reader for
     * @throws HistoryException If an error occurs while getting the history
     * @return A HistorReader that may be used to read out history data for a
     *  named file
     */
    public HistoryReader getHistoryReader(File file) throws HistoryException {
        History history = getHistory(file, false);
        return history == null ? null : new HistoryReader(history);
    }

    /**
     * Get the history for the specified file.
     *
     * @param file the file to get the history for
     * @return history for the file
     * @throws HistoryException on error when accessing the history
     */
    public History getHistory(File file) throws HistoryException {
        return getHistory(file, true, false);
    }

    public History getHistory(File file, boolean withFiles) throws HistoryException {
        return getHistory(file, true, false);
    }

    /**
     * Get history for the specified file (called from the web app).
     *
     * @param file the file to get the history for
     * @return history for the file
     * @throws HistoryException on error when accessing the history
     */
    public History getHistoryUI(File file) throws HistoryException {
        return getHistory(file, true, true);
    }

    /**
     * Get the history for the specified file.
     *
     * @param file the file to get the history for
     * @param withFiles whether or not the returned history should contain
     * a list of files touched by each changeset (the file list may be skipped
     * if false, but it doesn't have to)
     * @param ui called from the webapp
     * @return history for the file
     * @throws HistoryException on error when accessing the history
     */
    public History getHistory(File file, boolean withFiles, boolean ui)
            throws HistoryException {
        final File dir = file.isDirectory() ? file : file.getParentFile();
        final Repository repo = getRepository(dir);

        History history = null;
        RemoteSCM rscm = RuntimeEnvironment.getInstance().getRemoteScmSupported();
        boolean doRemote = (ui && (rscm == RemoteSCM.UIONLY))
            || (rscm == RemoteSCM.ON)
            || (ui || ((rscm == RemoteSCM.DIRBASED) && (repo != null) && repo.hasHistoryForDirectories()));

        if (repo != null && repo.isWorking() && repo.fileHasHistory(file)
            && (!repo.isRemote() || doRemote)) {

            if (useCache() && historyCache.supportsRepository(repo)) {
                history = historyCache.get(file, repo, withFiles);
            } else {
                history = repo.getHistory(file);
            }
        }

        return history;
    }

    /**
     * Get a named revision of the specified file.
     * @param parent The directory containing the file
     * @param basename The name of the file
     * @param rev The revision to get
     * @return An InputStream containing the named revision of the file.
     */
    public InputStream getRevision(String parent, String basename, String rev) {
        InputStream ret = null;

        Repository rep = getRepository(new File(parent));
        if (rep != null) {
            ret = rep.getHistoryGet(parent, basename, rev);
        }
        return ret;
    }

    /**
     * Does this directory contain files with source control information?
     * @param file The name of the directory
     * @return true if the files in this directory have associated revision
     * history
     */
    public boolean hasHistory(File file) {
        Repository repo = getRepository(file);

        if (repo == null) {
            return false;
        }

        // This should return true for Annotate view.
        return repo.isWorking() && repo.fileHasHistory(file)
                && ((RuntimeEnvironment.getInstance().getRemoteScmSupported() == RemoteSCM.ON)
                    || (RuntimeEnvironment.getInstance().getRemoteScmSupported() == RemoteSCM.UIONLY)
                    || (RuntimeEnvironment.getInstance().getRemoteScmSupported() == RemoteSCM.DIRBASED)
                    || !repo.isRemote());
    }

    /**
     * Check if we can annotate the specified file.
     *
     * @param file the file to check
     * @return <code>true</code> if the file is under version control and the
     * version control system supports annotation
     */
    public boolean hasAnnotation(File file) {
        if (!file.isDirectory()) {
            Repository repos = getRepository(file);
            if (repos != null && repos.isWorking()) {
                return repos.fileHasAnnotation(file);
            }
        }

        return false;
    }

    /**
     * Get the last modified times for all files and subdirectories in the
     * specified directory.
     *
     * @param directory the directory whose files to check
     * @return a map from file names to modification times for the files that
     *   the history cache has information about
     */
    public Map<String, Date> getLastModifiedTimes(File directory)
            throws HistoryException {
        Repository repository = getRepository(directory);
        if (repository != null && useCache()) {
            return historyCache.getLastModifiedTimes(directory, repository);
        }
        return Collections.emptyMap();
    }

    private void addRepositories(File[] files, Collection<RepositoryInfo> repos,
            IgnoredNames ignoredNames, int depth)
    {
        addRepositories(files, repos, ignoredNames, true, depth);
    }

    /**
     * recursivelly search for repositories with a depth limit
     * @param files list of files to check if they contain a repo
     * @param repos list of found repos
     * @param ignoredNames what files to ignore
     * @param recursiveSearch whether to use recursive search
     * @param depth current depth - using global scanningDepth - one can limit
     *  this to improve scanning performance
     */
    private void addRepositories(File[] files, Collection<RepositoryInfo> repos,
            IgnoredNames ignoredNames, boolean recursiveSearch, int depth) {
        for (File file : files) {
            Repository repository = null;
            if (file.getName().equals(".opengrok_skip_history")) {
                log.log(Level.INFO, "Skipping history cache creation for "
                        + file.getParentFile().getAbsolutePath()
                        + " and it's subdirectories");
                return;
            }
        }
        for (File file : files) {
            Repository repository = null;
            try {
                repository = RepositoryFactory.getRepository(file);
            } catch (InstantiationException ie) {
                log.log(Level.WARNING, "Could not create repoitory for '"
                    + file + "', could not instantiate the repository.", ie);
            } catch (IllegalAccessException iae) {
                log.log(Level.WARNING, "Could not create repoitory for '"
                    + file + "', missing access rights.", iae);
            }
            if (repository == null) {
                // Not a repository, search it's sub-dirs
                if (file.isDirectory() && !ignoredNames.ignore(file)) {
                    File subFiles[] = file.listFiles();
                    if (subFiles == null) {
                        log.log(Level.WARNING,
                            "Failed to get sub directories for '"
                            + file.getAbsolutePath()
                            + "', check access permissions.");
                    } else if (depth<=scanningDepth) {
                        addRepositories(subFiles, repos, ignoredNames, depth+1);
                    }
                }
            } else {
                try {
                    String path = file.getCanonicalPath();
                    repository.setDirectoryName(path);
                    if (RuntimeEnvironment.getInstance().isVerbose()) {
                        log.log(Level.CONFIG, "Adding <{0}> repository: <{1}>",
                            new Object[]{repository.getClass().getName(), path});
                    }

                    repos.add(new RepositoryInfo(repository));

                    // @TODO: Search only for one type of repository - the one found here
                    if (recursiveSearch && repository.supportsSubRepositories()) {
                        File subFiles[] = file.listFiles();
                        if (subFiles == null) {
                            log.log(Level.WARNING,
                                "Failed to get sub directories for '"
                                + file.getAbsolutePath()
                                + "', check access permissions.");
                        } else if (depth<=scanningDepth) {
                            // Search only one level down - if not: too much
                            // stat'ing for huge Mercurial repositories
                            addRepositories(subFiles, repos, ignoredNames,
                                false, depth+1);
                        }
                    }

                } catch (IOException exp) {
                    log.log(Level.WARNING, "Failed to get canonical path for "
                        + file.getAbsolutePath() + ": " + exp.getMessage());
                    log.log(Level.WARNING, "Repository will be ignored...", exp);
                }
            }
        }
    }

    /**
     * Search through the all of the directories and add all of the source
     * repositories found.
     *
     * @param dir the root directory to start the search in.
     */
    public void addRepositories(String dir) {
        List<RepositoryInfo> repos = new ArrayList<RepositoryInfo>();
        addRepositories(new File[] {new File(dir)}, repos,
                RuntimeEnvironment.getInstance().getIgnoredNames(),0);
        RuntimeEnvironment.getInstance().setRepositories(repos);
        invalidateRepositories(repos);
    }

    /**
     * Update the source the contents in the source repositories.
     */
    public void updateRepositories() {
        boolean verbose = RuntimeEnvironment.getInstance().isVerbose();

        for (Map.Entry<String, Repository> entry : repositories.entrySet()) {
            Repository repository = entry.getValue();

            String path = entry.getKey();
            String type = repository.getClass().getSimpleName();

            if (repository.isWorking()) {
                if (verbose) {
                    log.info(String.format("Update %s repository in %s",
                        type, path));
                }

                try {
                    repository.update();
                } catch (UnsupportedOperationException e) {
                    log.warning(String.format("Skipping update of %s repository"
                        + " in %s: Not implemented", type, path));
                } catch (Exception e) {
                    log.log(Level.WARNING, "An error occured while updating "
                        + path + " (" + type + ")", e);
                }
            } else {
                log.warning(String.format("Skipping update of %s repository in "
                    + "%s: Missing SCM dependencies?", type, path));
            }
        }
    }

    /**
     * Update the source the contents in the source repositories.
     * @param paths A list of files/directories to update
     */
    public void updateRepositories(Collection<String> paths) {
        boolean verbose = RuntimeEnvironment.getInstance().isVerbose();

        List<Repository> repos = getReposFromString(paths);

        for (Repository repository : repos) {
            String type = repository.getClass().getSimpleName();

            if (repository.isWorking()) {
                if (verbose) {
                    log.info(String.format("Update %s repository in %s", type,
                        repository.getDirectoryName()));
                }

                try {
                    repository.update();
                } catch (UnsupportedOperationException e) {
                    log.warning(String.format("Skipping update of %s repository"
                        + " in %s: Not implemented", type,
                        repository.getDirectoryName()));
                } catch (Exception e) {
                    log.log(Level.WARNING, "An error occured while updating "
                        + repository.getDirectoryName() + " (" + type + ")", e);
                }
            } else {
                log.warning(String.format("Skipping update of %s repository in"
                    + " %s: Missing SCM dependencies?", type,
                    repository.getDirectoryName()));
            }
        }
    }

    private void createCache(Repository repository, String sinceRevision) {
        String path = repository.getDirectoryName();
        String type = repository.getClass().getSimpleName();

        if (repository.isWorking()) {
            boolean verbose = RuntimeEnvironment.getInstance().isVerbose();
            Statistics elapsed = new Statistics();

            if (verbose) {
                log.log(Level.INFO, "Creating historycache for {0} ({1})",
                    new Object[]{path, type});
            }

            try {
                repository.createCache(historyCache, sinceRevision);
            } catch (Exception e) {
                log.log(Level.WARNING,
                    "An error occured while creating cache for " + path + " ("
                    + type + ")", e);
            }

            if (verbose) {
                elapsed.report(log, "Done historycache for " + path);
            }
        } else {
            log.log(Level.WARNING, "Skipping creation of historycache of "
                + type + " repository in " + path + ": Missing SCM dependencies?");
        }
    }

    private void createCacheReal(Collection<Repository> repositories) {
        Statistics elapsed = new Statistics();
        ExecutorService executor = RuntimeEnvironment.getHistoryExecutor();
        // Since we know each repository object from the repositories
        // collection is unique, we can abuse HashMap to create a list of
        // repository,revision tuples with repository as key (as the revision
        // string does not have to be unique - surely it is not unique
        // for the initial index case).
        HashMap<Repository,String> repos2process = new HashMap<>();

        // Collect the list of <latestRev,repo> pairs first so that we
        // do not have to deal with latch decrementing in the cycle below.
        for (final Repository repo : repositories) {
            final String latestRev;

            try {
                latestRev = historyCache.getLatestCachedRevision(repo);
                repos2process.put(repo, latestRev);
            } catch (HistoryException he) {
                log.log(Level.WARNING,
                        String.format(
                        "Failed to retrieve latest cached revision for %s",
                        repo.getDirectoryName()), he);
            }
        }

        log.log(Level.INFO, "Creating historycache for {0} repositories",
            repos2process.size());
        final CountDownLatch latch = new CountDownLatch(repos2process.size());
        for (final Map.Entry<Repository,String> entry : repos2process.entrySet()) {
            executor.submit(new Runnable() {
                @Override
                public void run() {
                    try {
                        createCache(entry.getKey(), entry.getValue());
                    } catch (Exception ex) {
                        // We want to catch any exception since we are in thread.
                        log.log(Level.WARNING,
                            "createCacheReal() got exception" + ex);
                    } finally {
                        latch.countDown();
                    }
                }
            });
        }

        /*
         * Wait until the history of all repositories is done. This is necessary
         * since the next phase of generating index will need the history to
         * be ready as it is recorded in Lucene index.
         */
        try {
            latch.await();
        } catch (InterruptedException ex) {
            OpenGrokLogger.getLogger().log(Level.SEVERE,
                "latch exception" + ex);
        }

        executor.shutdown();
        while (!executor.isTerminated()) {
            try {
                // Wait forever
                executor.awaitTermination(999,TimeUnit.DAYS);
            } catch (InterruptedException exp) {
                OpenGrokLogger.getLogger().log(Level.WARNING,
                    "Received interrupt while waiting for executor to finish", exp);
            }
        }
        RuntimeEnvironment.freeHistoryExecutor();
        try {
            /* Thread pool for handling renamed files needs to be destroyed too. */
            RuntimeEnvironment.destroyRenamedHistoryExecutor();
        } catch (InterruptedException ex) {
            OpenGrokLogger.getLogger().log(Level.SEVERE,
                "destroying of renamed thread pool failed", ex);
        }

        // The cache has been populated. Now, optimize how it is stored on
        // disk to enhance performance and save space.
        try {
            historyCache.optimize();
        } catch (HistoryException he) {
            OpenGrokLogger.getLogger().log(Level.WARNING,
                    "Failed optimizing the history cache database", he);
        }
        elapsed.report(log, "Done historycache for all repositories");
        historyCache.setHistoryIndexDone();
    }

    public void createCache(Collection<String> repositories) {
        if (!useCache()) {
            return;
        }
        createCacheReal(getReposFromString(repositories));
    }

    public void removeCache(Collection<String> repositories) throws HistoryException {
        List<Repository> repos = getReposFromString(repositories);
        HistoryCache cache = historyCache;
        if (cache == null) {
            if (RuntimeEnvironment.getInstance().storeHistoryCacheInDB()) {
                cache = new JDBCHistoryCache();
                cache.initialize();
            } else {
                cache = new FileHistoryCache();
            }
        }
        for (Repository r : repos) {
            try {
                cache.clear(r);
                log.info("History cache for " + r.getDirectoryName() + " cleared.");
            } catch (HistoryException e) {
                log.warning("Clearing history cache for repository " +
                    r.getDirectoryName() + " failed: " + e.getLocalizedMessage());
            }
        }
        invalidateRepositories(repos);
    }

    /**
     * Create the history cache for all of the repositories
     */
    public void createCache() {
        if (!useCache()) {
            return;
        }

        createCacheReal(repositories.values());
    }

    private List<Repository> getReposFromString(Collection<String> repositories) {
        ArrayList<Repository> repos = new ArrayList<Repository>();
        File root = RuntimeEnvironment.getInstance().getSourceRootFile();
        for (String file : repositories) {
            File f = new File(root, file);
            Repository r = getRepository(f);
            if (r == null) {
                log.log(Level.WARNING, "Could not locate a repository for {0}",
                    f.getAbsolutePath());
            } else if (!repos.contains(r)){
                repos.add(r);
            }
        }
        return repos;
    }

    /**
     * Ensure that we have a directory in the cache. If it's not there, fetch
     * its history and populate the cache. If it's already there, and the
     * cache is able to tell how recent it is, attempt to update it to the
     * most recent revision.
     *
     * @param file the root path to test
     * @throws HistoryException if an error occurs while accessing the
     * history cache
     */
    public void ensureHistoryCacheExists(File file) throws HistoryException {
        if (!useCache()) {
            return;
        }

        Repository repository = getRepository(file);

        if (repository == null) {
            // no repository -> no history :(
            return;
        }

        String sinceRevision = null;

        if (historyCache.hasCacheForDirectory(file, repository)) {
            sinceRevision = historyCache.getLatestCachedRevision(repository);
            if (sinceRevision == null) {
                // Cache already exists, but we don't know how recent it is,
                // so don't do anything.
                return;
            }
        }

        // Create cache from the beginning if it doesn't exist, or update it
        // incrementally otherwise.
        createCache(getRepository(file), sinceRevision);
    }

    protected Repository getRepository(File path) {
        Map<String, Repository> repos = repositories;

        File file = path;
        try {
            file = path.getCanonicalFile();
        } catch (IOException e) {
            log.log(Level.WARNING, "Failed to get canonical path for " + path, e);
            return null;
        }
        while (file != null) {
            Repository r = repos.get(file.getAbsolutePath());
            if (r != null) {
                return r;
            }
            file = file.getParentFile();
        }

        return null;
    }

    /**
     * Invalidate the current list of known repositories!
     *
     * @param repos The new repositories
     */
    public void invalidateRepositories(Collection<? extends RepositoryInfo> repos)
    {
        if (repos == null || repos.isEmpty()) {
            repositories.clear();
        } else {
            Map<String, Repository> nrep =
                new HashMap<String, Repository>(repos.size());
            for (RepositoryInfo i : repos) {
                try {
                    Repository r = RepositoryFactory.getRepository(i);
                    if (r == null) {
                        log.log(Level.WARNING,
                            "Failed to instanciate internal repository data for "
                            + i.getType() + " in " + i.getDirectoryName());
                    } else {
                        nrep.put(r.getDirectoryName(), r);
                    }
                } catch (InstantiationException ex) {
                    log.log(Level.WARNING, "Could not create " + i.getType()
                        + " for '" + i.getDirectoryName()
                        + "', could not instantiate the repository.", ex);
                } catch (IllegalAccessException iae) {
                    log.log(Level.WARNING, "Could not create " + i.getType()
                        + " for '" + i.getDirectoryName()
                        + "', missing access rights.", iae);
                }
            }
            repositories = nrep;
        }
    }
}
TOP

Related Classes of org.opensolaris.opengrok.history.HistoryGuru

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.