Package org.opensolaris.opengrok.util

Examples of org.opensolaris.opengrok.util.Statistics


        String path = repository.getDirectoryName();
        String type = repository.getClass().getSimpleName();

        if (repository.isWorking()) {
            boolean verbose = RuntimeEnvironment.getInstance().isVerbose();
            Statistics elapsed = new Statistics();

            if (verbose) {
                log.log(Level.INFO, "Creating historycache for {0} ({1})",
                    new Object[]{path, type});
            }

            try {
                repository.createCache(historyCache, sinceRevision);
            } catch (Exception e) {
                log.log(Level.WARNING,
                    "An error occured while creating cache for " + path + " ("
                    + type + ")", e);
            }

            if (verbose) {
                elapsed.report(log, "Done historycache for " + path);
            }
        } else {
            log.log(Level.WARNING, "Skipping creation of historycache of "
                + type + " repository in " + path + ": Missing SCM dependencies?");
        }
View Full Code Here


                + type + " repository in " + path + ": Missing SCM dependencies?");
        }
    }

    private void createCacheReal(Collection<Repository> repositories) {
        Statistics elapsed = new Statistics();
        ExecutorService executor = RuntimeEnvironment.getHistoryExecutor();
        // Since we know each repository object from the repositories
        // collection is unique, we can abuse HashMap to create a list of
        // repository,revision tuples with repository as key (as the revision
        // string does not have to be unique - surely it is not unique
        // for the initial index case).
        HashMap<Repository,String> repos2process = new HashMap<>();

        // Collect the list of <latestRev,repo> pairs first so that we
        // do not have to deal with latch decrementing in the cycle below.
        for (final Repository repo : repositories) {
            final String latestRev;

            try {
                latestRev = historyCache.getLatestCachedRevision(repo);
                repos2process.put(repo, latestRev);
            } catch (HistoryException he) {
                log.log(Level.WARNING,
                        String.format(
                        "Failed to retrieve latest cached revision for %s",
                        repo.getDirectoryName()), he);
            }
        }

        log.log(Level.INFO, "Creating historycache for {0} repositories",
            repos2process.size());
        final CountDownLatch latch = new CountDownLatch(repos2process.size());
        for (final Map.Entry<Repository,String> entry : repos2process.entrySet()) {
            executor.submit(new Runnable() {
                @Override
                public void run() {
                    try {
                        createCache(entry.getKey(), entry.getValue());
                    } catch (Exception ex) {
                        // We want to catch any exception since we are in thread.
                        log.log(Level.WARNING,
                            "createCacheReal() got exception" + ex);
                    } finally {
                        latch.countDown();
                    }
                }
            });
        }

        /*
         * Wait until the history of all repositories is done. This is necessary
         * since the next phase of generating index will need the history to
         * be ready as it is recorded in Lucene index.
         */
        try {
            latch.await();
        } catch (InterruptedException ex) {
            OpenGrokLogger.getLogger().log(Level.SEVERE,
                "latch exception" + ex);
        }

        executor.shutdown();
        while (!executor.isTerminated()) {
            try {
                // Wait forever
                executor.awaitTermination(999,TimeUnit.DAYS);
            } catch (InterruptedException exp) {
                OpenGrokLogger.getLogger().log(Level.WARNING,
                    "Received interrupt while waiting for executor to finish", exp);
            }
        }
        RuntimeEnvironment.freeHistoryExecutor();
        try {
            /* Thread pool for handling renamed files needs to be destroyed too. */
            RuntimeEnvironment.destroyRenamedHistoryExecutor();
        } catch (InterruptedException ex) {
            OpenGrokLogger.getLogger().log(Level.SEVERE,
                "destroying of renamed thread pool failed", ex);
        }

        // The cache has been populated. Now, optimize how it is stored on
        // disk to enhance performance and save space.
        try {
            historyCache.optimize();
        } catch (HistoryException he) {
            OpenGrokLogger.getLogger().log(Level.WARNING,
                    "Failed optimizing the history cache database", he);
        }
        elapsed.report(log, "Done historycache for all repositories");
        historyCache.setHistoryIndexDone();
    }
View Full Code Here

     *
     * @param argv argument vector
     */
    @SuppressWarnings("PMD.UseStringBufferForStringAppends")
    public static void main(String argv[]) {
        Statistics stats = new Statistics(); //this won't count JVM creation though
        boolean runIndex = true;
        boolean update = true;
        boolean optimizedChanged = false;
        ArrayList<String> zapCache = new ArrayList<>();
        CommandLineOptions cmdOptions = new CommandLineOptions();       

        if (argv.length == 0) {
            System.err.println(cmdOptions.getUsage());
            System.exit(1);
        } else {
            Executor.registerErrorHandler();
            boolean searchRepositories = false;
            ArrayList<String> subFiles = new ArrayList<>();
            ArrayList<String> repositories = new ArrayList<>();
            HashSet<String> allowedSymlinks = new HashSet<>();
            String configFilename = null;
            String configHost = null;
            boolean addProjects = false;
            boolean refreshHistory = false;
            String defaultProject = null;
            boolean listFiles = false;
            boolean listRepos = false;
            boolean createDict = false;
            int noThreads = 2 + (2 * Runtime.getRuntime().availableProcessors());

            // Parse command line options:
            Getopt getopt = new Getopt(argv, cmdOptions.getCommandString());

            try {
                getopt.parse();
            } catch (ParseException ex) {
                System.err.println("OpenGrok: " + ex.getMessage());
                System.err.println(cmdOptions.getUsage());
                System.exit(1);
            }

            try {
                Configuration cfg = null;
                int cmd;

                // We need to read the configuration file first, since we
                // will try to overwrite options..
                while ((cmd = getopt.getOpt()) != -1) {
                    if (cmd == 'R') {
                        cfg = Configuration.read(new File(getopt.getOptarg()));
                        break;
                    }
                }

                if (cfg == null) {
                    cfg = new Configuration();
                }

                String databaseDriver = cfg.getDatabaseDriver();
                String databaseURL = cfg.getDatabaseUrl();

                // Now we can handle all the other options..
                getopt.reset();
                while ((cmd = getopt.getOpt()) != -1) {
                    switch (cmd) {
                        case 'q':
                            cfg.setVerbose(false);
                            OpenGrokLogger.setOGConsoleLogLevel(Level.WARNING);
                            break;
                        case 'e':
                            cfg.setGenerateHtml(false);
                            break;
                        case 'P':
                            addProjects = true;
                            break;
                        case 'p':
                            defaultProject = getopt.getOptarg();
                            break;
                        case 'c':
                            cfg.setCtags(getopt.getOptarg());
                            break;
                        case 'w': {
                            String webapp = getopt.getOptarg();
                            if (webapp.charAt(0) != '/' && !webapp.startsWith("http")) {
                                webapp = "/" + webapp;
                            }
                            if (webapp.endsWith("/")) {
                                cfg.setUrlPrefix(webapp + "s?");
                            } else {
                                cfg.setUrlPrefix(webapp + "/s?");
                            }
                        }
                        break;
                        case 'W':
                            configFilename = getopt.getOptarg();
                            break;
                        case 'U':
                            configHost = getopt.getOptarg();
                            break;
                        case 'R':
                            // already handled
                            break;
                        case 'N':
                            allowedSymlinks.add(getopt.getOptarg());
                            break;
                        case 'n':
                            runIndex = false;
                            break;
                        case 'G':
                            cfg.setTagsEnabled(true);
                            break;
                        case 'H':
                            refreshHistory = true;
                            break;
                        case 'h':
                            repositories.add(getopt.getOptarg());
                            break;
                        case 'D':
                            cfg.setHistoryCacheInDB(true);
                            break;
                        case 'j':
                            databaseDriver = getopt.getOptarg();
                            // Should be a full class name, but we also accept
                            // the shorthands "client" and "embedded". Expand
                            // the shorthands here.
                            switch (databaseDriver) {
                                case "client":
                                    databaseDriver = DERBY_CLIENT_DRIVER;
                                    break;
                                case "embedded":
                                    databaseDriver = DERBY_EMBEDDED_DRIVER;
                                    break;
                            }
                            break;
                        case 'u':
                            databaseURL = getopt.getOptarg();
                            break;
                        case 'r':
                            if (getopt.getOptarg().equalsIgnoreCase(ON)) {
                                cfg.setRemoteScmSupported(Configuration.RemoteSCM.ON);
                            } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
                                cfg.setRemoteScmSupported(Configuration.RemoteSCM.OFF);
                            } else if (getopt.getOptarg().equalsIgnoreCase(DIRBASED)) {
                                cfg.setRemoteScmSupported(Configuration.RemoteSCM.DIRBASED);
                            } else if (getopt.getOptarg().equalsIgnoreCase(UIONLY)) {
                                cfg.setRemoteScmSupported(Configuration.RemoteSCM.UIONLY);
                            } else {
                                System.err.println("ERROR: You should pass either \"on\" or \"off\" or \"uionly\" as argument to -r");
                                System.err.println("       Ex: \"-r on\" will allow retrieval for remote SCM systems");
                                System.err.println("           \"-r off\" will ignore SCM for remote systems");
                                System.err.println("           \"-r dirbased\" will allow retrieval during history index "
                                    + "only for repositories which allow getting history for directories");
                                System.err.println("           \"-r uionly\" will support remote SCM for UI only");
                            }
                            break;
                        case 'o':
                            String CTagsExtraOptionsFile = getopt.getOptarg();
                            File CTagsFile = new File(CTagsExtraOptionsFile);
                            if (!(CTagsFile.isFile() && CTagsFile.canRead())) {
                                System.err.println("ERROR: File '"
                                        + CTagsExtraOptionsFile
                                        + "' not found for the -o option");
                                System.exit(1);
                            }
                            System.err.println("INFO: file with extra "
                                    + "options for ctags: " + CTagsExtraOptionsFile);
                            cfg.setCTagsExtraOptionsFile(CTagsExtraOptionsFile);
                            break;
                        case 'O': {
                            boolean oldval = cfg.isOptimizeDatabase();
                            if (getopt.getOptarg().equalsIgnoreCase(ON)) {
                                cfg.setOptimizeDatabase(true);
                            } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
                                cfg.setOptimizeDatabase(false);
                            } else {
                                System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -O");
                                System.err.println("       Ex: \"-O on\" will optimize the database as part of the index generation");
                                System.err.println("           \"-O off\" disable optimization of the index database");
                            }
                            if (oldval != cfg.isOptimizeDatabase()) {
                                optimizedChanged = true;
                            }
                            break;
                        }
                        case 'v':
                            cfg.setVerbose(true);
                            OpenGrokLogger.setOGConsoleLogLevel(Level.INFO);
                            break;
                        case 'C':
                            cfg.setPrintProgress(true);
                            break;

                        case 's': {
                            File sourceRoot = new File(getopt.getOptarg());
                            if (!sourceRoot.isDirectory()) {
                                System.err.println("ERROR: Source root must be a directory");
                                System.exit(1);
                            }
                            cfg.setSourceRoot(sourceRoot.getCanonicalPath());
                            break;
                        }
                        case 'd': {
                            File dataRoot = new File(getopt.getOptarg());
                            if (!dataRoot.exists() && !dataRoot.mkdirs()) {
                                System.err.println("ERROR: Cannot create data root");
                                System.exit(1);
                            }
                            if (!dataRoot.isDirectory()) {
                                System.err.println("ERROR: Data root must be a directory");
                                System.exit(1);
                            }
                            cfg.setDataRoot(dataRoot.getCanonicalPath());
                            break;
                        }
                        case 'i':
                            cfg.getIgnoredNames().add(getopt.getOptarg());
                            break;
                        case 'I':
                            cfg.getIncludedNames().add(getopt.getOptarg());
                            break;
                        case 'S':
                            searchRepositories = true;
                            break;
                        case 'Q':
                            if (getopt.getOptarg().equalsIgnoreCase(ON)) {
                                cfg.setQuickContextScan(true);
                            } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
                                cfg.setQuickContextScan(false);
                            } else {
                                System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -Q");
                                System.err.println("       Ex: \"-Q on\" will just scan a \"chunk\" of the file and insert \"[..all..]\"");
                                System.err.println("           \"-Q off\" will try to build a more accurate list by reading the complete file.");
                            }

                            break;
                        case 'm': {
                            try {
                                cfg.setRamBufferSize(Double.parseDouble(getopt.getOptarg()));
                            } catch (NumberFormatException exp) {
                                System.err.println("ERROR: Failed to parse argument to \"-m\": " + exp.getMessage());
                                System.exit(1);
                            }
                            break;
                        }
                        case 'a':
                            if (getopt.getOptarg().equalsIgnoreCase(ON)) {
                                cfg.setAllowLeadingWildcard(true);
                            } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
                                cfg.setAllowLeadingWildcard(false);
                            } else {
                                System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -a");
                                System.err.println("       Ex: \"-a on\" will allow a search to start with a wildcard");
                                System.err.println("           \"-a off\" will disallow a search to start with a wildcard");
                                System.exit(1);
                            }

                            break;

                        case 'A': {
                            String[] arg = getopt.getOptarg().split(":");
                            boolean prefix = false;

                            if (arg.length != 2) {
                                A_usage();
                            }

                            if (arg[0].endsWith(".")) {
                                arg[0] = arg[0].substring(0, arg[0].lastIndexOf('.')).toUpperCase();
                                prefix = true;
                            } else if (arg[0].startsWith(".")) {
                                arg[0] = arg[0].substring(arg[0].lastIndexOf('.') + 1).toUpperCase();
                            } else {
                                A_usage();
                            }

                            if (arg[1].equals("-")) {
                                if (prefix) {
                                    AnalyzerGuru.addPrefix(arg[0], null);
                                } else {
                                    AnalyzerGuru.addExtension(arg[0], null);
                                }
                                break;
                            }

                            if (prefix) {
                                try {
                                    AnalyzerGuru.addPrefix(
                                            arg[0],
                                            AnalyzerGuru.findFactory(arg[1]));
                                } catch (Exception e) {
                                    log.log(Level.SEVERE, "Unable to use {0} as a FileAnalyzerFactory", arg[1]);
                                    log.log(Level.SEVERE, "Stack: ", e.fillInStackTrace());
                                    System.exit(1);
                                }
                            } else {
                                try {
                                    AnalyzerGuru.addExtension(
                                            arg[0],
                                            AnalyzerGuru.findFactory(arg[1]));
                                } catch (Exception e) {
                                    log.log(Level.SEVERE, "Unable to use {0} as a FileAnalyzerFactory", arg[1]);
                                    log.log(Level.SEVERE, "Stack: ", e.fillInStackTrace());
                                    System.exit(1);
                                }
          }
                        }
                        break;
                        case 'L':
                            cfg.setWebappLAF(getopt.getOptarg());
                            break;
                        case 'T':
                            try {
                                noThreads = Integer.parseInt(getopt.getOptarg());
                            } catch (NumberFormatException exp) {
                                System.err.println("ERROR: Failed to parse argument to \"-T\": " + exp.getMessage());
                                System.exit(1);
                            }
                            break;
                        case 'z':
                            try {
                                cfg.setScanningDepth(Integer.parseInt(getopt.getOptarg()));
                            } catch (NumberFormatException exp) {
                                System.err.println("ERROR: Failed to parse argument to \"-z\": " + exp.getMessage());
                                System.exit(1);
                            }
                            break;
                        case 'l':
                            if (getopt.getOptarg().equalsIgnoreCase(ON)) {
                                cfg.setUsingLuceneLocking(true);
                            } else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
                                cfg.setUsingLuceneLocking(false);
                            } else {
                                System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -l");
                                System.err.println("       Ex: \"-l on\" will enable locks in Lucene");
                                System.err.println("           \"-l off\" will disable locks in Lucene");
                            }
                            break;
                        case 'B':
                            cfg.setUserPage(getopt.getOptarg());
                            break;
                        case 'X':
                            cfg.setUserPageSuffix(getopt.getOptarg());
                            break;
                        case 'V':
                            System.out.println(Info.getFullVersion());
                            System.exit(0);
                            break;
                        case 'k':
                            zapCache.add(getopt.getOptarg());
                            break;
                        case 'K':
                            listRepos = true;
                            break;
                        case '?':
                            System.err.println(cmdOptions.getUsage());
                            System.exit(0);
                            break;
                        case 't':
                            try {
                                int tmp = Integer.parseInt(getopt.getOptarg());
                                cfg.setTabSize(tmp);
                            } catch (NumberFormatException exp) {
                                System.err.println("ERROR: Failed to parse argument to \"-t\": " + exp.getMessage());
                                System.exit(1);
                            }
                            break;
                        default:
                            System.err.println("Internal Error - Unimplemented cmdline option: " + (char) cmd);
                            System.exit(1);
                    }
                }

                List<Class<? extends Repository>> repositoryClasses =
                        RepositoryFactory.getRepositoryClasses();
                for (Class<? extends Repository> clazz : repositoryClasses) {
                    try {
                        Field f = clazz.getDeclaredField("CMD_PROPERTY_KEY");
                        Object key = f.get(null);
                        if (key != null) {
                            cfg.setRepoCmd(clazz.getCanonicalName(),
                                    System.getProperty(key.toString()));
                        }
                    } catch (Exception e) {
                        // don't care
                    }
                }
                int optind = getopt.getOptind();
                if (optind != -1) {
                    while (optind < argv.length) {
                        subFiles.add(argv[optind]);
                        ++optind;
                    }
                }

                //logging starts here
                if (cfg.isVerbose()) {
                    String fn = LogManager.getLogManager().getProperty("java.util.logging.FileHandler.pattern");
                    if (fn != null) {
                        System.out.println("Logging filehandler pattern: " + fn);
                    }
                }

                if (cfg.isHistoryCacheInDB()) {
                    // The default database driver is Derby's client driver.
                    if (databaseDriver == null) {
                        databaseDriver = DERBY_CLIENT_DRIVER;
                    }

                    // The default URL depends on the database driver.
                    if (databaseURL == null) {
                        StringBuilder defaultURL = new StringBuilder();
                        defaultURL.append("jdbc:derby:");
                        if (databaseDriver.equals(DERBY_EMBEDDED_DRIVER)) {
                            defaultURL.append(cfg.getDataRoot())
                                    .append(File.separator);
                        } else {
                            defaultURL.append("//localhost/");
                        }
                        defaultURL.append("cachedb;create=true");
                        databaseURL = defaultURL.toString();
                    }
                }

                cfg.setDatabaseDriver(databaseDriver);
                cfg.setDatabaseUrl(databaseURL);

                // automatically allow symlinks that are directly in source root
                String file = cfg.getSourceRoot();
                if (file != null) {
                    File sourceRootFile = new File(file);
                    File[] projectDirs = sourceRootFile.listFiles();
                    if (projectDirs != null) {
                        for (File projectDir : projectDirs) {
                            if (!projectDir.getCanonicalPath().equals(projectDir.getAbsolutePath())) {
                                allowedSymlinks.add(projectDir.getAbsolutePath());
                            }
                        }
                    }
                }

                allowedSymlinks.addAll(cfg.getAllowedSymlinks());
                cfg.setAllowedSymlinks(allowedSymlinks);

                // Set updated configuration in RuntimeEnvironment.
                RuntimeEnvironment env = RuntimeEnvironment.getInstance();
                env.setConfiguration(cfg);

                // Issue a warning when JDBC is used with renamed file handling.
                // This causes heavy slowdown when used with JavaDB (issue #774).
                if (RuntimeEnvironment.isRenamedFilesEnabled() && cfg.isHistoryCacheInDB()) {
                    System.out.println("History stored in DB and renamed file handling is on - possible performance degradation");
                }

                getInstance().prepareIndexer(env, searchRepositories, addProjects,
                        defaultProject, configFilename, refreshHistory,
                        listFiles, createDict, subFiles, repositories,
                        zapCache, listRepos);
                if (listRepos || !zapCache.isEmpty()) {
                    return;
                }
                if (runIndex || (optimizedChanged && env.isOptimizeDatabase())) {
                    IndexChangedListener progress = new DefaultIndexChangedListener();
                    getInstance().doIndexerExecution(update, noThreads, subFiles,
                            progress);
                }
                getInstance().sendToConfigHost(env, configHost);
            } catch (IndexerException ex) {
                log.log(Level.SEVERE, "Exception running indexer", ex);
                System.err.println(cmdOptions.getUsage());
                System.exit(1);
            } catch (Throwable e) {
                System.err.println("Exception: " + e.getLocalizedMessage());
                log.log(Level.SEVERE, "Unexpected Exception", e);
                System.exit(1);
            }
            finally {
                stats.report(log);
            }
        }

    }
View Full Code Here

     * if any).
     */
    public void doIndexerExecution(final boolean update, int noThreads, List<String> subFiles,
            IndexChangedListener progress)
            throws IOException {
        Statistics elapsed = new Statistics();
        RuntimeEnvironment env = RuntimeEnvironment.getInstance().register();
        log.info("Starting indexing");

        ExecutorService executor = Executors.newFixedThreadPool(noThreads);

        if (subFiles == null || subFiles.isEmpty()) {
            if (update) {
                IndexDatabase.updateAll(executor, progress);
            } else if (env.isOptimizeDatabase()) {
                IndexDatabase.optimizeAll(executor);
            }
        } else {
            List<IndexDatabase> dbs = new ArrayList<>();

            for (String path : subFiles) {
                Project project = Project.getProject(path);
                if (project == null && env.hasProjects()) {
                    log.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
                } else {
                    IndexDatabase db;
                    if (project == null) {
                        db = new IndexDatabase();
                    } else {
                        db = new IndexDatabase(project);
                    }
                    int idx = dbs.indexOf(db);
                    if (idx != -1) {
                        db = dbs.get(idx);
                    }

                    if (db.addDirectory(path)) {
                        if (idx == -1) {
                            dbs.add(db);
                        }
                    } else {
                        log.log(Level.WARNING, "Directory does not exist \"{0}\"", path);
                    }
                }
            }

            for (final IndexDatabase db : dbs) {
                final boolean optimize = env.isOptimizeDatabase();
                db.addIndexChangedListener(progress);
                executor.submit(new Runnable() {
                    @Override
                    public void run() {
                        try {
                            if (update) {
                                db.update();
                            } else if (optimize) {
                                db.optimize();
                            }
                        } catch (Throwable e) {
                            log.log(Level.SEVERE, "An error occured while "
                                    + (update ? "updating" : "optimizing")
                                    + " index", e);
                        }
                    }
                });
            }
        }

        executor.shutdown();
        while (!executor.isTerminated()) {
            try {
                // Wait forever
                executor.awaitTermination(999, TimeUnit.DAYS);
            } catch (InterruptedException exp) {
                log.log(Level.WARNING, "Received interrupt while waiting for executor to finish", exp);
            }
        }
        try {
            // It can happen that history index is not done in prepareIndexer()
            // but via db.update() above in which case we must make sure the
            // thread pool for renamed file handling is destroyed.
            RuntimeEnvironment.destroyRenamedHistoryExecutor();
        } catch (InterruptedException ex) {
            log.log(Level.SEVERE,
                "destroying of renamed thread pool failed", ex);
        }
        elapsed.report(log, "Done indexing data of all repositories");
    }
View Full Code Here

TOP

Related Classes of org.opensolaris.opengrok.util.Statistics

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.