Package org.archive.util.bdbje

Examples of org.archive.util.bdbje.EnhancedEnvironment


    private static int copyPersistEnv(File sourceDir, StoredSortedMap<String,Map> historyMap)
    throws DatabaseException {
        int count = 0;

        // open the source env history DB, copying entries to target env
        EnhancedEnvironment sourceEnv = setupCopyEnvironment(sourceDir, true);
        StoredClassCatalog sourceClassCatalog = sourceEnv.getClassCatalog();
        DatabaseConfig historyDbConfig = HISTORY_DB_CONFIG.toDatabaseConfig();
        historyDbConfig.setReadOnly(true);
        Database sourceHistoryDB = sourceEnv.openDatabase(
                null, URI_HISTORY_DBNAME, historyDbConfig);
        StoredSortedMap<String,Map> sourceHistoryMap = new StoredSortedMap<String,Map>(sourceHistoryDB,
                new StringBinding(), new SerialBinding<Map>(sourceClassCatalog,
                        Map.class), true);

        Iterator<Entry<String,Map>> iter = sourceHistoryMap.entrySet().iterator();
        while (iter.hasNext()) {
            Entry<String,Map> item = iter.next();
            if (logger.isLoggable(Level.FINE)) {
                logger.fine(item.getKey() + " " + new JSONObject(item.getValue()));
            }
           
            if (historyMap != null) {
                historyMap.put(item.getKey(), item.getValue());
            }
            count++;
        }
        StoredIterator.close(iter);
        sourceHistoryDB.close();
        sourceEnv.close();
       
        return count;
    }
View Full Code Here


     */
    public static int populatePersistEnv(String sourcePath, File envFile)
        throws IOException {
        int count = 0;
        StoredSortedMap<String,Map> historyMap = null;
        EnhancedEnvironment targetEnv = null;
        StoredClassCatalog classCatalog = null;
        Database historyDB = null;

        if (envFile != null) {
            // set up target environment
            FileUtils.ensureWriteableDirectory(envFile);
            targetEnv = setupCopyEnvironment(envFile);
            classCatalog = targetEnv.getClassCatalog();
            historyDB = targetEnv.openDatabase(null, URI_HISTORY_DBNAME,
                    HISTORY_DB_CONFIG.toDatabaseConfig());
            historyMap = new StoredSortedMap<String,Map>(historyDB,
                    new StringBinding(), new SerialBinding<Map>(classCatalog,
                        Map.class), true);
        }

        try {
            count = copyPersistSourceToHistoryMap(new File(sourcePath), historyMap);
        } finally {
            // in finally block so that we unlock the target env even if we
            // failed to populate it
            if (envFile != null) {
                logger.info(count + " records imported from " + sourcePath + " to BDB env " + envFile);
                historyDB.sync();
                historyDB.close();
                targetEnv.close();
            } else {
                logger.info(count + " records found in " + sourcePath);
            }
        }

View Full Code Here

    public static EnhancedEnvironment setupCopyEnvironment(File env, boolean readOnly) throws DatabaseException {
        EnvironmentConfig envConfig = new EnvironmentConfig();
        envConfig.setAllowCreate(true);
        envConfig.setReadOnly(readOnly);
        try {
            return new EnhancedEnvironment(env, envConfig);
        } catch (IllegalArgumentException iae) {
            throw new IllegalArgumentException("problem with specified environment "+env+"; is it already open?", iae);
        }
    }
View Full Code Here

            // prevent BDB's cleaner from deleting log files
            config.setConfigParam("je.cleaner.expunge", "false");
        } // else leave whatever other setting was already in place

        org.archive.util.FileUtils.ensureWriteableDirectory(f);
        this.bdbEnvironment = new EnhancedEnvironment(f, config);
        this.classCatalog = this.bdbEnvironment.getClassCatalog();
        if(!create) {
            // freeze last log file -- so that originating checkpoint isn't fouled
            DbBackup dbBackup = new DbBackup(bdbEnvironment);
            dbBackup.startBackup();
View Full Code Here

        org.archive.util.FileUtils.ensureWriteableDirectory(this.envDir);
        try {
            EnvironmentConfig envConfig = new EnvironmentConfig();
            envConfig.setTransactional(false);
            envConfig.setAllowCreate(true);
            env = new EnhancedEnvironment(envDir,envConfig);
            BdbModule.BdbConfig dbConfig = StoredQueue.databaseConfig();
            db = env.openDatabase(null, "StoredMapTest", dbConfig.toDatabaseConfig());
        } catch (DatabaseException e) {
            throw new RuntimeException(e);
        }
View Full Code Here

        File source = new File(args[0]);
        File env = new File(args[1]);
        FileUtils.ensureWriteableDirectory(env);
       
        // setup target environment
        EnhancedEnvironment targetEnv = PersistProcessor.setupCopyEnvironment(env);
        StoredClassCatalog classCatalog = targetEnv.getClassCatalog();
        Database historyDB = targetEnv.openDatabase(
                null,
                PersistProcessor.URI_HISTORY_DBNAME,
                PersistProcessor.HISTORY_DB_CONFIG.toDatabaseConfig());
        @SuppressWarnings({ "rawtypes", "unchecked" })
        StoredSortedMap<String, Object> historyMap = new StoredSortedMap<String, Object>(historyDB,
                new StringBinding(), new SerialBinding(classCatalog, Map.class), true);
       
        int count = 0;
       
        if(source.isFile()) {
            // scan log, writing to database
            BufferedReader br = ArchiveUtils.getBufferedReader(source);
            Iterator<String> iter = new LineReadingIterator(br);
            while(iter.hasNext()) {
                String line = (String) iter.next();
                String[] splits = line.split("\\s");
                String uri = splits[0];
                if(!uri.matches("\\w+:.*")) {
                    // prepend "http://"
                    uri = "http://"+uri;
                }
                String key = PersistProcessor.persistKeyFor(uri);
                int precedence = Integer.parseInt(splits[1]);
                @SuppressWarnings("unchecked")
                Map<String, Object> map = (Map<String, Object>)historyMap.get(key);
                if (map==null) {
                    map = new HashMap<String, Object>();
                }
                map.put(A_PRECALC_PRECEDENCE, precedence);
                historyMap.put(key,map);
                count++;
                if(count % 100000 == 0) {
                    System.out.print(count+"... ");
                }
            }
            br.close();
            System.out.println();
            System.out.println(count+" entries loaded");
        } else {
            // error
            System.err.println("unacceptable source file");
            return;
        }
       
        // cleanup
        historyDB.sync();
        historyDB.close();
        targetEnv.close();
        System.out.println(count+" records imported from "+source+" to BDB env "+env);
    }
View Full Code Here

TOP

Related Classes of org.archive.util.bdbje.EnhancedEnvironment

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.