Examples of krati.store.DynamicDataStore

krati.store.DynamicDataStore
DynamicDataStore is implemented using Linear Hashing. Its capacity grows as needed. The key-value pairs are stored in the underlying DataArray using the following format:
```
 [count:int][key-length:int][key:bytes][value-length:int][value:bytes][key-length:int][key:bytes][value-length:int][value:bytes]... +------------------ key-value pair 1 ---------------------+------------------- key-value pair 2 -------------------+ 
```
@author jwu
06/04, 2011 - Added support for Closeable
06/08, 2011 - Scale to the Integer.MAX_VALUE capacity
06/25, 2011 - Added constructor using StoreConfig
08/24, 2012 - Disable full rehashing on open/close
09/05, 2012 - Expand capacity on first-time creation

            int numSyncBatches,
            int segmentFileSizeMB,
            SegmentFactory segmentFactory,
            double segmentCompactFactor,
            double hashLoadFactor) throws Exception {
        return new DynamicDataStore(
                homeDir,
                StoreParams.getDynamicStoreInitialLevel(initialCapacity),
                batchSize,
                numSyncBatches,
                segmentFileSizeMB,

View Full Code Here

    protected SegmentFactory getSegmentFactory() {
        return new krati.core.segment.MemorySegmentFactory();
    }
    
    protected DynamicDataStore getDynamicDataStore(File storeDir, int initLevel, int segFileSizeMB) throws Exception {
        return new DynamicDataStore(storeDir, initLevel, 10000, 5, segFileSizeMB, getSegmentFactory(), 0.75, new HashFunctionInteger());
    }

View Full Code Here

    public void testCapacityGrowth() throws Exception {
        StatsLog.logger.info(">>> testCapacityGrowth");
        
        // Create DynamicDataStore 1
        File storeDir = getHomeDirectory();
        DynamicDataStore dynStore1 = getDynamicDataStore(storeDir, 0, 32);
        
        StatsLog.logger.info("create dynStore1"); 
        StatsLog.logger.info("level=" + dynStore1.getLevel() +
                             " split=" + dynStore1.getSplit() +
                             " capacity=" + dynStore1.capacity() +
                             " loadFactor=" + dynStore1.getLoadFactor());
        
        int keyStart;
        int keyCount;
        int unitCapacity = dynStore1.getUnitCapacity();
        
        keyStart = 0;
        keyCount = unitCapacity;
        write(keyStart, keyCount, dynStore1);
        StatsLog.logger.info("update keyStart=" + keyStart + " keyCount=" + keyCount);
        StatsLog.logger.info("level=" + dynStore1.getLevel() +
                             " split=" + dynStore1.getSplit() +
                             " capacity=" + dynStore1.capacity() +
                             " loadFactor=" + dynStore1.getLoadFactor());
        if ((unitCapacity * 2) != dynStore1.capacity())
            throw new RuntimeException("capacity expected: " + (unitCapacity * 2));
        
        keyStart += keyCount;
        write(keyStart, keyCount, dynStore1);
        StatsLog.logger.info("update keyStart=" + keyStart + " keyCount=" + keyCount);
        StatsLog.logger.info("level=" + dynStore1.getLevel() +
                             " split=" + dynStore1.getSplit() +
                             " capacity=" + dynStore1.capacity() +
                             " loadFactor=" + dynStore1.getLoadFactor());
        if ((unitCapacity * 3) != dynStore1.capacity())
            throw new RuntimeException("capacity expected: " + (unitCapacity * 3));
        
        keyStart += keyCount;
        write(keyStart, keyCount, dynStore1);
        StatsLog.logger.info("update keyStart=" + keyStart + " keyCount=" + keyCount);
        StatsLog.logger.info("level=" + dynStore1.getLevel() +
                             " split=" + dynStore1.getSplit() +
                             " capacity=" + dynStore1.capacity() +
                             " loadFactor=" + dynStore1.getLoadFactor());
        if ((unitCapacity * 4) != dynStore1.capacity())
            throw new RuntimeException("capacity expected: " + (unitCapacity * 4));
        
        keyStart += keyCount;
        write(keyStart, keyCount, dynStore1);
        StatsLog.logger.info("update keyStart=" + keyStart + " keyCount=" + keyCount);
        StatsLog.logger.info("level=" + dynStore1.getLevel() +
                             " split=" + dynStore1.getSplit() +
                             " capacity=" + dynStore1.capacity() +
                             " loadFactor=" + dynStore1.getLoadFactor());
        if ((unitCapacity * 5) != dynStore1.capacity())
            throw new RuntimeException("capacity expected: " + (unitCapacity * 5));
        
        keyStart += keyCount;
        write(keyStart, keyCount, dynStore1);
        StatsLog.logger.info("update keyStart=" + keyStart + " keyCount=" + keyCount);
        StatsLog.logger.info("level=" + dynStore1.getLevel() +
                             " split=" + dynStore1.getSplit() +
                             " capacity=" + dynStore1.capacity() +
                             " loadFactor=" + dynStore1.getLoadFactor());
        if ((unitCapacity * 6) != dynStore1.capacity())
            throw new RuntimeException("capacity expected: " + (unitCapacity * 6));
        
        dynStore1.rehash();
        StatsLog.logger.info("rehash()");
        StatsLog.logger.info("level=" + dynStore1.getLevel() +
                             " split=" + dynStore1.getSplit() +
                             " capacity=" + dynStore1.capacity() +
                             " loadFactor=" + dynStore1.getLoadFactor());
        if(dynStore1.getLevel() != 3)
            throw new RuntimeException("level expected: " + 3);
        if(dynStore1.getSplit() != 0)
            throw new RuntimeException("split expected: " + 0);
        if(dynStore1.getLevelCapacity() != dynStore1.capacity())
            throw new RuntimeException("capacity expected: " + dynStore1.getLevelCapacity());
        
        keyStart = 0;
        keyCount = unitCapacity * 16;
        write(keyStart, keyCount, dynStore1);
        StatsLog.logger.info("update keyStart=" + keyStart + " keyCount=" + keyCount);
        StatsLog.logger.info("level=" + dynStore1.getLevel() +
                             " split=" + dynStore1.getSplit() +
                             " capacity=" + dynStore1.capacity() +
                             " loadFactor=" + dynStore1.getLoadFactor());
        if ((unitCapacity * 18) != dynStore1.capacity())
            throw new RuntimeException("capacity expected: " + (unitCapacity * 18));
        
        keyStart = 0;
        keyCount = unitCapacity * 8;
        write(keyStart, keyCount, dynStore1);
        StatsLog.logger.info("update keyStart=" + keyStart + " keyCount=" + keyCount);
        StatsLog.logger.info("level=" + dynStore1.getLevel() +
                             " split=" + dynStore1.getSplit() +
                             " capacity=" + dynStore1.capacity() +
                             " loadFactor=" + dynStore1.getLoadFactor());
        if ((unitCapacity * 26) != dynStore1.capacity())
            throw new RuntimeException("capacity expected: " + (unitCapacity * 26));
        
        dynStore1.rehash();
        StatsLog.logger.info("rehash()");
        StatsLog.logger.info("level=" + dynStore1.getLevel() +
                             " split=" + dynStore1.getSplit() +
                             " capacity=" + dynStore1.capacity() +
                             " loadFactor=" + dynStore1.getLoadFactor());
        if(dynStore1.getLevel() != 5)
            throw new RuntimeException("level expected: " + 5);
        if(dynStore1.getSplit() != 0)
            throw new RuntimeException("split expected: " + 0);
        if ((dynStore1.getLevelCapacity()) != dynStore1.capacity())
            throw new RuntimeException("capacity expected: " + dynStore1.getLevelCapacity());
        
        // Bring loadFactor up to 75%
        keyStart = unitCapacity << 5;
        keyCount = (int)(dynStore1.capacity() * 0.75);
        write(keyStart, keyCount, dynStore1);
        StatsLog.logger.info("update keyStart=" + keyStart + " keyCount=" + keyCount);
        StatsLog.logger.info("level=" + dynStore1.getLevel() +
                             " split=" + dynStore1.getSplit() +
                             " capacity=" + dynStore1.capacity() +
                             " loadFactor=" + dynStore1.getLoadFactor());
        if ((unitCapacity << 5) != dynStore1.capacity())
            throw new RuntimeException("capacity expected: " + (unitCapacity << 5));
        
        dynStore1.sync();
        int capacity1 = dynStore1.capacity();
        
        // Create DynamicDataStore 2
        DynamicDataStore dynStore2 = getDynamicDataStore(storeDir, 0, 32);
        StatsLog.logger.info("create dynStore2");
        StatsLog.logger.info("level=" + dynStore2.getLevel() +
                             " split=" + dynStore2.getSplit() +
                             " capacity=" + dynStore2.capacity() +
                             " loadFactor=" + dynStore2.getLoadFactor());
        int capacity2 = dynStore2.capacity();
        
        // Check capacity
        if (capacity1 != capacity2)
            throw new RuntimeException("DynamicDataStore 1 and 2 have different capacities: " + capacity1 + ":" + capacity2);

View Full Code Here

    
    public void testStoreConfig() throws Exception {
        File dir = FileUtils.getTestDir(getClass().getSimpleName());
        
        StoreConfig config;
        DynamicDataStore store;
        
        config = new StoreConfig(dir, 10000);
        config.setSegmentFileSizeMB(32);
        assertTrue(config.getDataHandler() == null);
        config.setDataHandler(createDataStoreHandler());
        assertTrue(config.getDataHandler() != null);
        
        byte[] key = nextKey();
        byte[] value = nextValue();
        store = new DynamicDataStore(config);
        store.put(key, value);
        assertTrue(Arrays.equals(value, store.get(key)));
        store.close();
        
        StoreConfig config2 = StoreConfig.newInstance(dir);
        if(config.getDataHandler().getClass() != DefaultDataStoreHandler.class) {
            assertTrue(config2.getDataHandler() != null);
            assertEquals(config.getDataHandler().getClass(), config2.getDataHandler().getClass());
        }
        
        store = new DynamicDataStore(config2);
        assertTrue(Arrays.equals(value, store.get(key)));
        store.close();
        
        FileUtils.deleteDirectory(dir);
    }

View Full Code Here

    public void testUpdates() throws Exception {
        StatsLog.logger.info(">>> testUpdates");
        
        // Create DynamicDataStore
        File storeDir = getHomeDirectory();
        DynamicDataStore dynStore = getDynamicDataStore(storeDir, 0, 32);


        checkRandomPuts(dynStore, 1.0);
        checkRandomPuts(dynStore, 0.1);
        checkRandomPuts(dynStore, 0.2);
        checkRandomPuts(dynStore, 0.5);

View Full Code Here

    public void testClear() throws Exception {
        StatsLog.logger.info(">>> testClear");
        
        // Create DynamicDataStore
        File storeDir = getHomeDirectory();
        DynamicDataStore dynStore = getDynamicDataStore(storeDir, 0, 32);
        
        checkRandomPuts(dynStore, 0.1);
        checkRandomPuts(dynStore, 0.1);
        dynStore.clear();
        
        StatsLog.logger.info(dynStore.getStatus());
        
        checkRandomPuts(dynStore, 0.1);
        checkRandomPuts(dynStore, 0.3);
        checkRandomPuts(dynStore, 0.3);
        dynStore.sync();
        dynStore.clear();
        
        StatsLog.logger.info(dynStore.getStatus());
        
        checkRandomPuts(dynStore, 0.3);
        checkRandomPuts(dynStore, 0.5);
        checkRandomDeletes(dynStore, 0.3);
        dynStore.sync();
        dynStore.clear();
        
        cleanTestOutput();
    }

View Full Code Here

    protected DynamicDataStore create(int initialCapacity) throws Exception {
        File storeDir = DirUtils.getTestDir(getClass());
        StoreConfig config = new StoreConfig(storeDir, initialCapacity);
        config.setSegmentFactory(new ChannelSegmentFactory());
        config.setSegmentFileSizeMB(16);
        return new DynamicDataStore(config);
    }

View Full Code Here

            e.printStackTrace();
        }
    }
    
    protected void doInitialCapacity(int initialCapacity) throws Exception {
        DynamicDataStore store = create(initialCapacity);
        int capacity = store.capacity();
        if(initialCapacity == DynamicConstants.SUB_ARRAY_SIZE) {
            assertEquals(initialCapacity, capacity);
        } else {
            assertTrue(capacity >= initialCapacity);
        }
        store.close();


        int initialCapacity2 = capacity + _rand.nextInt(capacity);
        DynamicDataStore store2 = create(initialCapacity2);
        assertEquals(capacity, store2.capacity());
        store2.close();


        int initialCapacity3 = capacity - _rand.nextInt(capacity);
        DynamicDataStore store3 = create(initialCapacity3);
        assertEquals(capacity, store3.capacity());
        store3.close();
    }

View Full Code Here

        doInitialCapacity(DynamicConstants.SUB_ARRAY_SIZE - _rand.nextInt(10000));
    }
    
    protected void doInitialCapacityLevel(int initLevel) throws Exception {
        int initialCapacity = DynamicConstants.SUB_ARRAY_SIZE << initLevel;
        DynamicDataStore store = create(initialCapacity);
        int capacity = store.capacity();
        assertEquals(initialCapacity, capacity);
        store.close();
        
        int initialCapacity2 = capacity + _rand.nextInt(capacity);
        DynamicDataStore store2 = create(initialCapacity2);
        assertEquals(capacity, store2.capacity());
        store2.close();
        
        int initialCapacity3 = capacity - _rand.nextInt(capacity);
        DynamicDataStore store3 = create(initialCapacity3);
        assertEquals(capacity, store3.capacity());
        store3.close();
    }

View Full Code Here

        try {
            StoreConfig storeConfig = new StoreConfig(homeDir, initialCapacity);
            storeConfig.setSegmentFactory(segmentFactory);
            storeConfig.setHashFunction(hashFunction);
            storeConfig.setSegmentFileSizeMB(segmentFileSize);
            DataStore dynamicDataStore = new DynamicDataStore(storeConfig);
            result = new SerializableObjectStore(dynamicDataStore, keySerializer, valueSerializer);
        } catch (Exception e) {
            throw new RuntimeCamelException("Failed to create Krati DataStore.", e);
        }
        return result;

View Full Code Here

0 1 2

TOP

Related Classes of krati.store.DynamicDataStore

krati.core.array.AddressArray

krati.core.array.AddressArrayFactory

krati.core.array.SimpleDataArray

krati.core.segment.SegmentManager

krati.core.StoreConfig

krati.core.StoreFactory

krati.store.index.HashIndex

krati.util.FnvHashFunction

krati.util.LinearHashing

org.apache.camel.component.krati.KratiHelper

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.