Examples of org.xtreemfs.babudb.index.LSMTree

org.xtreemfs.babudb.index.LSMTree

        
        for (int i = 0; i < cfg.getIndices().length; i++) {
            
            int index = cfg.getIndices()[i];
            
            final LSMTree tree = trees.get(index);
            final String newFileName = directory + "/" + getSnapshotFilename(index, 0, 0);
            
            File dir = new File(directory);
            if (!dir.exists() && !dir.mkdirs())
                throw new IOException("Directory doesnt exist and cannot be created:'" + directory + "'");
            
            tree.materializeSnapshot(newFileName, snapIds[i], index, cfg);
        }
    }

View Full Code Here

     */
    public void cleanupSnapshot(final int viewId, final long sequenceNo) throws IOException {
        
        for (int index = 0; index < trees.size(); index++) {
            
            final LSMTree tree = trees.get(index);
            
            Logging.logMessage(Logging.LEVEL_INFO, this, "linking to snapshot " + databaseDir
                + File.separator + getSnapshotFilename(index, viewId, sequenceNo) + ", dbName="
                + databaseName + ", index=" + index);
            
            // catch any I/O exception that may occur while re-linking the
            // snapshot; this is done to ensure that old checkpoints are
            // properly cleaned up, and the database remains in a consistent
            // state
            IOException exception = null;
            try {
                tree.linkToSnapshot(databaseDir + File.separator
                    + getSnapshotFilename(index, viewId, sequenceNo));
            } catch (ClosedByInterruptException exc) {
                Logging.logError(Logging.LEVEL_DEBUG, this, exc);
            } catch (IOException exc) {
                Logging.logError(Logging.LEVEL_ERROR, this, exc);

View Full Code Here

                    }
                }
                
                // insert into the in-memory-tree
                for (InsertRecord ir : irg.getInserts()) {
                    LSMTree index = lsmDB.getIndex(ir.getIndexId());
                    
                    if (ir.getValue() != null) {
                        index.insert(ir.getKey(), ir.getValue());
                    } else {
                        index.delete(ir.getKey());
                    }
                }
                
                return null;
            }

View Full Code Here

        
        final DefaultByteRangeComparator comp = DefaultByteRangeComparator.getInstance();
        final byte[] value = "value".getBytes();
        final String[] keys = { "a", "v", "blub", "blubber", "ertz", "yagga", "zwum", "x" };
        
        LSMTree tree = new LSMTree(null, comp, COMPRESSION, 16, 1024 * 1024 * 512, MMAP, -1);
        for (String k : keys)
            tree.insert(k.getBytes(), value);
        
        int snapId = tree.createSnapshot();
        tree.materializeSnapshot(SNAP_FILE, snapId, 0, new DefaultSnapshotConfig("blub", new int[] { 0 },
            new byte[][][] { { "a".getBytes(), "bl".getBytes(), "zwum".getBytes() } }, null));
        tree.linkToSnapshot(SNAP_FILE);
        
        assertEquals(value, tree.lookup("a".getBytes()));
        assertEquals(null, tree.lookup("v".getBytes()));
        assertEquals(value, tree.lookup("blub".getBytes()));
        assertEquals(value, tree.lookup("blubber".getBytes()));
        assertEquals(null, tree.lookup("ertz".getBytes()));
        assertEquals(null, tree.lookup("yaggaa".getBytes()));
        assertEquals(value, tree.lookup("zwum".getBytes()));
        assertEquals(null, tree.lookup("x".getBytes()));
        
        Iterator<Entry<byte[], byte[]>> it = tree.prefixLookup(new byte[0]);
        int i = 0;
        for (; it.hasNext(); i++)
            it.next();
        
        assertEquals(4, i);

View Full Code Here

        byte[][] keys = new byte[][] { "00001".getBytes(), "00002".getBytes(), "00003".getBytes(),
            "00004".getBytes(), "00005".getBytes() };
        byte[][] vals = new byte[][] { "1".getBytes(), "2".getBytes(), "3".getBytes(), "4".getBytes(),
            "5".getBytes() };
        
        LSMTree tree = new LSMTree(null, DefaultByteRangeComparator.getInstance(), COMPRESSION, 16,
            1024 * 1024 * 512, MMAP, -1);
        TreeMap<byte[], byte[]> map = new TreeMap<byte[], byte[]>(new DefaultByteRangeComparator());
        
        // insert some key-value pairs
        for (int i = 0; i < keys.length; i++) {
            tree.insert(keys[i], vals[i]);
            map.put(keys[i], vals[i]);
        }
        
        for (byte[] key : map.keySet())
            assertEquals(new String(map.get(key)), new String(tree.lookup(key)));
        assertEquals(new String(map.firstKey()), new String(tree.firstEntry().getKey()));
        
        // create, materialize and link a new snapshot
        int snapId = tree.createSnapshot();
        tree.materializeSnapshot(SNAP_FILE, snapId);
        tree.linkToSnapshot(SNAP_FILE);
        
        for (byte[] key : map.keySet())
            assertEquals(new String(map.get(key)), new String(tree.lookup(key)));
        assertEquals(new String(map.firstKey()), new String(tree.firstEntry().getKey()));
        
        // insert some more keys-value pairs
        byte[][] newKeys = new byte[][] { "00001".getBytes(), "00006".getBytes(), "00002".getBytes(),
            "00007".getBytes(), "00008".getBytes(), "00009".getBytes(), "00003".getBytes() };
        byte[][] newVals = new byte[][] { null, "gf".getBytes(), "werr".getBytes(), "afds".getBytes(),
            "sdaew".getBytes(), "hf".getBytes(), null };
        
        for (int i = 0; i < newKeys.length; i++) {
            tree.insert(newKeys[i], newVals[i]);
            if (newVals[i] != null)
                map.put(newKeys[i], newVals[i]);
            else
                map.remove(newKeys[i]);
        }
        
        for (byte[] key : map.keySet())
            assertEquals(new String(map.get(key)), new String(tree.lookup(key)));
        assertEquals(new String(map.firstKey()), new String(tree.firstEntry().getKey()));
        
        // perform a prefix lookup
        Iterator<Entry<byte[], byte[]>> it = tree.prefixLookup(new byte[0]);
        Iterator<Entry<byte[], byte[]>> entries = map.entrySet().iterator();
        
        int count = 0;
        for (; it.hasNext(); count++) {
            
            Entry<byte[], byte[]> entry = it.next();
            Entry<byte[], byte[]> mapEntry = entries.next();
            
            assertEquals(new String(entry.getKey()), new String(mapEntry.getKey()));
            assertEquals(new String(entry.getValue()), new String(mapEntry.getValue()));
        }
        
        // perform reverse a prefix lookup
        it = tree.prefixLookup(new byte[0], false);
        entries = map.descendingMap().entrySet().iterator();
        
        count = 0;
        for (; it.hasNext(); count++) {
            
            Entry<byte[], byte[]> entry = it.next();
            Entry<byte[], byte[]> mapEntry = entries.next();
            
            assertEquals(new String(entry.getKey()), new String(mapEntry.getKey()));
            assertEquals(new String(entry.getValue()), new String(mapEntry.getValue()));
        }
        
        assertEquals(map.size(), count);
        
        // create, materialize and link a new snapshot
        snapId = tree.createSnapshot();
        tree.materializeSnapshot(SNAP_FILE2, snapId);
        tree.linkToSnapshot(SNAP_FILE2);
        
        for (byte[] key : map.keySet())
            assertEquals(new String(map.get(key)), new String(tree.lookup(key)));
        assertEquals(new String(map.firstKey()), new String(tree.firstEntry().getKey()));
        
        // delete all entries
        for (byte[] key : keys) {
            map.remove(key);
            tree.delete(key);
        }
        
        for (byte[] key : newKeys) {
            map.remove(key);
            tree.delete(key);
        }
        
        assertEquals(0, map.size());
        assertFalse(tree.prefixLookup(new byte[0]).hasNext());
        
        // create, materialize and link a new snapshot
        snapId = tree.createSnapshot();
        tree.materializeSnapshot(SNAP_FILE3, snapId);
        tree.linkToSnapshot(SNAP_FILE3);
        
        it = tree.prefixLookup(new byte[0]);
        assertFalse(it.hasNext());
        
        tree.insert("test".getBytes(), "test".getBytes());
        tree.delete("test".getBytes());
        
        snapId = tree.createSnapshot();
        tree.materializeSnapshot(SNAP_FILE4, snapId);
        tree.linkToSnapshot(SNAP_FILE4);
        
        assertEquals(null, tree.lookup("test".getBytes()));
        
        tree.destroy();
    }

View Full Code Here

        // randomly insert 200 elements in a map
        
        final int numElements = 200;
        final DefaultByteRangeComparator comp = DefaultByteRangeComparator.getInstance();
        
        LSMTree tree = new LSMTree(null, comp, COMPRESSION, 16, 1024 * 1024 * 512, MMAP, -1);
        
        final TreeMap<byte[], byte[]> map1 = new TreeMap<byte[], byte[]>(comp);
        for (int i = 0x10; i < numElements; i++) {
            byte[] key = Integer.toHexString(i).getBytes();
            byte[] val = Integer.toHexString((int) (Math.random() * Integer.MAX_VALUE)).getBytes();
            map1.put(key, val);
            tree.insert(key, val);
        }
        
        int snap1 = tree.createSnapshot();
        final TreeMap<byte[], byte[]> map2 = new TreeMap<byte[], byte[]>(map1);
        
        for (int i = 0x10; i < numElements; i += 2) {
            byte[] key = Integer.toHexString(i).getBytes();
            tree.insert(key, null);
            map2.remove(key);
        }
        
        int snap2 = tree.createSnapshot();
        final TreeMap<byte[], byte[]> map3 = new TreeMap<byte[], byte[]>(map2);
        
        for (int i = 0x10; i < numElements; i += 5) {
            byte[] key = Integer.toHexString(i).getBytes();
            byte[] val = Integer.toHexString((int) (Math.random() * Integer.MAX_VALUE)).getBytes();
            tree.insert(key, val);
            map3.put(key, val);
        }
        
        // peform prefix lookups
        
        // current tree, ascending
        Iterator<Entry<byte[], byte[]>> it = tree.prefixLookup(new byte[0], true);
        Iterator<byte[]> itExpected = map3.values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // current tree, descending
        it = tree.prefixLookup(new byte[0], false);
        itExpected = map3.descendingMap().values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // snapshot 2, ascending
        it = tree.prefixLookup(new byte[0], snap2, true);
        itExpected = map2.values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // snapshot 2, descending
        it = tree.prefixLookup(new byte[0], snap2, false);
        itExpected = map2.descendingMap().values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // snapshot 1, ascending
        it = tree.prefixLookup(new byte[0], snap1, true);
        itExpected = map1.values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // snapshot 1, descending
        it = tree.prefixLookup(new byte[0], snap1, false);
        itExpected = map1.descendingMap().values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // real prefix lookup, current tree, ascending
        it = tree.prefixLookup("4".getBytes(), true);
        itExpected = map3.subMap("4".getBytes(), true, "5".getBytes(), false).values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // real prefix lookup, current tree, descending
        it = tree.prefixLookup("4".getBytes(), false);
        itExpected = map3.descendingMap().subMap("5".getBytes(), false, "4".getBytes(), true).values()
                .iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // real (empty) prefix lookup, current tree, ascending
        it = tree.prefixLookup("XXXXXXXX".getBytes(), true);
        itExpected = map3.subMap("XXXXXXXX".getBytes(), true, "XXXXXXXY".getBytes(), false).values()
                .iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // real (empty) prefix lookup, current tree, descending
        it = tree.prefixLookup("XXXXXXXX".getBytes(), false);
        itExpected = map3.descendingMap().subMap("XXXXXXXY".getBytes(), false, "XXXXXXXX".getBytes(), true)
                .values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // first entry, current tree
        assertEquals(map3.firstEntry().getKey(), tree.firstEntry().getKey());
        assertEquals(map3.firstEntry().getValue(), tree.firstEntry().getValue());
        
        // last entry, current tree
        assertEquals(map3.lastEntry().getKey(), tree.lastEntry().getKey());
        assertEquals(map3.lastEntry().getValue(), tree.lastEntry().getValue());
        
        // first entry, snapshot 2
        assertEquals(map2.firstEntry().getKey(), tree.firstEntry(snap2).getKey());
        assertEquals(map2.firstEntry().getValue(), tree.firstEntry(snap2).getValue());
        
        // last entry, snapshot 2
        assertEquals(map2.lastEntry().getKey(), tree.lastEntry(snap2).getKey());
        assertEquals(map2.lastEntry().getValue(), tree.lastEntry(snap2).getValue());
        
        // first entry, snapshot 1
        assertEquals(map1.firstEntry().getKey(), tree.firstEntry(snap1).getKey());
        assertEquals(map1.firstEntry().getValue(), tree.firstEntry(snap1).getValue());
        
        // last entry, snapshot 1
        assertEquals(map1.lastEntry().getKey(), tree.lastEntry(snap1).getKey());
        assertEquals(map1.lastEntry().getValue(), tree.lastEntry(snap1).getValue());
        
    }

View Full Code Here

        // randomly insert 200 elements in a map
        
        final int numElements = 200;
        final DefaultByteRangeComparator comp = DefaultByteRangeComparator.getInstance();
        
        LSMTree tree = new LSMTree(null, comp, COMPRESSION, 16, 1024 * 1024 * 512, MMAP, -1);
        
        // insert all 200 keys
        final TreeMap<byte[], byte[]> map1 = new TreeMap<byte[], byte[]>(comp);
        for (int i = 0x10; i < numElements; i++) {
            byte[] key = Integer.toHexString(i).getBytes();
            byte[] val = Integer.toHexString((int) (Math.random() * Integer.MAX_VALUE)).getBytes();
            map1.put(key, val);
            tree.insert(key, val);
        }
        
        // take a snapshot
        int snap1 = tree.createSnapshot();
        final TreeMap<byte[], byte[]> map2 = new TreeMap<byte[], byte[]>(map1);
        
        // delete every second key
        for (int i = 0x10; i < numElements; i += 2) {
            byte[] key = Integer.toHexString(i).getBytes();
            tree.insert(key, null);
            map2.remove(key);
        }
        
        // take a snapshot
        int snap2 = tree.createSnapshot();
        final TreeMap<byte[], byte[]> map3 = new TreeMap<byte[], byte[]>(map2);
        
        // overwrite every 5th key
        for (int i = 0x10; i < numElements; i += 5) {
            byte[] key = Integer.toHexString(i).getBytes();
            byte[] val = Integer.toHexString((int) (Math.random() * Integer.MAX_VALUE)).getBytes();
            tree.insert(key, val);
            map3.put(key, val);
        }
        
        // peform range lookups
        
        // current tree, ascending
        Iterator<Entry<byte[], byte[]>> it = tree.rangeLookup(new byte[0], new byte[0], true);
        Iterator<byte[]> itExpected = map3.values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // current tree, descending
        it = tree.rangeLookup(new byte[0], new byte[0], false);
        itExpected = map3.descendingMap().values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // snapshot 2, ascending
        it = tree.rangeLookup(new byte[0], new byte[0], snap2, true);
        itExpected = map2.values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // snapshot 2, descending
        it = tree.rangeLookup(new byte[0], new byte[0], snap2, false);
        itExpected = map2.descendingMap().values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // snapshot 1, ascending
        it = tree.rangeLookup(new byte[0], new byte[0], snap1, true);
        itExpected = map1.values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // snapshot 1, descending
        it = tree.rangeLookup(new byte[0], new byte[0], snap1, false);
        itExpected = map1.descendingMap().values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // real range lookup, current tree, ascending
        it = tree.rangeLookup("2".getBytes(), "5".getBytes(), true);
        itExpected = map3.subMap("2".getBytes(), "5".getBytes()).values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // real range lookup, current tree, descending
        it = tree.rangeLookup("5".getBytes(), "2".getBytes(), false);
        itExpected = map3.descendingMap().subMap("5".getBytes(), "2".getBytes()).values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // real (empty) range lookup, current tree, ascending
        it = tree.rangeLookup("XXXXXXXX".getBytes(), "XXXXXXXY".getBytes(), true);
        itExpected = map3.subMap("XXXXXXXX".getBytes(), true, "XXXXXXXY".getBytes(), true).values()
                .iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // real (empty) range lookup, current tree, descending
        it = tree.rangeLookup("XXXXXXXY".getBytes(), "XXXXXXXX".getBytes(), false);
        itExpected = map3.descendingMap().subMap("XXXXXXXY".getBytes(), true, "XXXXXXXX".getBytes(), true)
                .values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // real range lookup, snapshot 1, ascending
        it = tree.rangeLookup("2".getBytes(), "5".getBytes(), snap1, true);
        itExpected = map1.subMap("2".getBytes(), "5".getBytes()).values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());
        
        // real range lookup, snapshot 1, descending
        it = tree.rangeLookup("5".getBytes(), "2".getBytes(), snap1, false);
        itExpected = map1.descendingMap().subMap("5".getBytes(), "2".getBytes()).values().iterator();
        while (it.hasNext())
            assertEquals(itExpected.next(), it.next().getValue());
        assertFalse(itExpected.hasNext());

View Full Code Here

                
            });
        }
        
        // read the LSM tree
        LSMTree tree = new LSMTree(path, new DefaultByteRangeComparator(), false, 16, 1024 * 1024 * 512, true, -1);
        
        System.out.println("inserting " + inserts + " random elements ...");
        for (int i = 0; i < inserts; i++) {
            byte[] entry = createRandomString(minChar, maxChar, minStrLen, maxStrLen).getBytes();
            tree.insert(entry, entry);
        }
        
        // Iterator<Entry<ReusableBuffer, ReusableBuffer>> it =
        // tree.prefixLookup(null);
        // while (it.hasNext())
        // System.out.println(new String(it.next().getKey().array()));
        //        
        System.out.println("performing " + lookups + " random lookups ...");
        
        // look up each element
        int hits = 0;
        long t0 = System.currentTimeMillis();
        for (int i = 0; i < lookups; i++) {
            
            byte[] key = createRandomString(minChar, maxChar, minStrLen, maxStrLen).getBytes();
            byte[] result = tree.lookup(key);
            if (result != null)
                hits++;
            
            if (i % 100000 == 0)
                System.out.println(i);

View Full Code Here

0 1

TOP

Related Classes of org.xtreemfs.babudb.index.LSMTree

org.xtreemfs.babudb.index.LSMTreeTest

org.xtreemfs.babudb.index.overlay.MultiOverlayBufferTree

org.xtreemfs.babudb.index.reader.DiskIndex

org.xtreemfs.babudb.index.reader.InternalDiskIndexIterator

org.xtreemfs.babudb.index.reader.InternalMergeIterator

org.xtreemfs.babudb.index.writer.DiskIndexWriter

org.xtreemfs.babudb.lsmdb.DatabaseManagerImpl

org.xtreemfs.babudb.lsmdb.LSMDatabase

org.xtreemfs.babudb.lsmdb.LSMLookupInterface

org.xtreemfs.babudb.sandbox.LSMTreePerformanceTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.