Examples of org.xtreemfs.babudb.index.writer.DiskIndexWriter

Package org.xtreemfs.babudb.index.writer

Examples of org.xtreemfs.babudb.index.writer.DiskIndexWriter

org.xtreemfs.babudb.index.writer.DiskIndexWriter
Writes an index to a set of files on disk. A file will not be larger than the given max file size. Multiple files are necessary to handle databases larger than 2 GB on 32-bit systems due to MMap limitations. The index has two parts, a sorted list of blocks containing key/value-pairs and a block index. The block index is a sparse index pointing to the sorted blocks. @author stender @author hoegqvist

     * @throws IOException
     *             if an I/O error occurs while writing the snapshot
     */
    public void materializeSnapshot(String targetFile, int snapId) throws IOException {
        
        DiskIndexWriter writer = new DiskIndexWriter(targetFile, maxEntriesPerBlock, compressed,
            maxBlockFileSize);
        
        InternalMergeIterator it = internalPrefixLookup(null, snapId, true);
        writer.writeIndex(it);
        it.free();
    }

View Full Code Here

     *             if an I/O error occurs while writing the snapshot
     */
    public void materializeSnapshot(String targetFile, final int snapId, final int indexId,
        final SnapshotConfig snap) throws IOException {
        
        DiskIndexWriter writer = new DiskIndexWriter(targetFile, maxEntriesPerBlock, compressed,
            maxBlockFileSize);
        writer.writeIndex(new ResultSet<Object, Object>() {
            
            private ResultSet<Object, Object>[] iterators;
            
            private Entry<Object, Object>             next;

View Full Code Here

        
        // delete old index file
        FSUtils.delTree(new File(PATH1));
        
        // write the map to a disk index
        DiskIndexWriter index = new DiskIndexWriter(PATH1, MAX_BLOCK_ENTRIES, COMPRESSED, MAX_BLOCK_FILE_SIZE);
        index.writeIndex(getBufferIterator(map.entrySet().iterator()));
        
        // read the disk index
        DiskIndex diskIndex = new DiskIndex(PATH1, DefaultByteRangeComparator.getInstance(), COMPRESSED,
            MMAPED);

View Full Code Here

        for (int i = 0; i < keys.length; i++)
            testMap.put(keys[i].getBytes(), vals[i].getBytes());
        
        // write the map to a disk index
        FSUtils.delTree(new File(PATH2));
        DiskIndexWriter index = new DiskIndexWriter(PATH2, 4, COMPRESSED, MAX_BLOCK_FILE_SIZE);
        index.writeIndex(getBufferIterator(testMap.entrySet().iterator()));
        
        // read the disk index
        DiskIndex diskIndex = new DiskIndex(PATH2, DefaultByteRangeComparator.getInstance(), COMPRESSED,
            MMAPED);
        
        // create an iterator w/ matching start and end buffers
        Iterator<Entry<byte[], byte[]>> it = diskIndex.rangeLookup("brabbel".getBytes(), "yagga".getBytes(),
            true);
        assertIterator(it, keys, vals, 1, 4);
        
        // create an iterator with matching start buffer
        it = diskIndex.rangeLookup("brabbel".getBytes(), "g".getBytes(), true);
        assertIterator(it, keys, vals, 1, 2);
        
        // create an iterator with matching end buffer
        it = diskIndex.rangeLookup("b".getBytes(), "brabbel".getBytes(), true);
        assertIterator(it, keys, vals, 0, 0);
        
        // create an iterator w/o matching start and end buffers
        it = diskIndex.rangeLookup("blu".getBytes(), "yyz".getBytes(), true);
        assertIterator(it, keys, vals, 1, 6);
        
        // check ranges outside the boundaries; should be empty
        it = diskIndex.rangeLookup("A".getBytes(), "Z".getBytes(), true);
        assertFalse(it.hasNext());
        
        it = diskIndex.rangeLookup("1".getBytes(), "2".getBytes(), true);
        assertFalse(it.hasNext());
        
        diskIndex.destroy();
        
        // create a disk index from an empty index file
        FSUtils.delTree(new File(PATH1));
        index = new DiskIndexWriter(PATH1, 4, COMPRESSED, MAX_BLOCK_FILE_SIZE);
        index.writeIndex(EMPTY_RESULT_SET);
        
        diskIndex = new DiskIndex(PATH1, new DefaultByteRangeComparator(), COMPRESSED, MMAPED);
        
        // check ranges; should all be empty
        it = diskIndex.rangeLookup(new byte[0], new byte[0], true);

View Full Code Here

        for (int i = 0; i < keys.length; i++)
            testMap.put(keys[i].getBytes(), vals[i].getBytes());
        
        // write the map to a disk index
        FSUtils.delTree(new File(PATH2));
        DiskIndexWriter index = new DiskIndexWriter(PATH2, 4, COMPRESSED, MAX_BLOCK_FILE_SIZE);
        index.writeIndex(getBufferIterator(testMap.entrySet().iterator()));
        
        // read the disk index
        DiskIndex diskIndex = new DiskIndex(PATH2, DefaultByteRangeComparator.getInstance(), COMPRESSED,
            MMAPED);
        
        // create an iterator w/ matching start and end buffers
        Iterator<Entry<byte[], byte[]>> it = diskIndex.rangeLookup("brabbel".getBytes(), "yagga".getBytes(),
            false);
        assertIterator(it, keys, vals, 5, 2);
        
        // create an iterator with matching start buffer
        it = diskIndex.rangeLookup("brabbel".getBytes(), "g".getBytes(), false);
        assertIterator(it, keys, vals, 2, 2);
        
        // create an iterator with matching end buffer
        it = diskIndex.rangeLookup("b".getBytes(), "brabbel".getBytes(), false);
        assertIterator(it, keys, vals, 1, 0);
        
        // create an iterator w/o matching start and end buffers
        it = diskIndex.rangeLookup("blu".getBytes(), "yyz".getBytes(), false);
        assertIterator(it, keys, vals, 6, 1);
        
        // check ranges outside the boundaries; should be empty
        it = diskIndex.rangeLookup("A".getBytes(), "Z".getBytes(), false);
        assertFalse(it.hasNext());
        
        it = diskIndex.rangeLookup("1".getBytes(), "2".getBytes(), false);
        assertFalse(it.hasNext());
        
        diskIndex.destroy();
        
        // create a disk index from an empty index file
        FSUtils.delTree(new File(PATH1));
        index = new DiskIndexWriter(PATH1, 4, COMPRESSED, MAX_BLOCK_FILE_SIZE);
        index.writeIndex(EMPTY_RESULT_SET);
        
        diskIndex = new DiskIndex(PATH1, new DefaultByteRangeComparator(), COMPRESSED, MMAPED);
        
        // check ranges; should all be empty
        it = diskIndex.rangeLookup(new byte[0], new byte[0], false);

View Full Code Here

        
        // delete old index file
        FSUtils.delTree(new File(PATH1));
        
        // write the map to a disk index
        DiskIndexWriter index = new DiskIndexWriter(PATH1, MAX_BLOCK_ENTRIES, COMPRESSED, MAX_BLOCK_FILE_SIZE);
        index.writeIndex(getBufferIterator(map.entrySet().iterator()));
        
        // read the disk index
        DiskIndex diskIndex = new DiskIndex(PATH1, DefaultByteRangeComparator.getInstance(), COMPRESSED,
            MMAPED);

View Full Code Here

        
        // delete old index file
        FSUtils.delTree(new File(PATH1));
        
        // write the map to a disk index
        DiskIndexWriter index = new DiskIndexWriter(PATH1, MAX_BLOCK_ENTRIES, COMPRESSED, MAX_BLOCK_FILE_SIZE);
        index.writeIndex(getBufferIterator(map.entrySet().iterator()));
        
        // read the disk index
        DiskIndex diskIndex = new DiskIndex(PATH1, DefaultByteRangeComparator.getInstance(), COMPRESSED,
            MMAPED);

View Full Code Here

        for (int i = 0; i < entries.length; i++)
            testMap.put(entries[i], entries[i]);
        
        // write the map to a disk index
        FSUtils.delTree(new File(PATH2));
        DiskIndexWriter index = new DiskIndexWriter(PATH2, 2, COMPRESSED, MAX_BLOCK_FILE_SIZE);
        index.writeIndex(getBufferIterator(testMap.entrySet().iterator()));
    }

View Full Code Here

            new File(path).delete();
            
            System.out.println("creating new database with " + size + " random entries ...");
            
            // write the map to a disk index
            DiskIndexWriter index = new DiskIndexWriter(path, entriesPerBlock, false, blockFileSize);
            index.writeIndex(new ResultSet<Object, Object>() {
                
                private int    count;
                
                private String next = minChar + "";

View Full Code Here

        if (!(new File(path).exists()) || overwrite) {
            if(verbose)
              System.out.println("creating new database with " + size + " random entries ...");
            
            // write the map to a disk index
            DiskIndexWriter index = new DiskIndexWriter(path, entriesPerBlock.intValue(), compress, blockFileSize);
            
            if(!input.equals(DEFAULT_DATAGEN)) {
              /* note that the iterator must return the items sorted */
              
              index.writeIndex(DataGenerator.fileIterator(lookupHits, size, hitrate, input));
            } else {
              index.writeIndex(DataGenerator.randomIterator(lookupHits, size, hitrate, minStrLen, maxStrLen, minChar, maxChar));
            }
        } else {
          //   populate the lookup-hits table
          DiskIndex diskIndexTmp = new DiskIndex(path, new DefaultByteRangeComparator(), compress, mmap);
          Iterator<Entry<byte[], byte[]>> itTmp = diskIndexTmp.rangeLookup(null, null, true);

View Full Code Here

TOP

Related Classes of org.xtreemfs.babudb.index.writer.DiskIndexWriter

org.xtreemfs.babudb.index.ByteRange

org.xtreemfs.babudb.index.DiskIndexTest

org.xtreemfs.babudb.index.LSMTree

org.xtreemfs.babudb.sandbox.DiskIndexPerformanceTest

org.xtreemfs.babudb.sandbox.LSMTreePerformanceTest

org.xtreemfs.foundation.buffer.ReusableBuffer

java.nio.ByteBuffer

java.io.IOException

java.io.File

java.io.FileOutputStream

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.