Examples of krati.core.segment.SegmentIndexBuffer

krati.core.segment.SegmentIndexBuffer
SegmentIndexBuffer @author jwu @since 08/25, 2012

     * @throws IOException if this operation can not be finished properly.
     */
    private boolean compact(Segment segment, Segment segTarget, final boolean sibEnabled) throws IOException {
        // Optimization: use the source segment index buffer file if it is available for compaction.
        if(sibEnabled) {
            SegmentIndexBuffer sibSource =
                    _dataArray.getSegmentManager()
                    .loadSegmentIndexBuffer(segment.getSegmentId(), segment.getLastForcedTime());
            if(sibSource != null) {
                return compact(segment, sibSource, segTarget);
            }
        }
        
        Segment segSource = segment; 
        int segSourceId = segSource.getSegmentId();
        int segTargetId = segTarget.getSegmentId();
        
        Chronos c = new Chronos();
        if(!segment.canReadFromBuffer() && segment.getLoadFactor() > 0.1) {
            segSource = new BufferedSegment(segment, getByteBuffer((int)segment.getInitialSize()));
            _log.info("buffering: " + c.tick() + " ms");
        }
        
        // Open the segment index buffer for the target segment
        SegmentIndexBuffer sibTarget = sibEnabled ? _dataArray.getSegmentManager().openSegmentIndexBuffer(segTargetId) : null;
        
        long sizeLimit = segTarget.getInitialSize();
        long bytesTransferred = 0;
        boolean succ = true;
        
        try {
            AddressFormat addrFormat = _dataArray._addressFormat;
            
            for(int index = 0, cnt = _dataArray.length(); index < cnt; index++) {
                long oldAddress = _dataArray.getAddress(index);
                int oldSegPos = addrFormat.getOffset(oldAddress);
                int oldSegInd = addrFormat.getSegment(oldAddress);
                int length = addrFormat.getDataSize(oldAddress);
                
                if (oldSegInd == segSourceId && oldSegPos >= Segment.dataStartPosition) {
                    if(length == 0) length = segSource.readInt(oldSegPos);
                    int byteCnt = 4 + length;
                    long newSegPos = segTarget.getAppendPosition();
                    long newAddress = addrFormat.composeAddress((int)newSegPos, segTargetId, length);
                    
                    if(segTarget.getAppendPosition() + byteCnt >= sizeLimit) {
                        succ = false;
                        break;
                    }
                    
                    // Transfer bytes from source to target
                    segSource.transferTo(oldSegPos, byteCnt, segTarget);
                    bytesTransferred += byteCnt;
                    
                    if(sibTarget != null) sibTarget.add(index, (int)newSegPos);
                    _updateManager.addUpdate(index, byteCnt, newAddress, oldAddress, segTarget);
                }
            }
            
            // Push whatever left into update queue

View Full Code Here

            segSource = new BufferedSegment(segment, getByteBuffer((int)segment.getInitialSize()));
            _log.info("buffering: " + c.tick() + " ms");
        }
        
        // Open the segment index buffer for the target segment
        SegmentIndexBuffer sibTarget = _dataArray.getSegmentManager().openSegmentIndexBuffer(segTargetId);
        
        long sizeLimit = segTarget.getInitialSize();
        long bytesTransferred = 0;
        boolean succ = true;
        
        try {
            AddressFormat addrFormat = _dataArray._addressFormat;
            SegmentIndexBuffer.IndexOffset reuse = new SegmentIndexBuffer.IndexOffset();
            
            for(int i = 0, cnt = sibSource.size(); i < cnt; i++) {
                sibSource.get(i, reuse);
                int index = reuse.getIndex();
                int sibSegPos = reuse.getOffset();
                long oldAddress = _dataArray.getAddress(index);
                int oldSegPos = addrFormat.getOffset(oldAddress);
                
                if(sibSegPos != oldSegPos) continue;
                
                int oldSegInd = addrFormat.getSegment(oldAddress);
                int length = addrFormat.getDataSize(oldAddress);
                
                if (oldSegInd == segSourceId && oldSegPos >= Segment.dataStartPosition) {
                    if(length == 0) length = segSource.readInt(oldSegPos);
                    int byteCnt = 4 + length;
                    long newSegPos = segTarget.getAppendPosition();
                    long newAddress = addrFormat.composeAddress((int)newSegPos, segTargetId, length);
                    
                    if(segTarget.getAppendPosition() + byteCnt >= sizeLimit) {
                        succ = false;
                        break;
                    }
                    
                    // Transfer bytes from source to target
                    segSource.transferTo(oldSegPos, byteCnt, segTarget);
                    bytesTransferred += byteCnt;
                    
                    sibTarget.add(index, (int)newSegPos);
                    _updateManager.addUpdate(index, byteCnt, newAddress, oldAddress, segTarget);
                }
            }
            
            // Push whatever left into update queue

View Full Code Here

public class TestSegmentIndexBuffer extends TestCase {
    protected Random _rand = new Random();
    protected SegmentIndexBuffer _sib;
    
    protected SegmentIndexBuffer create() {
        return new SegmentIndexBuffer();
    }

View Full Code Here

    protected Random _rand = new Random();
    protected SegmentIndexBuffer _sib;
    protected SegmentIndexBufferIO _sibIO;
    
    protected SegmentIndexBuffer create() {
        return new SegmentIndexBuffer();
    }

View Full Code Here

        
        File sibFile = new File(FileUtils.getTestDir(getClass().getSimpleName()), segId + ".sib");
        int writeLength = _sibIO.write(_sib, sibFile);
        
        // Read without validating lastForcedTime
        SegmentIndexBuffer sibRead = new SegmentIndexBuffer();
        int readLength = _sibIO.read(sibRead, sibFile);
        
        assertEquals(writeLength, readLength);
        assertEquals(_sib.getSegmentId(), sibRead.getSegmentId());
        assertEquals(_sib.getSegmentLastForcedTime(), sibRead.getSegmentLastForcedTime());
        assertEquals(_sib.size(), sibRead.size());
        
        for(int i = 0; i < size; i++) {
            SegmentIndexBuffer.IndexOffset o1 = _sib.get(i);
            SegmentIndexBuffer.IndexOffset o2 = sibRead.get(i);
            assertEquals(o1.getIndex(), o2.getIndex());
            assertEquals(o1.getOffset(), o2.getOffset());
        }


        // Read with validating lastForcedTime
        sibRead = new SegmentIndexBuffer();
        readLength = _sibIO.read(sibRead, sibFile, lastForcedTime);
        
        assertEquals(writeLength, readLength);
        assertEquals(_sib.getSegmentId(), sibRead.getSegmentId());
        assertEquals(_sib.getSegmentLastForcedTime(), sibRead.getSegmentLastForcedTime());
        assertEquals(_sib.size(), sibRead.size());
        
        for(int i = 0; i < size; i++) {
            SegmentIndexBuffer.IndexOffset o1 = _sib.get(i);
            SegmentIndexBuffer.IndexOffset o2 = sibRead.get(i);
            assertEquals(o1.getIndex(), o2.getIndex());
            assertEquals(o1.getOffset(), o2.getOffset());
        }
    }

View Full Code Here

TOP

Related Classes of krati.core.segment.SegmentIndexBuffer

krati.core.array.SimpleDataArrayCompactor

test.store.segment.TestSegmentIndexBuffer

test.store.segment.TestSegmentIndexBufferIO

java.security.MessageDigest

java.io.IOException

java.nio.ByteBuffer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.