Package krati.core.segment

Examples of krati.core.segment.Segment


        long startTime, endTime, diffTime;
        System.out.printf("Read multi-segments (%d)%n", segCount);

        try {
            while (segManager.getSegmentCount() < segCount) {
                Segment segment = segManager.nextSegment();
                testAppend(segment);
            }

            Set<SegmentReader> runnerSet = new HashSet<SegmentReader>();
            SegmentReader[] runnerArray = new SegmentReader[segManager.getSegmentCount()];
View Full Code Here


        long startTime, endTime, diffTime;
        System.out.printf("Read multi-segment (%d) with threads%n", segCount);
       
        try {
            while(segManager.getSegmentCount() < segCount) {
                Segment segment = segManager.nextSegment();
                testAppend(segment);
            }
           
            Set<SegmentReader> runnerSet = new HashSet<SegmentReader>();
            SegmentReader[] runnerArray = new SegmentReader[segManager.getSegmentCount()];
View Full Code Here

    private void freeCompactedSegments() {
        SegmentManager segManager = _dataArray.getSegmentManager();
        if(segManager == null) return;
       
        while(!_freeQueue.isEmpty()) {
            Segment seg = _freeQueue.remove();
            try {
                segManager.freeSegment(seg);
            } catch(Exception e) {
                _log.error("failed to free Segment " + seg.getSegmentId() + ": " + seg.getStatus(), e);
            }
        }
    }
View Full Code Here

    private boolean inspect() {
        SegmentManager segManager = _dataArray.getSegmentManager();
        if(segManager == null) return false;
       
        synchronized(segManager) {
            Segment segCurrent = _dataArray.getCurrentSegment();
           
            /*
             * Find source segments that are least loaded.
             * The source segments must be in the READ_ONLY mode.
             */
            ArrayList<Segment> recycleList = new ArrayList<Segment>();
            int cnt = segManager.getSegmentCount();
            for(int i = 0; i < cnt; i++) {
                Segment seg = segManager.getSegment(i);
                if(seg != null && seg.getMode() == Segment.Mode.READ_ONLY && seg != segCurrent) {
                    if (seg.getLoadFactor() < _compactLoadFactor && !_ignoredSegs.contains(seg)) {
                        recycleList.add(seg);
                    }
                }
            }
           
            // No segment need compaction
            if (recycleList.size() == 0) {
                _segPermits.set(0);
                return false;
            }
           
            // Sort recycleList in ascending order of load size
            Collections.sort(recycleList, _segmentLoadCmp);
           
            // Compact no more than 3 segments per compaction cycle.
            // The total of segment load factors need to be less than
            // 0.8 to allow 20% inaccuracy (for safety).
            double totalFactor = 0;
            for(int i = 0, len = Math.min(3, recycleList.size()); i < len; i++) {
                Segment seg = recycleList.get(i);
                if(totalFactor < 0.8) {
                    totalFactor += Math.max(0, seg.getLoadFactor());
                    if(totalFactor < 0.8) {
                        _segSourceList.add(seg);
                    }
                } else {
                    break;
                }
            }
           
            try {
                for(Segment seg : _segSourceList) {
                    _log.info("Segment " + seg.getSegmentId() + " load factor=" + ((long)(seg.getLoadFactor() * 10000) / 10000.0));
                }
            } catch(ConcurrentModificationException e) {
                _segPermits.set(0);
                _segSourceList.clear();
                return false;
View Full Code Here

            if(sibSource != null) {
                return compact(segment, sibSource, segTarget);
            }
        }
       
        Segment segSource = segment;
        int segSourceId = segSource.getSegmentId();
        int segTargetId = segTarget.getSegmentId();
       
        Chronos c = new Chronos();
        if(!segment.canReadFromBuffer() && segment.getLoadFactor() > 0.1) {
            segSource = new BufferedSegment(segment, getByteBuffer((int)segment.getInitialSize()));
            _log.info("buffering: " + c.tick() + " ms");
        }
       
        // Open the segment index buffer for the target segment
        SegmentIndexBuffer sibTarget = sibEnabled ? _dataArray.getSegmentManager().openSegmentIndexBuffer(segTargetId) : null;
       
        long sizeLimit = segTarget.getInitialSize();
        long bytesTransferred = 0;
        boolean succ = true;
       
        try {
            AddressFormat addrFormat = _dataArray._addressFormat;
           
            for(int index = 0, cnt = _dataArray.length(); index < cnt; index++) {
                long oldAddress = _dataArray.getAddress(index);
                int oldSegPos = addrFormat.getOffset(oldAddress);
                int oldSegInd = addrFormat.getSegment(oldAddress);
                int length = addrFormat.getDataSize(oldAddress);
               
                if (oldSegInd == segSourceId && oldSegPos >= Segment.dataStartPosition) {
                    if(length == 0) length = segSource.readInt(oldSegPos);
                    int byteCnt = 4 + length;
                    long newSegPos = segTarget.getAppendPosition();
                    long newAddress = addrFormat.composeAddress((int)newSegPos, segTargetId, length);
                   
                    if(segTarget.getAppendPosition() + byteCnt >= sizeLimit) {
                        succ = false;
                        break;
                    }
                   
                    // Transfer bytes from source to target
                    segSource.transferTo(oldSegPos, byteCnt, segTarget);
                    bytesTransferred += byteCnt;
                   
                    if(sibTarget != null) sibTarget.add(index, (int)newSegPos);
                    _updateManager.addUpdate(index, byteCnt, newAddress, oldAddress, segTarget);
                }
            }
           
            // Push whatever left into update queue
            _updateManager.endUpdate(segTarget);
            _log.info("bytes transferred from " + segSource.getSegmentId() + ": " + bytesTransferred + " time: " + c.tick() + " ms");
           
            return succ;
        } finally {
            if(segSource.getClass() == BufferedSegment.class) {
                segSource.close(false);
                segSource = null;
            }
        }
    }
View Full Code Here

     * @return <code>true</code> if the source Segment is compacted successfully.
     *         Otherwise, <code>false</code>.
     * @throws IOException if this operation can not be finished properly.
     */
    private boolean compact(Segment segment, SegmentIndexBuffer sibSource, Segment segTarget) throws IOException {
        Segment segSource = segment;
        int segSourceId = segSource.getSegmentId();
        int segTargetId = segTarget.getSegmentId();
       
        Chronos c = new Chronos();
        if(!segment.canReadFromBuffer() && segment.getLoadFactor() > 0.1) {
            segSource = new BufferedSegment(segment, getByteBuffer((int)segment.getInitialSize()));
            _log.info("buffering: " + c.tick() + " ms");
        }
       
        // Open the segment index buffer for the target segment
        SegmentIndexBuffer sibTarget = _dataArray.getSegmentManager().openSegmentIndexBuffer(segTargetId);
       
        long sizeLimit = segTarget.getInitialSize();
        long bytesTransferred = 0;
        boolean succ = true;
       
        try {
            AddressFormat addrFormat = _dataArray._addressFormat;
            SegmentIndexBuffer.IndexOffset reuse = new SegmentIndexBuffer.IndexOffset();
           
            for(int i = 0, cnt = sibSource.size(); i < cnt; i++) {
                sibSource.get(i, reuse);
                int index = reuse.getIndex();
                int sibSegPos = reuse.getOffset();
                long oldAddress = _dataArray.getAddress(index);
                int oldSegPos = addrFormat.getOffset(oldAddress);
               
                if(sibSegPos != oldSegPos) continue;
               
                int oldSegInd = addrFormat.getSegment(oldAddress);
                int length = addrFormat.getDataSize(oldAddress);
               
                if (oldSegInd == segSourceId && oldSegPos >= Segment.dataStartPosition) {
                    if(length == 0) length = segSource.readInt(oldSegPos);
                    int byteCnt = 4 + length;
                    long newSegPos = segTarget.getAppendPosition();
                    long newAddress = addrFormat.composeAddress((int)newSegPos, segTargetId, length);
                   
                    if(segTarget.getAppendPosition() + byteCnt >= sizeLimit) {
                        succ = false;
                        break;
                    }
                   
                    // Transfer bytes from source to target
                    segSource.transferTo(oldSegPos, byteCnt, segTarget);
                    bytesTransferred += byteCnt;
                   
                    sibTarget.add(index, (int)newSegPos);
                    _updateManager.addUpdate(index, byteCnt, newAddress, oldAddress, segTarget);
                }
            }
           
            // Push whatever left into update queue
            _updateManager.endUpdate(segTarget);
            _log.info("bytes fastscanned from " + segSource.getSegmentId() + ": " + bytesTransferred + " time: " + c.tick() + " ms");
           
            return succ;
        } finally {
            if(segSource.getClass() == BufferedSegment.class) {
                segSource.close(false);
                segSource = null;
            }
        }
    }
View Full Code Here

         * the hwMark of the underlying array file. If hwMark is being used, the current
         * redo entry is discarded upon crash. Upon restart, the hwMark from the underlying
         * array file may prevent updates in the lost redo entry from being resent.  
         */
        long updateScn = _addressArray.getLWMark();
        Segment segTarget = updateBatch.getTargetSegment();
       
        for(int i = 0; i < updateCount; i++) {
            int index = updateBatch.getUpdateIndex(i);
            long origAddr = updateBatch.getOriginDataAddr(i);
            long currAddr = getAddress(index);
           
            if(currAddr == 0 ||      /* data at the given index is deleted by writer */
               currAddr != origAddr  /* data at the given index is updated by writer */) {
                /*
                 * The address generated by the compactor is obsolete.
                 */
                int updateBytes = updateBatch.getUpdateDataSize(i);
                totalIgnoreBytes += updateBytes;
                ignoreCount++;
            } else {
                /*
                 * The address generated by the compactor has not been touched by the writer.
                 * Update the address array directly.
                 */
                setCompactionAddress(index, updateBatch.getUpdateDataAddr(i), updateScn);
            }
        }
       
        int consumeCount = updateCount - ignoreCount;
        int totalConsumeBytes = totalUpdateBytes - totalIgnoreBytes;
       
        if(_log.isTraceEnabled()) {
            _log.trace("consumed compaction batch " +
                       updateBatch.getDescriptiveId() +
                       " updates " + consumeCount + "/" + updateCount +
                       " bytes " + totalConsumeBytes + "/" + totalUpdateBytes);
        }
       
        // Update segment load size
        segTarget.decrLoadSize(totalIgnoreBytes);
    }
View Full Code Here

     * Sync with the Segment compactor to consume all the produced {@link CompactionUpdateBatch}(es).
     */
    protected void syncCompactor() {
        ConcurrentLinkedQueue<Segment> queue = _compactor.getCompactedQueue();
        while(!queue.isEmpty()) {
            Segment seg = queue.remove();
            consumeCompactionBatches();
            _compactor.getFreeQueue().offer(seg);
        }
       
        consumeCompactionBatches();
View Full Code Here

            int segInd = _addressFormat.getSegment(address);
            int length = _addressFormat.getDataSize(address);
           
            if (segPos >= Segment.dataStartPosition) {
                // get data segment
                Segment seg = _segmentManager.getSegment(segInd);
               
                // read data length
                if(seg != null) seg.decrLoadSize(4 + ((length == 0) ? seg.readInt(segPos) : length));
            }
        }
        catch(IOException e1) {}
        catch(IndexOutOfBoundsException e2) {}
    }
View Full Code Here

       
        // no data found
        if(segPos < Segment.dataStartPosition) return false;
       
        // get data segment
        Segment seg = _segmentManager.getSegment(segInd);
        if(seg == null) return false;
       
        return true;
    }
View Full Code Here

TOP

Related Classes of krati.core.segment.Segment

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.