Examples of org.iq80.leveldb.util.PureJavaCrc32C

org.iq80.leveldb.util.Slice
A pure-java implementation of the CRC32 checksum that uses the CRC32-C polynomial, the same polynomial used by iSCSI and implemented on many Intel chipsets supporting SSE4.2.

    }


    public static int getChunkChecksum(int chunkTypeId, byte[] buffer, int offset, int length)
    {
        // Compute the crc of the record type and the payload.
        PureJavaCrc32C crc32C = new PureJavaCrc32C();
        crc32C.update(chunkTypeId);
        crc32C.update(buffer, offset, length);
        return crc32C.getMaskedValue();
    }

View Full Code Here

        closed = true;
    }


    public static int crc32c(Slice data, CompressionType type)
    {
        PureJavaCrc32C crc32c = new PureJavaCrc32C();
        crc32c.update(data.getRawArray(), data.getRawOffset(), data.length());
        crc32c.update(type.persistentId() & 0xFF);
        return crc32c.getMaskedValue();
    }

View Full Code Here

        }


        long bytes = 0;
        int crc = 0;
        while (bytes < 1000 * 1048576) {
            PureJavaCrc32C checksum = new PureJavaCrc32C();
            checksum.update(data, 0, blockSize);
            crc = checksum.getMaskedValue();
            finishedSingleOp();
            bytes += blockSize;
        }
        System.out.printf("... crc=0x%x\r", crc);

View Full Code Here

        return comparator.name();
    }


    @Override
    public Slice findShortestSeparator(Slice start, Slice limit) {
        return new Slice(comparator.findShortestSeparator(start.getBytes(), limit.getBytes()));
    }

View Full Code Here

        return new Slice(comparator.findShortestSeparator(start.getBytes(), limit.getBytes()));
    }


    @Override
    public Slice findShortSuccessor(Slice key) {
        return new Slice(comparator.findShortSuccessor(key.getBytes()));
    }

View Full Code Here


        // If we just wrote a block, we can now add the handle to index block
        if (pendingIndexEntry) {
            Preconditions.checkState(dataBlockBuilder.isEmpty(), "Internal error: Table has a pending index entry but data block builder is empty");


            Slice shortestSeparator = userComparator.findShortestSeparator(lastKey, key);


            Slice handleEncoding = BlockHandle.writeBlockHandle(pendingHandle);
            indexBlockBuilder.add(shortestSeparator, handleEncoding);
            pendingIndexEntry = false;
        }


        lastKey = key;

View Full Code Here


    private BlockHandle writeBlock(BlockBuilder blockBuilder)
            throws IOException
    {
        // close the block
        Slice raw = blockBuilder.finish();


        // attempt to compress the block
        Slice blockContents = raw;
        CompressionType blockCompressionType = CompressionType.NONE;
        if (compressionType == CompressionType.SNAPPY) {
            ensureCompressedOutputCapacity(maxCompressedLength(raw.length()));
            try {
                int compressedSize = Snappy.compress(raw.getRawArray(), raw.getRawOffset(), raw.length(), compressedOutput.getRawArray(), 0);


                // Don't use the compressed data if compressed less than 12.5%,
                if (compressedSize < raw.length() - (raw.length() / 8)) {
                    blockContents = compressedOutput.slice(0, compressedSize);
                    blockCompressionType = CompressionType.SNAPPY;
                }
            }
            catch (IOException ignored) {
                // compression failed, so just store uncompressed form
            }
        }


        // create block trailer
        BlockTrailer blockTrailer = new BlockTrailer(blockCompressionType, crc32c(blockContents, blockCompressionType));
        Slice trailer = BlockTrailer.writeBlockTrailer(blockTrailer);


        // create a handle to this block
        BlockHandle blockHandle = new BlockHandle(position, blockContents.length());


        // write data and trailer
        position += fileChannel.write(new ByteBuffer[]{blockContents.toByteBuffer(), trailer.toByteBuffer()});


        // clean up state
        blockBuilder.reset();


        return blockHandle;

View Full Code Here

        // TODO(postrelease): Add stats and other meta blocks
        BlockHandle metaindexBlockHandle = writeBlock(metaIndexBlockBuilder);


        // add last handle to index block
        if (pendingIndexEntry) {
            Slice shortSuccessor = userComparator.findShortSuccessor(lastKey);


            Slice handleEncoding = BlockHandle.writeBlockHandle(pendingHandle);
            indexBlockBuilder.add(shortSuccessor, handleEncoding);
            pendingIndexEntry = false;
        }


        // write index block
        BlockHandle indexBlockHandle = writeBlock(indexBlockBuilder);


        // write footer
        Footer footer = new Footer(metaindexBlockHandle, indexBlockHandle);
        Slice footerEncoding = Footer.writeFooter(footer);
        position += fileChannel.write(footerEncoding.toByteBuffer());
    }

View Full Code Here

        return new BlockHandle(offset, (int) size);
    }


    public static Slice writeBlockHandle(BlockHandle blockHandle)
    {
        Slice slice = Slices.allocate(MAX_ENCODED_LENGTH);
        SliceOutput sliceOutput = slice.output();
        writeBlockHandleTo(blockHandle, sliceOutput);
        return slice.slice();
    }

View Full Code Here

        if (sharedBytes < Math.min(start.length(), limit.length())) {
            // if we can add one to the last shared byte without overflow and the two keys differ by more than
            // one increment at this location.
            int lastSharedByte = start.getUnsignedByte(sharedBytes);
            if (lastSharedByte < 0xff && lastSharedByte + 1 < limit.getUnsignedByte(sharedBytes)) {
                Slice result = start.copySlice(0, sharedBytes + 1);
                result.setByte(sharedBytes, lastSharedByte + 1);


                assert (compare(result, limit) < 0) : "start must be less than last limit";
                return result;
            }
        }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.iq80.leveldb.util.PureJavaCrc32C

co.cask.cdap.data2.dataset2.lib.table.leveldb.LevelDBOrderedTableCore

co.cask.cdap.data2.dataset2.lib.table.leveldb.LevelDBOrderedTableService

com.alexkasko.springjdbc.compress.SnappyCompressor

com.cloudhopper.datastore.leveldb.LevelDbDataStore

com.fasterxml.storemate.backend.leveldb.LevelDBBuilder

com.google.common.util.concurrent.ThreadFactoryBuilder

com.nabalive.server.web.DataUtil

com.nabalive.server.web.TokenUtil

com.saasovation.common.port.adapter.persistence.eventsourcing.leveldb.LevelDBJournal

com.saasovation.common.port.adapter.persistence.eventsourcing.leveldb.LevelDBJournalRepairTool

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.