Examples of org.apache.cassandra.utils.PureJavaCrc32

org.apache.cassandra.utils.PureJavaCrc32
A pure-java implementation of the CRC32 checksum that uses the same polynomial as the built-in native CRC32. This is to avoid the JNI overhead for certain uses of Checksumming where many small pieces of data are checksummed in succession. The current version is ~10x to 1.8x as fast as Sun's native java.util.zip.CRC32 in Java 1.6 @see java.util.zip.CRC32 This class is copied from hadoop-commons project. (The initial patch added PureJavaCrc32 was HADOOP-6148)

        this.futures = new ArrayList<Future<?>>();
        this.buffer = new byte[4096];
        this.invalidMutations = new HashMap<UUID, AtomicInteger>();
        // count the number of replayed mutation. We don't really care about atomicity, but we need it to be a reference.
        this.replayedCount = new AtomicInteger();
        this.checksum = new PureJavaCrc32();


        // compute per-CF and global replay positions
        cfPositions = new HashMap<UUID, ReplayPosition>();
        Ordering<ReplayPosition> replayPositionOrdering = Ordering.from(ReplayPosition.comparator);
        Map<UUID, ReplayPosition> truncationPositions = SystemTable.getTruncationPositions();

View Full Code Here

    {
        assert !closed;
        ReplayPosition repPos = getContext();
        markDirty(rowMutation, repPos);


        Checksum checksum = new PureJavaCrc32();
        byte[] serializedRow = rowMutation.getSerializedBuffer(MessagingService.current_version);


        checksum.update(serializedRow.length);
        buffer.putInt(serializedRow.length);
        buffer.putLong(checksum.getValue());


        buffer.put(serializedRow);
        checksum.update(serializedRow, 0, serializedRow.length);
        buffer.putLong(checksum.getValue());


        if (buffer.remaining() >= 4)
        {
            // writes end of segment marker and rewinds back to position where it starts
            buffer.putInt(CommitLog.END_OF_SEGMENT_MARKER);

View Full Code Here

    {
        assert !closed;
        ReplayPosition repPos = getContext();
        markDirty(rowMutation, repPos);


        Checksum checksum = new PureJavaCrc32();
        byte[] serializedRow = rowMutation.getSerializedBuffer(MessagingService.version_);


        checksum.update(serializedRow.length);
        buffer.putInt(serializedRow.length);
        buffer.putLong(checksum.getValue());


        buffer.put(serializedRow);
        checksum.update(serializedRow, 0, serializedRow.length);
        buffer.putLong(checksum.getValue());


        if (buffer.remaining() >= 4)
        {
            // writes end of segment marker and rewinds back to position where it starts
            buffer.putInt(CommitLog.END_OF_SEGMENT_MARKER);

View Full Code Here

    {
        assert !closed;
        ReplayPosition repPos = getContext();
        markDirty(rowMutation, repPos);


        Checksum checksum = new PureJavaCrc32();
        byte[] serializedRow = rowMutation.getSerializedBuffer(MessagingService.version_);


        checksum.update(serializedRow.length);
        buffer.putInt(serializedRow.length);
        buffer.putLong(checksum.getValue());


        buffer.put(serializedRow);
        checksum.update(serializedRow, 0, serializedRow.length);
        buffer.putLong(checksum.getValue());


        if (buffer.remaining() >= 4)
        {
            // writes end of segment marker and rewinds back to position where it starts
            buffer.putInt(CommitLog.END_OF_SEGMENT_MARKER);

View Full Code Here

            // list (otherwise we'll just start replay from the first flush position that we do have, which is not correct).
            ReplayPosition rp = ReplayPosition.getReplayPosition(cfs.getSSTables());
            cfPositions.put(cfs.metadata.cfId, rp);
        }
        this.globalPosition = Ordering.from(ReplayPosition.comparator).min(cfPositions.values());
        this.checksum = new PureJavaCrc32();
    }

View Full Code Here

    {
        assert !closed;
        ReplayPosition repPos = getContext();
        markDirty(rowMutation, repPos);


        Checksum checksum = new PureJavaCrc32();
        byte[] serializedRow = FBUtilities.serialize(rowMutation, RowMutation.serializer, MessagingService.current_version);


        checksum.update(serializedRow.length);
        buffer.putInt(serializedRow.length);
        buffer.putLong(checksum.getValue());


        buffer.put(serializedRow);
        checksum.update(serializedRow, 0, serializedRow.length);
        buffer.putLong(checksum.getValue());


        if (buffer.remaining() >= 4)
        {
            // writes end of segment marker and rewinds back to position where it starts
            buffer.putInt(CommitLog.END_OF_SEGMENT_MARKER);

View Full Code Here

        this.futures = new ArrayList<Future<?>>();
        this.buffer = new byte[4096];
        this.invalidMutations = new HashMap<UUID, AtomicInteger>();
        // count the number of replayed mutation. We don't really care about atomicity, but we need it to be a reference.
        this.replayedCount = new AtomicInteger();
        this.checksum = new PureJavaCrc32();


        // compute per-CF and global replay positions
        cfPositions = new HashMap<UUID, ReplayPosition>();
        Ordering<ReplayPosition> replayPositionOrdering = Ordering.from(ReplayPosition.comparator);
        Map<UUID, ReplayPosition> truncationPositions = SystemTable.getTruncationPositions();

View Full Code Here

            assert nextMarker > lastSyncedOffset;


            // write previous sync marker to point to next sync marker
            // we don't chain the crcs here to ensure this method is idempotent if it fails
            int offset = lastSyncedOffset;
            final PureJavaCrc32 crc = new PureJavaCrc32();
            crc.update((int) (id & 0xFFFFFFFFL));
            crc.update((int) (id >>> 32));
            crc.update(offset);
            buffer.putInt(offset, nextMarker);
            buffer.putLong(offset + 4, crc.getValue());


            // zero out the next sync marker so replayer can cleanly exit
            if (nextMarker < buffer.capacity())
            {
                buffer.putInt(nextMarker, 0);

View Full Code Here

        }


        allocator.allocate(mutation, (int) totalSize, alloc);
        try
        {
            PureJavaCrc32 checksum = new PureJavaCrc32();
            final ByteBuffer buffer = alloc.getBuffer();
            DataOutputStream dos = new DataOutputStream(new ChecksummedOutputStream(new ByteBufferOutputStream(buffer), checksum));


            // checksummed length
            dos.writeInt((int) size);
            buffer.putLong(checksum.getValue());


            // checksummed mutation
            Mutation.serializer.serialize(mutation, dos, MessagingService.current_version);
            buffer.putLong(checksum.getValue());
        }
        catch (IOException e)
        {
            throw new FSWriteError(e, alloc.getSegment().getPath());
        }

View Full Code Here

            assert nextMarker > lastSyncedOffset;


            // write previous sync marker to point to next sync marker
            // we don't chain the crcs here to ensure this method is idempotent if it fails
            int offset = lastSyncedOffset;
            final PureJavaCrc32 crc = new PureJavaCrc32();
            crc.update((int) (id & 0xFFFFFFFFL));
            crc.update((int) (id >>> 32));
            crc.update(offset);
            buffer.putInt(offset, nextMarker);
            buffer.putLong(offset + 4, crc.getValue());


            // zero out the next sync marker so replayer can cleanly exit
            if (nextMarker < buffer.capacity())
            {
                buffer.putInt(nextMarker, 0);

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.cassandra.utils.PureJavaCrc32

org.apache.cassandra.db.commitlog.CommitLog

org.apache.cassandra.db.commitlog.CommitLogDescriptor

org.apache.cassandra.db.commitlog.CommitLogReplayer

org.apache.cassandra.db.commitlog.CommitLogSegment

org.apache.cassandra.io.util.DataIntegrityMetadata$ChecksumValidator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.