Examples of CRC32

com.jcraft.jzlib.CRC32
java.util.zip.CRC32
A class that can be used to compute the CRC-32 of a data stream. @see Checksum @version 1.32, 11/17/05 @author David Connelly
org.apache.kafka.common.utils.Crc32
This class was taken from Hadoop org.apache.hadoop.util.PureJavaCrc32 A pure-java implementation of the CRC32 checksum that uses the same polynomial as the built-in native CRC32. This is to avoid the JNI overhead for certain uses of Checksumming where many small pieces of data are checksummed in succession. The current version is ~10x to 1.8x as fast as Sun's native java.util.zip.CRC32 in Java 1.6 @see java.util.zip.CRC32

Examples of java.util.zip.CRC32

    FileInputStream inStream = null;
    try
    {
      inStream = new FileInputStream(file);
      long size = file.length();
      CRC32 crc32 = null;
      MessageDigest md = null;
      Adler32 adler32 = null;
      if (algo.equalsIgnoreCase("CRC32"))
        crc32 = new CRC32();
      else if (algo.equalsIgnoreCase("Adler32"))
        adler32 = new Adler32();
      else md = MessageDigest.getInstance(algo);
      byte [] buf = new byte[BUF_SIZE];
      int readed = -1;
      if (listener != null) listener.notify(new ProgNotify(ProgNotify.START, 0, new Long(size).toString()));
      long total = size;
      boolean bAbort = false;
      while(size != 0)
      {
        if ((readed = inStream.read(buf)) >= 0)
        {
          if (crc32 != null) crc32.update(buf, 0, readed);
          else if (adler32 != null)  adler32.update(buf, 0, readed);
          else md.update(buf, 0, readed);
          
          assert size <= readed;
          size -= readed;
          if (listener != null)
          {
            if (bAbort = listener.notify(new ProgNotify(ProgNotify.RUN, (int)(1000L - (size*1000L)/total), null)))
              break;
          }
        }
        else break;
      }
      if (bAbort)
      {
        if (listener != null) listener.notify(new ProgNotify(ProgNotify.ABORT, 0, null));
      }
      else
      {
        if (crc32 != null) m_byteArray = longToByteArray4(crc32.getValue());
        else if (adler32 != null) m_byteArray = longToByteArray4(adler32.getValue());
        else m_byteArray = md.digest();
      
        int index = 0;
        StringBuilder str = new StringBuilder();

View Full Code Here

Examples of java.util.zip.CRC32

     * @param adler the number to use as starting point for the CRC-32 algorithm
     */
    public CRC32Ext(final int crc) {
        super();
        this.crc=crc;
        this.intern = new CRC32();
        setCRCRef(this.crc);
    }

View Full Code Here

Examples of java.util.zip.CRC32

     * Get the CRC32 checksum of a stream, reading
     * its contents entirely and closing it.
     */
    private long getStreamCheckSum(InputStream in)
    throws Exception {
        CRC32 sum = new CRC32();


        byte[] buf = new byte[32*1024];


        for (;;) {
            int read = in.read(buf);
            if (read == -1)
                break;
            sum.update(buf, 0, read);
        }
        in.close();
        return sum.getValue();
    }

View Full Code Here

Examples of java.util.zip.CRC32

                {
                    if (logger.isDebugEnabled())
                        logger.debug("Reading mutation at " + reader.getFilePointer());


                    long claimedCRC32;
                    Checksum checksum = new CRC32();
                    int serializedSize;
                    try
                    {
                        // any of the reads may hit EOF
                        serializedSize = reader.readInt();
                        // RowMutation must be at LEAST 10 bytes:
                        // 3 each for a non-empty Table and Key (including the 2-byte length from
                        // writeUTF/writeWithShortLength) and 4 bytes for column count.
                        // This prevents CRC by being fooled by special-case garbage in the file; see CASSANDRA-2128
                        if (serializedSize < 10)
                            break;
                        long claimedSizeChecksum = reader.readLong();
                        checksum.update(serializedSize);
                        if (checksum.getValue() != claimedSizeChecksum)
                            break; // entry wasn't synced correctly/fully.  that's ok.


                        if (serializedSize > bytes.length)
                            bytes = new byte[(int) (1.2 * serializedSize)];
                        reader.readFully(bytes, 0, serializedSize);
                        claimedCRC32 = reader.readLong();
                    }
                    catch(EOFException eof)
                    {
                        break; // last CL entry didn't get completely written.  that's ok.
                    }


                    checksum.update(bytes, 0, serializedSize);
                    if (claimedCRC32 != checksum.getValue())
                    {
                        // this entry must not have been fsynced.  probably the rest is bad too,
                        // but just in case there is no harm in trying them (since we still read on an entry boundary)
                        continue;
                    }

View Full Code Here

Examples of java.util.zip.CRC32


    static class CommitLogHeaderSerializer implements ICompactSerializer2<CommitLogHeader>
    {
        public void serialize(CommitLogHeader clHeader, DataOutput dos) throws IOException
        {
            Checksum checksum = new CRC32();


            // write the first checksum after the fixed-size part, so we won't read garbage lastFlushedAt data.
            dos.writeInt(clHeader.cfDirtiedAt.size()); // 4
            checksum.update(clHeader.cfDirtiedAt.size());
            dos.writeLong(checksum.getValue());


            // write the 2nd checksum after the lastflushedat map
            for (Map.Entry<Integer, Integer> entry : clHeader.cfDirtiedAt.entrySet())
            {
                dos.writeInt(entry.getKey()); // 4
                checksum.update(entry.getKey());
                dos.writeInt(entry.getValue()); // 4
                checksum.update(entry.getValue());
            }
            dos.writeLong(checksum.getValue());
        }

View Full Code Here

Examples of java.util.zip.CRC32

            dos.writeLong(checksum.getValue());
        }


        public CommitLogHeader deserialize(DataInput dis) throws IOException
        {
            Checksum checksum = new CRC32();


            int lastFlushedAtSize = dis.readInt();
            checksum.update(lastFlushedAtSize);
            if (checksum.getValue() != dis.readLong())
            {
                throw new IOException("Invalid or corrupt commitlog header");
            }
            Map<Integer, Integer> lastFlushedAt = new HashMap<Integer, Integer>();
            for (int i = 0; i < lastFlushedAtSize; i++)
            {
                int key = dis.readInt();
                checksum.update(key);
                int value = dis.readInt();
                checksum.update(value);
                lastFlushedAt.put(key, value);
            }
            if (checksum.getValue() != dis.readLong())
            {
                throw new IOException("Invalid or corrupt commitlog header");
            }


            return new CommitLogHeader(lastFlushedAt);

View Full Code Here

Examples of java.util.zip.CRC32

    }


    @Test
    public void testRecoveryWithBadSizeChecksum() throws Exception
    {
        Checksum checksum = new CRC32();
        checksum.update(100);
        testRecoveryWithBadSizeArgument(100, 100, ~checksum.getValue());
    }

View Full Code Here

Examples of java.util.zip.CRC32

        // Note: this can actually happen (in periodic mode) when data is flushed
        // before it had time to hit the commitlog (since the header is flushed by the system)
        // see https://issues.apache.org/jira/browse/CASSANDRA-2285
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        DataOutputStream dos = new DataOutputStream(out);
        Checksum checksum = new CRC32();


        // write the first checksum after the fixed-size part, so we won't read garbage lastFlushedAt data.
        dos.writeInt(1);
        checksum.update(1);
        dos.writeLong(checksum.getValue());
        dos.writeInt(0);
        checksum.update(0);
        dos.writeInt(200);
        checksum.update(200);
        dos.writeLong(checksum.getValue());
        dos.close();


        testRecovery(out.toByteArray(), new byte[0]);
    }

View Full Code Here

Examples of java.util.zip.CRC32

        testRecovery(out.toByteArray(), new byte[0]);
    }


    protected void testRecoveryWithBadSizeArgument(int size, int dataSize) throws Exception
    {
        Checksum checksum = new CRC32();
        checksum.update(size);
        testRecoveryWithBadSizeArgument(size, dataSize, checksum.getValue());
    }

View Full Code Here

Examples of java.util.zip.CRC32

  IndexInput main;
  Checksum digest;


  public ChecksumIndexInput(IndexInput main) {
    this.main = main;
    digest = new CRC32();
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.