Examples of voldemort.store.readonly.checksum.CheckSum

Package voldemort.store.readonly.checksum

Examples of voldemort.store.readonly.checksum.CheckSum

voldemort.store.readonly.checksum.CheckSum

        File indexFile = new File(testSourceDirectory, "0_0.index");
        byte[] indexBytes = TestUtils.randomBytes(100);
        FileUtils.writeByteArrayToFile(indexFile, indexBytes);


        final Path source = new Path(indexFile.getAbsolutePath());
        CheckSum fileCheckSumGenerator = CheckSum.getInstance(CheckSumType.MD5);


        fileCheckSumGenerator.update(indexBytes);
        byte[] checksumCalculated = calculateCheckSumForFile(source);


        HdfsFetcher fetcher = new HdfsFetcher();


        Configuration config = new Configuration();


        FileSystem fs = source.getFileSystem(config);


        FileSystem spyfs = Mockito.spy(fs);
        CopyStats stats = new CopyStats(testSourceDirectory.getAbsolutePath(), sizeOfPath(fs,
                                                                                          source));


        File destination = new File(testDestinationDirectory.getAbsolutePath() + "1");
        Utils.mkdirs(destination);
        File copyLocation = new File(destination, "0_0.index");


        Mockito.doThrow(new IOException())
               .doAnswer(Mockito.CALLS_REAL_METHODS)
               .when(spyfs)
               .open(source);


        Object[] params = { spyfs, source, copyLocation, stats, CheckSumType.MD5 };


        CheckSum ckSum = (CheckSum) this.invokePrivateMethod(fetcher,
                                                             "copyFileWithCheckSum",
                                                             params);


        assertEquals(Arrays.equals(ckSum.getCheckSum(), checksumCalculated), true);


    }

View Full Code Here

        File indexFile = new File(testSourceDirectory, "0_0.index");
        byte[] indexBytes = TestUtils.randomBytes(VoldemortConfig.DEFAULT_BUFFER_SIZE * 3);
        FileUtils.writeByteArrayToFile(indexFile, indexBytes);


        final Path source = new Path(indexFile.getAbsolutePath());
        CheckSum fileCheckSumGenerator = CheckSum.getInstance(CheckSumType.MD5);


        fileCheckSumGenerator.update(indexBytes);
        byte[] checksumCalculated = calculateCheckSumForFile(source);


        HdfsFetcher fetcher = new HdfsFetcher();


        Configuration config = new Configuration();


        FileSystem fs = source.getFileSystem(config);


        FileSystem spyfs = Mockito.spy(fs);
        CopyStats stats = new CopyStats(testSourceDirectory.getAbsolutePath(), sizeOfPath(fs,
                                                                                          source));


        File destination = new File(testDestinationDirectory.getAbsolutePath() + "1");
        Utils.mkdirs(destination);
        File copyLocation = new File(destination, "0_0.index");


        FSDataInputStream input = null;


        input = fs.open(source);
        FSDataInputStream spyinput = Mockito.spy(input);


        Mockito.doAnswer(Mockito.CALLS_REAL_METHODS)
               .doThrow(new EofException())
               .when(spyinput)
               .read();


        Mockito.doReturn(spyinput).doReturn(input).when(spyfs).open(source);


        Object[] params = { spyfs, source, copyLocation, stats, CheckSumType.MD5 };


        CheckSum ckSum = (CheckSum) this.invokePrivateMethod(fetcher,
                                                             "copyFileWithCheckSum",
                                                             params);


        assertEquals(Arrays.equals(ckSum.getCheckSum(), checksumCalculated), true);


    }

View Full Code Here

        File indexFile = new File(testSourceDirectory, "0_0.index");
        byte[] indexBytes = TestUtils.randomBytes(100);
        FileUtils.writeByteArrayToFile(indexFile, indexBytes);


        final Path source = new Path(indexFile.getAbsolutePath());
        CheckSum fileCheckSumGenerator = CheckSum.getInstance(CheckSumType.MD5);


        fileCheckSumGenerator.update(indexBytes);


        HdfsFetcher fetcher = new HdfsFetcher();


        Configuration config = new Configuration();


        FileSystem fs = source.getFileSystem(config);


        FileSystem spyfs = Mockito.spy(fs);
        CopyStats stats = new CopyStats(testSourceDirectory.getAbsolutePath(), sizeOfPath(fs,
                                                                                          source));


        File destination = new File(testDestinationDirectory.getAbsolutePath() + "1");
        Utils.mkdirs(destination);
        File copyLocation = new File(destination, "0_0.index");


        Mockito.doThrow(new RuntimeException())
               .doAnswer(Mockito.CALLS_REAL_METHODS)
               .when(spyfs)
               .open(source);


        Object[] params = { spyfs, source, copyLocation, stats, CheckSumType.MD5 };


        CheckSum ckSum = (CheckSum) this.invokePrivateMethod(fetcher,
                                                             "copyFileWithCheckSum",
                                                             params);


    }

View Full Code Here


    /*
     * Helper method to calculate checksum for a single file
     */
    private byte[] calculateCheckSumForFile(Path source) throws Exception {
        CheckSum fileCheckSumGenerator = CheckSum.getInstance(CheckSumType.MD5);
        byte[] buffer = new byte[VoldemortConfig.DEFAULT_BUFFER_SIZE];


        FSDataInputStream input = null;


        Configuration config = new Configuration();


        FileSystem fs = source.getFileSystem(config);
        input = fs.open(source);


        while(true) {
            int read = input.read(buffer);
            if(read < 0) {
                break;
            }
            // Update the per file checksum
            if(fileCheckSumGenerator != null) {
                fileCheckSumGenerator.update(buffer, 0, read);
            }


        }


        return fileCheckSumGenerator.getCheckSum();
    }

View Full Code Here

                Arrays.sort(statuses, new IndexFileLastComparator());
                byte[] origCheckSum = null;
                CheckSumType checkSumType = CheckSumType.NONE;


                // Do a checksum of checksum - Similar to HDFS
                CheckSum checkSumGenerator = null;
                CheckSum fileCheckSumGenerator = null;


                for(FileStatus status: statuses) {


                    // Kept for backwards compatibility
                    if(status.getPath().getName().contains("checkSum.txt")) {


                        // Ignore old checksum files


                    } else if(status.getPath().getName().contains(".metadata")) {


                        logger.debug("Reading .metadata");
                        // Read metadata into local file
                        File copyLocation = new File(dest, status.getPath().getName());
                        copyFileWithCheckSum(fs, status.getPath(), copyLocation, stats, null);


                        // Open the local file to initialize checksum
                        ReadOnlyStorageMetadata metadata;
                        try {
                            metadata = new ReadOnlyStorageMetadata(copyLocation);
                        } catch(IOException e) {
                            logger.error("Error reading metadata file ", e);
                            throw new VoldemortException(e);
                        }


                        // Read checksum
                        String checkSumTypeString = (String) metadata.get(ReadOnlyStorageMetadata.CHECKSUM_TYPE);
                        String checkSumString = (String) metadata.get(ReadOnlyStorageMetadata.CHECKSUM);


                        if(checkSumTypeString != null && checkSumString != null) {


                            try {
                                origCheckSum = Hex.decodeHex(checkSumString.toCharArray());
                            } catch(DecoderException e) {
                                logger.error("Exception reading checksum file. Ignoring checksum ",
                                             e);
                                continue;
                            }


                            logger.debug("Checksum from .metadata "
                                         + new String(Hex.encodeHex(origCheckSum)));


                            // Define the Global checksum generator
                            checkSumType = CheckSum.fromString(checkSumTypeString);
                            checkSumGenerator = CheckSum.getInstance(checkSumType);
                        }


                    } else if(!status.getPath().getName().startsWith(".")) {


                        // Read other (.data , .index files)
                        File copyLocation = new File(dest, status.getPath().getName());
                        fileCheckSumGenerator = copyFileWithCheckSum(fs,
                                                                     status.getPath(),
                                                                     copyLocation,
                                                                     stats,
                                                                     checkSumType);


                        if(fileCheckSumGenerator != null && checkSumGenerator != null) {
                            byte[] checkSum = fileCheckSumGenerator.getCheckSum();
                            if(logger.isDebugEnabled()) {
                                logger.debug("Checksum for " + status.getPath() + " - "
                                             + new String(Hex.encodeHex(checkSum)));
                            }
                            checkSumGenerator.update(checkSum);

View Full Code Here

    private CheckSum copyFileWithCheckSum(FileSystem fs,
                                          Path source,
                                          File dest,
                                          CopyStats stats,
                                          CheckSumType checkSumType) throws Throwable {
        CheckSum fileCheckSumGenerator = null;
        logger.debug("Starting copy of " + source + " to " + dest);
        FSDataInputStream input = null;
        OutputStream output = null;


        for(int attempt = 0; attempt < maxAttempts; attempt++) {
            boolean success = true;
            long totalBytesRead = 0;
            boolean fsOpened = false;
            try {


                // Create a per file checksum generator
                if(checkSumType != null) {
                    fileCheckSumGenerator = CheckSum.getInstance(checkSumType);
                }


                logger.info("Attempt " + attempt + " at copy of " + source + " to " + dest);


                input = fs.open(source);
                fsOpened = true;


                output = new BufferedOutputStream(new FileOutputStream(dest));
                byte[] buffer = new byte[bufferSize];
                while(true) {
                    int read = input.read(buffer);
                    if(read < 0) {
                        break;
                    } else {
                        output.write(buffer, 0, read);
                    }


                    // Update the per file checksum
                    if(fileCheckSumGenerator != null) {
                        fileCheckSumGenerator.update(buffer, 0, read);
                    }


                    // Check if we need to throttle the fetch
                    if(throttler != null) {
                        throttler.maybeThrottle(read);

View Full Code Here

    private CheckSum copyFileWithCheckSum(RestFileSystem rfs,
                                          String source,
                                          File dest,
                                          CopyStats stats,
                                          CheckSumType checkSumType) throws Throwable {
        CheckSum fileCheckSumGenerator = null;
        logger.info("Starting copy of " + source + " to " + dest);
        BufferedInputStream input = null;
        OutputStream output = null;


        for(int attempt = 0; attempt < maxAttempts; attempt++) {
            boolean success = true;
            long totalBytesRead = 0;
            boolean fsOpened = false;


            try {
                // Create a per file checksum generator
                if(checkSumType != null) {
                    fileCheckSumGenerator = CheckSum.getInstance(checkSumType);
                }


                logger.info("Attempt " + attempt + " at copy of " + source + " to " + dest);


                input = new BufferedInputStream(rfs.openFile(source).getInputStream());
                fsOpened = true;


                output = new BufferedOutputStream(new FileOutputStream(dest));
                byte[] buffer = new byte[bufferSize];
                while(true) {
                    int read = input.read(buffer);
                    if(read < 0) {
                        break;
                    } else {
                        output.write(buffer, 0, read);
                    }


                    // Update the per file checksum
                    if(fileCheckSumGenerator != null) {
                        fileCheckSumGenerator.update(buffer, 0, read);
                    }


                    // Check if we need to throttle the fetch
                    if(throttler != null) {
                        throttler.maybeThrottle(read);

View Full Code Here

                Arrays.sort(statuses, new IndexFileLastComparator());
                byte[] origCheckSum = null;
                CheckSumType checkSumType = CheckSumType.NONE;


                // Do a checksum of checksum - Similar to HDFS
                CheckSum checkSumGenerator = null;
                CheckSum fileCheckSumGenerator = null;


                for(RestFileStatus status: statuses) {
                    String fileNameWithAbsolutePath = status.getAbsolutePath();
                    String shortFileName = status.getPathSuffix();
                    logger.info("fetching file: " + fileNameWithAbsolutePath);
                    // Kept for backwards compatibility
                    if(shortFileName.contains("checkSum.txt")) {


                        logger.warn("Found checksum file in old format: " + shortFileName);


                    } else if(shortFileName.contains(".metadata")) {


                        logger.debug("Reading .metadata");
                        // Read metadata into local file
                        File copyLocation = new File(dest, shortFileName);
                        copyFileWithCheckSum(rfs,
                                             fileNameWithAbsolutePath,
                                             copyLocation,
                                             stats,
                                             null);


                        // Open the local file to initialize checksum
                        ReadOnlyStorageMetadata metadata;
                        try {
                            metadata = new ReadOnlyStorageMetadata(copyLocation);
                        } catch(IOException e) {
                            logger.error("Error reading metadata file ", e);
                            throw new VoldemortException(e);
                        }


                        // Read checksum
                        String checkSumTypeString = (String) metadata.get(ReadOnlyStorageMetadata.CHECKSUM_TYPE);
                        String checkSumString = (String) metadata.get(ReadOnlyStorageMetadata.CHECKSUM);


                        if(checkSumTypeString != null && checkSumString != null) {


                            try {
                                origCheckSum = Hex.decodeHex(checkSumString.toCharArray());
                            } catch(DecoderException e) {
                                logger.error("Exception reading checksum file. Ignoring checksum ",
                                             e);
                                continue;
                            }


                            logger.debug("Checksum from .metadata "
                                         + new String(Hex.encodeHex(origCheckSum)));


                            // Define the Global checksum generator
                            checkSumType = CheckSum.fromString(checkSumTypeString);
                            checkSumGenerator = CheckSum.getInstance(checkSumType);
                        }


                    } else if(!shortFileName.startsWith(".")) {


                        // Read other (.data , .index files)
                        File copyLocation = new File(dest, shortFileName);
                        fileCheckSumGenerator = copyFileWithCheckSum(rfs,
                                                                     fileNameWithAbsolutePath,
                                                                     copyLocation,
                                                                     stats,
                                                                     checkSumType);


                        if(fileCheckSumGenerator != null && checkSumGenerator != null) {
                            byte[] checkSum = fileCheckSumGenerator.getCheckSum();
                            if(logger.isDebugEnabled()) {
                                logger.debug("Checksum for " + shortFileName + " - "
                                             + new String(Hex.encodeHex(checkSum)));
                            }
                            checkSumGenerator.update(checkSum);

View Full Code Here

                                + counters.getCounter(KeyValueWriter.CollisionCounter.MAX_COLLISIONS));
                }
            }


            // Do a CheckSumOfCheckSum - Similar to HDFS
            CheckSum checkSumGenerator = CheckSum.getInstance(this.checkSumType);
            if(!this.checkSumType.equals(CheckSumType.NONE) && checkSumGenerator == null) {
                throw new VoldemortException("Could not generate checksum digest for type "
                                             + this.checkSumType);
            }


            // Check if all folder exists and with format file
            for(Node node: cluster.getNodes()) {


                ReadOnlyStorageMetadata metadata = new ReadOnlyStorageMetadata();


                if(saveKeys) {
                    metadata.add(ReadOnlyStorageMetadata.FORMAT,
                                 ReadOnlyStorageFormat.READONLY_V2.getCode());
                } else {
                    metadata.add(ReadOnlyStorageMetadata.FORMAT,
                                 ReadOnlyStorageFormat.READONLY_V1.getCode());
                }


                Path nodePath = new Path(outputDir.toString(), "node-" + node.getId());


                if(!outputFs.exists(nodePath)) {
                    logger.info("No data generated for node " + node.getId()
                                + ". Generating empty folder");
                    outputFs.mkdirs(nodePath); // Create empty folder
                    outputFs.setPermission(nodePath, new FsPermission(HADOOP_FILE_PERMISSION));
                    logger.info("Setting permission to 755 for " + nodePath);
                }


                if(checkSumType != CheckSumType.NONE) {


                    FileStatus[] storeFiles = outputFs.listStatus(nodePath, new PathFilter() {


                        public boolean accept(Path arg0) {
                            if(arg0.getName().endsWith("checksum")
                               && !arg0.getName().startsWith(".")) {
                                return true;
                            }
                            return false;
                        }
                    });


                    if(storeFiles != null && storeFiles.length > 0) {
                        Arrays.sort(storeFiles, new IndexFileLastComparator());
                        FSDataInputStream input = null;


                        for(FileStatus file: storeFiles) {
                            try {
                                input = outputFs.open(file.getPath());
                                byte fileCheckSum[] = new byte[CheckSum.checkSumLength(this.checkSumType)];
                                input.read(fileCheckSum);
                                logger.debug("Checksum for file " + file.toString() + " - "
                                             + new String(Hex.encodeHex(fileCheckSum)));
                                checkSumGenerator.update(fileCheckSum);
                            } catch(Exception e) {
                                logger.error("Error while reading checksum file " + e.getMessage(),
                                             e);
                            } finally {
                                if(input != null)
                                    input.close();
                            }
                            outputFs.delete(file.getPath(), false);
                        }


                        metadata.add(ReadOnlyStorageMetadata.CHECKSUM_TYPE,
                                     CheckSum.toString(checkSumType));


                        String checkSum = new String(Hex.encodeHex(checkSumGenerator.getCheckSum()));
                        logger.info("Checksum for node " + node.getId() + " - " + checkSum);


                        metadata.add(ReadOnlyStorageMetadata.CHECKSUM, checkSum);
                    }
                }

View Full Code Here

TOP

Related Classes of voldemort.store.readonly.checksum.CheckSum

voldemort.server.protocol.hadoop.RestHadoopFetcher

voldemort.store.readonly.fetcher.HdfsFetcher

voldemort.store.readonly.fetcher.HDFSFetcherAdvancedTest

voldemort.store.readonly.mr.HadoopStoreBuilder

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.