Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.MD5MD5CRC32FileChecksum

446
447
448
449
450
451
452
453
454
455
456
          .header("Access-Control-Allow-Origin", "*")
          .build();
    }
    case GETFILECHECKSUM:
    {
      MD5MD5CRC32FileChecksum checksum = null;
      DFSClient dfsclient = newDfsClient(nnId, conf);
      try {
        checksum = dfsclient.getFileChecksum(fullpath, Long.MAX_VALUE);
        dfsclient.close();
        dfsclient = null;
View Full Code Here
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
      }
      out1.close();
      out2.close();

      // the two checksums must be different.
      MD5MD5CRC32FileChecksum sum1 =
          (MD5MD5CRC32FileChecksum)dfs.getFileChecksum(path1);
      MD5MD5CRC32FileChecksum sum2 =
          (MD5MD5CRC32FileChecksum)dfs.getFileChecksum(path2);
      assertFalse(sum1.equals(sum2));

      // check the individual params
      assertEquals(DataChecksum.Type.CRC32C, sum1.getCrcType());
      assertEquals(DataChecksum.Type.CRC32,  sum2.getCrcType());

    } finally {
      if (cluster != null) {
        cluster.getFileSystem().delete(testBasePath, true);
        cluster.shutdown();
View Full Code Here
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
    final byte[] bytes = StringUtils.hexStringToByte((String)m.get("bytes"));

    final DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes));
    final DataChecksum.Type crcType =
        MD5MD5CRC32FileChecksum.getCrcTypeFromAlgorithmName(algorithm);
    final MD5MD5CRC32FileChecksum checksum;

    // Recreate what DFSClient would have returned.
    switch(crcType) {
      case CRC32:
        checksum = new MD5MD5CRC32GzipFileChecksum();
        break;
      case CRC32C:
        checksum = new MD5MD5CRC32CastagnoliFileChecksum();
        break;
      default:
        throw new IOException("Unknown algorithm: " + algorithm);
    }
    checksum.readFields(in);

    //check algorithm name
    if (!checksum.getAlgorithmName().equals(algorithm)) {
      throw new IOException("Algorithm not matched. Expected " + algorithm
          + ", Received " + checksum.getAlgorithmName());
    }
    //check length
    if (length != checksum.getLength()) {
      throw new IOException("Length not matched: length=" + length
          + ", checksum.getLength()=" + checksum.getLength());
    }

    return checksum;
  }
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
     
      try {
        final DFSClient dfs = DatanodeJspHelper.getDFSClient(request,
            datanode, conf, getUGI(request, conf));
        final ClientProtocol nnproxy = dfs.getNamenode();
        final MD5MD5CRC32FileChecksum checksum = DFSClient.getFileChecksum(
            path, nnproxy, socketFactory, socketTimeout, dfs.getDataEncryptionKey(), false);
        MD5MD5CRC32FileChecksum.write(xml, checksum);
      } catch(IOException ioe) {
        writeXml(ioe, path, xml);
      } catch (InterruptedException e) {
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
          UnixUserGroupInformation.UGI_PROPERTY_NAME, ugi);
      final ProtocolProxy<ClientProtocol> nnproxy =
        DFSClient.createRPCNamenode(conf);

      try {
        final MD5MD5CRC32FileChecksum checksum = DFSClient.getFileChecksum(
            DataTransferProtocol.DATA_TRANSFER_VERSION,
            filename, nnproxy.getProxy(), nnproxy, socketFactory, socketTimeout);
        MD5MD5CRC32FileChecksum.write(xml, checksum);
      } catch(IOException ioe) {
        new RemoteException(ioe.getClass().getName(), ioe.getMessage()
View Full Code Here
1727
1728
1729
1730
1731
1732
1733
      }
    }

    //compute file MD5
    final MD5Hash fileMD5 = MD5Hash.digest(md5out.getData());
    return new MD5MD5CRC32FileChecksum(bytesPerCRC, crcPerBlock, fileMD5);
  }
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
        new HdfsConfiguration(datanode.getConf());
     
      try {
        final DFSClient dfs = DatanodeJspHelper.getDFSClient(request,
            datanode, conf, getUGI(request, conf));
        final MD5MD5CRC32FileChecksum checksum = dfs.getFileChecksum(path, Long.MAX_VALUE);
        MD5MD5CRC32FileChecksum.write(xml, checksum);
      } catch(IOException ioe) {
        writeXml(ioe, path, xml);
      } catch (InterruptedException e) {
        writeXml(e, path, xml);
View Full Code Here
368
369
370
371
372
373
374
375
376
377
378
      return Response.ok(streaming).type(
          MediaType.APPLICATION_OCTET_STREAM).build();
    }
    case GETFILECHECKSUM:
    {
      MD5MD5CRC32FileChecksum checksum = null;
      DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
      try {
        checksum = dfsclient.getFileChecksum(fullpath);
        dfsclient.close();
        dfsclient = null;
View Full Code Here
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
      }
      out1.close();
      out2.close();

      // the two checksums must be different.
      MD5MD5CRC32FileChecksum sum1 =
          (MD5MD5CRC32FileChecksum)dfs.getFileChecksum(path1);
      MD5MD5CRC32FileChecksum sum2 =
          (MD5MD5CRC32FileChecksum)dfs.getFileChecksum(path2);
      assertFalse(sum1.equals(sum2));

      // check the individual params
      assertEquals(DataChecksum.Type.CRC32C, sum1.getCrcType());
      assertEquals(DataChecksum.Type.CRC32,  sum2.getCrcType());

    } finally {
      if (cluster != null) {
        cluster.getFileSystem().delete(testBasePath, true);
        cluster.shutdown();
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
          public ClientProtocol run() throws IOException {
            return DFSClient.createNamenode(conf);
          }
        });
       
        final MD5MD5CRC32FileChecksum checksum = DFSClient.getFileChecksum(
            filename, nnproxy, socketFactory, socketTimeout);
        MD5MD5CRC32FileChecksum.write(xml, checksum);
      } catch(IOException ioe) {
        writeXml(ioe, filename, xml);
      } catch (InterruptedException e) {
View Full Code Here
TOP

Related Classes of org.apache.hadoop.fs.MD5MD5CRC32FileChecksum

  • com.tripadvisor.hadoop.ExternalHDFSChecksumGenerator
  • org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum
  • org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum
  • org.apache.hadoop.fs.Options.ChecksumOpt
  • org.apache.hadoop.hdfs.DFSClient
  • org.apache.hadoop.hdfs.server.datanode.web.resources.DatanodeWebHdfsMethods
  • org.apache.hadoop.hdfs.server.namenode.FileChecksumServlets$GetServlet
  • org.apache.hadoop.hdfs.TestDistributedFileSystem
  • org.apache.hadoop.hdfs.web.JsonUtil
  • org.apache.hadoop.io.MD5Hash
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.