Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.MD5Hash

463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
      final int bytesPerCRC = checksum.getBytesPerChecksum();
      final long crcPerBlock = (metadataIn.getLength()
          - BlockMetadataHeader.getHeaderSize())/checksum.getChecksumSize();
     
      //compute block checksum
      final MD5Hash md5 = MD5Hash.digest(checksumIn);

      if (LOG.isDebugEnabled()) {
        LOG.debug("block=" + block + ", bytesPerCRC=" + bytesPerCRC
            + ", crcPerBlock=" + crcPerBlock + ", md5=" + md5);
      }

      //write reply
      out.writeShort(DataTransferProtocol.OP_STATUS_SUCCESS);
      out.writeInt(bytesPerCRC);
      out.writeLong(crcPerBlock);
      md5.write(out);
      out.flush();
    } finally {
      IOUtils.closeStream(out);
      IOUtils.closeStream(checksumIn);
      IOUtils.closeStream(metadataIn);
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
    DeleteDuplicates dedup = new DeleteDuplicates(conf);
    dedup.dedup(new Path[]{index2});
    FsDirectory dir = new FsDirectory(fs, new Path(index2, "part-0000"), false, conf);
    IndexReader reader = IndexReader.open(dir);
    assertEquals("only one doc left", reader.numDocs(), 1);
    MD5Hash hash = MD5Hash.digest("2");
    for (int i = 0; i < reader.maxDoc(); i++) {
      if (reader.isDeleted(i)) {
        System.out.println("-doc " + i + " deleted");
        continue;
      }
      Document doc = reader.document(i);
      // make sure we got the right one
      assertEquals("check hash", hash.toString(), doc.get("digest"));
      System.out.println(doc);
    }
    reader.close();
  }
View Full Code Here
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
    }
    reader.close();
    dir = new FsDirectory(fs, new Path(index2, "part-0000"), false, conf);
    reader = IndexReader.open(dir);
    assertEquals("only one doc left", reader.numDocs(), 1);
    MD5Hash hash = MD5Hash.digest("2");
    for (int i = 0; i < reader.maxDoc(); i++) {
      if (reader.isDeleted(i)) {
        System.out.println("-doc " + i + " deleted");
        continue;
      }
      Document doc = reader.document(i);
      // make sure we got the right one
      assertEquals("check hash", hash.toString(), doc.get("digest"));
      System.out.println(doc);
    }
    reader.close();
  }
View Full Code Here
446
447
448
449
450
451
452
453
454
455
456
      int read = 0;
      while ((read = imageIn.read(in)) > 0) {
        totalRead += read;
        LOG.debug("Computing fsimage checksum. Read " + totalRead + " bytes so far.");
      }
      return new MD5Hash(digester.digest());
    } finally {
      if (imageIn != null) {
        imageIn.close();
      }
    }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
      } else if (key.equals("machine")) {
        machineName = pmap.get("machine")[0];
      } else if (key.equals("token")) {
        token = new CheckpointSignature(pmap.get("token")[0]);
      } else if (key.equals("newChecksum")) {
        newChecksum = new MD5Hash(pmap.get("newChecksum")[0]);
      }
    }

    int numGets = (isGetImage?1:0) + (isGetEdit?1:0);
    if ((numGets > 1) || (numGets == 0) && !isPutImage) {
View Full Code Here
206
207
208
209
210
211
212
            output[i].close();
          }
        }
      }
    }
    return digester == null ? null : new MD5Hash(digester.digest());
  }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
            // send edits
            TransferFsImage.getFileServer(response.getOutputStream(),
                nnImage.getFsEditName(), getThrottler(conf));
          } else if (ff.putImage()) {
            synchronized (fsImageTransferLock) {
              final MD5Hash expectedChecksum = ff.getNewChecksum();
              // issue a HTTP get request to download the new fsimage
              nnImage.validateCheckpointUpload(ff.getToken());
              reloginIfNecessary().doAs(new PrivilegedExceptionAction<Void>() {
                @Override
                public Void run() throws Exception {
                  MD5Hash actualChecksum = TransferFsImage.getFileClient(ff.getInfoServer(),
                      "getimage=1", nnImage.getFsImageNameCheckpoint(), true);
                  LOG.info("Downloaded new fsimage with checksum: " + actualChecksum);
                  if (!actualChecksum.equals(expectedChecksum)) {
                    throw new IOException("Actual checksum of transferred fsimage: "
                        + actualChecksum + " does not match expected checksum: "
                        + expectedChecksum);
                  }
                  return null;
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
              if (UserGroupInformation.isSecurityEnabled()) {
                UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
              }
             
              // issue a HTTP get request to download the new fsimage
              MD5Hash downloadImageDigest =
                TransferFsImage.downloadImageToStorage(
                        parsedParams.getInfoServer(), txid,
                        nnImage.getStorage(), true);
              nnImage.saveDigestAndRenameCheckpointImage(txid, downloadImageDigest);
             
View Full Code Here
281
282
283
284
285
286
287
288
289
290
   */
  public static void setVerificationHeaders(HttpServletResponse response, File file)
  throws IOException {
    response.setHeader(TransferFsImage.CONTENT_LENGTH,
        String.valueOf(file.length()));
    MD5Hash hash = MD5FileUtils.readStoredMd5ForFile(file);
    if (hash != null) {
      response.setHeader(TransferFsImage.MD5_HEADER, hash.toString());
    }
  }
View Full Code Here
436
437
438
439
440
441
442
443
444
445
446
            if (sig.mostRecentCheckpointTxId ==
                dstImage.getStorage().getMostRecentCheckpointTxId()) {
              LOG.info("Image has not changed. Will not download image.");
            } else {
              LOG.info("Image has changed. Downloading updated image from NN.");
              MD5Hash downloadedHash = TransferFsImage.downloadImageToStorage(
                  nnHostPort, sig.mostRecentCheckpointTxId, dstImage.getStorage(), true);
              dstImage.saveDigestAndRenameCheckpointImage(
                  sig.mostRecentCheckpointTxId, downloadedHash);
            }
       
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.MD5Hash

  • co.nubetech.hiho.dedup.HihoTuple
  • co.nubetech.hiho.dedup.TestHashUtility
  • co.nubetech.hiho.dedup.TestHihoTuple
  • com.tripadvisor.hadoop.ExternalHDFSChecksumGenerator
  • org.apache.hadoop.fs.MD5MD5CRC32FileChecksum
  • org.apache.hadoop.hdfs.DFSClient
  • org.apache.hadoop.hdfs.qjournal.client.TestImageUploadStream
  • org.apache.hadoop.hdfs.qjournal.client.TestQuorumJournalManagerManifest
  • org.apache.hadoop.hdfs.qjournal.server.GetJournalImageServlet
  • org.apache.hadoop.hdfs.qjournal.server.Journal
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.