Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.ChecksumException

300
301
302
303
304
305
306
307
308
309
310
          checksum.reset();
          int dLen = Math.min(dLeft, bytesPerChecksum);
          checksum.update(buf, dOff, dLen);
          if (!checksum.compare(buf, cOff)) {
            long failedPos = offset + len -dLeft;
            throw new ChecksumException("Checksum failed at " +
                                        failedPos, failedPos);
          }
          dLeft -= dLen;
          dOff += dLen;
          cOff += checksumSize;
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
      if (crc != null) {
        crc.reset();
        CheckedInputStream chk = new CheckedInputStream(in, crc);
        IOUtils.readFully(chk, buf.array(), 0, size);
        if (chk.getChecksum().getValue() != in.readLong()) {
          throw new ChecksumException("Checksum error reading spill index: " +
                                indexFileName, -1);
        }
      } else {
        IOUtils.readFully(in, buf.array(), 0, size);
      }
View Full Code Here
551
552
553
554
555
556
557
558
559
560
561
      checksum.reset();
      int dLen = Math.min(dLeft, chunkSize);
      checksum.update(buf, dOff, dLen);
      if (!checksum.compare(buf, cOff)) {
        long failedPos = offset + datalen - dLeft;
        throw new ChecksumException("Checksum failed at " + failedPos,
            failedPos);
      }
      dLeft -= dLen;
      dOff += dLen;
      cOff += checksumSize;
View Full Code Here
360
361
362
363
364
365
366
367
368
369
370
    while ((read = inputStream.read(buf)) > -1) {
      LOG.info(String.format("read %d bytes", read));
     
      if (!validateSequentialBytes(buf, (int) (startPos + numRead), read)) {
        LOG.error(String.format("invalid bytes: [%s]\n", Arrays.toString(buf)));
        throw new ChecksumException(
          String.format("unable to validate bytes"),
          startPos
        );
      }
     
View Full Code Here
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
          for (int i=0; i<numChunks; i++) {
            checksum.reset();
            int dLen = Math.min(dLeft, bytesPerChecksum);
            checksum.update(buf, dOff, dLen);
            if (!checksum.compare(buf, cOff)) {
              throw new ChecksumException("Checksum failed at " +
                                          (offset + len - dLeft), len);
            }
            dLeft -= dLen;
            dOff += dLen;
            cOff += checksumSize;
View Full Code Here
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
        throws IOException {
      if (checksum != null) {
        int calculatedChecksum = (int)checksum.getValue();
        int readChecksum = in.readInt(); // read in checksum
        if (readChecksum != calculatedChecksum) {
          throw new ChecksumException(
              "Transaction is corrupt. Calculated checksum is " +
              calculatedChecksum + " but read checksum " + readChecksum, txid);
        }
      }
    }
View Full Code Here
601
602
603
604
605
606
607
608
609
610
611
      checksum.reset();
      int dLen = Math.min(dLeft, chunkSize);
      checksum.update(buf, dOff, dLen);
      if (!checksum.compare(buf, cOff)) {
        long failedPos = offset + datalen - dLeft;
        throw new ChecksumException("Checksum failed at " + failedPos,
            failedPos);
      }
      dLeft -= dLen;
      dOff += dLen;
      cOff += checksumSize;
View Full Code Here
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
    }
   
    int bytesRead = in.read(b, off, len);

    if (bytesRead < 0) {
      throw new ChecksumException("Checksum Error", 0);
    }
   
    sum.update(b,off,bytesRead);

    currentOffset += bytesRead;

    if (disableChecksumValidation) {
      return bytesRead;
    }
   
    if (currentOffset == dataLength) {
      // The last four bytes are checksum. Strip them and verify
      csum = new byte[checksumSize];
      IOUtils.readFully(in, csum, 0, checksumSize);
      if (!sum.compare(csum, 0)) {
        throw new ChecksumException("Checksum Error", 0);
      }
    }
    return bytesRead;
  }
View Full Code Here
425
426
427
428
429
430
431
432
433
434
435
    while ((read = inputStream.read(buf)) > -1) {
      LOG.info(String.format("read %d bytes", read));

      if (!validateSequentialBytes(buf, (int) (startPos + numRead), read)) {
        LOG.error(String.format("invalid bytes: [%s]\n", Arrays.toString(buf)));
        throw new ChecksumException(
          String.format("unable to validate bytes"),
          startPos
        );
      }
View Full Code Here
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
        throws IOException {
      if (checksum != null) {
        int calculatedChecksum = (int)checksum.getValue();
        int readChecksum = in.readInt(); // read in checksum
        if (readChecksum != calculatedChecksum) {
          throw new ChecksumException(
              "Transaction is corrupt. Calculated checksum is " +
              calculatedChecksum + " but read checksum " + readChecksum, txid);
        }
      }
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.fs.ChecksumException

  • com.linkedin.camus.etl.kafka.mapred.EtlRecordReader
  • org.apache.hadoop.dfs.DataNode$BlockSender
  • org.apache.hadoop.hdfs.BlockReaderLocalInlineChecksum
  • org.apache.hadoop.hdfs.BlockReaderLocalWithChecksum
  • org.apache.hadoop.hdfs.notifier.server.TestTransactionalEditLogReader$TestTransactionalEditLogReaderHandler
  • org.apache.hadoop.hdfs.server.datanode.BlockInlineChecksumReader
  • org.apache.hadoop.hdfs.server.datanode.BlockSender
  • org.apache.hadoop.hdfs.server.datanode.BlockWithChecksumFileReader
  • org.apache.hadoop.hdfs.server.datanode.RaidBlockSender
  • org.apache.hadoop.hdfs.server.namenode.FSEditLog
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.