Examples of EditLogValidation


Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

      this.lastTxId = val.getEndTxId();
      this.hasCorruptHeader = val.hasCorruptHeader();
    }

    public void scanLog() throws IOException {
      EditLogValidation val = EditLogFileInputStream.scanEditLog(file);
      this.lastTxId = val.getEndTxId();
      this.hasCorruptHeader = val.hasCorruptHeader();
    }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

    logFile = testDir.listFiles()[0];
    long validLength = getNonTrailerLength(logFile);

    // Make sure that uncorrupted log has the expected length and number
    // of transactions.
    EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile);
    assertEquals(NUM_TXNS + 2, validation.getNumTransactions());
    assertEquals(validLength, validation.getValidLength());
   
    // Back up the uncorrupted log
    File logFileBak = new File(testDir, logFile.getName() + ".bak");
    Files.copy(logFile, logFileBak);

    // Corrupt the log file in various ways for each txn
    for (Map.Entry<Long, Long> entry : offsetToTxId.entrySet()) {
      long txOffset = entry.getKey();
      long txid = entry.getValue();
     
      // Restore backup, truncate the file exactly before the txn
      Files.copy(logFileBak, logFile);
      truncateFile(logFile, txOffset);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      assertEquals("Failed when truncating to length " + txOffset,
          txid - 1, validation.getNumTransactions());
      assertEquals(txOffset, validation.getValidLength());

      // Restore backup, truncate the file with one byte in the txn,
      // also isn't valid
      Files.copy(logFileBak, logFile);
      truncateFile(logFile, txOffset + 1);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      assertEquals("Failed when truncating to length " + (txOffset + 1),
          txid - 1, validation.getNumTransactions());
      assertEquals(txOffset, validation.getValidLength());

      // Restore backup, corrupt the txn opcode
      Files.copy(logFileBak, logFile);
      corruptByteInFile(logFile, txOffset);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      assertEquals("Failed when corrupting txn opcode at " + txOffset,
          txid - 1, validation.getNumTransactions());
      assertEquals(txOffset, validation.getValidLength());

      // Restore backup, corrupt a byte a few bytes into the txn
      Files.copy(logFileBak, logFile);
      corruptByteInFile(logFile, txOffset+5);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      assertEquals("Failed when corrupting txn data at " + (txOffset+5),
          txid - 1, validation.getNumTransactions());
      assertEquals(txOffset, validation.getValidLength());
    }
   
    // Corrupt the log at every offset to make sure that validation itself
    // never throws an exception, and that the calculated lengths are monotonically
    // increasing
    long prevNumValid = 0;
    for (long offset = 0; offset < validLength; offset++) {
      Files.copy(logFileBak, logFile);
      corruptByteInFile(logFile, offset);
      EditLogValidation val = EditLogFileInputStream.validateEditLog(logFile);
      assertTrue(String.format("%d should have been >= %d",
          val.getNumTransactions(), prevNumValid),
          val.getNumTransactions() >= prevNumValid);
      prevNumValid = val.getNumTransactions();
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

        if (lastTxId == HdfsConstants.INVALID_TXID || txid > lastTxId) {
          lastTxId = txid;
        }
        numValid++;
      }
      return new EditLogValidation(lastPos, lastTxId, false);
    } finally {
      IOUtils.closeStream(in);
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

    logFile = testDir.listFiles()[0];
    long validLength = getNonTrailerLength(logFile);

    // Make sure that uncorrupted log has the expected length and number
    // of transactions.
    EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile);
    assertEquals(NUM_TXNS + 2, validation.numTransactions);
    assertEquals(validLength, validation.validLength);
   
    // Back up the uncorrupted log
    File logFileBak = new File(testDir, logFile.getName() + ".bak");
    Files.copy(logFile, logFileBak);

    // Corrupt the log file in various ways for each txn
    for (Map.Entry<Long, Long> entry : offsetToTxId.entrySet()) {
      long txOffset = entry.getKey();
      long txid = entry.getValue();
     
      // Restore backup, truncate the file exactly before the txn
      Files.copy(logFileBak, logFile);
      truncateFile(logFile, txOffset);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      assertEquals("Failed when truncating to length " + txOffset,
          txid - 1, validation.numTransactions);
      assertEquals(txOffset, validation.validLength);

      // Restore backup, truncate the file with one byte in the txn,
      // also isn't valid
      Files.copy(logFileBak, logFile);
      truncateFile(logFile, txOffset + 1);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      assertEquals("Failed when truncating to length " + (txOffset + 1),
          txid - 1, validation.numTransactions);
      assertEquals(txOffset, validation.validLength);

      // Restore backup, corrupt the txn opcode
      Files.copy(logFileBak, logFile);
      corruptByteInFile(logFile, txOffset);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      assertEquals("Failed when corrupting txn opcode at " + txOffset,
          txid - 1, validation.numTransactions);
      assertEquals(txOffset, validation.validLength);

      // Restore backup, corrupt a byte a few bytes into the txn
      Files.copy(logFileBak, logFile);
      corruptByteInFile(logFile, txOffset+5);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      assertEquals("Failed when corrupting txn data at " + (txOffset+5),
          txid - 1, validation.numTransactions);
      assertEquals(txOffset, validation.validLength);
    }
   
    // Corrupt the log at every offset to make sure that validation itself
    // never throws an exception, and that the calculated lengths are monotonically
    // increasing
    long prevNumValid = 0;
    for (long offset = 0; offset < validLength; offset++) {
      Files.copy(logFileBak, logFile);
      corruptByteInFile(logFile, offset);
      EditLogValidation val = EditLogFileInputStream.validateEditLog(logFile);
      assertTrue(val.numTransactions >= prevNumValid);
      prevNumValid = val.numTransactions;
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

     * Find out where the edit log ends.
     * This will update the lastTxId of the EditLogFile or
     * mark it as corrupt if it is.
     */
    public void validateLog() throws IOException {
      EditLogValidation val = EditLogFileInputStream.validateEditLog(file);
      this.lastTxId = val.getEndTxId();
      this.hasCorruptHeader = val.hasCorruptHeader();
    }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

      this.lastTxId = val.getEndTxId();
      this.hasCorruptHeader = val.hasCorruptHeader();
    }

    public void scanLog() throws IOException {
      EditLogValidation val = EditLogFileInputStream.scanEditLog(file);
      this.lastTxId = val.getEndTxId();
      this.hasCorruptHeader = val.hasCorruptHeader();
    }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

      rwf.seek(0);
      rwf.writeLong(42); // corrupt header
    } finally {
      rwf.close();
    }
    EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile);
    assertTrue(validation.hasCorruptHeader());
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

    File logFile = prepareUnfinalizedTestEditLog(testDir, NUM_TXNS,
        offsetToTxId);
    // Back up the uncorrupted log
    File logFileBak = new File(testDir, logFile.getName() + ".bak");
    Files.copy(logFile, logFileBak);
    EditLogValidation validation =
        EditLogFileInputStream.validateEditLog(logFile);
    assertTrue(!validation.hasCorruptHeader());
    // We expect that there will be an OP_START_LOG_SEGMENT, followed by
    // NUM_TXNS opcodes, followed by an OP_END_LOG_SEGMENT.
    assertEquals(NUM_TXNS + 1, validation.getEndTxId());
    // Corrupt each edit and verify that validation continues to work
    for (Map.Entry<Long, Long> entry : offsetToTxId.entrySet()) {
      long txOffset = entry.getKey();
      long txId = entry.getValue();

      // Restore backup, corrupt the txn opcode
      Files.copy(logFileBak, logFile);
      corruptByteInFile(logFile, txOffset);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      long expectedEndTxId = (txId == (NUM_TXNS + 1)) ?
          NUM_TXNS : (NUM_TXNS + 1);
      assertEquals("Failed when corrupting txn opcode at " + txOffset,
          expectedEndTxId, validation.getEndTxId());
      assertTrue(!validation.hasCorruptHeader());
    }

    // Truncate right before each edit and verify that validation continues
    // to work
    for (Map.Entry<Long, Long> entry : offsetToTxId.entrySet()) {
      long txOffset = entry.getKey();
      long txId = entry.getValue();

      // Restore backup, corrupt the txn opcode
      Files.copy(logFileBak, logFile);
      truncateFile(logFile, txOffset);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      long expectedEndTxId = (txId == 0) ?
          HdfsConstants.INVALID_TXID : (txId - 1);
      assertEquals("Failed when corrupting txid " + txId + " txn opcode " +
        "at " + txOffset, expectedEndTxId, validation.getEndTxId());
      assertTrue(!validation.hasCorruptHeader());
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

    SortedMap<Long, Long> offsetToTxId = Maps.newTreeMap();
    File logFile = prepareUnfinalizedTestEditLog(testDir, 0, offsetToTxId);
    // Truncate the file so that there is nothing except the header and
    // layout flags section.
    truncateFile(logFile, 8);
    EditLogValidation validation =
        EditLogFileInputStream.validateEditLog(logFile);
    assertTrue(!validation.hasCorruptHeader());
    assertEquals(HdfsConstants.INVALID_TXID, validation.getEndTxId());
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

      rwf.seek(0);
      rwf.writeLong(42); // corrupt header
    } finally {
      rwf.close();
    }
    EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile);
    assertTrue(validation.hasCorruptHeader());
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.