Examples of EditLogValidation


Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

    File logFile = prepareUnfinalizedTestEditLog(testDir, NUM_TXNS,
        offsetToTxId);
    // Back up the uncorrupted log
    File logFileBak = new File(testDir, logFile.getName() + ".bak");
    Files.copy(logFile, logFileBak);
    EditLogValidation validation =
        EditLogFileInputStream.validateEditLog(logFile);
    assertTrue(!validation.hasCorruptHeader());
    // We expect that there will be an OP_START_LOG_SEGMENT, followed by
    // NUM_TXNS opcodes, followed by an OP_END_LOG_SEGMENT.
    assertEquals(NUM_TXNS + 1, validation.getEndTxId());
    // Corrupt each edit and verify that validation continues to work
    for (Map.Entry<Long, Long> entry : offsetToTxId.entrySet()) {
      long txOffset = entry.getKey();
      long txId = entry.getValue();

      // Restore backup, corrupt the txn opcode
      Files.copy(logFileBak, logFile);
      corruptByteInFile(logFile, txOffset);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      long expectedEndTxId = (txId == (NUM_TXNS + 1)) ?
          NUM_TXNS : (NUM_TXNS + 1);
      assertEquals("Failed when corrupting txn opcode at " + txOffset,
          expectedEndTxId, validation.getEndTxId());
      assertTrue(!validation.hasCorruptHeader());
    }

    // Truncate right before each edit and verify that validation continues
    // to work
    for (Map.Entry<Long, Long> entry : offsetToTxId.entrySet()) {
      long txOffset = entry.getKey();
      long txId = entry.getValue();

      // Restore backup, corrupt the txn opcode
      Files.copy(logFileBak, logFile);
      truncateFile(logFile, txOffset);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      long expectedEndTxId = (txId == 0) ?
          HdfsConstants.INVALID_TXID : (txId - 1);
      assertEquals("Failed when corrupting txid " + txId + " txn opcode " +
        "at " + txOffset, expectedEndTxId, validation.getEndTxId());
      assertTrue(!validation.hasCorruptHeader());
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

    File testDir = new File(TEST_DIR, "testValidateEmptyEditLog");
    SortedMap<Long, Long> offsetToTxId = Maps.newTreeMap();
    File logFile = prepareUnfinalizedTestEditLog(testDir, 0, offsetToTxId);
    // Truncate the file so that there is nothing except the header
    truncateFile(logFile, 4);
    EditLogValidation validation =
        EditLogFileInputStream.validateEditLog(logFile);
    assertTrue(!validation.hasCorruptHeader());
    assertEquals(HdfsConstants.INVALID_TXID, validation.getEndTxId());
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

    logFile = testDir.listFiles()[0];
    long validLength = getNonTrailerLength(logFile);

    // Make sure that uncorrupted log has the expected length and number
    // of transactions.
    EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile);
    assertEquals(NUM_TXNS + 2, validation.numTransactions);
    assertEquals(validLength, validation.validLength);
   
    // Back up the uncorrupted log
    File logFileBak = new File(testDir, logFile.getName() + ".bak");
    Files.copy(logFile, logFileBak);

    // Corrupt the log file in various ways for each txn
    for (Map.Entry<Long, Long> entry : offsetToTxId.entrySet()) {
      long txOffset = entry.getKey();
      long txid = entry.getValue();
     
      // Restore backup, truncate the file exactly before the txn
      Files.copy(logFileBak, logFile);
      truncateFile(logFile, txOffset);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      assertEquals("Failed when truncating to length " + txOffset,
          txid - 1, validation.numTransactions);
      assertEquals(txOffset, validation.validLength);

      // Restore backup, truncate the file with one byte in the txn,
      // also isn't valid
      Files.copy(logFileBak, logFile);
      truncateFile(logFile, txOffset + 1);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      assertEquals("Failed when truncating to length " + (txOffset + 1),
          txid - 1, validation.numTransactions);
      assertEquals(txOffset, validation.validLength);

      // Restore backup, corrupt the txn opcode
      Files.copy(logFileBak, logFile);
      corruptByteInFile(logFile, txOffset);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      assertEquals("Failed when corrupting txn opcode at " + txOffset,
          txid - 1, validation.numTransactions);
      assertEquals(txOffset, validation.validLength);

      // Restore backup, corrupt a byte a few bytes into the txn
      Files.copy(logFileBak, logFile);
      corruptByteInFile(logFile, txOffset+5);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      assertEquals("Failed when corrupting txn data at " + (txOffset+5),
          txid - 1, validation.numTransactions);
      assertEquals(txOffset, validation.validLength);
    }
   
    // Corrupt the log at every offset to make sure that validation itself
    // never throws an exception, and that the calculated lengths are monotonically
    // increasing
    long prevNumValid = 0;
    for (long offset = 0; offset < validLength; offset++) {
      Files.copy(logFileBak, logFile);
      corruptByteInFile(logFile, offset);
      EditLogValidation val = EditLogFileInputStream.validateEditLog(logFile);
      assertTrue(val.numTransactions >= prevNumValid);
      prevNumValid = val.numTransactions;
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

     * Find out where the edit log ends.
     * This will update the lastTxId of the EditLogFile or
     * mark it as corrupt if it is.
     */
    public void validateLog() throws IOException {
      EditLogValidation val = EditLogFileInputStream.validateEditLog(file);
      this.lastTxId = val.getEndTxId();
      this.hasCorruptHeader = val.hasCorruptHeader();
    }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

    File testDir = new File(TEST_DIR, "testValidateEmptyEditLog");
    SortedMap<Long, Long> offsetToTxId = Maps.newTreeMap();
    File logFile = prepareUnfinalizedTestEditLog(testDir, 0, offsetToTxId);
    // Truncate the file so that there is nothing except the header
    truncateFile(logFile, 4);
    EditLogValidation validation =
        EditLogFileInputStream.validateEditLog(logFile);
    assertTrue(!validation.hasCorruptHeader());
    assertEquals(HdfsConstants.INVALID_TXID, validation.getEndTxId());
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

      rwf.seek(0);
      rwf.writeLong(42); // corrupt header
    } finally {
      rwf.close();
    }
    EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile);
    assertTrue(validation.hasCorruptHeader());
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

    File logFile = prepareUnfinalizedTestEditLog(testDir, NUM_TXNS,
        offsetToTxId);
    // Back up the uncorrupted log
    File logFileBak = new File(testDir, logFile.getName() + ".bak");
    Files.copy(logFile, logFileBak);
    EditLogValidation validation =
        EditLogFileInputStream.validateEditLog(logFile);
    assertTrue(!validation.hasCorruptHeader());
    // We expect that there will be an OP_START_LOG_SEGMENT, followed by
    // NUM_TXNS opcodes, followed by an OP_END_LOG_SEGMENT.
    assertEquals(NUM_TXNS, validation.getEndTxId());
    // Corrupt each edit and verify that validation continues to work
    for (Map.Entry<Long, Long> entry : offsetToTxId.entrySet()) {
      long txOffset = entry.getKey();
      long txId = entry.getValue();

      // Restore backup, corrupt the txn opcode
      Files.copy(logFileBak, logFile);
      corruptByteInFile(logFile, txOffset);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      long expectedEndTxId = (txId == (NUM_TXNS)) ?
          NUM_TXNS - 1 : (NUM_TXNS);
      assertEquals("Failed when corrupting txn opcode at " + txOffset,
          expectedEndTxId, validation.getEndTxId());
      assertTrue(!validation.hasCorruptHeader());
    }

    // Truncate right before each edit and verify that validation continues
    // to work
    for (Map.Entry<Long, Long> entry : offsetToTxId.entrySet()) {
      long txOffset = entry.getKey();
      long txId = entry.getValue();

      // Restore backup, corrupt the txn opcode
      Files.copy(logFileBak, logFile);
      truncateFile(logFile, txOffset);
      validation = EditLogFileInputStream.validateEditLog(logFile);
      long expectedEndTxId = (txId == 0) ?
          HdfsConstants.INVALID_TXID : (txId - 1);
      assertEquals("Failed when corrupting txid " + txId + " txn opcode " +
        "at " + txOffset, expectedEndTxId, validation.getEndTxId());
      assertTrue(!validation.hasCorruptHeader());
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

    StorageDirectory sd = cluster.getNameNode().getFSImage()
      .storage.getStorageDir(0);
    File editLog = NNStorage.getInProgressEditsFile(sd, 0);

    EditLogValidation validation = EditLogFileInputStream.validateEditLog(editLog);
    assertEquals("Edit log should contain a header as valid length",
        HEADER_LEN, validation.getValidLength());
    assertEquals(1, validation.getNumTransactions());
    assertEquals("Edit log should have 1MB pre-allocated",
        PREALLOCATION_LENGTH, editLog.length());
   

    cluster.getFileSystem().mkdirs(new Path("/tmp"),
        new FsPermission((short)777));

    long oldLength = validation.getValidLength();
    validation = EditLogFileInputStream.validateEditLog(editLog);
    assertTrue("Edit log should have more valid data after writing a txn " +
        "(was: " + oldLength + " now: " + validation.getValidLength() + ")",
        validation.getValidLength() > oldLength);
    assertEquals(2, validation.getNumTransactions());

    assertEquals("Edit log should be 1MB long, plus 4 bytes for the version number",
        PREALLOCATION_LENGTH, editLog.length());
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation

     * Count the number of valid transactions in a log.
     * This will update the lastTxId of the EditLogFile or
     * mark it as corrupt if it is.
     */
    public void validateLog() throws IOException {
      EditLogValidation val = EditLogFileInputStream.validateEditLog(file);
      if (val.getNumTransactions() == 0) {
        markCorrupt();
      } else {
        this.lastTxId = val.getEndTxId();
      }
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.