Examples of FSImageFile


Examples of org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile

   * @throws IOException
   */
  boolean loadFSImage(FSNamesystem target, MetaRecoveryContext recovery)
      throws IOException {
    FSImageStorageInspector inspector = storage.readAndInspectDirs();
    FSImageFile imageFile = null;
   
    isUpgradeFinalized = inspector.isUpgradeFinalized();
    List<FSImageFile> imageFiles = inspector.getLatestImages();

    StartupProgress prog = NameNode.getStartupProgress();
    prog.beginPhase(Phase.LOADING_FSIMAGE);
    File phaseFile = imageFiles.get(0).getFile();
    prog.setFile(Phase.LOADING_FSIMAGE, phaseFile.getAbsolutePath());
    prog.setSize(Phase.LOADING_FSIMAGE, phaseFile.length());
    boolean needToSave = inspector.needToSave();

    Iterable<EditLogInputStream> editStreams = null;

    initEditLog();

    if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT,
                               getLayoutVersion())) {
      // If we're open for write, we're either non-HA or we're the active NN, so
      // we better be able to load all the edits. If we're the standby NN, it's
      // OK to not be able to read all of edits right now.
      long toAtLeastTxId = editLog.isOpenForWrite() ? inspector.getMaxSeenTxId() : 0;
      editStreams = editLog.selectInputStreams(
          imageFiles.get(0).getCheckpointTxId() + 1,
          toAtLeastTxId, recovery, false);
    } else {
      editStreams = FSImagePreTransactionalStorageInspector
        .getEditLogStreams(storage);
    }
    int maxOpSize = conf.getInt(DFSConfigKeys.
          DFS_NAMENODE_MAX_OP_SIZE_KEY,
        DFSConfigKeys.DFS_NAMENODE_MAX_OP_SIZE_DEFAULT);
    for (EditLogInputStream elis : editStreams) {
      elis.setMaxOpSize(maxOpSize);
    }
    for (EditLogInputStream l : editStreams) {
      LOG.debug("Planning to load edit log stream: " + l);
    }
    if (!editStreams.iterator().hasNext()) {
      LOG.info("No edit log streams selected.");
    }
   
    for (int i = 0; i < imageFiles.size(); i++) {
      try {
        imageFile = imageFiles.get(i);
        loadFSImageFile(target, recovery, imageFile);
        break;
      } catch (IOException ioe) {
        LOG.error("Failed to load image from " + imageFile, ioe);
        target.clear();
        imageFile = null;
      }
    }
    // Failed to load any images, error out
    if (imageFile == null) {
      FSEditLog.closeAllStreams(editStreams);
      throw new IOException("Failed to load an FSImage file!");
    }
    prog.endPhase(Phase.LOADING_FSIMAGE);
    long txnsAdvanced = loadEdits(editStreams, target, recovery);
    needToSave |= needsResaveBasedOnStaleCheckpoint(imageFile.getFile(),
                                                    txnsAdvanced);
    editLog.setNextTxId(lastAppliedTxId + 1);
    return needToSave;
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile

    }
    if (!editStreams.iterator().hasNext()) {
      LOG.info("No edit log streams selected.");
    }
   
    FSImageFile imageFile = null;
    for (int i = 0; i < imageFiles.size(); i++) {
      try {
        imageFile = imageFiles.get(i);
        loadFSImageFile(target, recovery, imageFile);
        break;
      } catch (IOException ioe) {
        LOG.error("Failed to load image from " + imageFile, ioe);
        target.clear();
        imageFile = null;
      }
    }
    // Failed to load any images, error out
    if (imageFile == null) {
      FSEditLog.closeAllStreams(editStreams);
      throw new IOException("Failed to load an FSImage file!");
    }
    prog.endPhase(Phase.LOADING_FSIMAGE);
   
    if (!rollingRollback) {
      long txnsAdvanced = loadEdits(editStreams, target, startOpt, recovery);
      needToSave |= needsResaveBasedOnStaleCheckpoint(imageFile.getFile(),
          txnsAdvanced);
      if (RollingUpgradeStartupOption.DOWNGRADE.matches(startOpt)) {
        // rename rollback image if it is downgrade
        renameCheckpoint(NameNodeFile.IMAGE_ROLLBACK, NameNodeFile.IMAGE);
      }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile

   
    List<File> imageFiles = new ArrayList<File>();
    for (File dir : dirs) {
      FSImageTransactionalStorageInspector inspector =
        inspectStorageDirectory(dir, NameNodeDirType.IMAGE);
      FSImageFile latestImage = inspector.getLatestImage();
      assertNotNull("No image in " + dir, latestImage);     
      long thisTxId = latestImage.getCheckpointTxId();
      if (imageTxId != -1 && thisTxId != imageTxId) {
        fail("Storage directory " + dir + " does not have the same " +
            "last image index " + imageTxId + " as another");
      }
      imageTxId = thisTxId;
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile

    FSImageTransactionalStorageInspector inspector =
      new FSImageTransactionalStorageInspector();
    inspector.inspectDirectory(sd);

    FSImageFile latestImage = inspector.getLatestImage();
    return (latestImage == null) ? null : latestImage.getFile();
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile

        "/foo/current/" + getInProgressEditsFileName(457));

    inspector.inspectDirectory(mockDir);
    assertEquals(2, inspector.foundImages.size());

    FSImageFile latestImage = inspector.getLatestImage();
    assertEquals(456, latestImage.txId);
    assertSame(mockDir, latestImage.sd);
    assertTrue(inspector.isUpgradeFinalized());
   
    assertEquals(new File("/foo/current/"+getImageFileName(456)),
        latestImage.getFile());
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile

   * @throws IOException
   */
  boolean loadFSImage(FSNamesystem target, MetaRecoveryContext recovery)
      throws IOException {
    FSImageStorageInspector inspector = storage.readAndInspectDirs();
    FSImageFile imageFile = null;
   
    isUpgradeFinalized = inspector.isUpgradeFinalized();
    List<FSImageFile> imageFiles = inspector.getLatestImages();

    StartupProgress prog = NameNode.getStartupProgress();
    prog.beginPhase(Phase.LOADING_FSIMAGE);
    File phaseFile = imageFiles.get(0).getFile();
    prog.setFile(Phase.LOADING_FSIMAGE, phaseFile.getAbsolutePath());
    prog.setSize(Phase.LOADING_FSIMAGE, phaseFile.length());
    boolean needToSave = inspector.needToSave();

    Iterable<EditLogInputStream> editStreams = null;

    initEditLog();

    if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT,
                               getLayoutVersion())) {
      // If we're open for write, we're either non-HA or we're the active NN, so
      // we better be able to load all the edits. If we're the standby NN, it's
      // OK to not be able to read all of edits right now.
      long toAtLeastTxId = editLog.isOpenForWrite() ? inspector.getMaxSeenTxId() : 0;
      editStreams = editLog.selectInputStreams(
          imageFiles.get(0).getCheckpointTxId() + 1,
          toAtLeastTxId, recovery, false);
    } else {
      editStreams = FSImagePreTransactionalStorageInspector
        .getEditLogStreams(storage);
    }
    int maxOpSize = conf.getInt(DFSConfigKeys.
          DFS_NAMENODE_MAX_OP_SIZE_KEY,
        DFSConfigKeys.DFS_NAMENODE_MAX_OP_SIZE_DEFAULT);
    for (EditLogInputStream elis : editStreams) {
      elis.setMaxOpSize(maxOpSize);
    }
    for (EditLogInputStream l : editStreams) {
      LOG.debug("Planning to load edit log stream: " + l);
    }
    if (!editStreams.iterator().hasNext()) {
      LOG.info("No edit log streams selected.");
    }
   
    for (int i = 0; i < imageFiles.size(); i++) {
      try {
        imageFile = imageFiles.get(i);
        loadFSImageFile(target, recovery, imageFile);
        break;
      } catch (IOException ioe) {
        LOG.error("Failed to load image from " + imageFile, ioe);
        target.clear();
        imageFile = null;
      }
    }
    // Failed to load any images, error out
    if (imageFile == null) {
      FSEditLog.closeAllStreams(editStreams);
      throw new IOException("Failed to load an FSImage file!");
    }
    prog.endPhase(Phase.LOADING_FSIMAGE);
    long txnsAdvanced = loadEdits(editStreams, target, recovery);
    needToSave |= needsResaveBasedOnStaleCheckpoint(imageFile.getFile(),
                                                    txnsAdvanced);
    editLog.setNextTxId(lastAppliedTxId + 1);
    return needToSave;
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile

        "/foo/current/" + getInProgressEditsFileName(457));

    inspector.inspectDirectory(mockDir);
    assertEquals(2, inspector.foundImages.size());

    FSImageFile latestImage = inspector.getLatestImages().get(0);
    assertEquals(456, latestImage.txId);
    assertSame(mockDir, latestImage.sd);
    assertTrue(inspector.isUpgradeFinalized());
   
    assertEquals(new File("/foo/current/"+getImageFileName(456)),
        latestImage.getFile());
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile

   
    List<File> imageFiles = new ArrayList<File>();
    for (File dir : dirs) {
      FSImageTransactionalStorageInspector inspector =
        inspectStorageDirectory(dir, NameNodeDirType.IMAGE);
      FSImageFile latestImage = inspector.getLatestImage();
      assertNotNull("No image in " + dir, latestImage);     
      long thisTxId = latestImage.getCheckpointTxId();
      if (imageTxId != -1 && thisTxId != imageTxId) {
        fail("Storage directory " + dir + " does not have the same " +
            "last image index " + imageTxId + " as another");
      }
      imageTxId = thisTxId;
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile

    FSImageTransactionalStorageInspector inspector =
      new FSImageTransactionalStorageInspector();
    inspector.inspectDirectory(sd);

    FSImageFile latestImage = inspector.getLatestImage();
    return (latestImage == null) ? null : latestImage.getFile();
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile

   
    assertEquals(2, inspector.foundEditLogs.size());
    assertEquals(2, inspector.foundImages.size());
    assertTrue(inspector.foundEditLogs.get(1).isInProgress());
   
    FSImageFile latestImage = inspector.getLatestImage();
    assertEquals(456, latestImage.txId);
    assertSame(mockDir, latestImage.sd);
    assertTrue(inspector.isUpgradeFinalized());
   
    LoadPlan plan = inspector.createLoadPlan();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.