Examples of org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress

Package org.apache.hadoop.hdfs.server.namenode.startupprogress

Examples of org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress

org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress
StartupProgress is used in various parts of the namenode codebase to indicate startup progress. Its methods provide ways to indicate begin and end of a {@link Phase} or {@link Step} within a phase. Additional methods provide waysto associate a step or phase with optional information, such as a file name or file size. It also provides counters, which can be incremented by the caller to indicate progress through a long-running task. This class is thread-safe. Any number of threads may call any methods, even for the same phase or step, without risk of corrupting internal state. For all begin/end methods and set methods, the last one in wins, overwriting any prior writes. Instances of {@link Counter} provide an atomic incrementoperation to prevent lost updates. After startup completes, the tracked data is frozen. Any subsequent updates or counter increments are no-ops. For read access, call {@link #createView()} to create a consistent view witha clone of the data.

   * This is where we apply edits that we've been writing to disk all
   * along.
   */
  long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId,
      StartupOption startOpt, MetaRecoveryContext recovery) throws IOException {
    StartupProgress prog = NameNode.getStartupProgress();
    Step step = createStartupProgressStep(edits);
    prog.beginStep(Phase.LOADING_EDITS, step);
    fsNamesys.writeLock();
    try {
      long startTime = now();
      FSImage.LOG.info("Start loading edits file " + edits.getName());
      long numEdits = loadEditRecords(edits, false, expectedStartingTxId,
          startOpt, recovery);
      FSImage.LOG.info("Edits file " + edits.getName() 
          + " of size " + edits.length() + " edits # " + numEdits 
          + " loaded in " + (now()-startTime)/1000 + " seconds");
      return numEdits;
    } finally {
      edits.close();
      fsNamesys.writeUnlock();
      prog.endStep(Phase.LOADING_EDITS, step);
    }
  }

View Full Code Here

    
    long expectedTxId = expectedStartingTxId;
    long numEdits = 0;
    long lastTxId = in.getLastTxId();
    long numTxns = (lastTxId - expectedStartingTxId) + 1;
    StartupProgress prog = NameNode.getStartupProgress();
    Step step = createStartupProgressStep(in);
    prog.setTotal(Phase.LOADING_EDITS, step, numTxns);
    Counter counter = prog.getCounter(Phase.LOADING_EDITS, step);
    long lastLogTime = now();
    long lastInodeId = fsNamesys.getLastInodeId();
    
    try {
      while (true) {

View Full Code Here

          + blockManager.numOfUnderReplicatedBlocks() + " blocks");


      startSecretManagerIfNecessary();


      // If startup has not yet completed, end safemode phase.
      StartupProgress prog = NameNode.getStartupProgress();
      if (prog.getStatus(Phase.SAFEMODE) != Status.COMPLETE) {
        prog.endStep(Phase.SAFEMODE, STEP_AWAITING_REPORTED_BLOCKS);
        prog.endPhase(Phase.SAFEMODE);
      }
    }

View Full Code Here

    private synchronized void incrementSafeBlockCount(short replication) {
      if (replication == safeReplication) {
        this.blockSafe++;


        // Report startup progress only if we haven't completed startup yet.
        StartupProgress prog = NameNode.getStartupProgress();
        if (prog.getStatus(Phase.SAFEMODE) != Status.COMPLETE) {
          if (this.awaitingReportedBlocksCounter == null) {
            this.awaitingReportedBlocksCounter = prog.getCounter(Phase.SAFEMODE,
              STEP_AWAITING_REPORTED_BLOCKS);
          }
          this.awaitingReportedBlocksCounter.increment();
        }

View Full Code Here

          + ", isRollingUpgrade=" + isRollingUpgrade() + ")");
      if (needToSave) {
        fsImage.saveNamespace(this);
      } else {
        // No need to save, so mark the phase done.
        StartupProgress prog = NameNode.getStartupProgress();
        prog.beginPhase(Phase.SAVING_CHECKPOINT);
        prog.endPhase(Phase.SAVING_CHECKPOINT);
      }
      // This will start a new log segment and write to the seen_txid file, so
      // we shouldn't do it when coming up in standby state
      if (!haEnabled || (haEnabled && startOpt == StartupOption.UPGRADE)) {
        fsImage.openEditLogForWrite();

View Full Code Here

    this.haContext = haContext;
    try {
      nnResourceChecker = new NameNodeResourceChecker(conf);
      checkAvailableResources();
      assert safeMode != null && !isPopulatingReplQueues();
      StartupProgress prog = NameNode.getStartupProgress();
      prog.beginPhase(Phase.SAFEMODE);
      prog.setTotal(Phase.SAFEMODE, STEP_AWAITING_REPORTED_BLOCKS,
        getCompleteBlocksTotal());
      setBlockTotal();
      blockManager.activate(conf);
    } finally {
      writeUnlock();

View Full Code Here

          + blockManager.numOfUnderReplicatedBlocks() + " blocks");


      startSecretManagerIfNecessary();


      // If startup has not yet completed, end safemode phase.
      StartupProgress prog = NameNode.getStartupProgress();
      if (prog.getStatus(Phase.SAFEMODE) != Status.COMPLETE) {
        prog.endStep(Phase.SAFEMODE, STEP_AWAITING_REPORTED_BLOCKS);
        prog.endPhase(Phase.SAFEMODE);
      }
    }

View Full Code Here

        fsImage.recoverTransitionRead(startOpt, this, recovery) && !haEnabled;
      if (needToSave) {
        fsImage.saveNamespace(this);
      } else {
        // No need to save, so mark the phase done.
        StartupProgress prog = NameNode.getStartupProgress();
        prog.beginPhase(Phase.SAVING_CHECKPOINT);
        prog.endPhase(Phase.SAVING_CHECKPOINT);
      }
      // This will start a new log segment and write to the seen_txid file, so
      // we shouldn't do it when coming up in standby state
      if (!haEnabled) {
        fsImage.openEditLogForWrite();

View Full Code Here

    try {
      nnResourceChecker = new NameNodeResourceChecker(conf);
      checkAvailableResources();
      assert safeMode != null &&
        !safeMode.isPopulatingReplQueues();
      StartupProgress prog = NameNode.getStartupProgress();
      prog.beginPhase(Phase.SAFEMODE);
      prog.setTotal(Phase.SAFEMODE, STEP_AWAITING_REPORTED_BLOCKS,
        getCompleteBlocksTotal());
      setBlockTotal();
      blockManager.activate(conf);
    } finally {
      writeUnlock();

View Full Code Here

    private synchronized void incrementSafeBlockCount(short replication) {
      if (replication == safeReplication) {
        this.blockSafe++;


        // Report startup progress only if we haven't completed startup yet.
        StartupProgress prog = NameNode.getStartupProgress();
        if (prog.getStatus(Phase.SAFEMODE) != Status.COMPLETE) {
          if (this.awaitingReportedBlocksCounter == null) {
            this.awaitingReportedBlocksCounter = prog.getCounter(Phase.SAFEMODE,
              STEP_AWAITING_REPORTED_BLOCKS);
          }
          this.awaitingReportedBlocksCounter.increment();
        }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress

org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager

org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager$SerializerCompat

org.apache.hadoop.hdfs.server.namenode.CacheManager

org.apache.hadoop.hdfs.server.namenode.CacheManager$SerializerCompat

org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader

org.apache.hadoop.hdfs.server.namenode.FSImage

org.apache.hadoop.hdfs.server.namenode.FSImageFormat$Loader

org.apache.hadoop.hdfs.server.namenode.FSImageFormat$Saver

org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf$Loader

org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf$Saver

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.