Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction$ReplicaUnderConstruction

Package org.apache.hadoop.hdfs.server.blockmanagement

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction$ReplicaUnderConstruction

org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction
ReplicaUnderConstruction contains information about replicas while they are under construction. The GS, the length and the state of the replica is as reported by the data-node. It is not guaranteed, but expected, that data-nodes actually have corresponding replicas.

      }
    } else { // the penultimate block is null
      Preconditions.checkState(oldBlocks == null || oldBlocks.length == 0);
    }
    // add the new block
    BlockInfo newBI = new BlockInfoUnderConstruction(
          newBlock, file.getBlockReplication());
    fsNamesys.getBlockManager().addBlockCollection(newBI, file);
    file.addBlock(newBI);
    fsNamesys.getBlockManager().processQueuedMessagesForBlock(newBlock);
  }

View Full Code Here

        BlockInfo newBI;
        if (!op.shouldCompleteLastBlock()) {
          // TODO: shouldn't this only be true for the last block?
          // what about an old-version fsync() where fsync isn't called
          // until several blocks in?
          newBI = new BlockInfoUnderConstruction(
              newBlock, file.getBlockReplication());
        } else {
          // OP_CLOSE should add finalized blocks. This code path
          // is only executed when loading edits written by prior
          // versions of Hadoop. Current versions always log

View Full Code Here

    final DatanodeStorageInfo[] targets = {};


    FSNamesystem namesystem = new FSNamesystem(conf, image);
    namesystem.setImageLoaded(true);
    FSNamesystem namesystemSpy = spy(namesystem);
    BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction(
        block, 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets);
    blockInfo.setBlockCollection(file);
    blockInfo.setGenerationStamp(genStamp);
    blockInfo.initializeBlockRecovery(genStamp);
    doReturn(true).when(file).removeLastBlock(any(Block.class));
    doReturn(true).when(file).isUnderConstruction();


    doReturn(blockInfo).when(namesystemSpy).getStoredBlock(any(Block.class));
    doReturn("").when(namesystemSpy).closeFileCommitBlocks(

View Full Code Here

          " Try again later.";
      NameNode.stateChangeLog.warn(message);
      throw new AlreadyBeingCreatedException(message);
    case UNDER_CONSTRUCTION:
    case UNDER_RECOVERY:
      final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)lastBlock;
      // setup the last block locations from the blockManager if not known
      if (uc.getNumExpectedLocations() == 0) {
        uc.setExpectedLocations(blockManager.getStorages(lastBlock));
      }


      if (uc.getNumExpectedLocations() == 0 && uc.getNumBytes() == 0) {
        // There is no datanode reported to this block.
        // may be client have crashed before writing data to pipeline.
        // This blocks doesn't need any recovery.
        // We can remove this block and close the file.
        pendingFile.removeLastBlock(lastBlock);
        finalizeINodeFileUnderConstruction(src, pendingFile,
            iip.getLatestSnapshotId());
        NameNode.stateChangeLog.warn("BLOCK* internalReleaseLease: "
            + "Removed empty last block and closed file.");
        return true;
      }
      // start recovery of the last block for this file
      long blockRecoveryId = nextGenerationStamp(isLegacyBlock(uc));
      lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
      uc.initializeBlockRecovery(blockRecoveryId);
      leaseManager.renewLease(lease);
      // Cannot close file right now, since the last block requires recovery.
      // This may potentially cause infinite loop in lease recovery
      // if there are no valid replicas on data-nodes.
      NameNode.stateChangeLog.warn(

View Full Code Here

      boolean logRetryCache)
      throws IOException {
    assert hasWriteLock();
    // check the vadility of the block and lease holder name
    final INodeFile pendingFile = checkUCBlock(oldBlock, clientName);
    final BlockInfoUnderConstruction blockinfo
        = (BlockInfoUnderConstruction)pendingFile.getLastBlock();


    // check new GS & length: this is not expected
    if (newBlock.getGenerationStamp() <= blockinfo.getGenerationStamp() ||
        newBlock.getNumBytes() < blockinfo.getNumBytes()) {
      String msg = "Update " + oldBlock + " (len = " + 
        blockinfo.getNumBytes() + ") to an older state: " + newBlock + 
        " (len = " + newBlock.getNumBytes() +")";
      LOG.warn(msg);
      throw new IOException(msg);
    }


    // Update old block with the new generation stamp and new length
    blockinfo.setNumBytes(newBlock.getNumBytes());
    blockinfo.setGenerationStampAndVerifyReplicas(newBlock.getGenerationStamp());


    // find the DatanodeDescriptor objects
    final DatanodeStorageInfo[] storages = blockManager.getDatanodeManager()
        .getDatanodeStorageInfos(newNodes, newStorageIDs);
    blockinfo.setExpectedLocations(storages);


    String src = pendingFile.getFullPathName();
    persistBlocks(src, pendingFile, logRetryCache);
  }

View Full Code Here

    FSImage image = new FSImage(conf);
    DatanodeDescriptor[] targets = new DatanodeDescriptor[0];


    FSNamesystem namesystem = new FSNamesystem(conf, image);
    FSNamesystem namesystemSpy = spy(namesystem);
    BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction(
        block, 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets);
    blockInfo.setBlockCollection(file);
    blockInfo.setGenerationStamp(genStamp);
    blockInfo.initializeBlockRecovery(genStamp);
    doReturn(true).when(file).removeLastBlock(any(Block.class));


    doReturn(blockInfo).when(namesystemSpy).getStoredBlock(any(Block.class));
    doReturn("").when(namesystemSpy).closeFileCommitBlocks(
        any(INodeFileUnderConstruction.class),

View Full Code Here


    @Override
    boolean checkNamenodeBeforeReturn() throws Exception {
      INodeFileUnderConstruction fileNode = (INodeFileUnderConstruction) cluster
          .getNamesystem(0).getFSDirectory().getINode4Write(file).asFile();
      BlockInfoUnderConstruction blkUC = 
          (BlockInfoUnderConstruction) (fileNode.getBlocks())[1];
      int datanodeNum = blkUC.getExpectedLocations().length;
      for (int i = 0; i < CHECKTIMES && datanodeNum != 2; i++) {
        Thread.sleep(1000);
        datanodeNum = blkUC.getExpectedLocations().length;
      }
      return datanodeNum == 2;
    }

View Full Code Here

          " Try again later.";
      NameNode.stateChangeLog.warn(message);
      throw new AlreadyBeingCreatedException(message);
    case UNDER_CONSTRUCTION:
    case UNDER_RECOVERY:
      final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)lastBlock;
      // setup the last block locations from the blockManager if not known
      if (uc.getNumExpectedLocations() == 0) {
        uc.setExpectedLocations(blockManager.getNodes(lastBlock));
      }
      // start recovery of the last block for this file
      long blockRecoveryId = nextGenerationStamp();
      lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
      uc.initializeBlockRecovery(blockRecoveryId);
      leaseManager.renewLease(lease);
      // Cannot close file right now, since the last block requires recovery.
      // This may potentially cause infinite loop in lease recovery
      // if there are no valid replicas on data-nodes.
      NameNode.stateChangeLog.warn(

View Full Code Here

      throws IOException {
    assert hasWriteLock();
    // check the vadility of the block and lease holder name
    final INodeFileUnderConstruction pendingFile
        = checkUCBlock(oldBlock, clientName);
    final BlockInfoUnderConstruction blockinfo
        = (BlockInfoUnderConstruction)pendingFile.getLastBlock();


    // check new GS & length: this is not expected
    if (newBlock.getGenerationStamp() <= blockinfo.getGenerationStamp() ||
        newBlock.getNumBytes() < blockinfo.getNumBytes()) {
      String msg = "Update " + oldBlock + " (len = " + 
        blockinfo.getNumBytes() + ") to an older state: " + newBlock + 
        " (len = " + newBlock.getNumBytes() +")";
      LOG.warn(msg);
      throw new IOException(msg);
    }


    // Update old block with the new generation stamp and new length
    blockinfo.setGenerationStamp(newBlock.getGenerationStamp());
    blockinfo.setNumBytes(newBlock.getNumBytes());


    // find the DatanodeDescriptor objects
    final DatanodeManager dm = getBlockManager().getDatanodeManager();
    DatanodeDescriptor[] descriptors = null;
    if (newNodes.length > 0) {
      descriptors = new DatanodeDescriptor[newNodes.length];
      for(int i = 0; i < newNodes.length; i++) {
        descriptors[i] = dm.getDatanode(newNodes[i]);
      }
    }
    blockinfo.setExpectedLocations(descriptors);


    String src = leaseManager.findPath(pendingFile);
    dir.persistBlocks(src, pendingFile);
  }

View Full Code Here

  throws IOException {
    if (blocks == null || blocks.length == 0) {
      throw new IOException("Trying to update non-existant block. " +
          "File is empty.");
    }
    BlockInfoUnderConstruction ucBlock =
      lastBlock.convertToBlockUnderConstruction(
          BlockUCState.UNDER_CONSTRUCTION, targets);
    ucBlock.setBlockCollection(this);
    setBlock(numBlocks()-1, ucBlock);
    return ucBlock;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction$ReplicaUnderConstruction

org.apache.hadoop.hdfs.server.namenode.FileUnderConstructionFeature

org.apache.hadoop.hdfs.server.namenode.FSDirectory

org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader

org.apache.hadoop.hdfs.server.namenode.FSImageFormat$Loader

org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode$Loader

org.apache.hadoop.hdfs.server.namenode.FSImageSerialization

org.apache.hadoop.hdfs.server.namenode.FSNamesystem

org.apache.hadoop.hdfs.server.namenode.ha.TestPipelinesFailover

org.apache.hadoop.hdfs.server.namenode.ha.TestRetryCacheWithHA$UpdatePipelineOp

org.apache.hadoop.hdfs.server.namenode.INodeFile

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.