Examples of BlockInfoUnderConstruction

org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction
Represents a block that is currently being constructed.
This is usually the last block of a file opened for write or append.

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

      throw new IOException(message);
    }


    // no we know that the last block is not COMPLETE, and
    // that the penultimate block if exists is either COMPLETE or COMMITTED
    BlockInfoUnderConstruction lastBlock = pendingFile.getLastBlock();
    BlockUCState lastBlockState = lastBlock.getBlockUCState();
    BlockInfo penultimateBlock = pendingFile.getPenultimateBlock();
    boolean penultimateBlockMinReplication;
    BlockUCState penultimateBlockState;
    if (penultimateBlock == null) {
      penultimateBlockState = BlockUCState.COMPLETE;
      // If penultimate block doesn't exist then its minReplication is met
      penultimateBlockMinReplication = true;
    } else {
      penultimateBlockState = BlockUCState.COMMITTED;
      penultimateBlockMinReplication = 
        blockManager.checkMinReplication(penultimateBlock);
    }
    assert penultimateBlockState == BlockUCState.COMPLETE ||
           penultimateBlockState == BlockUCState.COMMITTED :
           "Unexpected state of penultimate block in " + src;


    switch(lastBlockState) {
    case COMPLETE:
      assert false : "Already checked that the last block is incomplete";
      break;
    case COMMITTED:
      // Close file if committed blocks are minimally replicated
      if(penultimateBlockMinReplication &&
          blockManager.checkMinReplication(lastBlock)) {
        finalizeINodeFileUnderConstruction(src, pendingFile);
        NameNode.stateChangeLog.warn("BLOCK*"
          + " internalReleaseLease: Committed blocks are minimally replicated,"
          + " lease removed, file closed.");
        return true;  // closed!
      }
      // Cannot close file right now, since some blocks 
      // are not yet minimally replicated.
      // This may potentially cause infinite loop in lease recovery
      // if there are no valid replicas on data-nodes.
      String message = "DIR* NameSystem.internalReleaseLease: " +
          "Failed to release lease for file " + src +
          ". Committed blocks are waiting to be minimally replicated." +
          " Try again later.";
      NameNode.stateChangeLog.warn(message);
      throw new AlreadyBeingCreatedException(message);
    case UNDER_CONSTRUCTION:
    case UNDER_RECOVERY:
      // setup the last block locations from the blockManager if not known
      if(lastBlock.getNumExpectedLocations() == 0)
        lastBlock.setExpectedLocations(blockManager.getNodes(lastBlock));
      // start recovery of the last block for this file
      long blockRecoveryId = nextGenerationStamp();
      lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
      lastBlock.initializeBlockRecovery(blockRecoveryId);
      leaseManager.renewLease(lease);
      // Cannot close file right now, since the last block requires recovery.
      // This may potentially cause infinite loop in lease recovery
      // if there are no valid replicas on data-nodes.
      NameNode.stateChangeLog.warn(

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

      throws IOException {
    assert hasWriteLock();
    // check the vadility of the block and lease holder name
    final INodeFileUnderConstruction pendingFile = 
      checkUCBlock(oldBlock, clientName);
    final BlockInfoUnderConstruction blockinfo = pendingFile.getLastBlock();


    // check new GS & length: this is not expected
    if (newBlock.getGenerationStamp() <= blockinfo.getGenerationStamp() ||
        newBlock.getNumBytes() < blockinfo.getNumBytes()) {
      String msg = "Update " + oldBlock + " (len = " + 
        blockinfo.getNumBytes() + ") to an older state: " + newBlock + 
        " (len = " + newBlock.getNumBytes() +")";
      LOG.warn(msg);
      throw new IOException(msg);
    }


    // Update old block with the new generation stamp and new length
    blockinfo.setGenerationStamp(newBlock.getGenerationStamp());
    blockinfo.setNumBytes(newBlock.getNumBytes());


    // find the DatanodeDescriptor objects
    final DatanodeManager dm = getBlockManager().getDatanodeManager();
    DatanodeDescriptor[] descriptors = null;
    if (newNodes.length > 0) {
      descriptors = new DatanodeDescriptor[newNodes.length];
      for(int i = 0; i < newNodes.length; i++) {
        descriptors[i] = dm.getDatanode(newNodes[i]);
      }
    }
    blockinfo.setExpectedLocations(descriptors);


    // persist blocks only if append is supported
    String src = leaseManager.findPath(pendingFile);
    if (supportAppends) {
      dir.persistBlocks(src, pendingFile);

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

        INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
        file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
        if (blocks.length > 0) {
          BlockInfo lastBlk = file.getLastBlock();
          // replace the last block of file
          file.setBlock(file.numBlocks() - 1, new BlockInfoUnderConstruction(
              lastBlk, replication));
        }
      }
      return file;
    }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

                              HdfsServerConstants.BlockUCState lastState,
                              Path file, DatanodeDescriptor dnd,
                              PermissionStatus ps,
                              boolean setStoredBlock) throws IOException {
    BlockInfo b = mock(BlockInfo.class);
    BlockInfoUnderConstruction b1 = mock(BlockInfoUnderConstruction.class);
    when(b.getBlockUCState()).thenReturn(penUltState);
    when(b1.getBlockUCState()).thenReturn(lastState);
    BlockInfo[] blocks;


    FSDirectory fsDir = mock(FSDirectory.class);
    INodeFileUnderConstruction iNFmock = mock(INodeFileUnderConstruction.class);


    fsn.dir.close();
    fsn.dir = fsDir;
    FSImage fsImage = mock(FSImage.class);
    FSEditLog editLog = mock(FSEditLog.class);
                            
    when(fsn.getFSImage()).thenReturn(fsImage);
    when(fsn.getFSImage().getEditLog()).thenReturn(editLog);
    
    switch (fileBlocksNumber) {
      case 0:
        blocks = new BlockInfo[0];
        break;
      case 1:
        blocks = new BlockInfo[]{b1};
        when(iNFmock.getLastBlock()).thenReturn(b1);
        break;
      default:
        when(iNFmock.getPenultimateBlock()).thenReturn(b);
        when(iNFmock.getLastBlock()).thenReturn(b1);
        blocks = new BlockInfo[]{b, b1};
    }
    
    when(iNFmock.getBlocks()).thenReturn(blocks);
    when(iNFmock.numBlocks()).thenReturn(blocks.length);
    when(iNFmock.isUnderConstruction()).thenReturn(true);
    when(iNFmock.convertToInodeFile()).thenReturn(iNFmock);    
    fsDir.addFile(file.toString(), ps, (short)3, 1l, "test", 
      "test-machine", dnd, 1001l);


    fsn.leaseManager = mock(LeaseManager.class);
    fsn.leaseManager.addLease("mock-lease", file.toString());
    if (setStoredBlock) {
      when(b1.getINode()).thenReturn(iNFmock);
      fsn.getBlockManager().addINode(b1, iNFmock);
    }


    when(fsDir.getFileINode(anyString())).thenReturn(iNFmock);
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

        BlockInfo newBI;
        if (!op.shouldCompleteLastBlock()) {
          // TODO: shouldn't this only be true for the last block?
          // what about an old-version fsync() where fsync isn't called
          // until several blocks in?
          newBI = new BlockInfoUnderConstruction(
              newBlock, file.getBlockReplication());
        } else {
          // OP_CLOSE should add finalized blocks. This code path
          // is only executed when loading edits written by prior
          // versions of Hadoop. Current versions always log

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction


      // check quota limits and updated space consumed
      updateCount(inodesInPath, 0, fileINode.getBlockDiskspace(), true);


      // associate new last block for the file
      BlockInfoUnderConstruction blockInfo =
        new BlockInfoUnderConstruction(
            block,
            fileINode.getFileReplication(),
            BlockUCState.UNDER_CONSTRUCTION,
            targets);
      getBlockManager().addBlockCollection(blockInfo, fileINode);

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction


    @Override
    boolean checkNamenodeBeforeReturn() throws Exception {
      INodeFileUnderConstruction fileNode = (INodeFileUnderConstruction) cluster
          .getNamesystem(0).getFSDirectory().getINode4Write(file).asFile();
      BlockInfoUnderConstruction blkUC = 
          (BlockInfoUnderConstruction) (fileNode.getBlocks())[1];
      int datanodeNum = blkUC.getExpectedLocations().length;
      for (int i = 0; i < CHECKTIMES && datanodeNum != 2; i++) {
        Thread.sleep(1000);
        datanodeNum = blkUC.getExpectedLocations().length;
      }
      return datanodeNum == 2;
    }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

    FSImage image = new FSImage(conf);
    DatanodeDescriptor[] targets = new DatanodeDescriptor[0];


    FSNamesystem namesystem = new FSNamesystem(conf, image);
    FSNamesystem namesystemSpy = spy(namesystem);
    BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction(
        block, 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets);
    blockInfo.setBlockCollection(file);
    blockInfo.setGenerationStamp(genStamp);
    blockInfo.initializeBlockRecovery(genStamp);
    doReturn(true).when(file).removeLastBlock(any(Block.class));


    doReturn(blockInfo).when(namesystemSpy).getStoredBlock(any(Block.class));
    doReturn("").when(namesystemSpy).closeFileCommitBlocks(
        any(INodeFileUnderConstruction.class),

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

          if(!isClosed)
            throw new RecoveryInProgressException(
                "Failed to close file " + src +
                ". Lease recovery is in progress. Try again later.");
        } else {
          BlockInfoUnderConstruction lastBlock=pendingFile.getLastBlock();
          if(lastBlock != null && lastBlock.getBlockUCState() ==
            BlockUCState.UNDER_RECOVERY) {
            throw new RecoveryInProgressException(
              "Recovery in progress, file [" + src + "], " +
              "lease owner [" + lease.getHolder() + "]");
            } else {

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

      throw new IOException(message);
    }


    // no we know that the last block is not COMPLETE, and
    // that the penultimate block if exists is either COMPLETE or COMMITTED
    BlockInfoUnderConstruction lastBlock = pendingFile.getLastBlock();
    BlockUCState lastBlockState = lastBlock.getBlockUCState();
    BlockInfo penultimateBlock = pendingFile.getPenultimateBlock();
    boolean penultimateBlockMinReplication;
    BlockUCState penultimateBlockState;
    if (penultimateBlock == null) {
      penultimateBlockState = BlockUCState.COMPLETE;
      // If penultimate block doesn't exist then its minReplication is met
      penultimateBlockMinReplication = true;
    } else {
      penultimateBlockState = BlockUCState.COMMITTED;
      penultimateBlockMinReplication = 
        blockManager.checkMinReplication(penultimateBlock);
    }
    assert penultimateBlockState == BlockUCState.COMPLETE ||
           penultimateBlockState == BlockUCState.COMMITTED :
           "Unexpected state of penultimate block in " + src;


    switch(lastBlockState) {
    case COMPLETE:
      assert false : "Already checked that the last block is incomplete";
      break;
    case COMMITTED:
      // Close file if committed blocks are minimally replicated
      if(penultimateBlockMinReplication &&
          blockManager.checkMinReplication(lastBlock)) {
        finalizeINodeFileUnderConstruction(src, pendingFile);
        NameNode.stateChangeLog.warn("BLOCK*"
          + " internalReleaseLease: Committed blocks are minimally replicated,"
          + " lease removed, file closed.");
        return true;  // closed!
      }
      // Cannot close file right now, since some blocks 
      // are not yet minimally replicated.
      // This may potentially cause infinite loop in lease recovery
      // if there are no valid replicas on data-nodes.
      String message = "DIR* NameSystem.internalReleaseLease: " +
          "Failed to release lease for file " + src +
          ". Committed blocks are waiting to be minimally replicated." +
          " Try again later.";
      NameNode.stateChangeLog.warn(message);
      throw new AlreadyBeingCreatedException(message);
    case UNDER_CONSTRUCTION:
    case UNDER_RECOVERY:
      // setup the last block locations from the blockManager if not known
      if(lastBlock.getNumExpectedLocations() == 0)
        lastBlock.setExpectedLocations(blockManager.getNodes(lastBlock));
      // start recovery of the last block for this file
      long blockRecoveryId = nextGenerationStamp();
      lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
      lastBlock.initializeBlockRecovery(blockRecoveryId);
      leaseManager.renewLease(lease);
      // Cannot close file right now, since the last block requires recovery.
      // This may potentially cause infinite loop in lease recovery
      // if there are no valid replicas on data-nodes.
      NameNode.stateChangeLog.warn(

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.