Package org.apache.hadoop.hdfs.server.blockmanagement

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo


    BlockInfo[] blocks = inode.getBlocks();
    assertTrue("File does not have blocks: " + inode.toString(),
        blocks != null && blocks.length > 0);
   
    int idx = 0;
    BlockInfo curBlock;
    // all blocks but the last two should be regular blocks
    for(; idx < blocks.length - 2; idx++) {
      curBlock = blocks[idx];
      assertTrue("Block is not complete: " + curBlock,
          curBlock.isComplete());
      assertTrue("Block is not in BlocksMap: " + curBlock,
          ns.getBlockManager().getStoredBlock(curBlock) == curBlock);
    }

    // the penultimate block is either complete or
    // committed if the file is not closed
    if(idx > 0) {
      curBlock = blocks[idx-1]; // penultimate block
      assertTrue("Block " + curBlock +
          " isUnderConstruction = " + inode.isUnderConstruction() +
          " expected to be " + isFileOpen,
          (isFileOpen && curBlock.isComplete()) ||
          (!isFileOpen && !curBlock.isComplete() ==
            (curBlock.getBlockUCState() ==
              BlockUCState.COMMITTED)));
      assertTrue("Block is not in BlocksMap: " + curBlock,
          ns.getBlockManager().getStoredBlock(curBlock) == curBlock);
    }

    // The last block is complete if the file is closed.
    // If the file is open, the last block may be complete or not.
    curBlock = blocks[idx]; // last block
    if (!isFileOpen) {
      assertTrue("Block " + curBlock + ", isFileOpen = " + isFileOpen,
          curBlock.isComplete());
    }
    assertTrue("Block is not in BlocksMap: " + curBlock,
        ns.getBlockManager().getStoredBlock(curBlock) == curBlock);
  }
View Full Code Here


 
  public void logAddBlock(String path, INodeFile file) {
    Preconditions.checkArgument(file.isUnderConstruction());
    BlockInfo[] blocks = file.getBlocks();
    Preconditions.checkState(blocks != null && blocks.length > 0);
    BlockInfo pBlock = blocks.length > 1 ? blocks[blocks.length - 2] : null;
    BlockInfo lastBlock = blocks[blocks.length - 1];
    AddBlockOp op = AddBlockOp.getInstance(cache.get()).setPath(path)
        .setPenultimateBlock(pBlock).setLastBlock(lastBlock);
    logEdit(op);
  }
View Full Code Here

      short replication = (short) f.getReplication();
      LoaderContext state = parent.getLoaderContext();

      BlockInfo[] blocks = new BlockInfo[bp.size()];
      for (int i = 0, e = bp.size(); i < e; ++i) {
        blocks[i] = new BlockInfo(PBHelper.convert(bp.get(i)), replication);
      }
      final PermissionStatus permissions = loadPermission(f.getPermission(),
          parent.getLoaderContext().getStringTable());

      final INodeFile file = new INodeFile(n.getId(),
          n.getName().toByteArray(), permissions, f.getModificationTime(),
          f.getAccessTime(), blocks, replication, f.getPreferredBlockSize());

      if (f.hasAcl()) {
        file.addAclFeature(new AclFeature(loadAclEntries(f.getAcl(),
            state.getStringTable())));
      }
     
      if (f.hasXAttrs()) {
        file.addXAttrFeature(new XAttrFeature(
            loadXAttrs(f.getXAttrs(), state.getStringTable())));
      }

      // under-construction information
      if (f.hasFileUC()) {
        INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
        file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
        if (blocks.length > 0) {
          BlockInfo lastBlk = file.getLastBlock();
          // replace the last block of file
          file.setBlock(file.numBlocks() - 1, new BlockInfoUnderConstruction(
              lastBlk, replication));
        }
      }
View Full Code Here

                              HdfsServerConstants.BlockUCState penUltState,
                              HdfsServerConstants.BlockUCState lastState,
                              Path file, DatanodeDescriptor dnd,
                              PermissionStatus ps,
                              boolean setStoredBlock) throws IOException {
    BlockInfo b = mock(BlockInfo.class);
    BlockInfoUnderConstruction b1 = mock(BlockInfoUnderConstruction.class);
    when(b.getBlockUCState()).thenReturn(penUltState);
    when(b1.getBlockUCState()).thenReturn(lastState);
    BlockInfo[] blocks;

    FSDirectory fsDir = mock(FSDirectory.class);
    INodeFileUnderConstruction iNFmock = mock(INodeFileUnderConstruction.class);
View Full Code Here

      new PermissionStatus("test", "test", new FsPermission((short)0777));
   
    mockFileBlocks(2, HdfsServerConstants.BlockUCState.COMMITTED,
      HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, file, dnd, ps, false);
   
    BlockInfo lastBlock = fsn.dir.getFileINode(anyString()).getLastBlock();
    try {
      fsn.commitBlockSynchronization(fsn.getExtendedBlock(lastBlock),
        recoveryId, newSize, true, false, new DatanodeID[1]);
    } catch (IOException ioe) {
      assertTrue(ioe.getMessage().startsWith("Block (="));
View Full Code Here

      new PermissionStatus("test", "test", new FsPermission((short)0777));
   
    mockFileBlocks(2, HdfsServerConstants.BlockUCState.COMMITTED,
      HdfsServerConstants.BlockUCState.COMPLETE, file, dnd, ps, true);
   
    BlockInfo lastBlock = fsn.dir.getFileINode(anyString()).getLastBlock();
    when(lastBlock.isComplete()).thenReturn(true);
   
    try {
      fsn.commitBlockSynchronization(fsn.getExtendedBlock(lastBlock),
        recoveryId, newSize, true, false, new DatanodeID[1]);
    } catch (IOException ioe) {
View Full Code Here

      new PermissionStatus("test", "test", new FsPermission((short)0777));
   
    mockFileBlocks(2, HdfsServerConstants.BlockUCState.COMMITTED,
      HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, file, dnd, ps, true);
   
    BlockInfo lastBlock = fsn.dir.getFileINode(anyString()).getLastBlock();
    when(((BlockInfoUnderConstruction)lastBlock).getBlockRecoveryId()).thenReturn(recoveryId-100);
   
    try {
      fsn.commitBlockSynchronization(fsn.getExtendedBlock(lastBlock),
        recoveryId, newSize, true, false, new DatanodeID[1]);
View Full Code Here

      new PermissionStatus("test", "test", new FsPermission((short)0777));
   
    mockFileBlocks(2, HdfsServerConstants.BlockUCState.COMMITTED,
      HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, file, dnd, ps, true);
   
    BlockInfo lastBlock = fsn.dir.getFileINode(anyString()).getLastBlock();
    when(((BlockInfoUnderConstruction)lastBlock).getBlockRecoveryId()).thenReturn(recoveryId+100);
   
    try {          
      fsn.commitBlockSynchronization(fsn.getExtendedBlock(lastBlock),
        recoveryId, newSize, true, false, new DatanodeID[1]);
View Full Code Here

      new PermissionStatus("test", "test", new FsPermission((short)0777));
   
    mockFileBlocks(2, HdfsServerConstants.BlockUCState.COMMITTED,
      HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, file, dnd, ps, true);
   
    BlockInfo lastBlock = fsn.dir.getFileINode(anyString()).getLastBlock();
    when(((BlockInfoUnderConstruction)lastBlock).getBlockRecoveryId()).thenReturn(recoveryId);
   
    boolean recoveryChecked = false;
    try {
      fsn.commitBlockSynchronization(fsn.getExtendedBlock(lastBlock),
View Full Code Here

    // Are we only updating the last block's gen stamp.
    boolean isGenStampUpdate = oldBlocks.length == newBlocks.length;
   
    // First, update blocks in common
    for (int i = 0; i < oldBlocks.length && i < newBlocks.length; i++) {
      BlockInfo oldBlock = oldBlocks[i];
      Block newBlock = newBlocks[i];
     
      boolean isLastBlock = i == newBlocks.length - 1;
      if (oldBlock.getBlockId() != newBlock.getBlockId() ||
          (oldBlock.getGenerationStamp() != newBlock.getGenerationStamp() &&
              !(isGenStampUpdate && isLastBlock))) {
        throw new IOException("Mismatched block IDs or generation stamps, " +
            "attempting to replace block " + oldBlock + " with " + newBlock +
            " as block # " + i + "/" + newBlocks.length + " of " +
            path);
      }
     
      oldBlock.setNumBytes(newBlock.getNumBytes());
      boolean changeMade =
        oldBlock.getGenerationStamp() != newBlock.getGenerationStamp();
      oldBlock.setGenerationStamp(newBlock.getGenerationStamp());
     
      if (oldBlock instanceof BlockInfoUnderConstruction &&
          (!isLastBlock || op.shouldCompleteLastBlock())) {
        changeMade = true;
        fsNamesys.getBlockManager().forceCompleteBlock(
            (INodeFileUnderConstruction)file,
            (BlockInfoUnderConstruction)oldBlock);
      }
      if (changeMade) {
        // The state or gen-stamp of the block has changed. So, we may be
        // able to process some messages from datanodes that we previously
        // were unable to process.
        fsNamesys.getBlockManager().processQueuedMessagesForBlock(newBlock);
      }
    }
   
    if (newBlocks.length < oldBlocks.length) {
      // We're removing a block from the file, e.g. abandonBlock(...)
      if (!file.isUnderConstruction()) {
        throw new IOException("Trying to remove a block from file " +
            path + " which is not under construction.");
      }
      if (newBlocks.length != oldBlocks.length - 1) {
        throw new IOException("Trying to remove more than one block from file "
            + path);
      }
      Block oldBlock = oldBlocks[oldBlocks.length - 1];
      boolean removed = fsDir.unprotectedRemoveBlock(path,
          (INodeFileUnderConstruction) file, oldBlock);
      if (!removed && !(op instanceof UpdateBlocksOp)) {
        throw new IOException("Trying to delete non-existant block " + oldBlock);
      }
    } else if (newBlocks.length > oldBlocks.length) {
      // We're adding blocks
      for (int i = oldBlocks.length; i < newBlocks.length; i++) {
        Block newBlock = newBlocks[i];
        BlockInfo newBI;
        if (!op.shouldCompleteLastBlock()) {
          // TODO: shouldn't this only be true for the last block?
          // what about an old-version fsync() where fsync isn't called
          // until several blocks in?
          newBI = new BlockInfoUnderConstruction(
              newBlock, file.getBlockReplication());
        } else {
          // OP_CLOSE should add finalized blocks. This code path
          // is only executed when loading edits written by prior
          // versions of Hadoop. Current versions always log
          // OP_ADD operations as each block is allocated.
          newBI = new BlockInfo(newBlock, file.getBlockReplication());
        }
        fsNamesys.getBlockManager().addBlockCollection(newBI, file);
        file.addBlock(newBI);
        fsNamesys.getBlockManager().processQueuedMessagesForBlock(newBlock);
      }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.