Examples of BlockInfoUnderConstruction


Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

        BlockInfo newBI;
        if (!op.shouldCompleteLastBlock()) {
          // TODO: shouldn't this only be true for the last block?
          // what about an old-version fsync() where fsync isn't called
          // until several blocks in?
          newBI = new BlockInfoUnderConstruction(
              newBlock, file.getReplication());
        } else {
          // OP_CLOSE should add finalized blocks. This code path
          // is only executed when loading edits written by prior
          // versions of Hadoop. Current versions always log
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

          if(!isClosed)
            throw new RecoveryInProgressException(
                "Failed to close file " + src +
                ". Lease recovery is in progress. Try again later.");
        } else {
          BlockInfoUnderConstruction lastBlock=pendingFile.getLastBlock();
          if(lastBlock != null && lastBlock.getBlockUCState() ==
            BlockUCState.UNDER_RECOVERY) {
            throw new RecoveryInProgressException(
              "Recovery in progress, file [" + src + "], " +
              "lease owner [" + lease.getHolder() + "]");
            } else {
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

      throw new IOException(message);
    }

    // no we know that the last block is not COMPLETE, and
    // that the penultimate block if exists is either COMPLETE or COMMITTED
    BlockInfoUnderConstruction lastBlock = pendingFile.getLastBlock();
    BlockUCState lastBlockState = lastBlock.getBlockUCState();
    BlockInfo penultimateBlock = pendingFile.getPenultimateBlock();
    boolean penultimateBlockMinReplication;
    BlockUCState penultimateBlockState;
    if (penultimateBlock == null) {
      penultimateBlockState = BlockUCState.COMPLETE;
      // If penultimate block doesn't exist then its minReplication is met
      penultimateBlockMinReplication = true;
    } else {
      penultimateBlockState = BlockUCState.COMMITTED;
      penultimateBlockMinReplication =
        blockManager.checkMinReplication(penultimateBlock);
    }
    assert penultimateBlockState == BlockUCState.COMPLETE ||
           penultimateBlockState == BlockUCState.COMMITTED :
           "Unexpected state of penultimate block in " + src;

    switch(lastBlockState) {
    case COMPLETE:
      assert false : "Already checked that the last block is incomplete";
      break;
    case COMMITTED:
      // Close file if committed blocks are minimally replicated
      if(penultimateBlockMinReplication &&
          blockManager.checkMinReplication(lastBlock)) {
        finalizeINodeFileUnderConstruction(src, pendingFile);
        NameNode.stateChangeLog.warn("BLOCK*"
          + " internalReleaseLease: Committed blocks are minimally replicated,"
          + " lease removed, file closed.");
        return true// closed!
      }
      // Cannot close file right now, since some blocks
      // are not yet minimally replicated.
      // This may potentially cause infinite loop in lease recovery
      // if there are no valid replicas on data-nodes.
      String message = "DIR* NameSystem.internalReleaseLease: " +
          "Failed to release lease for file " + src +
          ". Committed blocks are waiting to be minimally replicated." +
          " Try again later.";
      NameNode.stateChangeLog.warn(message);
      throw new AlreadyBeingCreatedException(message);
    case UNDER_CONSTRUCTION:
    case UNDER_RECOVERY:
      // setup the last block locations from the blockManager if not known
      if(lastBlock.getNumExpectedLocations() == 0)
        lastBlock.setExpectedLocations(blockManager.getNodes(lastBlock));
      // start recovery of the last block for this file
      long blockRecoveryId = nextGenerationStamp();
      lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
      lastBlock.initializeBlockRecovery(blockRecoveryId);
      leaseManager.renewLease(lease);
      // Cannot close file right now, since the last block requires recovery.
      // This may potentially cause infinite loop in lease recovery
      // if there are no valid replicas on data-nodes.
      NameNode.stateChangeLog.warn(
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

      throws IOException {
    assert hasWriteLock();
    // check the vadility of the block and lease holder name
    final INodeFileUnderConstruction pendingFile =
      checkUCBlock(oldBlock, clientName);
    final BlockInfoUnderConstruction blockinfo = pendingFile.getLastBlock();

    // check new GS & length: this is not expected
    if (newBlock.getGenerationStamp() <= blockinfo.getGenerationStamp() ||
        newBlock.getNumBytes() < blockinfo.getNumBytes()) {
      String msg = "Update " + oldBlock + " (len = " +
        blockinfo.getNumBytes() + ") to an older state: " + newBlock +
        " (len = " + newBlock.getNumBytes() +")";
      LOG.warn(msg);
      throw new IOException(msg);
    }

    // Update old block with the new generation stamp and new length
    blockinfo.setGenerationStamp(newBlock.getGenerationStamp());
    blockinfo.setNumBytes(newBlock.getNumBytes());

    // find the DatanodeDescriptor objects
    final DatanodeManager dm = getBlockManager().getDatanodeManager();
    DatanodeDescriptor[] descriptors = null;
    if (newNodes.length > 0) {
      descriptors = new DatanodeDescriptor[newNodes.length];
      for(int i = 0; i < newNodes.length; i++) {
        descriptors[i] = dm.getDatanode(newNodes[i]);
      }
    }
    blockinfo.setExpectedLocations(descriptors);

    String src = leaseManager.findPath(pendingFile);
    dir.persistBlocks(src, pendingFile);
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

  throws IOException {
    if (blocks == null || blocks.length == 0) {
      throw new IOException("Trying to update non-existant block. " +
          "File is empty.");
    }
    BlockInfoUnderConstruction ucBlock =
      lastBlock.convertToBlockUnderConstruction(
          BlockUCState.UNDER_CONSTRUCTION, targets);
    ucBlock.setINode(this);
    setBlock(numBlocks()-1, ucBlock);
    return ucBlock;
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

                              HdfsServerConstants.BlockUCState lastState,
                              Path file, DatanodeDescriptor dnd,
                              PermissionStatus ps,
                              boolean setStoredBlock) throws IOException {
    BlockInfo b = mock(BlockInfo.class);
    BlockInfoUnderConstruction b1 = mock(BlockInfoUnderConstruction.class);
    when(b.getBlockUCState()).thenReturn(penUltState);
    when(b1.getBlockUCState()).thenReturn(lastState);
    BlockInfo[] blocks;

    FSDirectory fsDir = mock(FSDirectory.class);
    INodeFileUnderConstruction iNFmock = mock(INodeFileUnderConstruction.class);

    fsn.dir.close();
    fsn.dir = fsDir;
    FSImage fsImage = mock(FSImage.class);
    FSEditLog editLog = mock(FSEditLog.class);
                           
    when(fsn.getFSImage()).thenReturn(fsImage);
    when(fsn.getFSImage().getEditLog()).thenReturn(editLog);
   
    switch (fileBlocksNumber) {
      case 0:
        blocks = new BlockInfo[0];
        break;
      case 1:
        blocks = new BlockInfo[]{b1};
        when(iNFmock.getLastBlock()).thenReturn(b1);
        break;
      default:
        when(iNFmock.getPenultimateBlock()).thenReturn(b);
        when(iNFmock.getLastBlock()).thenReturn(b1);
        blocks = new BlockInfo[]{b, b1};
    }
   
    when(iNFmock.getBlocks()).thenReturn(blocks);
    when(iNFmock.numBlocks()).thenReturn(blocks.length);
    when(iNFmock.isUnderConstruction()).thenReturn(true);
    when(iNFmock.convertToInodeFile()).thenReturn(iNFmock);   
    fsDir.addFile(file.toString(), ps, (short)3, 1l, "test",
      "test-machine", dnd, 1001l);

    fsn.leaseManager = mock(LeaseManager.class);
    fsn.leaseManager.addLease("mock-lease", file.toString());
    if (setStoredBlock) {
      when(b1.getINode()).thenReturn(iNFmock);
      fsn.getBlockManager().addINode(b1, iNFmock);
    }

    when(fsDir.getFileINode(anyString())).thenReturn(iNFmock);
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

      blocks[i] = new BlockInfo(blk, blockReplication);
    }
    // last block is UNDER_CONSTRUCTION
    if(numBlocks > 0) {
      blk.readFields(in);
      blocks[i] = new BlockInfoUnderConstruction(
        blk, blockReplication, BlockUCState.UNDER_CONSTRUCTION, null);
    }
    PermissionStatus perm = PermissionStatus.read(in);
    String clientName = readString(in);
    String clientMachine = readString(in);
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

    BlockManager bm0 = nn.getNamesystem().getBlockManager();
    BlockInfo storedBlock = bm0.getStoredBlock(blk.getLocalBlock());
    assertTrue("Block " + blk + " should be under construction, " +
        "got: " + storedBlock,
        storedBlock instanceof BlockInfoUnderConstruction);
    BlockInfoUnderConstruction ucBlock =
      (BlockInfoUnderConstruction)storedBlock;
    // We expect that the first indexed replica will be the one
    // to be in charge of the synchronization / recovery protocol.
    DatanodeDescriptor expectedPrimary = ucBlock.getExpectedLocations()[0];
    return expectedPrimary;
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

    @Override
    boolean checkNamenodeBeforeReturn() throws Exception {
      INodeFileUnderConstruction fileNode = (INodeFileUnderConstruction) cluster
          .getNamesystem(0).getFSDirectory().getINode4Write(file).asFile();
      BlockInfoUnderConstruction blkUC =
          (BlockInfoUnderConstruction) (fileNode.getBlocks())[1];
      int datanodeNum = blkUC.getExpectedStorageLocations().length;
      for (int i = 0; i < CHECKTIMES && datanodeNum != 2; i++) {
        Thread.sleep(1000);
        datanodeNum = blkUC.getExpectedStorageLocations().length;
      }
      return datanodeNum == 2;
    }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction

          if(!isClosed)
            throw new RecoveryInProgressException(
                "Failed to close file " + src +
                ". Lease recovery is in progress. Try again later.");
        } else {
          BlockInfoUnderConstruction lastBlock=pendingFile.getLastBlock();
          if(lastBlock != null && lastBlock.getBlockUCState() ==
            BlockUCState.UNDER_RECOVERY) {
            throw new RecoveryInProgressException(
              "Recovery in progress, file [" + src + "], " +
              "lease owner [" + lease.getHolder() + "]");
            } else {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.