Examples of org.apache.hadoop.hdfs.protocol.BlockListAsLongs

org.apache.hadoop.hdfs.protocol.BlockListAsLongs
This class provides an interface for accessing list of blocks that has been implemented as long[]. This class is usefull for block report. Rather than send block reports as a Block[] we can send it as a long[].

    long startTime = now();
    if (startTime - lastBlockReport > dnConf.blockReportInterval) {


      // Create block report
      long brCreateStartTime = now();
      BlockListAsLongs bReport = dn.data.getBlockReport(getBlockPoolId());


      // Send block report
      long brSendStartTime = now();
      cmd = bpNamenode.blockReport(bpRegistration, getBlockPoolId(), bReport
          .getBlockListAsLongs());


      // Log the block report processing stats from Datanode perspective
      long brSendCost = now() - brSendStartTime;
      long brCreateCost = brSendStartTime - brCreateStartTime;
      dn.metrics.addBlockReport(brSendCost);
      LOG.info("BlockReport of " + bReport.getNumberOfBlocks()
          + " blocks took " + brCreateCost + " msec to generate and "
          + brSendCost + " msecs for RPC and NN processing");


      // If we have sent the first block report, then wait a random
      // time before we start the periodic block reports.

View Full Code Here

  }


  public DatanodeCommand blockReport(DatanodeRegistration nodeReg,
                                     long[] blocks) throws IOException {
    verifyRequest(nodeReg);
    BlockListAsLongs blist = new BlockListAsLongs(blocks);
    stateChangeLog.debug("*BLOCK* NameNode.blockReport: "
           +"from "+nodeReg.getName()+" "+blist.getNumberOfBlocks() +" blocks");


    namesystem.processReport(nodeReg, blist);
    if (getFSImage().isUpgradeFinalized())
      return DatanodeCommand.FINALIZE;
    return null;

View Full Code Here

   * also add the Inode file to DataNodeDesc
   */
  public void blocksBeingWrittenReport(DatanodeRegistration nodeReg,
      long[] blocks) throws IOException {
    verifyRequest(nodeReg);
    BlockListAsLongs blist = new BlockListAsLongs(blocks);
    namesystem.processBlocksBeingWrittenReport(nodeReg, blist);
    
    stateChangeLog.info("*BLOCK* NameNode.blocksBeingWrittenReport: "
           +"from "+nodeReg.getName()+" "+blist.getNumberOfBlocks() +" blocks");
    
  }

View Full Code Here

        blockTable = Arrays.copyOf(blockTable, count);
      }
    } else {
      blockTable = new Block[0];
    }
    return new BlockListAsLongs(
        new ArrayList<Block>(Arrays.asList(blockTable)), null);
  }

View Full Code Here

    }
  }


  public void testGetBlockReport() throws IOException {
    SimulatedFSDataset fsdataset = getSimulatedFSDataset(); 
    BlockListAsLongs blockReport = fsdataset.getBlockReport(bpid);
    assertEquals(0, blockReport.getNumberOfBlocks());
    addSomeBlocks(fsdataset);
    blockReport = fsdataset.getBlockReport(bpid);
    assertEquals(NUMBLOCKS, blockReport.getNumberOfBlocks());
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
    }
  }

View Full Code Here

    }
  }
  
  public void testInjectionEmpty() throws IOException {
    SimulatedFSDataset fsdataset = getSimulatedFSDataset(); 
    BlockListAsLongs blockReport = fsdataset.getBlockReport(bpid);
    assertEquals(0, blockReport.getNumberOfBlocks());
    int bytesAdded = addSomeBlocks(fsdataset);
    blockReport = fsdataset.getBlockReport(bpid);
    assertEquals(NUMBLOCKS, blockReport.getNumberOfBlocks());
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
    }
    
    // Inject blocks into an empty fsdataset
    //  - injecting the blocks we got above.
    SimulatedFSDataset sfsdataset = getSimulatedFSDataset();
    sfsdataset.injectBlocks(bpid, blockReport);
    blockReport = sfsdataset.getBlockReport(bpid);
    assertEquals(NUMBLOCKS, blockReport.getNumberOfBlocks());
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
      assertEquals(blockIdToLen(b.getBlockId()), sfsdataset
          .getLength(new ExtendedBlock(bpid, b)));

View Full Code Here

    assertEquals(sfsdataset.getCapacity()-bytesAdded, sfsdataset.getRemaining());
  }


  public void testInjectionNonEmpty() throws IOException {
    SimulatedFSDataset fsdataset = getSimulatedFSDataset(); 
    BlockListAsLongs blockReport = fsdataset.getBlockReport(bpid);
    assertEquals(0, blockReport.getNumberOfBlocks());
    int bytesAdded = addSomeBlocks(fsdataset);
    blockReport = fsdataset.getBlockReport(bpid);
    assertEquals(NUMBLOCKS, blockReport.getNumberOfBlocks());
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
    }
    fsdataset = null;
    
    // Inject blocks into an non-empty fsdataset
    //  - injecting the blocks we got above.
    SimulatedFSDataset sfsdataset = getSimulatedFSDataset();
    // Add come blocks whose block ids do not conflict with
    // the ones we are going to inject.
    bytesAdded += addSomeBlocks(sfsdataset, NUMBLOCKS+1);
    sfsdataset.getBlockReport(bpid);
    assertEquals(NUMBLOCKS, blockReport.getNumberOfBlocks());
    sfsdataset.getBlockReport(bpid);
    assertEquals(NUMBLOCKS, blockReport.getNumberOfBlocks());
    sfsdataset.injectBlocks(bpid, blockReport);
    blockReport = sfsdataset.getBlockReport(bpid);
    assertEquals(NUMBLOCKS*2, blockReport.getNumberOfBlocks());
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
      assertEquals(blockIdToLen(b.getBlockId()), sfsdataset
          .getLength(new ExtendedBlock(bpid, b)));

View Full Code Here

  }


  public DatanodeCommand blockReport(DatanodeRegistration nodeReg,
                                     long[] blocks) throws IOException {
    verifyRequest(nodeReg);
    BlockListAsLongs blist = new BlockListAsLongs(blocks);
    stateChangeLog.debug("*BLOCK* NameNode.blockReport: "
           +"from "+nodeReg.getName()+" "+blist.getNumberOfBlocks() +" blocks");


    namesystem.processReport(nodeReg, blist);
    if (getFSImage().isUpgradeFinalized())
      return DatanodeCommand.FINALIZE;
    return null;

View Full Code Here

   * also add the Inode file to DataNodeDesc
   */
  public void blocksBeingWrittenReport(DatanodeRegistration nodeReg,
      long[] blocks) throws IOException {
    verifyRequest(nodeReg);
    BlockListAsLongs blist = new BlockListAsLongs(blocks);
    namesystem.processBlocksBeingWrittenReport(nodeReg, blist);
    
    stateChangeLog.info("*BLOCK* NameNode.blocksBeingWrittenReport: "
           +"from "+nodeReg.getName()+" "+blist.getNumberOfBlocks() +" blocks");
    
  }

View Full Code Here


    void formBlockReport() {
      // fill remaining slots with blocks that do not exist
      for(int idx = blocks.size()-1; idx >= nrBlocks; idx--)
        blocks.set(idx, new Block(blocks.size() - idx, 0, 0));
      blockReportList = new BlockListAsLongs(blocks,null).getBlockListAsLongs();
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hdfs.protocol.BlockListAsLongs

org.apache.hadoop.hdfs.server.blockmanagement.BlockManager

org.apache.hadoop.hdfs.server.blockmanagement.TestBlockManager

org.apache.hadoop.hdfs.server.datanode.BlockReportTestBase

org.apache.hadoop.hdfs.server.datanode.BPOfferService

org.apache.hadoop.hdfs.server.datanode.BPServiceActor

org.apache.hadoop.hdfs.server.datanode.DataNode

org.apache.hadoop.hdfs.server.datanode.DataNode$BPOfferService

org.apache.hadoop.hdfs.server.datanode.FSDataset

org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl

org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.