Package org.apache.hadoop.hdfs.server.namenode

Examples of org.apache.hadoop.hdfs.server.namenode.CachedBlock


    // Read the existing cache commands.
    long[] blockIds = new long[length];
    int i = 0;
    for (Iterator<CachedBlock> iter = list.iterator();
            iter.hasNext(); ) {
      CachedBlock cachedBlock = iter.next();
      blockIds[i++] = cachedBlock.getBlockId();
    }
    return new BlockIdCommand(action, poolId, blockIds);
  }
View Full Code Here


      if (!blockInfo.getBlockUCState().equals(BlockUCState.COMPLETE)) {
        // We don't try to cache blocks that are under construction.
        continue;
      }
      Block block = new Block(blockInfo.getBlockId());
      CachedBlock ncblock = new CachedBlock(block.getBlockId(),
          directive.getReplication(), mark);
      CachedBlock ocblock = cachedBlocks.get(ncblock);
      if (ocblock == null) {
        cachedBlocks.put(ncblock);
      } else {
        // Update bytesUsed using the current replication levels.
        // Assumptions: we assume that all the blocks are the same length
        // on each datanode.  We can assume this because we're only caching
        // blocks in state COMMITTED.
        // Note that if two directives are caching the same block(s), they will
        // both get them added to their bytesCached.
        List<DatanodeDescriptor> cachedOn =
            ocblock.getDatanodes(Type.CACHED);
        long cachedByBlock = Math.min(cachedOn.size(),
            directive.getReplication()) * blockInfo.getNumBytes();
        cachedTotal += cachedByBlock;

        if ((mark != ocblock.getMark()) ||
            (ocblock.getReplication() < directive.getReplication())) {
          //
          // Overwrite the block's replication and mark in two cases:
          //
          // 1. If the mark on the CachedBlock is different from the mark for
          // this scan, that means the block hasn't been updated during this
          // scan, and we should overwrite whatever is there, since it is no
          // longer valid.
          //
          // 2. If the replication in the CachedBlock is less than what the
          // directive asks for, we want to increase the block's replication
          // field to what the directive asks for.
          //
          ocblock.setReplicationAndMark(directive.getReplication(), mark);
        }
      }
    }
    // Increment the "cached" statistics
    directive.addBytesCached(cachedTotal);
View Full Code Here

   */
  private void rescanCachedBlockMap() {
    for (Iterator<CachedBlock> cbIter = cachedBlocks.iterator();
        cbIter.hasNext(); ) {
      scannedBlocks++;
      CachedBlock cblock = cbIter.next();
      List<DatanodeDescriptor> pendingCached =
          cblock.getDatanodes(Type.PENDING_CACHED);
      List<DatanodeDescriptor> cached =
          cblock.getDatanodes(Type.CACHED);
      List<DatanodeDescriptor> pendingUncached =
          cblock.getDatanodes(Type.PENDING_UNCACHED);
      // Remove nodes from PENDING_UNCACHED if they were actually uncached.
      for (Iterator<DatanodeDescriptor> iter = pendingUncached.iterator();
          iter.hasNext(); ) {
        DatanodeDescriptor datanode = iter.next();
        if (!cblock.isInList(datanode.getCached())) {
          datanode.getPendingUncached().remove(cblock);
          iter.remove();
        }
      }
      BlockInfo blockInfo = blockManager.
            getStoredBlock(new Block(cblock.getBlockId()));
      String reason = findReasonForNotCaching(cblock, blockInfo);
      int neededCached = 0;
      if (reason != null) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("not caching " + cblock + " because it is " + reason);
        }
      } else {
        neededCached = cblock.getReplication();
      }
      int numCached = cached.size();
      if (numCached >= neededCached) {
        // If we have enough replicas, drop all pending cached.
        for (Iterator<DatanodeDescriptor> iter = pendingCached.iterator();
View Full Code Here

      }
      long pendingCapacity = datanode.getCacheRemaining();
      // Subtract pending cached blocks from effective capacity
      Iterator<CachedBlock> it = datanode.getPendingCached().iterator();
      while (it.hasNext()) {
        CachedBlock cBlock = it.next();
        BlockInfo info =
            blockManager.getStoredBlock(new Block(cBlock.getBlockId()));
        if (info != null) {
          pendingCapacity -= info.getNumBytes();
        }
      }
      it = datanode.getPendingUncached().iterator();
      // Add pending uncached blocks from effective capacity
      while (it.hasNext()) {
        CachedBlock cBlock = it.next();
        BlockInfo info =
            blockManager.getStoredBlock(new Block(cBlock.getBlockId()));
        if (info != null) {
          pendingCapacity += info.getNumBytes();
        }
      }
      if (pendingCapacity < blockInfo.getNumBytes()) {
View Full Code Here

    // Read the existing cache commands.
    long[] blockIds = new long[length];
    int i = 0;
    for (Iterator<CachedBlock> iter = list.iterator();
            iter.hasNext(); ) {
      CachedBlock cachedBlock = iter.next();
      blockIds[i++] = cachedBlock.getBlockId();
    }
    return new BlockIdCommand(action, poolId, blockIds);
  }
View Full Code Here

    // Read the existing cache commands.
    long[] blockIds = new long[length];
    int i = 0;
    for (Iterator<CachedBlock> iter = list.iterator();
            iter.hasNext(); ) {
      CachedBlock cachedBlock = iter.next();
      blockIds[i++] = cachedBlock.getBlockId();
    }
    return new BlockIdCommand(action, poolId, blockIds);
  }
View Full Code Here

  public void testSingleList() {
    DatanodeDescriptor dn = new DatanodeDescriptor(
      new DatanodeID("127.0.0.1", "localhost", "abcd",
        5000, 5001, 5002, 5003));
    CachedBlock[] blocks = new CachedBlock[] {
          new CachedBlock(0L, (short)1, true),
          new CachedBlock(1L, (short)1, true),
          new CachedBlock(2L, (short)1, true),
      };
    // check that lists are empty
    Assert.assertTrue("expected pending cached list to start off empty.",
        !dn.getPendingCached().iterator().hasNext());
    Assert.assertTrue("expected cached list to start off empty.",
View Full Code Here

        datanodes[1].getPendingUncached(),
    };
    final int NUM_BLOCKS = 8000;
    CachedBlock[] blocks = new CachedBlock[NUM_BLOCKS];
    for (int i = 0; i < NUM_BLOCKS; i++) {
      blocks[i] = new CachedBlock(i, (short)i, true);
    }
    Random r = new Random(654);
    for (CachedBlocksList list : lists) {
      testAddElementsToList(list, blocks);
    }
View Full Code Here

    // Read the existing cache commands.
    long[] blockIds = new long[length];
    int i = 0;
    for (Iterator<CachedBlock> iter = list.iterator();
            iter.hasNext(); ) {
      CachedBlock cachedBlock = iter.next();
      blockIds[i++] = cachedBlock.getBlockId();
    }
    return new BlockIdCommand(action, poolId, blockIds);
  }
View Full Code Here

              blockInfo.getBlockUCState() + ", not COMPLETE.");
        }
        continue;
      }
      Block block = new Block(blockInfo.getBlockId());
      CachedBlock ncblock = new CachedBlock(block.getBlockId(),
          directive.getReplication(), mark);
      CachedBlock ocblock = cachedBlocks.get(ncblock);
      if (ocblock == null) {
        cachedBlocks.put(ncblock);
        ocblock = ncblock;
      } else {
        // Update bytesUsed using the current replication levels.
        // Assumptions: we assume that all the blocks are the same length
        // on each datanode.  We can assume this because we're only caching
        // blocks in state COMMITTED.
        // Note that if two directives are caching the same block(s), they will
        // both get them added to their bytesCached.
        List<DatanodeDescriptor> cachedOn =
            ocblock.getDatanodes(Type.CACHED);
        long cachedByBlock = Math.min(cachedOn.size(),
            directive.getReplication()) * blockInfo.getNumBytes();
        cachedTotal += cachedByBlock;

        if ((mark != ocblock.getMark()) ||
            (ocblock.getReplication() < directive.getReplication())) {
          //
          // Overwrite the block's replication and mark in two cases:
          //
          // 1. If the mark on the CachedBlock is different from the mark for
          // this scan, that means the block hasn't been updated during this
          // scan, and we should overwrite whatever is there, since it is no
          // longer valid.
          //
          // 2. If the replication in the CachedBlock is less than what the
          // directive asks for, we want to increase the block's replication
          // field to what the directive asks for.
          //
          ocblock.setReplicationAndMark(directive.getReplication(), mark);
        }
      }
      if (LOG.isTraceEnabled()) {
        LOG.trace("Directive " + directive.getId() + ": setting replication " +
                "for block " + blockInfo + " to " + ocblock.getReplication());
      }
    }
    // Increment the "cached" statistics
    directive.addBytesCached(cachedTotal);
    if (cachedTotal == neededTotal) {
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.server.namenode.CachedBlock

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.