Examples of org.apache.hadoop.hdfs.client.ShortCircuitReplica

org.apache.hadoop.hdfs.client.ShortCircuitReplica
A ShortCircuitReplica object contains file descriptors for a block that we are reading via short-circuit local reads. The file descriptors can be shared between multiple threads because all the operations we perform are stateless-- i.e., we use pread instead of read, to avoid using the shared position state.

    switch (resp.getStatus()) {
    case SUCCESS:
      byte buf[] = new byte[1];
      FileInputStream fis[] = new FileInputStream[2];
      sock.recvFileInputStreams(fis, buf, 0, buf.length);
      ShortCircuitReplica replica = null;
      try {
        ExtendedBlockId key =
            new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId());
        replica = new ShortCircuitReplica(key, fis[0], fis[1], cache,
            Time.monotonicNow(), slot);
      } catch (IOException e) {
        // This indicates an error reading from disk, or a format error.  Since
        // it's not a socket communication problem, we return null rather than
        // throwing an exception.

View Full Code Here

          if (entry == null) break;
          evictionTimeNs = entry.getKey();
          long evictionTimeMs = 
              TimeUnit.MILLISECONDS.convert(evictionTimeNs, TimeUnit.NANOSECONDS);
          if (evictionTimeMs + maxNonMmappedEvictableLifespanMs >= curMs) break;
          ShortCircuitReplica replica = entry.getValue();
          if (LOG.isTraceEnabled()) {
            LOG.trace("CacheCleaner: purging " + replica + ": " + 
                  StringUtils.getStackTrace(Thread.currentThread()));
          }
          purge(replica);

View Full Code Here

        if (evictableMmapped.size() < maxEvictableMmapedSize) {
          break;
        }
        needMoreSpace = true;
      }
      ShortCircuitReplica replica = entry.getValue();
      if (LOG.isTraceEnabled()) {
        String rationale = needMoreSpace ? "because we need more space" : 
            "because it's too old";
        LOG.trace("demoteOldEvictable: demoting " + replica + ": " +
            rationale + ": " +

View Full Code Here

      long evictableSize = evictable.size();
      long evictableMmappedSize = evictableMmapped.size();
      if (evictableSize + evictableMmappedSize <= maxTotalSize) {
        return;
      }
      ShortCircuitReplica replica;
      if (evictableSize == 0) {
       replica = evictableMmapped.firstEntry().getValue();
      } else {
       replica = evictable.firstEntry().getValue();
      }

View Full Code Here

   */
  private void removeEvictable(ShortCircuitReplica replica,
      TreeMap<Long, ShortCircuitReplica> map) {
    Long evictableTimeNs = replica.getEvictableTimeNs();
    Preconditions.checkNotNull(evictableTimeNs);
    ShortCircuitReplica removed = map.remove(evictableTimeNs);
    Preconditions.checkState(removed == replica,
        "failed to make " + replica + " unevictable");
    replica.setEvictableTimeNs(null);
  }

View Full Code Here

    if (info.getInvalidTokenException() != null) {
      LOG.warn(this + ": could not get " + key + " due to InvalidToken " +
            "exception.", info.getInvalidTokenException());
      return info;
    }
    ShortCircuitReplica replica = info.getReplica();
    if (replica == null) {
      LOG.warn(this + ": failed to get " + key);
      return info;
    }
    if (replica.purged) {
      // Ignore replicas that have already been purged from the cache.
      throw new RetriableException("Ignoring purged replica " +
          replica + ".  Retrying.");
    }
    // Check if the replica is stale before using it.
    // If it is, purge it and retry.
    if (replica.isStale()) {
      LOG.info(this + ": got stale replica " + replica + ".  Removing " +
          "this replica from the replicaInfoMap and retrying.");
      // Remove the cache's reference to the replica.  This may or may not
      // trigger a close.
      purge(replica);

View Full Code Here

      public void visit(int numOutstandingMmaps,
          Map<ExtendedBlockId, ShortCircuitReplica> replicas,
          Map<ExtendedBlockId, InvalidToken> failedLoads, 
          Map<Long, ShortCircuitReplica> evictable,
          Map<Long, ShortCircuitReplica> evictableMmapped) {
        ShortCircuitReplica replica = replicas.get(
            new ExtendedBlockId(firstBlock.getBlockId(), firstBlock.getBlockPoolId()));
        Assert.assertNotNull(replica);
        Assert.assertTrue(replica.hasMmap());
        // The replica should not yet be evictable, since we have it open.
        Assert.assertNull(replica.getEvictableTimeNs());
      }
    });


    // Read more blocks.
    results[2] = fsIn.read(null, 4096,

View Full Code Here

              Map<ExtendedBlockId, ShortCircuitReplica> replicas,
              Map<ExtendedBlockId, InvalidToken> failedLoads,
              Map<Long, ShortCircuitReplica> evictable,
              Map<Long, ShortCircuitReplica> evictableMmapped) {
            Assert.assertEquals(expectedOutstandingMmaps, numOutstandingMmaps);
            ShortCircuitReplica replica =
                replicas.get(ExtendedBlockId.fromExtendedBlock(block));
            Assert.assertNotNull(replica);
            Slot slot = replica.getSlot();
            if ((expectedIsAnchorable != slot.isAnchorable()) ||
                (expectedIsAnchored != slot.isAnchored())) {
              LOG.info("replica " + replica + " has isAnchorable = " +
                slot.isAnchorable() + ", isAnchored = " + slot.isAnchored() + 
                ".  Waiting for isAnchorable = " + expectedIsAnchorable +

View Full Code Here

          new ShortCircuitReplicaCreator() {
        @Override
        public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
          try {
            return new ShortCircuitReplicaInfo(
                new ShortCircuitReplica(key,
                    pairs[iVal].getFileInputStreams()[0],
                    pairs[iVal].getFileInputStreams()[1],
                    cache, Time.monotonicNow() + (iVal * HOUR_IN_MS), null));
          } catch (IOException e) {
            throw new RuntimeException(e);

View Full Code Here

      public void visit(int numOutstandingMmaps,
          Map<ExtendedBlockId, ShortCircuitReplica> replicas,
          Map<ExtendedBlockId, InvalidToken> failedLoads,
          Map<Long, ShortCircuitReplica> evictable,
          Map<Long, ShortCircuitReplica> evictableMmapped) {
        ShortCircuitReplica replica = replicas.get(
            ExtendedBlockId.fromExtendedBlock(block));
        Assert.assertNotNull(replica);
        Assert.assertTrue(replica.getSlot().isValid());
      }
    });
    // Stop the Namenode.  This will close the socket keeping the client's
    // shared memory segment alive, and make it stale.
    cluster.getDataNodes().get(0).shutdown();
    cache.accept(new CacheVisitor() {
      @Override
      public void visit(int numOutstandingMmaps,
          Map<ExtendedBlockId, ShortCircuitReplica> replicas,
          Map<ExtendedBlockId, InvalidToken> failedLoads,
          Map<Long, ShortCircuitReplica> evictable,
          Map<Long, ShortCircuitReplica> evictableMmapped) {
        ShortCircuitReplica replica = replicas.get(
            ExtendedBlockId.fromExtendedBlock(block));
        Assert.assertNotNull(replica);
        Assert.assertFalse(replica.getSlot().isValid());
      }
    });
    cluster.shutdown();
  }

View Full Code Here

0 1

TOP

Related Classes of org.apache.hadoop.hdfs.client.ShortCircuitReplica

org.apache.hadoop.fs.TestEnhancedByteBufferAccess

org.apache.hadoop.hdfs.BlockReaderFactory

org.apache.hadoop.hdfs.client.ShortCircuitCache

org.apache.hadoop.hdfs.client.ShortCircuitCache$CacheCleaner

org.apache.hadoop.hdfs.TestBlockReaderLocal

org.apache.hadoop.hdfs.TestShortCircuitCache

org.apache.hadoop.hdfs.TestShortCircuitCache$SimpleReplicaCreator

java.nio.channels.FileChannel

java.io.IOException

java.nio.MappedByteBuffer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.