Package org.apache.hadoop.hdfs.ShortCircuitShm

Examples of org.apache.hadoop.hdfs.ShortCircuitShm.Slot

440
441
442
443
444
445
446
447
448
449
450
    while (true) {
      curPeer = nextDomainPeer();
      if (curPeer == null) break;
      if (curPeer.fromCache) remainingCacheTries--;
      DomainPeer peer = (DomainPeer)curPeer.peer;
      Slot slot = null;
      ShortCircuitCache cache = clientContext.getShortCircuitCache();
      try {
        MutableBoolean usedPeer = new MutableBoolean(false);
        slot = cache.allocShmSlot(datanode, peer, usedPeer,
            new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId()),
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
        return null;
      }
      Entry<ShmId, DfsClientShm> entry = notFull.firstEntry();
      DfsClientShm shm = entry.getValue();
      ShmId shmId = shm.getShmId();
      Slot slot = shm.allocAndRegisterSlot(blockId);
      if (shm.isFull()) {
        if (LOG.isTraceEnabled()) {
          LOG.trace(this + ": pulled the last slot " + slot.getSlotIdx() +
              " out of " + shm);
        }
        DfsClientShm removedShm = notFull.remove(shmId);
        Preconditions.checkState(removedShm == shm);
        full.put(shmId, shm);
      } else {
        if (LOG.isTraceEnabled()) {
          LOG.trace(this + ": pulled slot " + slot.getSlotIdx() +
              " out of " + shm);
        }
      }
      return slot;
    }
View Full Code Here
237
238
239
240
241
242
243
244
245
246
247
            LOG.trace(this + ": shared memory segment access is disabled.");
          }
          return null;
        }
        // Try to use an existing slot.
        Slot slot = allocSlotFromExistingShm(blockId);
        if (slot != null) {
          return slot;
        }
        // There are no free slots.  If someone is loading more slots, wait
        // for that to finish.
View Full Code Here
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
              Map<Long, ShortCircuitReplica> evictableMmapped) {
            Assert.assertEquals(expectedOutstandingMmaps, numOutstandingMmaps);
            ShortCircuitReplica replica =
                replicas.get(ExtendedBlockId.fromExtendedBlock(block));
            Assert.assertNotNull(replica);
            Slot slot = replica.getSlot();
            if ((expectedIsAnchorable != slot.isAnchorable()) ||
                (expectedIsAnchored != slot.isAnchored())) {
              LOG.info("replica " + replica + " has isAnchorable = " +
                slot.isAnchorable() + ", isAnchored = " + slot.isAnchored() +
                ".  Waiting for isAnchorable = " + expectedIsAnchorable +
                ", isAnchored = " + expectedIsAnchored);
              return;
            }
            result.setValue(true);
View Full Code Here
415
416
417
418
419
420
421
422
423
424
425
    MutableBoolean usedPeer = new MutableBoolean(false);
    ExtendedBlockId blockId = new ExtendedBlockId(123, "xyz");
    final DatanodeInfo datanode =
        new DatanodeInfo(cluster.getDataNodes().get(0).getDatanodeId());
    // Allocating the first shm slot requires using up a peer.
    Slot slot = cache.allocShmSlot(datanode, peer, usedPeer,
                    blockId, "testAllocShm_client");
    Assert.assertNotNull(slot);
    Assert.assertTrue(usedPeer.booleanValue());
    cache.getDfsClientShmManager().visit(new Visitor() {
      @Override
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
        factory.createDescriptor("testAllocateSlots", 4096);
    ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream);
    int numSlots = 0;
    ArrayList<Slot> slots = new ArrayList<Slot>();
    while (!shm.isFull()) {
      Slot slot = shm.allocAndRegisterSlot(new ExtendedBlockId(123L, "test_bp1"));
      slots.add(slot);
      numSlots++;
    }
    LOG.info("allocated " + numSlots + " slots before running out.");
    int slotIdx = 0;
    for (Iterator<Slot> iter = shm.slotIterator();
        iter.hasNext(); ) {
      Assert.assertTrue(slots.contains(iter.next()));
    }
    for (Slot slot : slots) {
      Assert.assertFalse(slot.addAnchor());
      Assert.assertEquals(slotIdx++, slot.getSlotIdx());
    }
    for (Slot slot : slots) {
      slot.makeAnchorable();
    }
    for (Slot slot : slots) {
      Assert.assertTrue(slot.addAnchor());
    }
    for (Slot slot : slots) {
      slot.removeAnchor();
    }
    for (Slot slot : slots) {
      shm.unregisterSlot(slot.getSlotIdx());
      slot.makeInvalid();
    }
    shm.free();
    stream.close();
    FileUtil.fullyDelete(path);
  }
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
125
126
    RegisteredShm removedShm = segments.remove(shm.getShmId());
    Preconditions.checkState(removedShm == shm,
        "failed to remove " + shm.getShmId());
    // Stop tracking the slots.
    for (Iterator<Slot> iter = shm.slotIterator(); iter.hasNext(); ) {
      Slot slot = iter.next();
      boolean removed = slots.remove(slot.getBlockId(), slot);
      Preconditions.checkState(removed);
      slot.makeInvalid();
    }
    // De-allocate the memory map and close the shared file.
    shm.free();
  }
View Full Code Here
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
    RegisteredShm shm = segments.get(shmId);
    if (shm == null) {
      throw new InvalidRequestException("there is no shared memory segment " +
          "registered with shmId " + shmId);
    }
    Slot slot = shm.registerSlot(slotId.getSlotIdx(), blockId);
    if (isCached) {
      slot.makeAnchorable();
    } else {
      slot.makeUnanchorable();
    }
    boolean added = slots.put(blockId, slot);
    Preconditions.checkState(added);
    if (LOG.isTraceEnabled()) {
      LOG.trace(this + ": registered " + blockId + " with slot " +
View Full Code Here
329
330
331
332
333
334
335
336
337
338
    RegisteredShm shm = segments.get(shmId);
    if (shm == null) {
      throw new InvalidRequestException("there is no shared memory segment " +
          "registered with shmId " + shmId);
    }
    Slot slot = shm.getSlot(slotId.getSlotIdx());
    slot.makeInvalid();
    shm.unregisterSlot(slotId.getSlotIdx());
    slots.remove(slot.getBlockId(), slot);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.ShortCircuitShm.Slot

  • org.apache.hadoop.fs.TestEnhancedByteBufferAccess
  • org.apache.hadoop.hdfs.BlockReaderFactory
  • org.apache.hadoop.hdfs.client.DfsClientShmManager$EndpointShmManager
  • org.apache.hadoop.hdfs.client.TestShortCircuitShm
  • org.apache.hadoop.hdfs.server.datanode.ShortCircuitRegistry
  • org.apache.hadoop.hdfs.TestShortCircuitCache
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.