Package org.apache.hadoop.hdfs.shortcircuit

Examples of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm


    SharedFileDescriptorFactory factory =
        SharedFileDescriptorFactory.create("shm_",
            new String[] { path.getAbsolutePath() } );
    FileInputStream stream =
        factory.createDescriptor("testStartupShutdown", 4096);
    ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream);
    shm.free();
    stream.close();
    FileUtil.fullyDelete(path);
  }
View Full Code Here


    SharedFileDescriptorFactory factory =
        SharedFileDescriptorFactory.create("shm_",
            new String[] { path.getAbsolutePath() });
    FileInputStream stream =
        factory.createDescriptor("testAllocateSlots", 4096);
    ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream);
    int numSlots = 0;
    ArrayList<Slot> slots = new ArrayList<Slot>();
    while (!shm.isFull()) {
      Slot slot = shm.allocAndRegisterSlot(new ExtendedBlockId(123L, "test_bp1"));
      slots.add(slot);
      numSlots++;
    }
    LOG.info("allocated " + numSlots + " slots before running out.");
    int slotIdx = 0;
    for (Iterator<Slot> iter = shm.slotIterator();
        iter.hasNext(); ) {
      Assert.assertTrue(slots.contains(iter.next()));
    }
    for (Slot slot : slots) {
      Assert.assertFalse(slot.addAnchor());
      Assert.assertEquals(slotIdx++, slot.getSlotIdx());
    }
    for (Slot slot : slots) {
      slot.makeAnchorable();
    }
    for (Slot slot : slots) {
      Assert.assertTrue(slot.addAnchor());
    }
    for (Slot slot : slots) {
      slot.removeAnchor();
    }
    for (Slot slot : slots) {
      shm.unregisterSlot(slot.getSlotIdx());
      slot.makeInvalid();
    }
    shm.free();
    stream.close();
    FileUtil.fullyDelete(path);
  }
View Full Code Here

    BlockReaderLocal blockReaderLocal = null;
    FSDataInputStream fsIn = null;
    byte original[] = new byte[BlockReaderLocalTest.TEST_LENGTH];
   
    FileSystem fs = null;
    ShortCircuitShm shm = null;
    RandomAccessFile raf = null;
    try {
      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
      cluster.waitActive();
      fs = cluster.getFileSystem();
      DFSTestUtil.createFile(fs, TEST_PATH,
          BlockReaderLocalTest.TEST_LENGTH, (short)1, RANDOM_SEED);
      try {
        DFSTestUtil.waitReplication(fs, TEST_PATH, (short)1);
      } catch (InterruptedException e) {
        Assert.fail("unexpected InterruptedException during " +
            "waitReplication: " + e);
      } catch (TimeoutException e) {
        Assert.fail("unexpected TimeoutException during " +
            "waitReplication: " + e);
      }
      fsIn = fs.open(TEST_PATH);
      IOUtils.readFully(fsIn, original, 0,
          BlockReaderLocalTest.TEST_LENGTH);
      fsIn.close();
      fsIn = null;
      ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, TEST_PATH);
      File dataFile = MiniDFSCluster.getBlockFile(0, block);
      File metaFile = MiniDFSCluster.getBlockMetadataFile(0, block);

      ShortCircuitCache shortCircuitCache =
          ClientContext.getFromConf(conf).getShortCircuitCache();
      cluster.shutdown();
      cluster = null;
      test.setup(dataFile, checksum);
      FileInputStream streams[] = {
          new FileInputStream(dataFile),
          new FileInputStream(metaFile)
      };
      dataIn = streams[0];
      metaIn = streams[1];
      ExtendedBlockId key = new ExtendedBlockId(block.getBlockId(),
          block.getBlockPoolId());
      raf = new RandomAccessFile(
          new File(sockDir.getDir().getAbsolutePath(),
            UUID.randomUUID().toString()), "rw");
      raf.setLength(8192);
      FileInputStream shmStream = new FileInputStream(raf.getFD());
      shm = new ShortCircuitShm(ShmId.createRandom(), shmStream);
      ShortCircuitReplica replica =
          new ShortCircuitReplica(key, dataIn, metaIn, shortCircuitCache,
              Time.now(), shm.allocAndRegisterSlot(
                  ExtendedBlockId.fromExtendedBlock(block)));
      blockReaderLocal = new BlockReaderLocal.Builder(
              new DFSClient.Conf(conf)).
          setFilename(TEST_PATH.getName()).
          setBlock(block).
          setShortCircuitReplica(replica).
          setCachingStrategy(new CachingStrategy(false, readahead)).
          setVerifyChecksum(checksum).
          build();
      dataIn = null;
      metaIn = null;
      test.doTest(blockReaderLocal, original);
      // BlockReaderLocal should not alter the file position.
      Assert.assertEquals(0, streams[0].getChannel().position());
      Assert.assertEquals(0, streams[1].getChannel().position());
    } finally {
      if (fsIn != null) fsIn.close();
      if (fs != null) fs.close();
      if (cluster != null) cluster.shutdown();
      if (dataIn != null) dataIn.close();
      if (metaIn != null) metaIn.close();
      if (blockReaderLocal != null) blockReaderLocal.close();
      if (shm != null) shm.free();
      if (raf != null) raf.close();
    }
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.