Examples of org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset$SimulatedStorage

Package org.apache.hadoop.hdfs.server.datanode

Examples of org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset$SimulatedStorage

org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset
This class implements a simulated FSDataset. Blocks that are created are recorded but their data (plus their CRCs) are discarded. Fixed data is returned when blocks are read; a null CRC meta file is created for such data. This FSDataset does not remember any block information across its restarts; it does however offer an operation to inject blocks (See the TestInectionForSImulatedStorage() for a usage example of injection. Note the synchronization is coarse grained - it is at each method.

      checkBlockDataAndSize(fsdataset, b, blockIdToLen(i));
    }
  }


  public void testGetBlockReport() throws IOException {
    SimulatedFSDataset fsdataset = getSimulatedFSDataset(); 
    BlockListAsLongs blockReport = fsdataset.getBlockReport(bpid);
    assertEquals(0, blockReport.getNumberOfBlocks());
    addSomeBlocks(fsdataset);
    blockReport = fsdataset.getBlockReport(bpid);
    assertEquals(NUMBLOCKS, blockReport.getNumberOfBlocks());
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
    }

View Full Code Here

      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
    }
  }
  
  public void testInjectionEmpty() throws IOException {
    SimulatedFSDataset fsdataset = getSimulatedFSDataset(); 
    BlockListAsLongs blockReport = fsdataset.getBlockReport(bpid);
    assertEquals(0, blockReport.getNumberOfBlocks());
    int bytesAdded = addSomeBlocks(fsdataset);
    blockReport = fsdataset.getBlockReport(bpid);
    assertEquals(NUMBLOCKS, blockReport.getNumberOfBlocks());
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
    }
    
    // Inject blocks into an empty fsdataset
    //  - injecting the blocks we got above.
    SimulatedFSDataset sfsdataset = getSimulatedFSDataset();
    sfsdataset.injectBlocks(bpid, blockReport);
    blockReport = sfsdataset.getBlockReport(bpid);
    assertEquals(NUMBLOCKS, blockReport.getNumberOfBlocks());
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
      assertEquals(blockIdToLen(b.getBlockId()), sfsdataset
          .getLength(new ExtendedBlock(bpid, b)));
    }
    assertEquals(bytesAdded, sfsdataset.getDfsUsed());
    assertEquals(sfsdataset.getCapacity()-bytesAdded, sfsdataset.getRemaining());
  }

View Full Code Here

    assertEquals(bytesAdded, sfsdataset.getDfsUsed());
    assertEquals(sfsdataset.getCapacity()-bytesAdded, sfsdataset.getRemaining());
  }


  public void testInjectionNonEmpty() throws IOException {
    SimulatedFSDataset fsdataset = getSimulatedFSDataset(); 
    BlockListAsLongs blockReport = fsdataset.getBlockReport(bpid);
    assertEquals(0, blockReport.getNumberOfBlocks());
    int bytesAdded = addSomeBlocks(fsdataset);
    blockReport = fsdataset.getBlockReport(bpid);
    assertEquals(NUMBLOCKS, blockReport.getNumberOfBlocks());
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
    }
    fsdataset = null;
    
    // Inject blocks into an non-empty fsdataset
    //  - injecting the blocks we got above.
    SimulatedFSDataset sfsdataset = getSimulatedFSDataset();
    // Add come blocks whose block ids do not conflict with
    // the ones we are going to inject.
    bytesAdded += addSomeBlocks(sfsdataset, NUMBLOCKS+1);
    sfsdataset.getBlockReport(bpid);
    assertEquals(NUMBLOCKS, blockReport.getNumberOfBlocks());
    sfsdataset.getBlockReport(bpid);
    assertEquals(NUMBLOCKS, blockReport.getNumberOfBlocks());
    sfsdataset.injectBlocks(bpid, blockReport);
    blockReport = sfsdataset.getBlockReport(bpid);
    assertEquals(NUMBLOCKS*2, blockReport.getNumberOfBlocks());
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
      assertEquals(blockIdToLen(b.getBlockId()), sfsdataset
          .getLength(new ExtendedBlock(bpid, b)));
    }
    assertEquals(bytesAdded, sfsdataset.getDfsUsed());
    assertEquals(sfsdataset.getCapacity()-bytesAdded,  sfsdataset.getRemaining());
    
    // Now test that the dataset cannot be created if it does not have sufficient cap
    conf.setLong(SimulatedFSDataset.CONFIG_PROPERTY_CAPACITY, 10);
 
    try {
      sfsdataset = getSimulatedFSDataset();
      sfsdataset.addBlockPool(bpid, conf);
      sfsdataset.injectBlocks(bpid, blockReport);
      assertTrue("Expected an IO exception", false);
    } catch (IOException e) {
      // ok - as expected
    }
  }

View Full Code Here

      assertTrue(fsdataset.isValidBlock(new ExtendedBlock(bpid, b)));
    }
  }
  
  private SimulatedFSDataset getSimulatedFSDataset() throws IOException {
    SimulatedFSDataset fsdataset = new SimulatedFSDataset(conf); 
    fsdataset.addBlockPool(bpid, conf);
    return fsdataset;
  }

View Full Code Here

  int addSomeBlocks(FSDatasetInterface fsdataset ) throws IOException {
    return addSomeBlocks(fsdataset, 1);
  }


  public void testGetMetaData() throws IOException {
    FSDatasetInterface fsdataset = new SimulatedFSDataset(conf); 
    Block b = new Block(1, 5, 0);
    try {
      assertFalse(fsdataset.metaFileExists(b));
      assertTrue("Expected an IO exception", false);
    } catch (IOException e) {
      // ok - as expected
    }
    addSomeBlocks(fsdataset); // Only need to add one but ....
    b = new Block(1, 0, 0);
    InputStream metaInput = fsdataset.getMetaDataInputStream(b);
    DataInputStream metaDataInput = new DataInputStream(metaInput);
    short version = metaDataInput.readShort();
    assertEquals(FSDataset.METADATA_VERSION, version);
    DataChecksum checksum = DataChecksum.newDataChecksum(metaDataInput);
    assertEquals(DataChecksum.CHECKSUM_NULL, checksum.getChecksumType());

View Full Code Here

    assertEquals(0, checksum.getChecksumSize());  
  }




  public void testStorageUsage() throws IOException {
    FSDatasetInterface fsdataset = new SimulatedFSDataset(conf); 
    assertEquals(fsdataset.getDfsUsed(), 0);
    assertEquals(fsdataset.getRemaining(), fsdataset.getCapacity());
    int bytesAdded = addSomeBlocks(fsdataset);
    assertEquals(bytesAdded, fsdataset.getDfsUsed());
    assertEquals(fsdataset.getCapacity()-bytesAdded,  fsdataset.getRemaining());
    
  }

View Full Code Here

    }
    assertEquals(expectedLen, lengthRead);
  }
  
  public void testWriteRead() throws IOException {
    FSDatasetInterface fsdataset = new SimulatedFSDataset(conf); 
    addSomeBlocks(fsdataset);
    for (int i=1; i <= NUMBLOCKS; ++i) {
      Block b = new Block(i, 0, 0);
      assertTrue(fsdataset.isValidBlock(b));
      assertEquals(blockIdToLen(i), fsdataset.getLength(b));
      checkBlockDataAndSize(fsdataset, b, blockIdToLen(i));
    }
  }

View Full Code Here

  }






  public void testGetBlockReport() throws IOException {
    FSDatasetInterface fsdataset = new SimulatedFSDataset(conf); 
    Block[] blockReport = fsdataset.getBlockReport();
    assertEquals(0, blockReport.length);
    int bytesAdded = addSomeBlocks(fsdataset);
    blockReport = fsdataset.getBlockReport();
    assertEquals(NUMBLOCKS, blockReport.length);
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
    }

View Full Code Here

      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
    }
  }
  public void testInjectionEmpty() throws IOException {
    FSDatasetInterface fsdataset = new SimulatedFSDataset(conf); 
    Block[] blockReport = fsdataset.getBlockReport();
    assertEquals(0, blockReport.length);
    int bytesAdded = addSomeBlocks(fsdataset);
    blockReport = fsdataset.getBlockReport();
    assertEquals(NUMBLOCKS, blockReport.length);
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
    }
    
    // Inject blocks into an empty fsdataset
    //  - injecting the blocks we got above.
  
   
    SimulatedFSDataset sfsdataset = new SimulatedFSDataset(conf);
    sfsdataset.injectBlocks(blockReport);
    blockReport = sfsdataset.getBlockReport();
    assertEquals(NUMBLOCKS, blockReport.length);
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
      assertEquals(blockIdToLen(b.getBlockId()), sfsdataset.getLength(b));
    }
    assertEquals(bytesAdded, sfsdataset.getDfsUsed());
    assertEquals(sfsdataset.getCapacity()-bytesAdded, sfsdataset.getRemaining());
  }

View Full Code Here

    assertEquals(bytesAdded, sfsdataset.getDfsUsed());
    assertEquals(sfsdataset.getCapacity()-bytesAdded, sfsdataset.getRemaining());
  }


  public void testInjectionNonEmpty() throws IOException {
    FSDatasetInterface fsdataset = new SimulatedFSDataset(conf); 
    
    Block[] blockReport = fsdataset.getBlockReport();
    assertEquals(0, blockReport.length);
    int bytesAdded = addSomeBlocks(fsdataset);
    blockReport = fsdataset.getBlockReport();
    assertEquals(NUMBLOCKS, blockReport.length);
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
    }
    fsdataset = null;
    
    // Inject blocks into an non-empty fsdataset
    //  - injecting the blocks we got above.
  
   
    SimulatedFSDataset sfsdataset = new SimulatedFSDataset(conf);
    // Add come blocks whose block ids do not conflict with
    // the ones we are going to inject.
    bytesAdded += addSomeBlocks(sfsdataset, NUMBLOCKS+1);
    Block[] blockReport2 = sfsdataset.getBlockReport();
    assertEquals(NUMBLOCKS, blockReport.length);
    blockReport2 = sfsdataset.getBlockReport();
    assertEquals(NUMBLOCKS, blockReport.length);
    sfsdataset.injectBlocks(blockReport);
    blockReport = sfsdataset.getBlockReport();
    assertEquals(NUMBLOCKS*2, blockReport.length);
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
      assertEquals(blockIdToLen(b.getBlockId()), sfsdataset.getLength(b));
    }
    assertEquals(bytesAdded, sfsdataset.getDfsUsed());
    assertEquals(sfsdataset.getCapacity()-bytesAdded,  sfsdataset.getRemaining());
    
    
    // Now test that the dataset cannot be created if it does not have sufficient cap


    conf.setLong(SimulatedFSDataset.CONFIG_PROPERTY_CAPACITY, 10);
 
    try {
      sfsdataset = new SimulatedFSDataset(conf);
      sfsdataset.injectBlocks(blockReport);
      assertTrue("Expected an IO exception", false);
    } catch (IOException e) {
      // ok - as expected
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset$SimulatedStorage

org.apache.hadoop.hdfs.MiniDFSCluster

org.apache.hadoop.hdfs.protocol.Block

org.apache.hadoop.hdfs.protocol.BlockListAsLongs

org.apache.hadoop.hdfs.protocol.ExtendedBlock

org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream

org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams

org.apache.hadoop.hdfs.server.datanode.TestSimulatedFSDataset

org.apache.hadoop.hdfs.server.protocol.BlockRecoveryInfo

org.apache.hadoop.hdfs.server.protocol.DatanodeStorage

org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.