Examples of org.apache.hadoop.hdfs.protocol.ExtendedBlock

org.apache.hadoop.hdfs.protocol.ExtendedBlock
Identifies a Block uniquely across the block pools

      }
    }


    private ExtendedBlock addBlocks(String fileName, String clientName)
    throws IOException {
      ExtendedBlock prevBlock = null;
      for(int jdx = 0; jdx < blocksPerFile; jdx++) {
        LocatedBlock loc = nameNodeProto.addBlock(fileName, clientName,
            prevBlock, null, INodeId.GRANDFATHER_INODE_ID, null);
        prevBlock = loc.getBlock();
        for(DatanodeInfo dnInfo : loc.getLocations()) {

View Full Code Here

      nodes = blks.get(0).getLocations();
      oldBlock = blks.get(0).getBlock();
      
      LocatedBlock newLbk = client.getNamenode().updateBlockForPipeline(
          oldBlock, client.getClientName());
      newBlock = new ExtendedBlock(oldBlock.getBlockPoolId(),
          oldBlock.getBlockId(), oldBlock.getNumBytes(), 
          newLbk.getBlock().getGenerationStamp());
    }

View Full Code Here

      
      // Corrupt a block by deleting it
      String[] fileNames = util.getFileNames(topDir);
      DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost",
                                          cluster.getNameNodePort()), conf);
      ExtendedBlock block = dfsClient.getNamenode().getBlockLocations(
          fileNames[0], 0, Long.MAX_VALUE).get(0).getBlock();
      for (int i=0; i<4; i++) {
        File blockFile = MiniDFSCluster.getBlockFile(i, block);
        if(blockFile != null && blockFile.exists()) {
          assertTrue(blockFile.delete());

View Full Code Here

    fs = cluster.getFileSystem();
    Path file1 = new Path("/testCorruptBlock");
    DFSTestUtil.createFile(fs, file1, 1024, factor, 0);
    // Wait until file replication has completed
    DFSTestUtil.waitReplication(fs, file1, factor);
    ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1);


    // Make sure filesystem is in healthy state
    outStr = runFsck(conf, 0, true, "/");
    System.out.println(outStr);
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));

View Full Code Here

      }
    }
    
    /* Send a block replace request to the output stream*/
    private void sendRequest(DataOutputStream out) throws IOException {
      final ExtendedBlock eb = new ExtendedBlock(nnc.blockpoolID, block.getBlock());
      final Token<BlockTokenIdentifier> accessToken = nnc.getAccessToken(eb);
      new Sender(out).replaceBlock(eb, accessToken,
          source.getStorageID(), proxySource.getDatanode());
    }

View Full Code Here

      if (!blockInfoSet.isEmpty()) {
        block = blockInfoSet.first();
      }
    }
    if ( block != null ) {
      verifyBlock(new ExtendedBlock(blockPoolId, block));
      processedBlocks.put(block.getBlockId(), 1);
    }
  }

View Full Code Here

      fsIn = fs.open(TEST_PATH);
      IOUtils.readFully(fsIn, original, 0,
          BlockReaderLocalTest.TEST_LENGTH);
      fsIn.close();
      fsIn = null;
      ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, TEST_PATH);
      File dataFile = MiniDFSCluster.getBlockFile(0, block);
      File metaFile = MiniDFSCluster.getBlockMetadataFile(0, block);


      ShortCircuitCache shortCircuitCache =
          ClientContext.getFromConf(conf).getShortCircuitCache();
      cluster.shutdown();
      cluster = null;
      test.setup(dataFile, checksum);
      FileInputStream streams[] = {
          new FileInputStream(dataFile),
          new FileInputStream(metaFile)
      };
      dataIn = streams[0];
      metaIn = streams[1];
      ExtendedBlockId key = new ExtendedBlockId(block.getBlockId(),
          block.getBlockPoolId());
      raf = new RandomAccessFile(
          new File(sockDir.getDir().getAbsolutePath(),
            UUID.randomUUID().toString()), "rw");
      raf.setLength(8192);
      FileInputStream shmStream = new FileInputStream(raf.getFD());

View Full Code Here

    Path file1 = new Path("/tmp/testBadBlockReportOnTransfer/file1");
    DFSTestUtil.createFile(fs, file1, 1024, replFactor, 0);
    DFSTestUtil.waitReplication(fs, file1, replFactor);
  
    // Corrupt the block belonging to the created file
    ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1);


    int blockFilesCorrupted = cluster.corruptBlockOnDataNodes(block);
    assertEquals("Corrupted too few blocks", replFactor, blockFilesCorrupted); 


    // Increase replication factor, this should invoke transfer request

View Full Code Here

      out.close();
      
      waitForBlockReplication(testFile, dfsClient.getNamenode(), numDataNodes, -1);


      // get first block of the file.
      ExtendedBlock block = dfsClient.getNamenode().getBlockLocations(testFile,
          0, Long.MAX_VALUE).get(0).getBlock();
      
      cluster.shutdown();
      cluster = null;

View Full Code Here

    final FileSystem fs = cluster.getFileSystem();
    final int fileLen = fs.getConf().getInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 512);
    DFSTestUtil.createFile(fs, fileName, fileLen, REPLICATION_FACTOR, 0);
    DFSTestUtil.waitReplication(fs, fileName, REPLICATION_FACTOR);


    ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, fileName);


    // Change the length of a replica
    for (int i=0; i<cluster.getDataNodes().size(); i++) {
      if (TestDatanodeBlockScanner.changeReplicaLength(block, i, lenDelta)) {
        break;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hdfs.protocol.ExtendedBlock

org.apache.hadoop.fs.TestEnhancedByteBufferAccess

org.apache.hadoop.hdfs.BlockReaderTestUtil

org.apache.hadoop.hdfs.BlockStorageLocationUtil

org.apache.hadoop.hdfs.DFSClient

org.apache.hadoop.hdfs.DFSInputStream

org.apache.hadoop.hdfs.DFSOutputStream

org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer

org.apache.hadoop.hdfs.DistributedFileSystem

org.apache.hadoop.hdfs.protocolPB.PBHelper

org.apache.hadoop.hdfs.protocolPB.TestPBHelper

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.