Examples of org.apache.hadoop.hdfs.protocol.Block

org.apache.hadoop.hdfs.protocol.Block
A Block is a Hadoop FS primitive, identified by a long.


      File blockFiles[] = dir.listFiles();
      for (int i = 0; i < blockFiles.length; i++) {
        if (Block.isBlockFilename(blockFiles[i])) {
          long genStamp = getGenerationStampFromFile(blockFiles, blockFiles[i]);
          blockSet.add(new Block(blockFiles[i], blockFiles[i].length(), genStamp));
        }
      }
    }

View Full Code Here


      File blockFiles[] = dir.listFiles();
      for (int i = 0; i < blockFiles.length; i++) {
        if (Block.isBlockFilename(blockFiles[i])) {
          long genStamp = getGenerationStampFromFile(blockFiles, blockFiles[i]);
          Block block = new Block(blockFiles[i], blockFiles[i].length(), genStamp);
          blockSet.add(new BlockAndFile(blockFiles[i].getAbsoluteFile(), block));
        }
      }
    }

View Full Code Here


      File blockFiles[] = dir.listFiles();
      for (int i = 0; i < blockFiles.length; i++) {
        if (Block.isBlockFilename(blockFiles[i])) {
          long genStamp = getGenerationStampFromFile(blockFiles, blockFiles[i]);
          volumeMap.put(new Block(blockFiles[i], blockFiles[i].length(), genStamp), 
                        new DatanodeBlockInfo(volume, blockFiles[i]));
        }
      }
    }

View Full Code Here

          }
          if (logVersion < -7) {
            blockSize = readLong(in);
          }
          // get blocks
          Block blocks[] = null;
          if (logVersion <= -14) {
            blocks = readBlocks(in);
          } else {
            BlockTwo oldblk = new BlockTwo();
            int num = in.readInt();
            blocks = new Block[num];
            for (int i = 0; i < num; i++) {
              oldblk.readFields(in);
              blocks[i] = new Block(oldblk.blkid, oldblk.len, 
                                    Block.GRANDFATHER_GENERATION_STAMP);
            }
          }


          // Older versions of HDFS does not store the block size in inode.

View Full Code Here


  static private Block[] readBlocks(DataInputStream in) throws IOException {
    int numBlocks = in.readInt();
    Block[] blocks = new Block[numBlocks];
    for (int i = 0; i < numBlocks; i++) {
      blocks[i] = new Block();
      blocks[i].readFields(in);
    }
    return blocks;
  }

View Full Code Here

  private BlockReader getBlockReader(
    int offset, int lenToRead) throws IOException {
    InetSocketAddress targetAddr = null;
    Socket s = null;
    BlockReader blockReader = null;
    Block block = testBlock.getBlock();
    DatanodeInfo[] nodes = testBlock.getLocations();
    targetAddr = NetUtils.createSocketAddr(nodes[0].getName());
    s = new Socket();
    s.connect(targetAddr, HdfsConstants.READ_TIMEOUT);
    s.setSoTimeout(HdfsConstants.READ_TIMEOUT);


    return DFSClient.BlockReader.newBlockReader(
      s, targetAddr.toString()+ ":" + block.getBlockId(),
      block.getBlockId(),
      testBlock.getBlockToken(),
      block.getGenerationStamp(),
      offset, lenToRead,
      conf.getInt("io.file.buffer.size", 4096));
  }

View Full Code Here

      
      // populate the cluster with a one block file
      final Path FILE_PATH = new Path("/testfile");
      DFSTestUtil.createFile(fs, FILE_PATH, 1L, REPLICATION_FACTOR, 1L);
      DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR);
      Block block = DFSTestUtil.getFirstBlock(fs, FILE_PATH);


      // keep a copy of all datanode descriptor
      DatanodeDescriptor[] datanodes = (DatanodeDescriptor[])
         namesystem.heartbeats.toArray(new DatanodeDescriptor[REPLICATION_FACTOR]);

View Full Code Here


    LocatedBlock lblocks[] = new LocatedBlock[2];


    // Find block in data stream.
    DFSClient.DFSDataInputStream dfsIn = (DFSClient.DFSDataInputStream) in;
    Block dataBlock = dfsIn.getCurrentBlock();
    if (dataBlock == null) {
      LOG.error("Error: Current block in data stream is null! ");
      return false;
    }
    DatanodeInfo[] dataNode = {dfsIn.getCurrentDatanode()}; 
    lblocks[0] = new LocatedBlock(dataBlock, dataNode);
    LOG.info("Found checksum error in data stream at block="
        + dataBlock + " on datanode="
        + dataNode[0].getName());


    // Find block in checksum stream
    DFSClient.DFSDataInputStream dfsSums = (DFSClient.DFSDataInputStream) sums;
    Block sumsBlock = dfsSums.getCurrentBlock();
    if (sumsBlock == null) {
      LOG.error("Error: Current block in checksum stream is null! ");
      return false;
    }
    DatanodeInfo[] sumsNode = {dfsSums.getCurrentDatanode()};

View Full Code Here

    int fileLen = Math.min(conf.getInt("dfs.block.size", 4096), 4096);
    
    createFile(fileSys, file, fileLen);


    // get the first blockid for the file
    Block firstBlock = DFSTestUtil.getFirstBlock(fileSys, file);
    long newBlockId = firstBlock.getBlockId() + 1;


    recvBuf.reset();
    sendBuf.reset();
    
    // bad version
    recvOut.writeShort((short)(DataTransferProtocol.DATA_TRANSFER_VERSION-1));
    sendOut.writeShort((short)(DataTransferProtocol.DATA_TRANSFER_VERSION-1));
    sendRecvData("Wrong Version", true);


    // bad ops
    sendBuf.reset();
    sendOut.writeShort((short)DataTransferProtocol.DATA_TRANSFER_VERSION);
    sendOut.writeByte((byte)(DataTransferProtocol.OP_WRITE_BLOCK-1));
    sendRecvData("Wrong Op Code", true);
    
    /* Test OP_WRITE_BLOCK */
    sendBuf.reset();
    sendOut.writeShort((short)DataTransferProtocol.DATA_TRANSFER_VERSION);
    sendOut.writeByte((byte)DataTransferProtocol.OP_WRITE_BLOCK);
    sendOut.writeLong(newBlockId); // block id
    sendOut.writeLong(0);          // generation stamp
    sendOut.writeInt(0);           // targets in pipeline 
    sendOut.writeBoolean(false);   // recoveryFlag
    Text.writeString(sendOut, "cl");// clientID
    sendOut.writeBoolean(false); // no src node info
    sendOut.writeInt(0);           // number of downstream targets
    BlockTokenSecretManager.DUMMY_TOKEN.write(sendOut);
    sendOut.writeByte((byte)DataChecksum.CHECKSUM_CRC32);
    
    // bad bytes per checksum
    sendOut.writeInt(-1-random.nextInt(oneMil));
    recvBuf.reset();
    recvOut.writeShort((short)DataTransferProtocol.OP_STATUS_ERROR);
    sendRecvData("wrong bytesPerChecksum while writing", true);


    sendBuf.reset();
    recvBuf.reset();
    sendOut.writeShort((short)DataTransferProtocol.DATA_TRANSFER_VERSION);
    sendOut.writeByte((byte)DataTransferProtocol.OP_WRITE_BLOCK);
    sendOut.writeLong(newBlockId);
    sendOut.writeLong(0);          // generation stamp
    sendOut.writeInt(0);           // targets in pipeline 
    sendOut.writeBoolean(false);   // recoveryFlag
    Text.writeString(sendOut, "cl");// clientID
    sendOut.writeBoolean(false); // no src node info


    // bad number of targets
    sendOut.writeInt(-1-random.nextInt(oneMil));
    recvOut.writeShort((short)DataTransferProtocol.OP_STATUS_ERROR);
    sendRecvData("bad targets len while writing block " + newBlockId, true);


    sendBuf.reset();
    recvBuf.reset();
    sendOut.writeShort((short)DataTransferProtocol.DATA_TRANSFER_VERSION);
    sendOut.writeByte((byte)DataTransferProtocol.OP_WRITE_BLOCK);
    sendOut.writeLong(++newBlockId);
    sendOut.writeLong(0);          // generation stamp
    sendOut.writeInt(0);           // targets in pipeline 
    sendOut.writeBoolean(false);   // recoveryFlag
    Text.writeString(sendOut, "cl");// clientID
    sendOut.writeBoolean(false); // no src node info
    sendOut.writeInt(0);
    BlockTokenSecretManager.DUMMY_TOKEN.write(sendOut);
    sendOut.writeByte((byte)DataChecksum.CHECKSUM_CRC32);
    sendOut.writeInt((int)512);
    sendOut.writeInt(4);           // size of packet
    sendOut.writeLong(0);          // OffsetInBlock
    sendOut.writeLong(100);        // sequencenumber
    sendOut.writeBoolean(false);   // lastPacketInBlock
    
    // bad data chunk length
    sendOut.writeInt(-1-random.nextInt(oneMil));
    recvOut.writeShort((short)DataTransferProtocol.OP_STATUS_SUCCESS);
    Text.writeString(recvOut, ""); // first bad node
    recvOut.writeLong(100);        // sequencenumber
    recvOut.writeShort((short)DataTransferProtocol.OP_STATUS_ERROR);
    sendRecvData("negative DATA_CHUNK len while writing block " + newBlockId, 
                 true);


    // test for writing a valid zero size block
    sendBuf.reset();
    recvBuf.reset();
    sendOut.writeShort((short)DataTransferProtocol.DATA_TRANSFER_VERSION);
    sendOut.writeByte((byte)DataTransferProtocol.OP_WRITE_BLOCK);
    sendOut.writeLong(++newBlockId);
    sendOut.writeLong(0);          // generation stamp
    sendOut.writeInt(0);           // targets in pipeline 
    sendOut.writeBoolean(false);   // recoveryFlag
    Text.writeString(sendOut, "cl");// clientID
    sendOut.writeBoolean(false); // no src node info
    sendOut.writeInt(0);
    BlockTokenSecretManager.DUMMY_TOKEN.write(sendOut);
    sendOut.writeByte((byte)DataChecksum.CHECKSUM_CRC32);
    sendOut.writeInt((int)512);    // checksum size
    sendOut.writeInt(8);           // size of packet
    sendOut.writeLong(0);          // OffsetInBlock
    sendOut.writeLong(100);        // sequencenumber
    sendOut.writeBoolean(true);    // lastPacketInBlock


    sendOut.writeInt(0);           // chunk length
    sendOut.writeInt(0);           // zero checksum
    //ok finally write a block with 0 len
    recvOut.writeShort((short)DataTransferProtocol.OP_STATUS_SUCCESS);
    Text.writeString(recvOut, ""); // first bad node
    new DataTransferProtocol.PipelineAck(100, 
        new short[]{DataTransferProtocol.OP_STATUS_SUCCESS}).write(recvOut);
    sendRecvData("Writing a zero len block blockid " + newBlockId, false);
    
    /* Test OP_READ_BLOCK */


    // bad block id
    sendBuf.reset();
    recvBuf.reset();
    sendOut.writeShort((short)DataTransferProtocol.DATA_TRANSFER_VERSION);
    sendOut.writeByte((byte)DataTransferProtocol.OP_READ_BLOCK);
    newBlockId = firstBlock.getBlockId()-1;
    sendOut.writeLong(newBlockId);
    sendOut.writeLong(firstBlock.getGenerationStamp());
    sendOut.writeLong(0L);
    sendOut.writeLong(fileLen);
    recvOut.writeShort((short)DataTransferProtocol.OP_STATUS_ERROR);
    Text.writeString(sendOut, "cl");
    BlockTokenSecretManager.DUMMY_TOKEN.write(sendOut);
    sendRecvData("Wrong block ID " + newBlockId + " for read", false); 


    // negative block start offset
    sendBuf.reset();
    sendOut.writeShort((short)DataTransferProtocol.DATA_TRANSFER_VERSION);
    sendOut.writeByte((byte)DataTransferProtocol.OP_READ_BLOCK);
    sendOut.writeLong(firstBlock.getBlockId());
    sendOut.writeLong(firstBlock.getGenerationStamp());
    sendOut.writeLong(-1L);
    sendOut.writeLong(fileLen);
    Text.writeString(sendOut, "cl");
    BlockTokenSecretManager.DUMMY_TOKEN.write(sendOut);
    sendRecvData("Negative start-offset for read for block " + 
                 firstBlock.getBlockId(), false);


    // bad block start offset
    sendBuf.reset();
    sendOut.writeShort((short)DataTransferProtocol.DATA_TRANSFER_VERSION);
    sendOut.writeByte((byte)DataTransferProtocol.OP_READ_BLOCK);
    sendOut.writeLong(firstBlock.getBlockId());
    sendOut.writeLong(firstBlock.getGenerationStamp());
    sendOut.writeLong(fileLen);
    sendOut.writeLong(fileLen);
    Text.writeString(sendOut, "cl");
    BlockTokenSecretManager.DUMMY_TOKEN.write(sendOut);
    sendRecvData("Wrong start-offset for reading block " +
                 firstBlock.getBlockId(), false);
    
    // negative length is ok. Datanode assumes we want to read the whole block.
    recvBuf.reset();
    recvOut.writeShort((short)DataTransferProtocol.OP_STATUS_SUCCESS);    
    sendBuf.reset();
    sendOut.writeShort((short)DataTransferProtocol.DATA_TRANSFER_VERSION);
    sendOut.writeByte((byte)DataTransferProtocol.OP_READ_BLOCK);
    sendOut.writeLong(firstBlock.getBlockId());
    sendOut.writeLong(firstBlock.getGenerationStamp());
    sendOut.writeLong(0);
    sendOut.writeLong(-1-random.nextInt(oneMil));
    Text.writeString(sendOut, "cl");
    BlockTokenSecretManager.DUMMY_TOKEN.write(sendOut);
    sendRecvData("Negative length for reading block " +
                 firstBlock.getBlockId(), false);
    
    // length is more than size of block.
    recvBuf.reset();
    recvOut.writeShort((short)DataTransferProtocol.OP_STATUS_ERROR);    
    sendBuf.reset();
    sendOut.writeShort((short)DataTransferProtocol.DATA_TRANSFER_VERSION);
    sendOut.writeByte((byte)DataTransferProtocol.OP_READ_BLOCK);
    sendOut.writeLong(firstBlock.getBlockId());
    sendOut.writeLong(firstBlock.getGenerationStamp());
    sendOut.writeLong(0);
    sendOut.writeLong(fileLen + 1);
    Text.writeString(sendOut, "cl");
    BlockTokenSecretManager.DUMMY_TOKEN.write(sendOut);
    sendRecvData("Wrong length for reading block " +
                 firstBlock.getBlockId(), false);
    
    //At the end of all this, read the file to make sure that succeeds finally.
    sendBuf.reset();
    sendOut.writeShort((short)DataTransferProtocol.DATA_TRANSFER_VERSION);
    sendOut.writeByte((byte)DataTransferProtocol.OP_READ_BLOCK);
    sendOut.writeLong(firstBlock.getBlockId());
    sendOut.writeLong(firstBlock.getGenerationStamp());
    sendOut.writeLong(0);
    sendOut.writeLong(fileLen);
    Text.writeString(sendOut, "cl");
    BlockTokenSecretManager.DUMMY_TOKEN.write(sendOut);
    readFile(fileSys, file, fileLen);

View Full Code Here

    //no-op
  }


  /** {@inheritDoc} */
  public Block getStoredBlock(long blkid) throws IOException {
    Block b = new Block(blkid);
    BInfo binfo = blockMap.get(b);
    if (binfo == null) {
      return null;
    }
    b.setGenerationStamp(binfo.getGenerationStamp());
    b.setNumBytes(binfo.getlength());
    return b;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hdfs.protocol.Block

org.apache.hadoop.hdfs.DataNodeCluster

org.apache.hadoop.hdfs.DFSInputStream

org.apache.hadoop.hdfs.DistributedFileSystem

org.apache.hadoop.hdfs.protocolPB.PBHelper

org.apache.hadoop.hdfs.protocolPB.TestPBHelper

org.apache.hadoop.hdfs.security.token.block.TestBlockToken

org.apache.hadoop.hdfs.server.balancer.Balancer

org.apache.hadoop.hdfs.server.balancer.TestBalancer

org.apache.hadoop.hdfs.server.blockmanagement.BlockManager

org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.