Package org.apache.hadoop.hdfs.protocol

Examples of org.apache.hadoop.hdfs.protocol.LocatedBlocksWithMetaInfo


    Path f = new Path("/testNameNodeFingerprintSent1.txt");
    DataOutputStream a_out = fs.create(f);
    a_out.writeBytes("something");
    a_out.close();
   
    LocatedBlocksWithMetaInfo locs = client.namenode.openAndFetchMetaInfo("/testNameNodeFingerprintSent.txt",
        0L, 0L);
    TestCase.assertEquals(locs.getMethodFingerPrint(), serverFp);   
  }
View Full Code Here


    try {
      Path path = new Path("/dir/file");
      DFSTestUtil.createFile(fs, path, 1, (short)1, 0L);
      DFSTestUtil.waitReplication(fs, path, (short)1);
      FileStatus status = fs.getFileStatus(path);
      LocatedBlocksWithMetaInfo blocks = namenode.openAndFetchMetaInfo(
          path.toString(), 0, status.getLen());
      Assert.assertEquals(1, blocks.getLocatedBlocks().size());
      LocatedBlock block = blocks.getLocatedBlocks().get(0);
      Assert.assertEquals(1, block.getLocations().length);
      DatanodeInfo source = block.getLocations()[0];
      Set<DatanodeInfo> excluded = new HashSet<DatanodeInfo>();
      for (DatanodeInfo d : datanodes) {
        excluded.add(d);
      }
      excluded.remove(source);
      DatanodeInfo target = excluded.iterator().next();
      excluded.add(source);
      excluded.remove(target);
      BlockMover.BlockMoveAction action =
          blockMover.new BlockMoveAction(block, source, excluded, 1,
              blocks.getDataProtocolVersion(), blocks.getNamespaceID());
      LOG.info("Start moving block from " + source + " to " + target);
      action.run();
      LOG.info("Done moving block");
      boolean blockMoved = false;
      long startTime = System.currentTimeMillis();
      while (!blockMoved && System.currentTimeMillis() - startTime < 60000) {
        blocks = namenode.openAndFetchMetaInfo(
            path.toString(), 0, status.getLen());
        block = blocks.getLocatedBlocks().get(0);
        if (block.getLocations().length == 1 &&
            block.getLocations()[0].equals((target))) {
          blockMoved = true;
          break;
        }
View Full Code Here

    cluster.setLeasePeriod(1000, 1000);
    AppendTestUtil.sleep(5000);

    //check block sizes
    final long len = fs.getFileStatus(pnew).getLen();
    final LocatedBlocksWithMetaInfo locatedblocks = fs.dfs.namenode.openAndFetchMetaInfo(pnew.toString(), 0L, len);
    final int numblock = locatedblocks.locatedBlockCount();
    for(int i = 0; i < numblock; i++) {
      final LocatedBlock lb = locatedblocks.get(i);
      final Block blk = lb.getBlock();
      final long size = lb.getBlockSize();
      if (i < numblock - 1) {
        assertEquals(BLOCK_SIZE, size);
      }
      for(DatanodeInfo datanodeinfo : lb.getLocations()) {
        final DataNode dn = cluster.getDataNode(datanodeinfo.getIpcPort());
        final BlockMetaDataInfo metainfo = dn.getBlockMetaDataInfo(
            locatedblocks.getNamespaceID(), blk);
        assertEquals(size, metainfo.getNumBytes());
      }
    }
  }
View Full Code Here

     
      VersionedLocatedBlocks locatedBlocks;
      int namespaceId = 0;
      int methodsFingerprint = 0;
      if (DFSClient.isMetaInfoSuppoted(fs.getClient().namenodeProtocolProxy)) {
        LocatedBlocksWithMetaInfo lbksm = fs.getClient().namenode.
                  openAndFetchMetaInfo(uriPath, 0, stat.getLen());
        namespaceId = lbksm.getNamespaceID();
        locatedBlocks = lbksm;
        methodsFingerprint = lbksm.getMethodFingerPrint();
      } else {
        locatedBlocks = fs.getClient().namenode.open(uriPath, 0, stat.getLen());
      }
      final int dataTransferVersion = locatedBlocks.getDataProtocolVersion();
View Full Code Here

        new LinkedList<LocatedBlockWithMetaInfo>();
      VersionedLocatedBlocks locatedBlocks;
      int namespaceId = 0;
      int methodFingerprint = 0;
      if (DFSClient.isMetaInfoSuppoted(fs.getClient().namenodeProtocolProxy)) {
        LocatedBlocksWithMetaInfo lbksm = fs.getClient().namenode.
                  openAndFetchMetaInfo(uriPath, 0, stat.getLen());
        namespaceId = lbksm.getNamespaceID();
        locatedBlocks = lbksm;
        methodFingerprint = lbksm.getMethodFingerPrint();
        fs.getClient().getNewNameNodeIfNeeded(methodFingerprint);
      } else {
        locatedBlocks = fs.getClient().namenode.open(uriPath, 0, stat.getLen());
      }
      final int dataTransferVersion = locatedBlocks.getDataProtocolVersion();
View Full Code Here

        new LinkedList<LocatedBlockWithMetaInfo>();
      VersionedLocatedBlocks locatedBlocks;
      int namespaceId = 0;
      int methodFingerprint = 0;
      if (DFSClient.isMetaInfoSuppoted(fs.getClient().namenodeProtocolProxy)) {
        LocatedBlocksWithMetaInfo lbksm = fs.getClient().namenode.
                  openAndFetchMetaInfo(uriPath, 0, stat.getLen());
        namespaceId = lbksm.getNamespaceID();
        locatedBlocks = lbksm;
        methodFingerprint = lbksm.getMethodFingerPrint();
        fs.getClient().getNewNameNodeIfNeeded(methodFingerprint);
      } else {
        locatedBlocks = fs.getClient().namenode.open(uriPath, 0, stat.getLen());
      }
      final int dataTransferVersion = locatedBlocks.getDataProtocolVersion();
View Full Code Here

      String src) throws IOException {
    return shell.getSnapshotNode().getLocatedBlocks(snapshotId, src);
  }

  public DFSInputStream open(String snapshotId, String src) throws IOException {
    LocatedBlocksWithMetaInfo blocks[] = getLocatedBlocks(snapshotId, src);

    // Not strictly correct. block.length = 1 could mean directory with
    // one file. Might want to add a file specific API.
    if (blocks == null || blocks.length != 1) {
      throw new IOException("File at " + src + " doesn't exist in snapshot");
View Full Code Here

    private Map<Long, LocatedBlockWithMetaInfo> createOffsetToLocatedBlockMap(
        VersionedLocatedBlocks lbs) {
      Map<Long, LocatedBlockWithMetaInfo> result =
          new HashMap<Long, LocatedBlockWithMetaInfo>();
      if (lbs instanceof LocatedBlocksWithMetaInfo) {
        LocatedBlocksWithMetaInfo lbsm = (LocatedBlocksWithMetaInfo)lbs;
        for (LocatedBlock lb : lbs.getLocatedBlocks()) {
          result.put(lb.getStartOffset(), new LocatedBlockWithMetaInfo(
              lb.getBlock(), lb.getLocations(), lb.getStartOffset(),
              lbsm.getDataProtocolVersion(), lbsm.getNamespaceID(),
              lbsm.getMethodFingerPrint()));
        }
      } else {
        for (LocatedBlock lb : lbs.getLocatedBlocks()) {
          result.put(lb.getStartOffset(), new LocatedBlockWithMetaInfo(
              lb.getBlock(), lb.getLocations(), lb.getStartOffset(),
View Full Code Here

        throw new IOException("Cannot obtain " + LocatedBlocks.class +
            " from " + fs.getClass().getSimpleName());
      }
      DistributedFileSystem dfs = (DistributedFileSystem) fs;
      if (DFSClient.isMetaInfoSuppoted(dfs.getClient().namenodeProtocolProxy)) {
        LocatedBlocksWithMetaInfo lbwmi =
        dfs.getClient().namenode.openAndFetchMetaInfo(
            file.toUri().getPath(), 0, Long.MAX_VALUE);
        dfs.getClient().getNewNameNodeIfNeeded(lbwmi.getMethodFingerPrint());
        return lbwmi;
      }
      return dfs.getClient().namenode.open(
          file.toUri().getPath(), 0, Long.MAX_VALUE);
    }
View Full Code Here

   
    try {
      if (dfsClient.namenodeProtocolProxy != null &&
          dfsClient.namenodeProtocolProxy.isMethodSupported("openAndFetchMetaInfo",
              String.class, long.class, long.class)) {
        LocatedBlocksWithMetaInfo locs =
            dfsClient.namenode.openAndFetchMetaInfo(src, start, length);
        if (locs != null) {
          dfsClient.updateNamespaceIdIfNeeded(locs.getNamespaceID());
          dfsClient.updateDataTransferProtocolVersionIfNeeded(locs.getDataProtocolVersion());
          dfsClient.getNewNameNodeIfNeeded(locs.getMethodFingerPrint());
        }
        checkLocatedBlocks(locs);
        return locs;
      } else if (dfsClient.namenodeProtocolProxy != null &&
          dfsClient.namenodeProtocolProxy.isMethodSupported("open", String.class,
              long.class, long.class)) {
        VersionedLocatedBlocks locs = dfsClient.namenode.open(src, start, length);
        if (locs != null) {
          dfsClient.updateDataTransferProtocolVersionIfNeeded(locs.getDataProtocolVersion());
        }
        checkLocatedBlocks(locs);
        return locs;
      } else {
        LocatedBlocks locs = dfsClient.namenode.getBlockLocations(src, start, length);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.protocol.LocatedBlocksWithMetaInfo

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.