Package org.apache.hadoop.hdfs.protocol

Examples of org.apache.hadoop.hdfs.protocol.LocatedBlocksWithMetaInfo


        int srcNamespaceId = 0;
        boolean supportFederation = false;
        if (srcNamenodeProtocolProxy.isMethodSupported(
            "openAndFetchMetaInfo", String.class, long.class, long.class)) {
          supportFederation = true;
          LocatedBlocksWithMetaInfo srcBlockWithMetaInfo =
            srcNamenode.openAndFetchMetaInfo(src, 0, Long.MAX_VALUE);
          srcNamespaceId = srcBlockWithMetaInfo.getNamespaceID();
          srcLocatedBlks = srcBlockWithMetaInfo;
        } else {
          srcLocatedBlks = srcNamenode.getBlockLocations(src, 0, Long.MAX_VALUE);
        }
        List<LocatedBlock> locatedBlocks = srcLocatedBlks.getLocatedBlocks();
View Full Code Here


    //get all block locations
    final LocatedBlocks locatedBlocks = callGetBlockLocations(
        namenode, src, 0, Long.MAX_VALUE, isMetaInfoSuppoted(namenodeProxy));
    int namespaceId = 0;
    if (locatedBlocks instanceof LocatedBlocksWithMetaInfo) {
      LocatedBlocksWithMetaInfo lBlocks = (LocatedBlocksWithMetaInfo)locatedBlocks;
      dataTransferVersion = lBlocks.getDataProtocolVersion();
      namespaceId = lBlocks.getNamespaceID();
    } else if (dataTransferVersion == -1) {
      dataTransferVersion = namenode.getDataTransferProtocolVersion();
    }
    final List<LocatedBlock> locatedblocks  = locatedBlocks.getLocatedBlocks();
    final DataOutputBuffer md5out = new DataOutputBuffer();
View Full Code Here

    Path f = new Path("/testNameNodeFingerprintSent1.txt");
    DataOutputStream a_out = fs.create(f);
    a_out.writeBytes("something");
    a_out.close();
   
    LocatedBlocksWithMetaInfo locs = client.namenode.openAndFetchMetaInfo("/testNameNodeFingerprintSent.txt",
        0L, 0L);
    TestCase.assertEquals(locs.getMethodFingerPrint(), serverFp);   
  }
View Full Code Here

    cluster.setLeasePeriod(1000, 1000);
    AppendTestUtil.sleep(5000);

    //check block sizes
    final long len = fs.getFileStatus(pnew).getLen();
    final LocatedBlocksWithMetaInfo locatedblocks = fs.dfs.namenode.openAndFetchMetaInfo(pnew.toString(), 0L, len);
    final int numblock = locatedblocks.locatedBlockCount();
    for(int i = 0; i < numblock; i++) {
      final LocatedBlock lb = locatedblocks.get(i);
      final Block blk = lb.getBlock();
      final long size = lb.getBlockSize();
      if (i < numblock - 1) {
        assertEquals(BLOCK_SIZE, size);
      }
      for(DatanodeInfo datanodeinfo : lb.getLocations()) {
        final DataNode dn = cluster.getDataNode(datanodeinfo.getIpcPort());
        final BlockMetaDataInfo metainfo = dn.getBlockMetaDataInfo(
            locatedblocks.getNamespaceID(), blk);
        assertEquals(size, metainfo.getNumBytes());
      }
    }
  }
View Full Code Here

  }

  public boolean verifyBlockLocations(String src, String destination,
      NameNode srcNameNode, NameNode dstNameNode, boolean hardlink)
      throws IOException {
    LocatedBlocksWithMetaInfo srcLocatedBlocks =
      srcNameNode.openAndFetchMetaInfo(src, 0,Long.MAX_VALUE);
    List<LocatedBlock> srcblocks = srcLocatedBlocks.getLocatedBlocks();
    LocatedBlocksWithMetaInfo dstLocatedBlocks =
      dstNameNode.openAndFetchMetaInfo(destination, 0, Long.MAX_VALUE);
    List<LocatedBlock> dstblocks = dstLocatedBlocks.getLocatedBlocks();

    assertEquals(srcblocks.size(), dstblocks.size());

    Iterator<LocatedBlock> srcIt = srcblocks.iterator();
    Iterator<LocatedBlock> dstIt = dstblocks.iterator();
    while (srcIt.hasNext()) {
      LocatedBlock srcBlock = srcIt.next();
      LocatedBlock dstBlock = dstIt.next();
      List<DatanodeInfo> srcLocations = Arrays.asList(srcBlock.getLocations());
      List<DatanodeInfo> dstLocations = Arrays.asList(dstBlock.getLocations());

      System.out.println("Locations for src block : " + srcBlock.getBlock()
          + " file : " + src);
      for (DatanodeInfo info : srcLocations) {
        System.out.println("Datanode : " + info.toString());
      }

      System.out.println("Locations for dst block : " + dstBlock.getBlock()
          + " file : " + destination);
      for (DatanodeInfo info : dstLocations) {
        System.out.println("Datanode : " + info.toString());
      }

      assertEquals(srcLocations.size(), dstLocations.size());

      if (srcNameNode.getNameNodeAddress().equals(
          dstNameNode.getNameNodeAddress())) {
        // Same FS copy, verify blocks are machine local.
        assertTrue(srcLocations.containsAll(dstLocations));
        assertTrue(dstLocations.containsAll(srcLocations));
      } else {
        // Verify blocks are rack local.
        Iterator<DatanodeInfo> sit = srcLocations.iterator();
        while (sit.hasNext()) {
          DatanodeInfo srcInfo = sit.next();

          // Verify location.
          Iterator<DatanodeInfo> dit = dstLocations.iterator();
          boolean flag = false;
          while (dit.hasNext() && !flag) {
            DatanodeInfo dstInfo = dit.next();
            if (dstInfo.getHostName().equals(srcInfo.getHostName())) {
              verifyHardLinks(srcInfo, dstInfo,
                  srcLocatedBlocks.getNamespaceID(), srcBlock.getBlock(),
                  dstLocatedBlocks.getNamespaceID(), dstBlock.getBlock(),
                  hardlink);
            }
            flag = srcInfo.getNetworkLocation().equals(
                dstInfo.getNetworkLocation());
          }
View Full Code Here

        int srcNamespaceId = 0;
        boolean supportFederation = false;
        if (srcNamenodeProtocolProxy.isMethodSupported(
            "openAndFetchMetaInfo", String.class, long.class, long.class)) {
          supportFederation = true;
          LocatedBlocksWithMetaInfo srcBlockWithMetaInfo =
            srcNamenode.openAndFetchMetaInfo(src, 0, Long.MAX_VALUE);
          srcNamespaceId = srcBlockWithMetaInfo.getNamespaceID();
          srcLocatedBlks = srcBlockWithMetaInfo;
        } else {
          srcLocatedBlks = srcNamenode.getBlockLocations(src, 0, Long.MAX_VALUE);
        }
        List<LocatedBlock> locatedBlocks = srcLocatedBlks.getLocatedBlocks();
View Full Code Here

 
  LocatedBlocks createLocatedBlocks(List<LocatedBlock> blocks,
      BlockMetaInfoType type,int namespaceid, int methodsFingerprint) {
    switch (type) {
    case VERSION_AND_NAMESPACEID:
      return new LocatedBlocksWithMetaInfo(
          computeContentSummary().getLength(), blocks,
          isUnderConstruction(), DataTransferProtocol.DATA_TRANSFER_VERSION,
          namespaceid, methodsFingerprint);
    case VERSION:
      return new VersionedLocatedBlocks(computeContentSummary().getLength(), blocks,
View Full Code Here

    throws IOException {
      try {
        if (namenodeProtocolProxy != null &&
              namenodeProtocolProxy.isMethodSupported("openAndFetchMetaInfo",
                String.class, long.class, long.class)) {
          LocatedBlocksWithMetaInfo locs =
            namenode.openAndFetchMetaInfo(src, start, length);
          if (locs != null) {
            this.namespaceId = locs.getNamespaceID();
            updateDataTransferProtocolVersionIfNeeded(locs.getDataProtocolVersion());
            getNewNameNodeIfNeeded(locs.getMethodFingerPrint());
          }
          checkLocatedBlocks(locs);
          return locs;
        } else if (namenodeProtocolProxy != null &&
            namenodeProtocolProxy.isMethodSupported("open", String.class,
                long.class, long.class)) {
          VersionedLocatedBlocks locs = namenode.open(src, start, length);
          if (locs != null) {
            updateDataTransferProtocolVersionIfNeeded(locs.getDataProtocolVersion());
          }
          checkLocatedBlocks(locs);
          return locs;
        } else {
          LocatedBlocks locs = namenode.getBlockLocations(src, start, length);
View Full Code Here

    this.out = checkAndMarkRunningBalancer();
    if (out == null) {
      throw new IOException("Another balancer is running");
    }
    // get namespace id
    LocatedBlocksWithMetaInfo locations = client.openAndFetchMetaInfo(BALANCER_ID_PATH.toString(), 0L, 1L);
    this.namespaceId = locations.getNamespaceID();
  }
View Full Code Here

    //get all block locations
    final LocatedBlocks locatedBlocks = callGetBlockLocations(
        namenode, src, 0, Long.MAX_VALUE, isMetaInfoSuppoted(namenodeProxy));
    int namespaceId = 0;
    if (locatedBlocks instanceof LocatedBlocksWithMetaInfo) {
      LocatedBlocksWithMetaInfo lBlocks = (LocatedBlocksWithMetaInfo)locatedBlocks;
      dataTransferVersion = lBlocks.getDataProtocolVersion();
      namespaceId = lBlocks.getNamespaceID();
    } else if (dataTransferVersion == -1) {
      dataTransferVersion = namenode.getDataTransferProtocolVersion();
    }
    final List<LocatedBlock> locatedblocks  = locatedBlocks.getLocatedBlocks();
    final DataOutputBuffer md5out = new DataOutputBuffer();
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.protocol.LocatedBlocksWithMetaInfo

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.