Examples of org.apache.hadoop.hdfs.protocol.LocatedBlockWithMetaInfo

org.apache.hadoop.hdfs.protocol.LocatedBlockWithMetaInfo
A block with its location and NameNode meta info like data transfer version number and namespace id.

      }
      final int dataTransferVersion = locatedBlocks.getDataProtocolVersion();
      for (LocatedBlock b: locatedBlocks.getLocatedBlocks()) {
        if (b.isCorrupt() ||
            (b.getLocations().length == 0 && b.getBlockSize() > 0)) {
          corrupt.add(new LocatedBlockWithMetaInfo(b.getBlock(),
              b.getLocations(), b.getStartOffset(),
              dataTransferVersion, namespaceId, methodFingerprint));
        }
      }
      return corrupt;

View Full Code Here

        boolean allDecommissioning = true;
        for (DatanodeInfo i : b.getLocations()) {
          allDecommissioning &= i.isDecommissionInProgress();
        }
        if (allDecommissioning) {
          decommissioning.add(new LocatedBlockWithMetaInfo(b.getBlock(),
              b.getLocations(), b.getStartOffset(),
              dataTransferVersion, namespaceId, methodFingerprint));
        }
      }
      return decommissioning;

View Full Code Here

            favoured[i] = new DatanodeInfo(
                   new DatanodeID(name, "", -1, getPort(name)));
          }
        }


        LocatedBlockWithMetaInfo val =  namenode.addBlockAndFetchMetaInfo(
                                 path.pathname, 
                                 clientName,
                                 excludes,
                                 favoured,
                                 startOffset,
                                 lastBlock != null ? convertBlock(lastBlock) : null);


        // convert from LocatedBlockWithMetaInfo to TLocatedBlock
        Block bb = val.getBlock();
  TLocatedBlock tblk = new TLocatedBlock();
        tblk.block = new TBlock(bb.getBlockId(), bb.getNumBytes(),
                                bb.getGenerationStamp());
        tblk.namespaceId = val.getNamespaceID();
        tblk.dataTransferVersion = val.getDataProtocolVersion();


        // fill up replica locations
        tblk.location = new ArrayList<TDatanodeID>();
        DatanodeInfo[] replicas = val.getLocations();
        for (int i = 0; i < replicas.length; i++) {
          tblk.location.add(i, new TDatanodeID(replicas[i].getName()));
        }
        return tblk;
      } catch (IOException e) {

View Full Code Here

                                      FileStatus srcFile,
                                      BlockAndDatanodeResolver resolver) 
          throws IOException {
    // check the block placement policy
    for (BlockInfo srcBlock : srcBlocks) {
      LocatedBlockWithMetaInfo locatedBlock = resolver.getLocatedBlock(srcBlock);
      DatanodeInfo[] datanodes = locatedBlock.getLocations();
      if (datanodes.length != DEFAULT_NON_RAID_FILE_REPLICATION) {
        continue;
      }
      
      // move the block if all the 3 replicas are in the same rack.
      if (blockMover.isOnSameRack(datanodes[0], datanodes[1]) 
          && blockMover.isOnSameRack(datanodes[1], datanodes[2])) {
        Set<DatanodeInfo> excludedNodes = new HashSet<DatanodeInfo>(Arrays.asList(datanodes));
        DatanodeInfo target = blockMover.chooseTargetNodes(excludedNodes);


        blockMover.move(locatedBlock, datanodes[2], target, excludedNodes, 3, 
            locatedBlock.getDataProtocolVersion(), locatedBlock.getNamespaceID());


        // log the move submit info
        if (LOG.isDebugEnabled()) {
          LOG.debug("Move src block : " + locatedBlock.getBlock().getBlockId() + " from " + 
            datanodes[2] + " to target " + target + ", priority: 3. Replica info: " + 
            datanodes[0] + ", " + datanodes[1] + ", " + datanodes[2] + 
            ". Src file: " + srcFile.getPath());
        }
      }

View Full Code Here

              skip = false;
              continue;
            }
            
            int priority = numBlocks;
            LocatedBlockWithMetaInfo lb = resolver.getLocatedBlock(block);
            processedNode.add(otherNode);
            DatanodeInfo target = blockMover.chooseTargetNodes(excludedDatanodes);
            excludedDatanodes.add(target);
            if (lb != null) {
              blockMover.move(lb, replicaNode, target, excludedDatanodes, priority,
                  lb.getDataProtocolVersion(), lb.getNamespaceID());
              
              // log the move submit info
              if (LOG.isDebugEnabled()) {
                String stripeStr = getStripeStr(srcFile, stripeBlocks, resolver);
                LOG.debug("Move block : " + lb.getBlock().getBlockId() + " from " + 
                    replicaNode + " to " + target + ", priority:" + priority +
                    ". Stripe info: " + stripeStr);
              }
            }
          }

View Full Code Here

                      BlockAndDatanodeResolver resolver) 
                          throws IOException {
    StringBuilder sb = new StringBuilder();
    sb.append("File: " + srcFile.getPath() + ", ");
    for (BlockInfo block : stripeBlocks) {
      LocatedBlockWithMetaInfo lb = resolver.getLocatedBlock(block);
      sb.append("Block: " + lb.getBlock().getBlockId() + ", nodes: ");
      for (DatanodeInfo node: lb.getLocations()) {
        sb.append(node).append(",");
      }
    }
    return sb.toString();
  }

View Full Code Here

      checkParityInitialized();
      initialize(blk.file.getPath(), srcFs);
      Map<Long, LocatedBlockWithMetaInfo> offsetToLocatedBlock =
          pathAndOffsetToLocatedBlock.get(blk.file.getPath());
      if (offsetToLocatedBlock != null) {
        LocatedBlockWithMetaInfo lb = offsetToLocatedBlock.get(
            blk.blockLocation.getOffset());
        if (lb != null) {
          return lb;
        }
      }

View Full Code Here

      Map<Long, LocatedBlockWithMetaInfo> result =
          new HashMap<Long, LocatedBlockWithMetaInfo>();
      if (lbs instanceof LocatedBlocksWithMetaInfo) {
        LocatedBlocksWithMetaInfo lbsm = (LocatedBlocksWithMetaInfo)lbs;
        for (LocatedBlock lb : lbs.getLocatedBlocks()) {
          result.put(lb.getStartOffset(), new LocatedBlockWithMetaInfo(
              lb.getBlock(), lb.getLocations(), lb.getStartOffset(),
              lbsm.getDataProtocolVersion(), lbsm.getNamespaceID(),
              lbsm.getMethodFingerPrint()));
        }
      } else {
        for (LocatedBlock lb : lbs.getLocatedBlocks()) {
          result.put(lb.getStartOffset(), new LocatedBlockWithMetaInfo(
              lb.getBlock(), lb.getLocations(), lb.getStartOffset(),
              lbs.getDataProtocolVersion(), 0, 0));
        }
      }
      return result;

View Full Code Here

        throw new IOException ("DataTransferVersion " + dataTransferVersion +
              "requires the method appendAndFetchOldGS is supported in Namenode");
      } else if (namenodeProtocolProxy != null
          && namenodeProtocolProxy.isMethodSupported(
              "appendAndFetchMetaInfo", String.class, String.class)) {
        LocatedBlockWithMetaInfo loc = namenode.appendAndFetchMetaInfo(src,
            clientName);
        lastBlock = loc;
        if (loc != null) {
          updateNamespaceIdIfNeeded(loc.getNamespaceID());
          updateDataTransferProtocolVersionIfNeeded(loc.getDataProtocolVersion());
          getNewNameNodeIfNeeded(loc.getMethodFingerPrint());
        }
      } else {
        lastBlock = namenode.append(src, clientName);
      }

View Full Code Here

              throw new IOException(
                  "Fastcopy is not allowed from "
                      + "a non-federeated HDFS cluster to a federated HDFS cluster!");
            }


            LocatedBlockWithMetaInfo dstLocatedBlockWithMetaInfo = dstNamenode
                .addBlockAndFetchMetaInfo(destination, clientName, null,
                    favoredNodes, startPos);
            destinationLocatedBlock = dstLocatedBlockWithMetaInfo;
          } else if (dstNamenodeProtocolProxy.isMethodSupported(
              "addBlockAndFetchMetaInfo", String.class, String.class,
              DatanodeInfo[].class, DatanodeInfo[].class)) {
            if (!supportFederation) {
              throw new IOException(
                  "Fastcopy is not allowed from "
                      + "a non-federeated HDFS cluster to a federated HDFS cluster!");
            }
            LocatedBlockWithMetaInfo dstLocatedBlockWithMetaInfo = dstNamenode
                .addBlockAndFetchMetaInfo(destination, clientName, null,
                    favoredNodes);
            destinationLocatedBlock = dstLocatedBlockWithMetaInfo;
          } else {
            if (supportFederation) {

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.hadoop.hdfs.protocol.LocatedBlockWithMetaInfo

org.apache.hadoop.hdfs.DFSClient

org.apache.hadoop.hdfs.DFSClient$DFSOutputStream

org.apache.hadoop.hdfs.DFSOutputStream

org.apache.hadoop.hdfs.server.datanode.TestCopyBlockAPI

org.apache.hadoop.hdfs.server.datanode.TestInterDatanodeProtocol

org.apache.hadoop.hdfs.server.namenode.FSNamesystem

org.apache.hadoop.hdfs.TestChecksumFile

org.apache.hadoop.hdfs.TestDFSClientUpdateNameNodeSignature

org.apache.hadoop.hdfs.TestLeaseRecovery

org.apache.hadoop.hdfs.tools.FastCopy$FastFileCopy

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.