Package org.apache.hadoop.hdfs.protocol

Examples of org.apache.hadoop.hdfs.protocol.LocatedBlockWithMetaInfo

710
711
712
713
714
715
716
717
718
719
720
      List<LocatedBlock> lb = locatedBlocks.getLocatedBlocks();
     
      for (LocatedBlock b : lb) {
        if (decomBlockHashes.get(b.getBlock().hashCode()) != null) {
          blocks.add(new LocatedBlockWithMetaInfo(b.getBlock(),
              b.getLocations(), b.getStartOffset(),
              dataTransferVersion, namespaceId, methodsFingerprint));
        }
      }
     
View Full Code Here
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
      FakeBlockAndDatanodeResolver resolver = new FakeBlockAndDatanodeResolver();
      long blockId = 0;
      List<LocatedBlockWithMetaInfo> srcBlockList = new LinkedList<LocatedBlockWithMetaInfo>();
      List<BlockInfo> srcInfoList = new LinkedList<BlockInfo>();
      for (DatanodeInfo d : sourceLocations) {
        LocatedBlockWithMetaInfo lb = new LocatedBlockWithMetaInfo(
            new Block(blockId++, 0, 0L), new DatanodeInfo[]{d}, 0L,
            0, 0, 0);
        BlockInfo info = createBlockInfo(src, lb);
        srcBlockList.add(lb);
        srcInfoList.add(info);
        resolver.addBlock(info, lb);
        resolver.addNode(d.getName(), d);
      }
      List<LocatedBlock> parityBlockList = new LinkedList<LocatedBlock>();
      List<BlockInfo> parityInfoList = new LinkedList<BlockInfo>();
      for (DatanodeInfo d : parityLocations) {
        LocatedBlockWithMetaInfo lb = new LocatedBlockWithMetaInfo(
            new Block(blockId++, 0, 0L), new DatanodeInfo[]{d}, 0L,
            0, 0, 0);
        parityBlockList.add(lb);
        BlockInfo info = createBlockInfo(parity, lb);
        parityInfoList.add(info);
View Full Code Here
704
705
706
707
708
709
710
711
712
713
714
      }
      final int dataTransferVersion = locatedBlocks.getDataProtocolVersion();
      for (LocatedBlock b: locatedBlocks.getLocatedBlocks()) {
        if (b.isCorrupt() ||
            (b.getLocations().length == 0 && b.getBlockSize() > 0)) {
          corrupt.add(new LocatedBlockWithMetaInfo(b.getBlock(),
              b.getLocations(), b.getStartOffset(),
              dataTransferVersion, namespaceId, methodFingerprint));
        }
      }
      return corrupt;
View Full Code Here
758
759
760
761
762
763
764
765
766
767
768
        boolean allDecommissioning = true;
        for (DatanodeInfo i : b.getLocations()) {
          allDecommissioning &= i.isDecommissionInProgress();
        }
        if (allDecommissioning) {
          decommissioning.add(new LocatedBlockWithMetaInfo(b.getBlock(),
              b.getLocations(), b.getStartOffset(),
              dataTransferVersion, namespaceId, methodFingerprint));
        }
      }
      return decommissioning;
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
      // get data node
      DataNode datanode = cluster.getDataNode(nodes[0].getIpcPort());
      assertTrue(datanode != null);

      // verifies checksum file is of length 0
      LocatedBlockWithMetaInfo locatedblock = TestInterDatanodeProtocol
          .getLastLocatedBlock(dfs.dfs.namenode, filestr);
      Block lastblock = locatedblock.getBlock();
      DataNode.LOG.info("newblocks=" + lastblock);
      BlockPathInfo blockPathInfo = datanode.getBlockPathInfo(lastblock);
      String blockPath = blockPathInfo.getBlockPath();
      String metaPath = blockPathInfo.getMetaPath();

      File f = new File(blockPath);
      File meta = new File(metaPath);
      assertEquals(0, f.length());
      // set the checksum file to 0
      meta.delete();
      DataOutputStream outs = new DataOutputStream(new FileOutputStream(
          metaPath, false));
      outs.close();

      // issue recovery and makit e sure it succeeds.
      int numTries = 500;
      for (int idxTry = 0; idxTry < numTries; idxTry++) {
        boolean success = dfs.recoverLease(filepath);
        if (success) {
          break;
        } else if (idxTry == numTries - 1) {
          TestCase.fail("Recovery lease failed");
        } else {
          Thread.sleep(10);
        }
      }

      // make sure the meta file is still empty
      locatedblock = TestInterDatanodeProtocol.getLastLocatedBlock(
          dfs.dfs.namenode, filestr);
      Block newBlock = locatedblock.getBlock();
      blockPathInfo = datanode.getBlockPathInfo(newBlock);
      assertEquals(0, blockPathInfo.getNumBytes());
      metaPath = blockPathInfo.getMetaPath();
      meta = new File(metaPath);
      assertEquals(0, meta.length());
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
119
120
121
    }
    long serverVersion = ClientProtocol.versionID;
    int serverFpFromNn = ProtocolSignature.getFingerprint(ProtocolSignature.getProtocolSignature(
        0, serverVersion, inter).getMethods());
   
    LocatedBlockWithMetaInfo loc = client.namenode.addBlockAndFetchMetaInfo("/testNameNodeFingerprintSent.txt",
        client.getClientName(), null, 0L);
   
    int serverFp = loc.getMethodFingerPrint();
    TestCase.assertEquals(serverFpFromNn, serverFp);   

    FileSystem fs = cluster.getFileSystem();
    Path f = new Path("/testNameNodeFingerprintSent1.txt");
    DataOutputStream a_out = fs.create(f);
View Full Code Here
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
            favoured[i] = new DatanodeInfo(
                   new DatanodeID(name, "", -1, getPort(name)));
          }
        }

        LocatedBlockWithMetaInfo val =  namenode.addBlockAndFetchMetaInfo(
                                 path.pathname,
                                 clientName,
                                 excludes,
                                 favoured,
                                 startOffset,
                                 convertBlock(lastBlock));

        // convert from LocatedBlockWithMetaInfo to TLocatedBlock
        Block bb = val.getBlock();
  TLocatedBlock tblk = new TLocatedBlock();
        tblk.block = new TBlock(bb.getBlockId(), bb.getNumBytes(),
                                bb.getGenerationStamp());
        tblk.namespaceId = val.getNamespaceID();
        tblk.dataTransferVersion = val.getDataProtocolVersion();

        // fill up replica locations
        tblk.location = new ArrayList<TDatanodeID>();
        DatanodeInfo[] replicas = val.getLocations();
        for (int i = 0; i < replicas.length; i++) {
          tblk.location.add(i, new TDatanodeID(replicas[i].getName()));
        }
        return tblk;
      } catch (IOException e) {
View Full Code Here
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
              // leave the first block where it is
              skip = false;
              break;
            }
            int priority = numberOfNeighborBlocks;
            LocatedBlockWithMetaInfo lb = resolver.getLocatedBlock(block);
            DatanodeInfo datanode = resolver.getDatanodeInfo(node);
            Set<DatanodeInfo> excludedDatanodes = new HashSet<DatanodeInfo>();
            for (String name : excludedNodes) {
              excludedDatanodes.add(resolver.getDatanodeInfo(name));
            }
            if (lb != null) {
              blockMover.move(lb, datanode, excludedDatanodes, priority,
                  lb.getDataProtocolVersion(), lb.getNamespaceID());
            }
            break;
          }
        }
      }
View Full Code Here
552
553
554
555
556
557
558
559
560
561
562
      checkParityInitialized();
      initialize(blk.file, srcFs);
      Map<Long, LocatedBlockWithMetaInfo> offsetToLocatedBlock =
          pathAndOffsetToLocatedBlock.get(blk.file);
      if (offsetToLocatedBlock != null) {
        LocatedBlockWithMetaInfo lb = offsetToLocatedBlock.get(
            blk.blockLocation.getOffset());
        if (lb != null) {
          return lb;
        }
      }
View Full Code Here
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
      Map<Long, LocatedBlockWithMetaInfo> result =
          new HashMap<Long, LocatedBlockWithMetaInfo>();
      if (lbs instanceof LocatedBlocksWithMetaInfo) {
        LocatedBlocksWithMetaInfo lbsm = (LocatedBlocksWithMetaInfo)lbs;
        for (LocatedBlock lb : lbs.getLocatedBlocks()) {
          result.put(lb.getStartOffset(), new LocatedBlockWithMetaInfo(
              lb.getBlock(), lb.getLocations(), lb.getStartOffset(),
              lbsm.getDataProtocolVersion(), lbsm.getNamespaceID(),
              lbsm.getMethodFingerPrint()));
        }
      } else {
        for (LocatedBlock lb : lbs.getLocatedBlocks()) {
          result.put(lb.getStartOffset(), new LocatedBlockWithMetaInfo(
              lb.getBlock(), lb.getLocations(), lb.getStartOffset(),
              lbs.getDataProtocolVersion(), 0, 0));
        }
      }
      return result;
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.protocol.LocatedBlockWithMetaInfo

  • org.apache.hadoop.hdfs.DFSClient
  • org.apache.hadoop.hdfs.DFSClient$DFSOutputStream
  • org.apache.hadoop.hdfs.DFSOutputStream
  • org.apache.hadoop.hdfs.server.datanode.TestCopyBlockAPI
  • org.apache.hadoop.hdfs.server.datanode.TestInterDatanodeProtocol
  • org.apache.hadoop.hdfs.server.namenode.FSNamesystem
  • org.apache.hadoop.hdfs.TestChecksumFile
  • org.apache.hadoop.hdfs.TestDFSClientUpdateNameNodeSignature
  • org.apache.hadoop.hdfs.TestLeaseRecovery
  • org.apache.hadoop.hdfs.tools.FastCopy$FastFileCopy
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.