Package org.apache.hadoop.hdfs.protocol

Examples of org.apache.hadoop.hdfs.protocol.LocatedBlockWithMetaInfo

146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
      DFSTestUtil.createFile(dfs, filepath, ORG_FILE_SIZE, REPLICATION_NUM, 0L);
      assertTrue(dfs.dfs.exists(filestr));
      DFSTestUtil.waitReplication(dfs, filepath, REPLICATION_NUM);

      //get block info for the last block
      LocatedBlockWithMetaInfo locatedblock = TestInterDatanodeProtocol.getLastLocatedBlock(
          dfs.dfs.namenode, filestr);
      int namespaceId = locatedblock.getNamespaceID();
      DatanodeInfo[] datanodeinfos = locatedblock.getLocations();
      assertEquals(REPLICATION_NUM, datanodeinfos.length);

      //connect to data nodes
      InterDatanodeProtocol[] idps = new InterDatanodeProtocol[REPLICATION_NUM];
      DataNode[] datanodes = new DataNode[REPLICATION_NUM];
      for(int i = 0; i < REPLICATION_NUM; i++) {
      idps[i] = DataNode.createInterDataNodeProtocolProxy(
    datanodeinfos[i], conf, 0);
        datanodes[i] = cluster.getDataNode(datanodeinfos[i].getIpcPort());
        assertTrue(datanodes[i] != null);
      }
     
      //verify BlockMetaDataInfo
      Block lastblock = locatedblock.getBlock();
      DataNode.LOG.info("newblocks=" + lastblock);
      for(int i = 0; i < REPLICATION_NUM; i++) {
        checkMetaInfo(namespaceId, lastblock, idps[i]);
      }
View Full Code Here
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
              throw new IOException(
                  "Fastcopy is not allowed from "
                      + "a non-federeated HDFS cluster to a federated HDFS cluster!");
            }

            LocatedBlockWithMetaInfo dstLocatedBlockWithMetaInfo = dstNamenode
                .addBlockAndFetchMetaInfo(destination, clientName, null,
                    favoredNodes, startPos);
            destinationLocatedBlock = dstLocatedBlockWithMetaInfo;
          } else if (dstNamenodeProtocolProxy.isMethodSupported(
              "addBlockAndFetchMetaInfo", String.class, String.class,
              DatanodeInfo[].class, DatanodeInfo[].class)) {
            if (!supportFederation) {
              throw new IOException(
                  "Fastcopy is not allowed from "
                      + "a non-federeated HDFS cluster to a federated HDFS cluster!");
            }
            LocatedBlockWithMetaInfo dstLocatedBlockWithMetaInfo = dstNamenode
                .addBlockAndFetchMetaInfo(destination, clientName, null,
                    favoredNodes);
            destinationLocatedBlock = dstLocatedBlockWithMetaInfo;
          } else {
            if (supportFederation) {
View Full Code Here
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
    try {
      stat = getFileInfo(src);
      if (namenodeProtocolProxy != null
          && namenodeProtocolProxy.isMethodSupported(
              "appendAndFetchMetaInfo", String.class, String.class)) {
        LocatedBlockWithMetaInfo loc = namenode.appendAndFetchMetaInfo(src,
            clientName);
        lastBlock = loc;
        if (loc != null) {
          namespaceId = loc.getNamespaceID();
          updateDataTransferProtocolVersionIfNeeded(loc.getDataProtocolVersion());
          getNewNameNodeIfNeeded(loc.getMethodFingerPrint());
        }
      } else {
        lastBlock = namenode.append(src, clientName);
      }
View Full Code Here
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
            return dfsClient.namenode.addBlock(src, dfsClient.clientName);
          }
          dfsClient.updateDataTransferProtocolVersionIfNeeded(
              loc.getDataProtocolVersion());
          if (loc instanceof LocatedBlockWithMetaInfo) {
            LocatedBlockWithMetaInfo metaLoc = (LocatedBlockWithMetaInfo)loc;
            this.namespaceId = metaLoc.getNamespaceID();
            dfsClient.getNewNameNodeIfNeeded(metaLoc.getMethodFingerPrint());
          }
          return loc;
        } catch (RemoteException e) {
          IOException ue =
            e.unwrapRemoteException(FileNotFoundException.class,
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
119
120
121
    }
    long serverVersion = ClientProtocol.versionID;
    int serverFpFromNn = ProtocolSignature.getFingerprint(ProtocolSignature.getProtocolSignature(
        0, serverVersion, inter).getMethods());
   
    LocatedBlockWithMetaInfo loc = client.namenode.addBlockAndFetchMetaInfo("/testNameNodeFingerprintSent.txt",
        client.getClientName(), null, 0L);
   
    int serverFp = loc.getMethodFingerPrint();
    TestCase.assertEquals(serverFpFromNn, serverFp);   

    FileSystem fs = cluster.getFileSystem();
    Path f = new Path("/testNameNodeFingerprintSent1.txt");
    DataOutputStream a_out = fs.create(f);
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
      DFSTestUtil.createFile(dfs, filepath, ORG_FILE_SIZE, REPLICATION_NUM, 0L);
      assertTrue(dfs.dfs.exists(filestr));
      DFSTestUtil.waitReplication(dfs, filepath, REPLICATION_NUM);

      //get block info for the last block
      LocatedBlockWithMetaInfo locatedblock = TestInterDatanodeProtocol.getLastLocatedBlock(
          dfs.dfs.namenode, filestr);
      int namespaceId = locatedblock.getNamespaceID();
      DatanodeInfo[] datanodeinfos = locatedblock.getLocations();
      assertEquals(REPLICATION_NUM, datanodeinfos.length);

      //connect to data nodes
      InterDatanodeProtocol[] idps = new InterDatanodeProtocol[REPLICATION_NUM];
      DataNode[] datanodes = new DataNode[REPLICATION_NUM];
      for(int i = 0; i < REPLICATION_NUM; i++) {
      idps[i] = DataNode.createInterDataNodeProtocolProxy(
    datanodeinfos[i], conf, 0);
        datanodes[i] = cluster.getDataNode(datanodeinfos[i].getIpcPort());
        assertTrue(datanodes[i] != null);
      }
     
      //verify BlockMetaDataInfo
      Block lastblock = locatedblock.getBlock();
      DataNode.LOG.info("newblocks=" + lastblock);
      for(int i = 0; i < REPLICATION_NUM; i++) {
        checkMetaInfo(namespaceId, lastblock, idps[i]);
      }
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
      // get data node
      DataNode datanode = cluster.getDataNode(nodes[0].getIpcPort());
      assertTrue(datanode != null);

      // verifies checksum file is of length 0
      LocatedBlockWithMetaInfo locatedblock = TestInterDatanodeProtocol
          .getLastLocatedBlock(dfs.dfs.namenode, filestr);
      Block lastblock = locatedblock.getBlock();
      DataNode.LOG.info("newblocks=" + lastblock);
      BlockPathInfo blockPathInfo = datanode.getBlockPathInfo(lastblock);
      String blockPath = blockPathInfo.getBlockPath();
      String metaPath = blockPathInfo.getMetaPath();

      File f = new File(blockPath);
      File meta = new File(metaPath);
      assertEquals(0, f.length());
      // set the checksum file to 0
      meta.delete();
      DataOutputStream outs = new DataOutputStream(new FileOutputStream(
          metaPath, false));
      outs.close();

      // issue recovery and makit e sure it succeeds.
      int numTries = 500;
      for (int idxTry = 0; idxTry < numTries; idxTry++) {
        boolean success = dfs.recoverLease(filepath);
        if (success) {
          break;
        } else if (idxTry == numTries - 1) {
          TestCase.fail("Recovery lease failed");
        } else {
          Thread.sleep(10);
        }
      }

      // make sure the meta file is still empty
      locatedblock = TestInterDatanodeProtocol.getLastLocatedBlock(
          dfs.dfs.namenode, filestr);
      Block newBlock = locatedblock.getBlock();
      blockPathInfo = datanode.getBlockPathInfo(newBlock);
      assertEquals(0, blockPathInfo.getNumBytes());
      metaPath = blockPathInfo.getMetaPath();
      meta = new File(metaPath);
      assertEquals(0, meta.length());
View Full Code Here
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
              throw new IOException(
                  "Fastcopy is not allowed from "
                      + "a non-federeated HDFS cluster to a federated HDFS cluster!");
            }

            LocatedBlockWithMetaInfo dstLocatedBlockWithMetaInfo = dstNamenode
                .addBlockAndFetchMetaInfo(destination, clientName, null,
                    favoredNodes, startPos);
            destinationLocatedBlock = dstLocatedBlockWithMetaInfo;
          } else if (dstNamenodeProtocolProxy.isMethodSupported(
              "addBlockAndFetchMetaInfo", String.class, String.class,
              DatanodeInfo[].class, DatanodeInfo[].class)) {
            if (!supportFederation) {
              throw new IOException(
                  "Fastcopy is not allowed from "
                      + "a non-federeated HDFS cluster to a federated HDFS cluster!");
            }
            LocatedBlockWithMetaInfo dstLocatedBlockWithMetaInfo = dstNamenode
                .addBlockAndFetchMetaInfo(destination, clientName, null,
                    favoredNodes);
            destinationLocatedBlock = dstLocatedBlockWithMetaInfo;
          } else {
            if (supportFederation) {
View Full Code Here
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
            } else {
              return namenode.addBlock(src, clientName);
            }
            updateDataTransferProtocolVersionIfNeeded(loc.getDataProtocolVersion());
            if (loc instanceof LocatedBlockWithMetaInfo) {
              LocatedBlockWithMetaInfo metaLoc = (LocatedBlockWithMetaInfo)loc;
              this.namespaceId = metaLoc.getNamespaceID();
              getNewNameNodeIfNeeded(metaLoc.getMethodFingerPrint());
            }
            return loc;
          } catch (RemoteException e) {
            IOException ue =
              e.unwrapRemoteException(FileNotFoundException.class,
View Full Code Here
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
    try {
      stat = getFileInfo(src);
      if (namenodeProtocolProxy != null
          && namenodeProtocolProxy.isMethodSupported(
              "appendAndFetchMetaInfo", String.class, String.class)) {
        LocatedBlockWithMetaInfo loc = namenode.appendAndFetchMetaInfo(src,
            clientName);
        lastBlock = loc;
        if (loc != null) {
          namespaceId = loc.getNamespaceID();
          updateDataTransferProtocolVersionIfNeeded(loc.getDataProtocolVersion());
          getNewNameNodeIfNeeded(loc.getMethodFingerPrint());
        }
      } else {
        lastBlock = namenode.append(src, clientName);
      }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.protocol.LocatedBlockWithMetaInfo

  • org.apache.hadoop.hdfs.DFSClient
  • org.apache.hadoop.hdfs.DFSClient$DFSOutputStream
  • org.apache.hadoop.hdfs.DFSOutputStream
  • org.apache.hadoop.hdfs.server.datanode.TestCopyBlockAPI
  • org.apache.hadoop.hdfs.server.datanode.TestInterDatanodeProtocol
  • org.apache.hadoop.hdfs.server.namenode.FSNamesystem
  • org.apache.hadoop.hdfs.TestChecksumFile
  • org.apache.hadoop.hdfs.TestDFSClientUpdateNameNodeSignature
  • org.apache.hadoop.hdfs.TestLeaseRecovery
  • org.apache.hadoop.hdfs.tools.FastCopy$FastFileCopy
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.