Package org.apache.hadoop.hdfs.server.datanode

Examples of org.apache.hadoop.hdfs.server.datanode.FSDataset$BlockPoolSlice

1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
    List<File> files = new ArrayList<File>();
    List<DataNode> datanodes = cluster.getDataNodes();
    int nsId = cluster.getNameNode().getNamespaceID();
    Block[][] blocks = cluster.getAllBlockReports(nsId);
    for(int i = 0; i < blocks.length; i++) {
      FSDataset ds = (FSDataset)datanodes.get(i).getFSDataset();
      for(Block b : blocks[i]) {
        files.add(ds.getBlockFile(nsId,b));
      }       
    }
    return files;
  }
View Full Code Here
172
173
174
175
176
177
178
179
180
181
182
183
184
    assertEquals(len1, lb.getBlockSize());

    DatanodeInfo[] datanodeinfos = lb.getLocations();
    assertEquals(repl, datanodeinfos.length);
    final DataNode dn = cluster.getDataNode(datanodeinfos[0].getIpcPort());
    final FSDataset data = (FSDataset)dn.getFSDataset();
    int nsId = cluster.getNameNode().getNamespaceID();
    final RandomAccessFile raf = new RandomAccessFile(data.getBlockFile(nsId, blk), "rw");
    AppendTestUtil.LOG.info("dn=" + dn + ", blk=" + blk + " (length=" + blk.getNumBytes() + ")");
    if (!dn.useInlineChecksum) {
      assertEquals(len1, raf.length());
    } else {
      int bytesPerChecksum = conf.getInt("io.bytes.per.checksum", 512);
View Full Code Here
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
    List<File> files = new ArrayList<File>();
    List<DataNode> datanodes = cluster.getDataNodes();
    int nsId = cluster.getNameNode().getNamespaceID();
    Block[][] blocks = cluster.getAllBlockReports(nsId);
    for(int i = 0; i < blocks.length; i++) {
      FSDataset ds = (FSDataset)datanodes.get(i).getFSDataset();
      for(Block b : blocks[i]) {
        files.add(ds.getBlockFile(nsId,b));
      }       
    }
    return files;
  }
View Full Code Here
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
         
          Set<Integer> portSet = new HashSet<Integer>();
          for (DatanodeInfo dnInfo : locatedblock.getLocations()) {
            portSet.add(dnInfo.getPort());
          }
          FSDataset fsd = null;
          for (DataNode dn : cluster.getDataNodes()) {
            if (portSet.contains(dn.getPort())) {
              fsd = (FSDataset) (dn.data);
              TestCase.assertNotNull(fsd);
              DatanodeBlockInfo binfo = fsd.getDatanodeBlockInfo(nsId,
                  locatedblock.getBlock());
              TestCase.assertNotNull(binfo);
              long waitTimeLeft = 20000;
              while (true) {
                if (binfo.hasBlockCrcInfo()) {
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
      assertEquals(1, locations.locatedBlockCount());
      LocatedBlock locatedblock = locations.getLocatedBlocks().get(0);
      int nsId = cluster.getNameNode().getNamespaceID();
      for(DatanodeInfo datanodeinfo: locatedblock.getLocations()) {
        DataNode datanode = cluster.getDataNode(datanodeinfo.ipcPort);
        FSDataset dataset = (FSDataset)datanode.data;
        Block b = dataset.getStoredBlock(nsId, locatedblock.getBlock().getBlockId());
        Block newBlock = new Block(b);
        newBlock.setGenerationStamp(6661);
        dataset.updateBlock(nsId, b, newBlock);

        Block newBlock1 = new Block(b);
        newBlock1.setGenerationStamp(6662);
        boolean hitException = false;
        try {
          dataset.updateBlock(nsId, b, newBlock1);
        } catch (IOException e) {
          hitException = true;
        }
        TestCase.assertTrue("Shouldn't allow update block when generation doesn't match", hitException);
        dataset.updateBlock(nsId, newBlock, newBlock1);
      }
    } finally {
      IOUtils.closeStream(dfs);
      cluster.shutdown();
    }
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
                  dn.length == 1);

      LocatedBlocks locations = client.namenode.getBlockLocations(
                                  file1.toString(), 0, Long.MAX_VALUE);
      List<LocatedBlock> blocks = locations.getLocatedBlocks();
      FSDataset dataset = (FSDataset) dn[0].data;

      //
      // Create hard links for a few of the blocks
      //
      for (int i = 0; i < blocks.size(); i = i + 2) {
        Block b = (Block) blocks.get(i).getBlock();
        FSDataset fsd = (FSDataset) dataset;
        File f = fsd.getFile(nsId, b);
        File link = new File(f.toString() + ".link");
        System.out.println("Creating hardlink for File " + f +
                           " to " + link);
        HardLink.createHardLink(f, link);
      }
View Full Code Here
822
823
824
825
826
827
828
829
830
831
832
833
834
      LocatedBlock locatedblock = locations.getLocatedBlocks().get(0);
      int successcount = 0;
      int nsId = cluster.getNameNode().getNamespaceID();
      for(DatanodeInfo datanodeinfo: locatedblock.getLocations()) {
        DataNode datanode = cluster.getDataNode(datanodeinfo.ipcPort);
        FSDataset dataset = (FSDataset)datanode.data;
        Block b = dataset.getStoredBlock(nsId, locatedblock.getBlock().getBlockId());
        File blockfile = dataset.getReplicaToRead(nsId, b).getDataFileToRead();
        System.out.println("blockfile=" + blockfile);
        if (blockfile != null) {
          BufferedReader in = new BufferedReader(new FileReader(blockfile));
          assertEquals("something", in.readLine());
          in.close();
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
    super.tearDown();
  }

  public void testParallelCheckDirs() throws Exception {
    final DataNode datanode = cluster.getDataNodes().get(0);
    FSDataset fsDataset = (FSDataset) datanode.data;
    datanode.data = spy(fsDataset);

    final Method checkDiskMethod = DataNode.class.getDeclaredMethod(
        "checkDiskError", Exception.class);
    checkDiskMethod.setAccessible(true);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.server.datanode.FSDataset$BlockPoolSlice

  • org.apache.hadoop.fs.DF
  • org.apache.hadoop.fs.DU
  • org.apache.hadoop.hdfs.protocol.Block
  • org.apache.hadoop.hdfs.protocol.BlockListAsLongs
  • org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo
  • org.apache.hadoop.hdfs.protocol.DatanodeInfo
  • org.apache.hadoop.hdfs.protocol.ExtendedBlock
  • org.apache.hadoop.hdfs.protocol.LocatedBlock
  • org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState
  • org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.