Package org.apache.hadoop.hdfs.DFSClient

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

506
507
508
509
510
511
512
513
514
515
    fs.delete(root, true);
    files = null;
  }
 
  public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
  } 
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
    UserGroupInformation ugi = UserGroupInformation.createUserForTesting(username,
                                 new String[] {"supergroup"});
   
    final FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, conf);
   
    final DFSDataInputStream in = (DFSDataInputStream)fs.open(p);

    //Check visible length
    Assert.assertTrue(in.getVisibleLength() >= expectedsize);

    //Able to read?
    for(int i = 0; i < expectedsize; i++) {
      Assert.assertEquals((byte)i, (byte)in.read()); 
    }

    in.close();
  }
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
    final Configuration conf = new Configuration();
    // create cluster
    conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 512);

    final MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
    DFSDataInputStream in = null;
    FSDataOutputStream out = null;
    DistributedFileSystem dfs = null;
    try {
      Path path = new Path(MiniDFSCluster.getBaseDir().getPath(), "test");
      dfs = (DistributedFileSystem) cluster.getFileSystem();
      out = dfs.create(path);
      int fileLength = 1030;
      out.write(new byte[fileLength]);
      out.sync();
      cluster.restartNameNode();
      cluster.waitActive();
      in = (DFSDataInputStream) dfs.open(path, 1024);
      // Verify the length when we just restart NN. DNs will register
      // immediately.
      Assert.assertEquals(fileLength, in.getVisibleLength());
      cluster.shutdownDataNodes();
      cluster.restartNameNode(false);
      // This is just for ensuring NN started.
      verifyNNIsInSafeMode(dfs);

      try {
        in = (DFSDataInputStream) dfs.open(path);
        Assert.fail("Expected IOException");
      } catch (IOException e) {
        Assert.assertTrue(e.getLocalizedMessage().indexOf(
            "Name node is in safe mode") >= 0);
      }

    } finally {
      if (null != in) {
        in.close();
      }
      if (null != dfs) {
        dfs.dfs.clientRunning = false;
      }
      cluster.shutdown();
View Full Code Here
249
250
251
252
253
254
255
256
257
258
    fs.delete(root, true);
    files = null;
  }
 
  public static Block getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
  } 
View Full Code Here
504
505
506
507
508
509
510
511
512
513
    fs.delete(root, true);
    files = null;
  }
 
  public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
  } 
View Full Code Here
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
    switch(op.getValue()) {
    case OPEN:
    {
      final int b = bufferSize.getValue(conf);
      final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
      DFSDataInputStream in = null;
      try {
        in = new DFSClient.DFSDataInputStream(
            dfsclient.open(fullpath, b, true));
        in.seek(offset.getValue());
      } catch(IOException ioe) {
        IOUtils.cleanup(LOG, in);
        IOUtils.cleanup(LOG, dfsclient);
        throw ioe;
      }
      final DFSDataInputStream dis = in;
      final StreamingOutput streaming = new StreamingOutput() {
        @Override
        public void write(final OutputStream out) throws IOException {
          final Long n = length.getValue();
          DFSDataInputStream dfsin = dis;
          DFSClient client = dfsclient;
          try {
            if (n == null) {
              IOUtils.copyBytes(dfsin, out, b);
            } else {
              IOUtils.copyBytes(dfsin, out, n, false);
            }
            dfsin.close();
            dfsin = null;
            dfsclient.close();
            client = null;
          } finally {
            IOUtils.cleanup(LOG, dfsin);
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
    UserGroupInformation ugi = UserGroupInformation.createUserForTesting(username,
                                 new String[] {"supergroup"});
   
    final FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, conf);
   
    final DFSDataInputStream in = (DFSDataInputStream)fs.open(p);

    //Check visible length
    Assert.assertTrue(in.getVisibleLength() >= expectedsize);

    //Able to read?
    for(int i = 0; i < expectedsize; i++) {
      Assert.assertEquals((byte)i, (byte)in.read()); 
    }

    in.close();
  }
View Full Code Here
455
456
457
458
459
460
461
462
463
464
    fs.delete(root, true);
    files = null;
  }
 
  public static Block getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
  } 
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
75
76
  public void testRead() throws Exception{
    for(int i = 0; i < TEST_FILE_NUM; ++i) {
      String file = "/tmp" + i +".txt";
      DFSTestUtil.createFile(fs, new Path(file), FILE_LEN, (short)5, 1L);
     
      DFSDataInputStream in = (DFSDataInputStream)fs.open(new Path(file));
      int numOfRead = 0;
      while(in.read() > 0){
        numOfRead ++;
      }
      assertEquals(FILE_LEN * (i+1),
          metrics.readSize.getCurrentIntervalValue());
      assertEquals(numOfRead * (i+1),
View Full Code Here
241
242
243
244
245
246
247
248
249
250
    fs.delete(root, true);
    files = null;
  }
 
  public static Block getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
  } 
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

  • org.apache.hadoop.hdfs.DFSTestUtil
  • org.apache.hadoop.hdfs.server.datanode.web.resources.DatanodeWebHdfsMethods
  • org.apache.hadoop.hdfs.TestAvailableForUnderConstruction
  • org.apache.hadoop.hdfs.TestDFSClientMetrics
  • org.apache.hadoop.hdfs.TestDistributedFileSystem
  • org.apache.hadoop.hdfs.TestFileLengthOnClusterRestart
  • org.apache.hadoop.hdfs.TestFileStatus
  • org.apache.hadoop.hdfs.TestIoprio
  • org.apache.hadoop.hdfs.TestReadWhileWriting
  • org.apache.hadoop.hdfs.TestShortCircuitLocalRead
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.