Examples of DFSDataInputStream

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

220
221
222
223
224
225
226
227
228
229
230
231
232
    switch(op.getValue()) {
    case OPEN:
    {
      final int b = bufferSize.getValue(conf);
      final DFSDataInputStream in = new DFSClient.DFSDataInputStream(
          dfsclient.open(fullpath, b, true, null));
      in.seek(offset.getValue());

      final StreamingOutput streaming = new StreamingOutput() {
        @Override
        public void write(final OutputStream out) throws IOException {
          final Long n = length.getValue();
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
    switch(op.getValue()) {
    case OPEN:
    {
      final int b = bufferSize.getValue(conf);
      final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
      DFSDataInputStream in = null;
      try {
        in = new DFSClient.DFSDataInputStream(
        dfsclient.open(fullpath, b, true, null));
        in.seek(offset.getValue());
      } catch(IOException ioe) {
        IOUtils.cleanup(LOG, in);
        IOUtils.cleanup(LOG, dfsclient);
        throw ioe;
      }
      final DFSDataInputStream dis = in;
      final StreamingOutput streaming = new StreamingOutput() {
        @Override
        public void write(final OutputStream out) throws IOException {
          final Long n = length.getValue();
          DFSDataInputStream dfsin = dis;
          DFSClient client = dfsclient;
          try {
            if (n == null) {
              IOUtils.copyBytes(dfsin, out, b);
            } else {
              IOUtils.copyBytes(dfsin, out, n, b, false);
            }
            dfsin.close();
            dfsin = null;
            client.close();
            client = null;
          } finally {
            IOUtils.cleanup(LOG, dfsin);
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
    switch(op.getValue()) {
    case OPEN:
    {
      final int b = bufferSize.getValue(conf);
      final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
      DFSDataInputStream in = null;
      try {
        in = new DFSClient.DFSDataInputStream(
            dfsclient.open(fullpath, b, true));
        in.seek(offset.getValue());
      } catch(IOException ioe) {
        IOUtils.cleanup(LOG, in);
        IOUtils.cleanup(LOG, dfsclient);
        throw ioe;
      }
     
      final long n = length.getValue() != null? length.getValue()
        : in.getVisibleLength() - offset.getValue();
      return Response.ok(new OpenEntity(in, n, dfsclient)).type(
          MediaType.APPLICATION_OCTET_STREAM).build();
    }
    case GETFILECHECKSUM:
    {
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
    UserGroupInformation ugi = UserGroupInformation.createUserForTesting(username,
                                 new String[] {"supergroup"});
   
    final FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, conf);
   
    final DFSDataInputStream in = (DFSDataInputStream)fs.open(p);

    //Check visible length
    Assert.assertTrue(in.getVisibleLength() >= expectedsize);

    //Able to read?
    for(int i = 0; i < expectedsize; i++) {
      Assert.assertEquals((byte)i, (byte)in.read()); 
    }

    in.close();
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

505
506
507
508
509
510
511
512
513
514
    fs.delete(root, true);
    files = null;
  }
 
  public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
  } 
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

506
507
508
509
510
511
512
513
514
515
    fs.delete(root, true);
    files = null;
  }
 
  public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
  } 
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
    UserGroupInformation ugi = UserGroupInformation.createUserForTesting(username,
                                 new String[] {"supergroup"});
   
    final FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, conf);
   
    final DFSDataInputStream in = (DFSDataInputStream)fs.open(p);

    //Check visible length
    Assert.assertTrue(in.getVisibleLength() >= expectedsize);

    //Able to read?
    for(int i = 0; i < expectedsize; i++) {
      Assert.assertEquals((byte)i, (byte)in.read()); 
    }

    in.close();
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
    final Configuration conf = new Configuration();
    // create cluster
    conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 512);

    final MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
    DFSDataInputStream in = null;
    FSDataOutputStream out = null;
    DistributedFileSystem dfs = null;
    try {
      Path path = new Path(MiniDFSCluster.getBaseDir().getPath(), "test");
      dfs = (DistributedFileSystem) cluster.getFileSystem();
      out = dfs.create(path);
      int fileLength = 1030;
      out.write(new byte[fileLength]);
      out.sync();
      cluster.restartNameNode();
      cluster.waitActive();
      in = (DFSDataInputStream) dfs.open(path, 1024);
      // Verify the length when we just restart NN. DNs will register
      // immediately.
      Assert.assertEquals(fileLength, in.getVisibleLength());
      cluster.shutdownDataNodes();
      cluster.restartNameNode(false);
      // This is just for ensuring NN started.
      verifyNNIsInSafeMode(dfs);

      try {
        in = (DFSDataInputStream) dfs.open(path);
        Assert.fail("Expected IOException");
      } catch (IOException e) {
        Assert.assertTrue(e.getLocalizedMessage().indexOf(
            "Name node is in safe mode") >= 0);
      }

    } finally {
      if (null != in) {
        in.close();
      }
      if (null != dfs) {
        dfs.dfs.clientRunning = false;
      }
      cluster.shutdown();
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

249
250
251
252
253
254
255
256
257
258
    fs.delete(root, true);
    files = null;
  }
 
  public static Block getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
  } 
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

504
505
506
507
508
509
510
511
512
513
    fs.delete(root, true);
    files = null;
  }
 
  public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
  } 
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.