Examples of DFSDataInputStream


Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

  /**
   * Verifies that reading a file with the direct read(ByteBuffer) api gives the expected set of bytes.
   */
  static void checkFileContentDirect(FileSystem fs, Path name, byte[] expected,
      int readOffset) throws IOException {
    DFSDataInputStream stm = (DFSDataInputStream)fs.open(name);

    ByteBuffer actual = ByteBuffer.allocate(expected.length - readOffset);

    IOUtils.skipFully(stm, readOffset);

    actual.limit(3);

    //Read a small number of bytes first.
    int nread = stm.read(actual);
    actual.limit(nread + 2);
    nread += stm.read(actual);

    // Read across chunk boundary
    actual.limit(Math.min(actual.capacity(), nread + 517));
    nread += stm.read(actual);
    checkData(actual.array(), readOffset, expected, nread, "A few bytes");
    //Now read rest of it
    actual.limit(actual.capacity());
    while (actual.hasRemaining()) {
      int nbytes = stm.read(actual);

      if (nbytes < 0) {
        throw new EOFException("End of file reached before reading fully.");
      }
      nread += nbytes;
    }
    checkData(actual.array(), readOffset, expected, "Read 3");
    stm.close();
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

    fs.delete(root, true);
    files = null;
  }
 
  public static Block getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
 
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

    fs.delete(root, true);
    files = null;
  }
 
  public static Block getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
 
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

    fs.delete(root, true);
    files = null;
  }
 
  public static Block getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
 
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

    fs.delete(root, true);
    files = null;
  }
 
  public static Block getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
 
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

    switch(op.getValue()) {
    case OPEN:
    {
      final int b = bufferSize.getValue(conf);
      final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
      DFSDataInputStream in = null;
      try {
        in = new DFSClient.DFSDataInputStream(
        dfsclient.open(fullpath, b, true, null));
        in.seek(offset.getValue());
      } catch(IOException ioe) {
        IOUtils.cleanup(LOG, in);
        IOUtils.cleanup(LOG, dfsclient);
        throw ioe;
      }
     
      final long n = length.getValue() != null? length.getValue()
          : in.getVisibleLength();
      return Response.ok(new OpenEntity(in, n, dfsclient)).type(
          MediaType.APPLICATION_OCTET_STREAM).build();
    }
    case GETFILECHECKSUM:
    {
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

    fs.delete(root, true);
    files = null;
  }
 
  public static Block getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
 
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

    UserGroupInformation ugi = UserGroupInformation.createUserForTesting(username,
                                 new String[] {"supergroup"});
   
    final FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, conf);
   
    final DFSDataInputStream in = (DFSDataInputStream)fs.open(p);

    //Check visible length
    Assert.assertTrue(in.getVisibleLength() >= expectedsize);

    //Able to read?
    for(int i = 0; i < expectedsize; i++) {
      Assert.assertEquals((byte)i, (byte)in.read())
    }

    in.close();
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

    switch(op.getValue()) {
    case OPEN:
    {
      final int b = bufferSize.getValue(conf);
      final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
      DFSDataInputStream in = null;
      try {
        in = new DFSClient.DFSDataInputStream(
            dfsclient.open(fullpath, b, true));
        in.seek(offset.getValue());
      } catch(IOException ioe) {
        IOUtils.cleanup(LOG, in);
        IOUtils.cleanup(LOG, dfsclient);
        throw ioe;
      }
      final DFSDataInputStream dis = in;
      final StreamingOutput streaming = new StreamingOutput() {
        @Override
        public void write(final OutputStream out) throws IOException {
          final Long n = length.getValue();
          DFSDataInputStream dfsin = dis;
          DFSClient client = dfsclient;
          try {
            if (n == null) {
              IOUtils.copyBytes(dfsin, out, b);
            } else {
              IOUtils.copyBytes(dfsin, out, n, false);
            }
            dfsin.close();
            dfsin = null;
            dfsclient.close();
            client = null;
          } finally {
            IOUtils.cleanup(LOG, dfsin);
View Full Code Here

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

      assertTrue(status.getLen() == fileSize);
      assertEquals(fs.makeQualified(file1).toString(),
          status.getPath().toString());

      // test getVisbileLen
      DFSDataInputStream fin = (DFSDataInputStream)fs.open(file1);
      assertEquals(status.getLen(), fin.getVisibleLength());
     
      // test listStatus on a file
      FileStatus[] stats = fs.listStatus(file1);
      assertEquals(1, stats.length);
      status = stats[0];
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.