Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FSInputStream


      if ( isDir ) {
        verifyDir(client, path);
      } else {
        // this is not a directory. Checksum the file data.
        CRC32 fileCRC = new CRC32();
        FSInputStream in = client.open(path);
        byte[] buf = new byte[4096];
        int nRead = 0;
        while ( (nRead = in.read(buf, 0, buf.length)) > 0 ) {
          fileCRC.update(buf, 0, nRead);
        }
       
        verifyChecksum(path, fileCRC.getValue());
      }
View Full Code Here


        long lastLong = randomDataGenerator.nextLong();
        randomDataGenerator = makeRandomDataGenerator(); // restart (make new) PRNG
        ListIterator li = testfilesList.listIterator();
        while (li.hasNext()) {
          testFileName = (UTF8) li.next();
          FSInputStream nis = dfsClient.open(testFileName);
          byte[] bufferGolden = new byte[bufferSize];
          int m = 42;
          try {
            while (m != -1) {
              m = nis.read(buffer);
              if (m == buffer.length) {
                randomDataGenerator.nextBytes(bufferGolden);
                assertBytesEqual(buffer, bufferGolden, buffer.length);
              } else if (m > 0) {
                byte[] bufferGoldenPartial = new byte[m];
                randomDataGenerator.nextBytes(bufferGoldenPartial);
                assertBytesEqual(buffer, bufferGoldenPartial, bufferGoldenPartial.length);
              }
            }
          } finally {
            nis.close();
          }
        }
        // verify last randomDataGenerator rand val to ensure last file length was checked
        long lastLongAgain = randomDataGenerator.nextLong();
        assertEquals(lastLong, lastLongAgain);
View Full Code Here

  }
 
  @Test
  public void testWriteTo() throws IOException {

    FSInputStream fsin = new MockFSInputStream();
    ByteArrayOutputStream os = new ByteArrayOutputStream();

    // new int[]{s_1, c_1, s_2, c_2, ..., s_n, c_n} means to test
    // reading c_i bytes starting at s_i
    int[] pairs = new int[]{ 0, 10000,
View Full Code Here

    return InclusiveByteRange.satisfiableRanges(e, contentLength);
  }
 
  @Test
  public void testSendPartialData() throws IOException {
    FSInputStream in = new MockFSInputStream();
    ByteArrayOutputStream os = new ByteArrayOutputStream();

    // test if multiple ranges, then 416
    {
      List<InclusiveByteRange> ranges = strToRanges("0-,10-300", 500);
View Full Code Here

  public void testCopyStreamTarget() throws Exception {
    FSDataOutputStream out = mock(FSDataOutputStream.class);
    whenFsCreate().thenReturn(out);
    when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
    when(mockFs.rename(eq(tmpPath), eq(path))).thenReturn(true);
    FSInputStream in = mock(FSInputStream.class);
    when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1);
   
    tryCopyStream(in, true);
    verify(mockFs, never()).delete(eq(path), anyBoolean());
    verify(mockFs).rename(eq(tmpPath), eq(path));
    verify(mockFs, never()).delete(eq(tmpPath), anyBoolean());
View Full Code Here

    target.refreshStatus(); // so it's updated as existing
    cmd.setOverwrite(true);
    when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
    when(mockFs.delete(eq(path), eq(false))).thenReturn(true);
    when(mockFs.rename(eq(tmpPath), eq(path))).thenReturn(true);
    FSInputStream in = mock(FSInputStream.class);
    when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1);
   
    tryCopyStream(in, true);
    verify(mockFs).delete(eq(path), anyBoolean());
    verify(mockFs).rename(eq(tmpPath), eq(path));
    verify(mockFs, never()).delete(eq(tmpPath), anyBoolean());
View Full Code Here

  @Test
  public void testInterruptedCopyBytes() throws Exception {
    FSDataOutputStream out = mock(FSDataOutputStream.class);
    whenFsCreate().thenReturn(out);
    when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
    FSInputStream in = mock(FSInputStream.class);
    // make IOUtils.copyBytes fail
    when(in.read(any(byte[].class), anyInt(), anyInt())).thenThrow(
        new InterruptedIOException());
   
    tryCopyStream(in, false);
    verify(mockFs).delete(eq(tmpPath), anyBoolean());
    verify(mockFs, never()).rename(any(Path.class), any(Path.class));
View Full Code Here

    FSDataOutputStream out = mock(FSDataOutputStream.class);
    whenFsCreate().thenReturn(out);
    when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
    when(mockFs.rename(eq(tmpPath), eq(path))).thenThrow(
        new InterruptedIOException());
    FSInputStream in = mock(FSInputStream.class);
    when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1);
   
    tryCopyStream(in, false);
    verify(mockFs).delete(eq(tmpPath), anyBoolean());
    verify(mockFs).rename(eq(tmpPath), eq(path));
    verify(mockFs, never()).delete(eq(path), anyBoolean());
View Full Code Here

    setupSslProps(conf);
    HttpURLConnection connection = null;
    connection = openConnection(hostname, port, path);
    connection.connect();
    final InputStream in = connection.getInputStream();
    return new FSDataInputStream(new FSInputStream() {
      public int read() throws IOException {
        return in.read();
      }

      public int read(byte[] b, int off, int len) throws IOException {
View Full Code Here

      if ( isDir ) {
        verifyDir(dfs, path);
      } else {
        // this is not a directory. Checksum the file data.
        CRC32 fileCRC = new CRC32();
        FSInputStream in = dfs.dfs.open(pathName);
        byte[] buf = new byte[4096];
        int nRead = 0;
        while ( (nRead = in.read(buf, 0, buf.length)) > 0 ) {
          fileCRC.update(buf, 0, nRead);
        }
       
        verifyChecksum(pathName, fileCRC.getValue());
      }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.fs.FSInputStream

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.