Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FSInputStream

281
282
283
284
285
286
287
288
289
290
291
    final long filelength = cl == null? -1: Long.parseLong(cl);
    if (LOG.isDebugEnabled()) {
      LOG.debug("filelength = " + filelength);
    }

    return new FSDataInputStream(new FSInputStream() {
        long currentPos = 0;

        private void update(final boolean isEOF, final int n
            ) throws IOException {
          if (!isEOF) {
View Full Code Here
326
327
328
329
330
331
332
333
334
335
336
    final long filelength = cl == null? -1: Long.parseLong(cl);
    if (LOG.isDebugEnabled()) {
      LOG.debug("filelength = " + filelength);
    }

    return new FSDataInputStream(new FSInputStream() {
        long currentPos = 0;

        private void update(final boolean isEOF, final int n
            ) throws IOException {
          if (!isEOF) {
View Full Code Here
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
        long lastLong = randomDataGenerator.nextLong();
        randomDataGenerator = makeRandomDataGenerator(); // restart (make new) PRNG
        ListIterator li = testfilesList.listIterator();
        while (li.hasNext()) {
          testFileName = (UTF8) li.next();
          FSInputStream nis = dfsClient.open(testFileName);
          byte[] bufferGolden = new byte[bufferSize];
          int m = 42;
          try {
            while (m != -1) {
              m = nis.read(buffer);
              if (m == buffer.length) {
                randomDataGenerator.nextBytes(bufferGolden);
                assertBytesEqual(buffer, bufferGolden, buffer.length);
              } else if (m > 0) {
                byte[] bufferGoldenPartial = new byte[m];
                randomDataGenerator.nextBytes(bufferGoldenPartial);
                assertBytesEqual(buffer, bufferGoldenPartial, bufferGoldenPartial.length);
              }
            }
          } finally {
            nis.close();
          }
        }
        // verify last randomDataGenerator rand val to ensure last file length was checked
        long lastLongAgain = randomDataGenerator.nextLong();
        assertEquals(lastLong, lastLongAgain);
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
      IOException ie = new IOException("invalid url");
      ie.initCause(e);
      throw ie;
    }
    final InputStream in = connection.getInputStream();
    return new FSDataInputStream(new FSInputStream() {
        public int read() throws IOException {
          return in.read();
        }
        public int read(byte[] b, int off, int len) throws IOException {
          return in.read(b, off, len);
View Full Code Here
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
        long lastLong = randomDataGenerator.nextLong();
        randomDataGenerator = makeRandomDataGenerator(); // restart (make new) PRNG
        ListIterator li = testfilesList.listIterator();
        while (li.hasNext()) {
          testFileName = (UTF8) li.next();
          FSInputStream nis = dfsClient.open(testFileName);
          byte[] bufferGolden = new byte[bufferSize];
          int m = 42;
          try {
            while (m != -1) {
              m = nis.read(buffer);
              if (m == buffer.length) {
                randomDataGenerator.nextBytes(bufferGolden);
                assertBytesEqual(buffer, bufferGolden, buffer.length);
              } else if (m > 0) {
                byte[] bufferGoldenPartial = new byte[m];
                randomDataGenerator.nextBytes(bufferGoldenPartial);
                assertBytesEqual(buffer, bufferGoldenPartial, bufferGoldenPartial.length);
              }
            }
          } finally {
            nis.close();
          }
        }
        // verify last randomDataGenerator rand val to ensure last file length was checked
        long lastLongAgain = randomDataGenerator.nextLong();
        assertEquals(lastLong, lastLongAgain);
View Full Code Here
147
148
149
150
151
152
153
154
155
156
157
158
159
160
      if ( isDir ) {
        verifyDir(client, path);
      } else {
        // this is not a directory. Checksum the file data.
        CRC32 fileCRC = new CRC32();
        FSInputStream in = client.open(new UTF8(path));
        byte[] buf = new byte[4096];
        int nRead = 0;
        while ( (nRead = in.read(buf, 0, buf.length)) > 0 ) {
          fileCRC.update(buf, 0, nRead);
        }
       
        verifyChecksum(path, fileCRC.getValue());
      }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
      actual[idx] = 0;
    }
  }
 
  private void seekReadFile(FileSystem fileSys, Path name) throws IOException {
    FSInputStream stmRaw = fileSys.openRaw(name);
    FSDataInputStream stm = new FSDataInputStream(stmRaw, 4096);
    byte[] expected = new byte[ONEMB];
    Random rand = new Random(seed);
    rand.nextBytes(expected);
   
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
     *
     * @return the number of exceptions caught
     */
    static int openRead() {
      int exceptions = 0;
      FSInputStream in = null;
      for (int index = 0; index < numFiles; index++) {
        try {
          in = fileSys.openRaw(new Path(taskDir, "" + index));
          long toBeRead = bytesPerFile;
          while (toBeRead > 0) {
            int nbytes = (int) Math.min(buffer.length, toBeRead);
            toBeRead -= nbytes;
            try { // only try once
              in.read(buffer, 0, nbytes);
            } catch (IOException ioe) {
              exceptions++;
            }
          }
          in.close();
        } catch (IOException ioe) {
          exceptions++;
        }
      }
      return exceptions;
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
   * For blocks that reside on the nodes that are down, verify that their
   * replication factor is 1 more than the specified one.
   */
  private void checkFile(FileSystem fileSys, Path name, int repl,
                         String[] downnodes) throws IOException {
    FSInputStream is = fileSys.openRaw(name);
    DFSClient.DFSInputStream dis = (DFSClient.DFSInputStream) is;
    DatanodeInfo[][] dinfo = dis.getDataNodes();

    for (int blk = 0; blk < dinfo.length; blk++) { // for each block
      int hasdown = 0;
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
154
155
156
   
    assertEquals("Block size", Math.min(len, BLOCK_SIZE), s3FileSystem.getBlockSize(path));

    assertEquals("Length", len, s3FileSystem.getLength(path));

    FSInputStream in = s3FileSystem.openRaw(path);
    byte[] buf = new byte[len];

    in.readFully(0, buf);

    assertEquals(len, buf.length);
    for (int i = 0; i < buf.length; i++) {
      assertEquals("Position " + i, data[i], buf[i]);
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.fs.FSInputStream

  • org.apache.hadoop.dfs.ClusterTestDFS
  • org.apache.hadoop.dfs.HftpFileSystem
  • org.apache.hadoop.dfs.NNBench
  • org.apache.hadoop.dfs.TestDecommission
  • org.apache.hadoop.dfs.TestDFSUpgradeFromImage
  • org.apache.hadoop.dfs.TestSeekBug
  • org.apache.hadoop.dfs.TestSmallBlock
  • org.apache.hadoop.fs.s3.S3FileSystemBaseTest
  • org.apache.hadoop.fs.shell.TestCopy
  • org.apache.hadoop.hdfs.HftpFileSystem
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.