Package eu.stratosphere.core.fs

Examples of eu.stratosphere.core.fs.FSDataInputStream

697
698
699
700
701
702
703
704
705
706
707
708
709
710
      // Write out the length of the file
      final FileStatus file = fs.getFileStatus(jar);
      out.writeLong(file.getLen());

      // Now write the jar file
      final FSDataInputStream inStream = fs.open(this.userJars.get(i));
      final byte[] buf = new byte[BUFFERSIZE];
      int read = inStream.read(buf, 0, buf.length);
      while (read > 0) {
        out.write(buf, 0, read);
        read = inStream.read(buf, 0, buf.length);
      }
    }
  }
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
      try {
        if (!lfs.exists(tmp)) {
          FSDataOutputStream lfsOutput = lfs.create(tmp, false);
          Path distributedPath = new Path(filePath);
          FileSystem fs = distributedPath.getFileSystem();
          FSDataInputStream fsInput = fs.open(distributedPath);
          IOUtils.copyBytes(fsInput, lfsOutput);
        }
      } catch (IOException e1) {
        throw new RuntimeException("Error copying a file from hdfs to the local fs", e1);
      }
View Full Code Here
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
      final FileStatus status = fs.getFileStatus(storePath);

      StringRecord.writeString(out, libraryFileName);
      out.writeLong(status.getLen());

      final FSDataInputStream inStream = fs.open(storePath);
      final byte[] buf = new byte[8192]; // 8K Buffer*/
      int read = inStream.read(buf, 0, buf.length);
      while (read > 0) {
        out.write(buf, 0, read);
        read = inStream.read(buf, 0, buf.length);
      }

      inStream.close();
    }
  }
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
      long start = split.getStart();
      long length = split.getLength();

      final FileSystem fs = FileSystem.get(split.getPath().toUri());

      final FSDataInputStream fdis = fs.open(split.getPath());

      final LineReader lineReader = new LineReader(fdis, start, length, (1024 * 1024));

      byte[] line = lineReader.readLine();

View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
      final long start = split.getStart();
      final long length = split.getLength();

      final FileSystem fs = FileSystem.get(split.getPath().toUri());

      final FSDataInputStream fdis = fs.open(split.getPath());

      final LineReader lineReader = new LineReader(fdis, start, length, (1024 * 1024));

      byte[] line = lineReader.readLine();

View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
        pw.append("line\n");
      }
      pw.close();

      LocalFileSystem lfs = new LocalFileSystem();
      FSDataInputStream fis = lfs.open(pathtotestfile);

      // first, we test under "usual" conditions
      final LineReader lr = new LineReader(fis, 0, testfile.length(), 256);

      byte[] buffer;
View Full Code Here
337
338
339
340
341
342
343
344
345
346
347
348
    final FSDataOutputStream outputStream = fs.create(objectPath, false);
    generateTestData(outputStream, fileSize);
    outputStream.close();

    // Now read the same file back from S3
    final FSDataInputStream inputStream = fs.open(objectPath);
    testReceivedData(inputStream, fileSize);
    inputStream.close();

    // Delete test bucket
    fs.delete(bucketPath, true);
  }
View Full Code Here
755
756
757
758
759
760
761
762
763
764
765
766
767
        final FileSystem fs = FileSystem.get(this.split.getPath().toUri());
        this.fdis = fs.open(this.split.getPath());
       
        // check for canceling and close the stream in that case, because no one will obtain it
        if (this.aborted) {
          final FSDataInputStream f = this.fdis;
          this.fdis = null;
          f.close();
        }
      }
      catch (Throwable t) {
        this.error = t;
      }
View Full Code Here
810
811
812
813
814
815
816
817
818
819
820
821
822
    /**
     * Double checked procedure setting the abort flag and closing the stream.
     */
    private final void abortWait() {
      this.aborted = true;
      final FSDataInputStream inStream = this.fdis;
      this.fdis = null;
      if (inStream != null) {
        try {
          inStream.close();
        } catch (Throwable t) {}
      }
    }
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
151
152
      final FileOutputStream fosfile2 = new FileOutputStream(testfile2);
      fosfile2.write(testbytes);
      fosfile2.close();

      testbytestest = new byte[5];
      final FSDataInputStream lfsinput2 = lfs.open(pathtotestfile2);
      assertEquals(lfsinput2.read(testbytestest), 5);
      lfsinput2.close();
      assertTrue(Arrays.equals(testbytes, testbytestest));

      // does lfs see two files?
      assertEquals(lfs.listStatus(pathtotmpdir).length, 2);
View Full Code Here
TOP

Related Classes of eu.stratosphere.core.fs.FSDataInputStream

  • eu.stratosphere.api.common.io.BinaryInputFormat
  • eu.stratosphere.api.common.io.FileInputFormat$InputSplitOpenThread
  • eu.stratosphere.core.fs.local.LocalFileSystemTest
  • eu.stratosphere.nephele.execution.librarycache.LibraryCacheManager
  • eu.stratosphere.nephele.jobgraph.JobGraph
  • eu.stratosphere.nephele.jobmanager.DoubleSourceTask
  • eu.stratosphere.nephele.util.FileLineReader
  • eu.stratosphere.pact.runtime.cache.FileCache$CopyProcess
  • eu.stratosphere.runtime.fs.LineReaderTest
  • eu.stratosphere.runtime.fs.s3.S3FileSystemTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.