Examples of eu.stratosphere.core.fs.FileStatus

eu.stratosphere.core.fs.FileStatus
Interface that represents the client side information for a file independent of the file system.

  
  protected FileBaseStatistics getFileStats(FileBaseStatistics cachedStats, Path filePath, FileSystem fs,
      ArrayList<FileStatus> files) throws IOException {
    
    // get the file info and check whether the cached statistics are still valid.
    final FileStatus file = fs.getFileStatus(filePath);
    long latestModTime = file.getModificationTime();


    // enumerate all files and check their modification time stamp.
    if (file.isDir()) {
      FileStatus[] fss = fs.listStatus(filePath);
      files.ensureCapacity(fss.length);
      
      for (FileStatus s : fss) {
        if (!s.isDir()) {

View Full Code Here

    // get all the files that are involved in the splits
    List<FileStatus> files = new ArrayList<FileStatus>();
    long totalLength = 0;


    final FileSystem fs = path.getFileSystem();
    final FileStatus pathFile = fs.getFileStatus(path);


    if(!acceptFile(pathFile)) {
      throw new IOException("The given file does not pass the file-filter");
    }
    if (pathFile.isDir()) {
      // input is directory. list all contained files
      final FileStatus[] dir = fs.listStatus(path);
      for (int i = 0; i < dir.length; i++) {
        if (!dir[i].isDir() && acceptFile(dir[i])) {
          files.add(dir[i]);
          totalLength += dir[i].getLen();
          // as soon as there is one deflate file in a directory, we can not split it
          testForUnsplittable(dir[i]);
        }
      }
    } else {
      testForUnsplittable(pathFile);
      
      files.add(pathFile);
      totalLength += pathFile.getLen();
    }
    // returns if unsplittable
    if(unsplittable) {
      int splitNum = 0;
      for (final FileStatus file : files) {

View Full Code Here




  @Override
  public FSDataInputStream open(final Path f) throws IOException {


    final FileStatus fileStatus = getFileStatus(f); // Will throw FileNotFoundException if f does not exist


    // Make sure f is not a directory
    if (fileStatus.isDir()) {
      throw new IOException("Cannot open " + f.toUri() + " because it is a directory");
    }


    final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f);
    if (!bop.hasBucket() || !bop.hasObject()) {

View Full Code Here


  @Override
  public boolean delete(Path f, boolean recursive) throws IOException {


    try {
      final FileStatus fileStatus = getFileStatus(f); // Will throw a FileNotFoundException if f is invalid
      final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f);


      if (fileStatus.isDir()) {


        boolean retVal = false;
        final FileStatus[] dirContent = listStatus(f);
        if (dirContent.length > 0) {
          // Directory is not empty

View Full Code Here

      assertFalse(lfs.exists(pathtotmpdir));
      tempdir.mkdirs();


      // check that local file system recognizes file..
      assertTrue(lfs.exists(pathtotmpdir));
      final FileStatus localstatus1 = lfs.getFileStatus(pathtotmpdir);


      // check that lfs recognizes directory..
      assertTrue(localstatus1.isDir());


      // get status for files in this (empty) directory..
      final FileStatus[] statusforfiles = lfs.listStatus(pathtotmpdir);


      // no files in there.. hence, must be zero

View Full Code Here

    inputFormat.setFilePath(normalizedPath);
    inputFormat.setOpenTimeout(0);
    inputFormat.configure(configuration);


    final FileSystem fs = FileSystem.get(normalizedPath.toUri());
    FileStatus fileStatus = fs.getFileStatus(normalizedPath);


    BlockLocation[] blocks = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
    inputFormat.open(new FileInputSplit(0, new Path(path), 0, fileStatus.getLen(), blocks[0].getHosts()));
    return inputFormat;
  }

View Full Code Here

  @SuppressWarnings("unchecked")
  public static <T, F extends FileInputFormat<T>> List<F> openAllInputs(
      Class<F> inputFormatClass, String path, Configuration configuration) throws IOException {
    Path nephelePath = new Path(path);
    FileSystem fs = nephelePath.getFileSystem();
    FileStatus fileStatus = fs.getFileStatus(nephelePath);
    if (!fileStatus.isDir()) {
      return Arrays.asList(openInput(inputFormatClass, path, configuration));
    }
    FileStatus[] list = fs.listStatus(nephelePath);
    List<F> formats = new ArrayList<F>();
    for (int index = 0; index < list.length; index++) {

View Full Code Here


    if (inputSplits.size() < minNumSplits) {
      LOG.warn(String.format(
        "With the given block size %d, the file %s cannot be split into %d blocks. Filling up with empty splits...",
        blockSize, this.filePath, minNumSplits));
      FileStatus last = files.get(files.size() - 1);
      final BlockLocation[] blocks = fs.getFileBlockLocations(last, 0, last.getLen());
      for (int index = files.size(); index < minNumSplits; index++) {
        inputSplits.add(new FileInputSplit(index, last.getPath(), last.getLen(), 0, blocks[0].getHosts()));
      }
    }


    return inputSplits.toArray(new FileInputSplit[0]);
  }

View Full Code Here

  protected List<FileStatus> getFiles() throws IOException {
    // get all the files that are involved in the splits
    List<FileStatus> files = new ArrayList<FileStatus>();


    final FileSystem fs = this.filePath.getFileSystem();
    final FileStatus pathFile = fs.getFileStatus(this.filePath);


    if (pathFile.isDir()) {
      // input is directory. list all contained files
      final FileStatus[] partials = fs.listStatus(this.filePath);
      for (int i = 0; i < partials.length; i++) {
        if (!partials[i].isDir()) {
          files.add(partials[i]);

View Full Code Here

      int samplesTaken = 0;


      // take the samples
      while (samplesTaken < numSamples && fileNum < allFiles.size()) {
        // make a split for the sample and use it to read a record
        FileStatus file = allFiles.get(fileNum);
        FileInputSplit split = new FileInputSplit(0, file.getPath(), offset, file.getLen() - offset, null);


        // we open the split, read one line, and take its length
        try {
          open(split);
          if (readLine()) {
            totalNumBytes += this.currLen + this.delimiter.length;
            samplesTaken++;
          }
        } finally {
          // close the file stream, do not release the buffers
          super.close();
        }


        offset += stepSize;


        // skip to the next file, if necessary
        while (fileNum < allFiles.size() && offset >= (file = allFiles.get(fileNum)).getLen()) {
          offset -= file.getLen();
          fileNum++;
        }
      }
      
      // we have the width, store it

View Full Code Here

0 1

TOP

Related Classes of eu.stratosphere.core.fs.FileStatus

eu.stratosphere.api.common.io.BinaryInputFormat

eu.stratosphere.api.common.io.DelimitedInputFormat

eu.stratosphere.api.common.io.FileInputFormat

eu.stratosphere.api.common.io.FormatUtil

eu.stratosphere.api.java.record.io.avro.AvroRecordInputFormat

eu.stratosphere.core.fs.local.LocalFileSystemTest

eu.stratosphere.nephele.execution.librarycache.LibraryCacheManager

eu.stratosphere.nephele.jobgraph.JobFileInputVertex

eu.stratosphere.nephele.jobgraph.JobFileOutputVertex

eu.stratosphere.nephele.jobgraph.JobGraph

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.