Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FileStatus


        //reading directories is also progress
        reporter.progress();
      }
      else {
        FileSystem srcFs = srcPath.getFileSystem(conf);
        FileStatus srcStatus = srcFs.getFileStatus(srcPath);
        FSDataInputStream input = srcFs.open(srcStatus.getPath());
        reporter.setStatus("Copying file " + srcStatus.getPath() +
            " to archive.");
        copyData(srcStatus.getPath(), input, partStream, reporter);
        towrite = relPath.toString() + " file " + partname + " " + startPos
        + " " + srcStatus.getLen() + " ";
      }
      out.collect(new IntWritable(hash), new Text(towrite));
    }
View Full Code Here


                         + " replicas.");
      checkFile(fs, file1, 1);
      System.out.println("Path : \"" + file1 + "\"");

      // test getFileStatus on a file
      FileStatus status = fs.getFileStatus(file1);
      assertTrue(file1 + " should be a file",
          status.isDir() == false);
      assertTrue(status.getBlockSize() == blockSize);
      assertTrue(status.getReplication() == 1);
      assertTrue(status.getLen() == fileSize);
      assertEquals(fs.makeQualified(file1).toString(),
          status.getPath().toString());

      // test getVisbileLen
      DFSDataInputStream fin = (DFSDataInputStream)fs.open(file1);
      assertEquals(status.getLen(), fin.getVisibleLength());
     
      // test listStatus on a file
      FileStatus[] stats = fs.listStatus(file1);
      assertEquals(1, stats.length);
      status = stats[0];
      assertTrue(file1 + " should be a file",
          status.isDir() == false);
      assertTrue(status.getBlockSize() == blockSize);
      assertTrue(status.getReplication() == 1);
      assertTrue(status.getLen() == fileSize);
      assertEquals(fs.makeQualified(file1).toString(),
          status.getPath().toString());

      // test file status on a directory
      Path dir = new Path("/test/mkdirs");

      // test listStatus on a non-existent file/directory
      stats = fs.listStatus(dir);
      assertEquals(null, stats);
      try {
        status = fs.getFileStatus(dir);
        fail("getFileStatus of non-existent path should fail");
      } catch (FileNotFoundException fe) {
        assertTrue(fe.getMessage().startsWith("File does not exist"));
      }
     
      // create the directory
      assertTrue(fs.mkdirs(dir));
      assertTrue(fs.exists(dir));
      System.out.println("Dir : \"" + dir + "\"");

      // test getFileStatus on an empty directory
      status = fs.getFileStatus(dir);
      assertTrue(dir + " should be a directory", status.isDir());
      assertTrue(dir + " should be zero size ", status.getLen() == 0);
      assertEquals(fs.makeQualified(dir).toString(),
          status.getPath().toString());

      // test listStatus on an empty directory
      stats = fs.listStatus(dir);
      assertEquals(dir + " should be empty", 0, stats.length);
      assertEquals(dir + " should be zero size ",
          0, fs.getContentSummary(dir).getLength());
      assertEquals(dir + " should be zero size using hftp",
          0, hftpfs.getContentSummary(dir).getLength());
      assertTrue(dir + " should be zero size ",
                 fs.getFileStatus(dir).getLen() == 0);
      System.out.println("Dir : \"" + dir + "\"");

      // create another file that is smaller than a block.
      //
      Path file2 = new Path(dir, "filestatus2.dat");
      writeFile(fs, file2, 1, blockSize/4, blockSize);
      System.out.println("Created file filestatus2.dat with one "
                         + " replicas.");
      checkFile(fs, file2, 1);
      System.out.println("Path : \"" + file2 + "\"");

      // verify file attributes
      status = fs.getFileStatus(file2);
      assertTrue(status.getBlockSize() == blockSize);
      assertTrue(status.getReplication() == 1);
      file2 = fs.makeQualified(file2);
      assertEquals(file2.toString(), status.getPath().toString());

      // create another file in the same directory
      Path file3 = new Path(dir, "filestatus3.dat");
      writeFile(fs, file3, 1, blockSize/4, blockSize);
      System.out.println("Created file filestatus3.dat with one "
View Full Code Here

    TaskInfo taskInfo = null;
    boolean isTempFolderExists = false;
    String localTaskDir = null;
    TTClient ttClient = null;
    TaskID tID = null;
    FileStatus filesStatus [] = null;
    Path inputDir = new Path("input");
    Path outputDir = new Path("output");
    Configuration conf = new Configuration(cluster.getConf());
    JobConf jconf = new JobConf(conf);
    jconf.setJobName("Word Count");
View Full Code Here

   */
  @Test
  public void testDirCleanupAfterTaskFailed() throws IOException,
          InterruptedException {
    TTClient ttClient = null;
    FileStatus filesStatus [] = null;
    String localTaskDir = null;
    TaskInfo taskInfo = null;
    TaskID tID = null;
    boolean isTempFolderExists = false;
    Path inputDir = new Path("input");
View Full Code Here

    } else {
      url = info.getHistoryUrl();
    }
    Path p = new Path(url);
    if (p.toUri().getScheme().equals("file:/")) {
      FileStatus st = getFileStatus(url, true);
      Assert.assertNotNull("Job History file for " + jobId + " not present " +
          "when job is completed" , st);
    } else {
      FileStatus st = getFileStatus(url, false);
      Assert.assertNotNull("Job History file for " + jobId + " not present " +
          "when job is completed" , st);
    }
    LOG.info("Verified the job history for the jobId : " + jobId);
  }
View Full Code Here

    LOG.info("seed: " + seed);
    rand.setSeed(seed);
    SplittableCompressionCodec codec =
      ReflectionUtils.newInstance(codecClass, conf);
    final FileSystem fs = FileSystem.getLocal(conf);
    final FileStatus infile =
      fs.getFileStatus(writeSplitTestFile(fs, rand, codec, DEFLBYTES));
    if (infile.getLen() > Integer.MAX_VALUE) {
      fail("Unexpected compression: " + DEFLBYTES + " -> " + infile.getLen());
    }
    final int flen = (int) infile.getLen();
    final Text line = new Text();
    final Decompressor dcmp = CodecPool.getDecompressor(codec);
    try {
      for (int pos = 0; pos < infile.getLen(); pos += rand.nextInt(flen / 8)) {
        // read from random positions, verifying that there exist two sequential
        // lines as written in writeSplitTestFile
        final SplitCompressionInputStream in =
          codec.createInputStream(fs.open(infile.getPath()), dcmp,
              pos, flen, SplittableCompressionCodec.READ_MODE.BYBLOCK);
        if (in.getAdjustedStart() >= flen) {
          break;
        }
        LOG.info("SAMPLE " + in.getAdjustedStart() + "," + in.getAdjustedEnd());
        final LineReader lreader = new LineReader(in);
        lreader.readLine(line); // ignore; likely partial
        if (in.getPos() >= flen) {
          break;
        }
        lreader.readLine(line);
        final int seq1 = readLeadingInt(line);
        lreader.readLine(line);
        if (in.getPos() >= flen) {
          break;
        }
        final int seq2 = readLeadingInt(line);
        assertEquals("Mismatched lines", seq1 + 1, seq2);
      }
    } finally {
      CodecPool.returnDecompressor(dcmp);
    }
    // remove on success
    fs.delete(infile.getPath().getParent(), true);
  }
View Full Code Here

          //Checking for username directory to check if it has the
          //proper permissions
          localDir = localDir + Path.SEPARATOR +
                  TaskTracker.getPrivateDistributedCacheDir(userName);

          FileStatus fileStatusMapredLocalDirUserName = null;

          try {
            fileStatusMapredLocalDirUserName = tClient.
                            getFileStatus(localDirOnly, true);
          } catch (Exception e) {
            LOG.info("LocalDirOnly :" + localDirOnly + " not found");
            FileNotPresentForThisDirectoryPath = true;
          }

          //File will only be stored under one of the mapred.lcoal.dir
          //If other paths were hit, just continue 
          if (FileNotPresentForThisDirectoryPath)
            continue;

          Path pathMapredLocalDirUserName =
              fileStatusMapredLocalDirUserName.getPath();
          FsPermission fsPermMapredLocalDirUserName =
              fileStatusMapredLocalDirUserName.getPermission();
          Assert.assertTrue("Directory Permission is not 700",
            fsPermMapredLocalDirUserName.equals(new FsPermission("700")));

          //Get file status of all the directories
          //and files under that path.
View Full Code Here

    }
   
    // Create the lock directory if it does not exist
    Path launcherDirPath = new Path(launcherDir);
    try {
      FileStatus status = fs.getFileStatus(launcherDirPath);
      if (!status.isDir()) {
        fatalError(launcherDir + " exists, but is not a directory");
      }
    } catch (FileNotFoundException e) {
      if (!fs.mkdirs(launcherDirPath)) {
        fatalError("failed to create " + launcherDir);
      }
    }
   
    // Try to connect to, or launch, a JobTracker for our given tag
    Path lockPath = new Path(launcherDirPath, tag + ".lock");
    Path jtPath = new Path(launcherDirPath, tag + ".jobtracker");
    while (true) {
      try {
        FileStatus stat = fs.getFileStatus(lockPath);
        // If getFileStatus succeeds, the lock file exists; check if it's valid
        if (stat.getAccessTime() + leaseTimeout < System.currentTimeMillis()) {
          // Lease expired; delete the file so that we can attempt to create it
          // on the next iteration of the while loop, but sleep a bit before
          // continuing in case other launchers are trying to delete the file.
          fs.delete(lockPath, false);
          try {
View Full Code Here

   */
  private Path localizeJobConfFile(Path jobFile, String user, FileSystem userFs, JobID jobId)
  throws IOException {
    // Get sizes of JobFile and JarFile
    // sizes are -1 if they are not present.
    FileStatus status = null;
    long jobFileSize = -1;
    try {
      status = userFs.getFileStatus(jobFile);
      jobFileSize = status.getLen();
    } catch(FileNotFoundException fe) {
      jobFileSize = -1;
    }
    Path localJobFile =
      lDirAlloc.getLocalPathForWrite(getLocalJobConfFile(user,
View Full Code Here

  private void localizeJobJarFile(String user, JobID jobId, FileSystem userFs,
      JobConf localJobConf)
  throws IOException {
    // copy Jar file to the local FS and unjar it.
    String jarFile = localJobConf.getJar();
    FileStatus status = null;
    long jarFileSize = -1;
    if (jarFile != null) {
      Path jarFilePath = new Path(jarFile);
      try {
        status = userFs.getFileStatus(jarFilePath);
        jarFileSize = status.getLen();
      } catch (FileNotFoundException fe) {
        jarFileSize = -1;
      }
      // Here we check for five times the size of jarFileSize to accommodate for
      // unjarring the jar file in the jars directory
View Full Code Here

TOP

Related Classes of org.apache.hadoop.fs.FileStatus

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.