Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FileSystem.deleteOnExit()


    // create the partitions file
    FileSystem fs = FileSystem.get(job.getConfiguration());
    Path partitionsPath = new Path("/tmp", "partitions_" + UUID.randomUUID());
    fs.makeQualified(partitionsPath);
    fs.deleteOnExit(partitionsPath);
    writePartitions(job.getConfiguration(), partitionsPath, splitPoints);

    // configure job to use it
    job.setPartitionerClass(TotalOrderPartitioner.class);
    TotalOrderPartitioner.setPartitionFile(job.getConfiguration(), partitionsPath);
View Full Code Here


    }

    public void rmr(String folder) throws IOException {
        Path path = new Path(folder);
        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);
        fs.deleteOnExit(path);
        System.out.println("Delete: " + folder);
        fs.close();
    }

    public void rename(String src, String dst) throws IOException {
View Full Code Here

    Path lockPath = new Path(launcherDir, tag + ".lock");
    Path jtPath = new Path(launcherDir, tag + ".jobtracker");
    Path webuiPath = new Path(launcherDir, tag + ".webui");
   
    // Delete the .lock, .jobtracker and .webui files on exit
    fs.deleteOnExit(lockPath);
   
    // Start the JobTracker
    try {
      // Create a new JobTracker ID - make sure to include the username
      // Note: The task status server on the TT seems to break if we add an
View Full Code Here

      // Start it
      JobTracker jt = JobTracker.startTracker(jtConf, jtId);
      // After startTracker, the JT has updated its internal variables to
      // include its real RPC and HTTP addresses; write them to our files
      writeTextFile(fs, jtPath, host + ":" + jt.getTrackerPort());
      fs.deleteOnExit(jtPath);
      writeTextFile(fs, webuiPath, host + ":" + jt.getInfoPort());
      fs.deleteOnExit(webuiPath);
      // Launch a daemon thread to touch the lock file and maintain our lease
      new LeaseRenewalThread(fs, lockPath, leaseTimeout, jt).start();
      // Run the JT
View Full Code Here

      // After startTracker, the JT has updated its internal variables to
      // include its real RPC and HTTP addresses; write them to our files
      writeTextFile(fs, jtPath, host + ":" + jt.getTrackerPort());
      fs.deleteOnExit(jtPath);
      writeTextFile(fs, webuiPath, host + ":" + jt.getInfoPort());
      fs.deleteOnExit(webuiPath);
      // Launch a daemon thread to touch the lock file and maintain our lease
      new LeaseRenewalThread(fs, lockPath, leaseTimeout, jt).start();
      // Run the JT
      jt.offerService();
    } catch (UnknownHostException e) {
View Full Code Here

      FileSystem fs = path.getFileSystem(HBaseConfiguration.create());
      Path dst = new Path(System.getProperty("java.io.tmpdir") +
          java.io.File.separator +"." + pathPrefix +
          "." + className + "." + System.currentTimeMillis() + ".jar");
      fs.copyToLocalFile(path, dst);
      fs.deleteOnExit(dst);

      // TODO: code weaving goes here

      // TODO: wrap heap allocations and enforce maximum usage limits
View Full Code Here

      stm3.close();

      // set delete on exit flag on files.
      fs.deleteOnExit(file1);
      fs.deleteOnExit(file2);
      localfs.deleteOnExit(file3);

      // close the file system. This should make the above files
      // disappear.
      fs.close();
      localfs.close();
View Full Code Here

    VectorWritable inputVW = new VectorWritable(v);
    inputVectorPathWriter.append(NullWritable.get(), inputVW);
    inputVectorPathWriter.close();
    URI ivpURI = inputVectorPath.toUri();
    DistributedCache.setCacheFiles(new URI[] {ivpURI}, conf);
    fs.deleteOnExit(inputVectorPath);

    conf.set(INPUT_VECTOR, ivpURI.toString());
    conf.setBoolean(IS_SPARSE_OUTPUT, !(v instanceof DenseVector));
    conf.setInt(OUTPUT_VECTOR_DIMENSION, outputVectorDim);
    FileInputFormat.addInputPath(conf, matrixInputPath);
View Full Code Here

    NullWritable n = NullWritable.get();
    VectorWritable v = new VectorWritable();
    reader.next(n,v);
    Vector vector = v.get();
    reader.close();
    fs.deleteOnExit(outputFile);
    return vector;
  }

  public static class TimesSquaredMapper<T extends WritableComparable> extends MapReduceBase
      implements Mapper<T,VectorWritable, NullWritable,VectorWritable> {
View Full Code Here

      dataTestDirOnTestFS = new Path(dataTestDir.getAbsolutePath());
    } else {
      Path base = getBaseTestDirOnTestFS();
      String randomStr = UUID.randomUUID().toString();
      dataTestDirOnTestFS = new Path(base, randomStr);
      if (deleteOnExit()) fs.deleteOnExit(dataTestDirOnTestFS);
    }
  }

  /**
   * Cleans the test data directory on the test filesystem.
View Full Code Here

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.