Examples of org.apache.hadoop.fs.Path

org.apache.hadoop.fs.Path
Names a file or directory in a {@link FileSystem}. Path strings use slash as the directory separator. A path string is absolute if it begins with a slash.

      fs = FileSystem.get(c);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    FSTableDescriptors fstd =
      new FSTableDescriptors(fs, new Path(c.get(HConstants.HBASE_DIR)));
    try {
      return fstd.get(this.tableName);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }

View Full Code Here

      fs = FileSystem.get(c);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    FSTableDescriptors fstd =
      new FSTableDescriptors(fs, new Path(c.get(HConstants.HBASE_DIR)));
    try {
      fstd.add(newDesc);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }

View Full Code Here

    // Get snapshot files
    SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
      new SnapshotReferenceUtil.FileVisitor() {
        public void storeFile (final String region, final String family, final String hfile)
            throws IOException {
          Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile));
          long size = new HFileLink(conf, path).getFileStatus(fs).getLen();
          files.add(new Pair<Path, Long>(path, size));
        }


        public void recoveredEdits (final String region, final String logfile)
            throws IOException {
          // copied with the snapshot referenecs
        }


        public void logFile (final String server, final String logfile)
            throws IOException {
          long size = new HLogLink(conf, server, logfile).getFileStatus(fs).getLen();
          files.add(new Pair<Path, Long>(new Path(server, logfile), size));
        }
    });


    return files;
  }

View Full Code Here

  }


  private static Path getInputFolderPath(final FileSystem fs, final Configuration conf)
      throws IOException, InterruptedException {
    String stagingName = "exportSnapshot-" + EnvironmentEdgeManager.currentTimeMillis();
    Path stagingDir = new Path(conf.get(CONF_TMP_DIR), stagingName);
    fs.mkdirs(stagingDir);
    return new Path(stagingDir, INPUT_FOLDER_PREFIX +
      String.valueOf(EnvironmentEdgeManager.currentTimeMillis()));
  }

View Full Code Here

   */
  private static Path[] createInputFiles(final Configuration conf,
      final List<Pair<Path, Long>> snapshotFiles, int mappers)
      throws IOException, InterruptedException {
    FileSystem fs = FileSystem.get(conf);
    Path inputFolderPath = getInputFolderPath(fs, conf);
    LOG.debug("Input folder location: " + inputFolderPath);


    List<List<Path>> splits = getBalancedSplits(snapshotFiles, mappers);
    Path[] inputFiles = new Path[splits.size()];


    Text key = new Text();
    for (int i = 0; i < inputFiles.length; i++) {
      List<Path> files = splits.get(i);
      inputFiles[i] = new Path(inputFolderPath, String.format("export-%d.seq", i));
      SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, inputFiles[i],
        Text.class, NullWritable.class);
      LOG.debug("Input split: " + i);
      try {
        for (Path file: files) {

View Full Code Here

  public int run(String[] args) throws Exception {
    boolean verifyChecksum = true;
    String snapshotName = null;
    String filesGroup = null;
    String filesUser = null;
    Path outputRoot = null;
    int filesMode = 0;
    int mappers = getConf().getInt("mapreduce.job.maps", 1);


    // Process command line args
    for (int i = 0; i < args.length; i++) {
      String cmd = args[i];
      try {
        if (cmd.equals("-snapshot")) {
          snapshotName = args[++i];
        } else if (cmd.equals("-copy-to")) {
          outputRoot = new Path(args[++i]);
        } else if (cmd.equals("-no-checksum-verify")) {
          verifyChecksum = false;
        } else if (cmd.equals("-mappers")) {
          mappers = Integer.parseInt(args[++i]);
        } else if (cmd.equals("-chuser")) {
          filesUser = args[++i];
        } else if (cmd.equals("-chgroup")) {
          filesGroup = args[++i];
        } else if (cmd.equals("-chmod")) {
          filesMode = Integer.parseInt(args[++i], 8);
        } else if (cmd.equals("-h") || cmd.equals("--help")) {
          printUsageAndExit();
        } else {
          System.err.println("UNEXPECTED: " + cmd);
          printUsageAndExit();
        }
      } catch (Exception e) {
        printUsageAndExit();
      }
    }


    // Check user options
    if (snapshotName == null) {
      System.err.println("Snapshot name not provided.");
      printUsageAndExit();
    }


    if (outputRoot == null) {
      System.err.println("Destination file-system not provided.");
      printUsageAndExit();
    }


    Configuration conf = getConf();
    Path inputRoot = FSUtils.getRootDir(conf);
    FileSystem inputFs = FileSystem.get(conf);
    FileSystem outputFs = FileSystem.get(outputRoot.toUri(), conf);


    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
    Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshotName, outputRoot);
    Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, outputRoot);


    // Check if the snapshot already exists
    if (outputFs.exists(outputSnapshotDir)) {
      System.err.println("The snapshot '" + snapshotName +
        "' already exists in the destination: " + outputSnapshotDir);

View Full Code Here


  @Override
  public HTableDescriptor remove(final String tablename)
  throws IOException {
    if (!this.fsreadonly) {
      Path tabledir = FSUtils.getTablePath(this.rootdir, tablename);
      if (this.fs.exists(tabledir)) {
        if (!this.fs.delete(tabledir, true)) {
          throw new IOException("Failed delete of " + tabledir.toString());
        }
      }
    }
    TableDescriptorModtime tdm = this.cache.remove(tablename);
    return tdm == null ? null : tdm.getTableDescriptor();

View Full Code Here

  }


  private static FileStatus getTableInfoPath(final FileSystem fs,
      final Path rootdir, final String tableName)
  throws IOException {
    Path tabledir = FSUtils.getTablePath(rootdir, tableName);
    return getTableInfoPath(fs, tabledir);
  }

View Full Code Here

    if (status == null || status.length < 1) return null;
    Arrays.sort(status, new FileStatusFileNameComparator());
    if (status.length > 1) {
      // Clean away old versions of .tableinfo
      for (int i = 1; i < status.length; i++) {
        Path p = status[i].getPath();
        // Clean up old versions
        if (!fs.delete(p, false)) {
          LOG.warn("Failed cleanup of " + status);
        } else {
          LOG.debug("Cleaned up old tableinfo file " + p);

View Full Code Here

   * @param tabledir
   * @param sequenceid
   * @return Name of tableinfo file.
   */
  static Path getTableInfoFileName(final Path tabledir, final int sequenceid) {
    return new Path(tabledir,
      TABLEINFO_NAME + "." + formatTableInfoSequenceId(sequenceid));
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.fs.Path

co.nubetech.hiho.common.HihoTestCase

co.nubetech.hiho.dedup.TestDedupJob

co.nubetech.hiho.job.TestDBQueryInputJobWithCluster

co.nubetech.hiho.mapreduce.lib.output.AppendSequenceFileOutputFormat

co.nubetech.hiho.mapreduce.lib.output.AppendTextOutputFormat

co.nubetech.hiho.merge.TestMergeJob

com.alimama.mdrill.utils.HadoopBaseUtils

com.alimama.mdrill.utils.IndexUtils

com.alimama.mdrill.utils.ZipUtils

com.cloudera.circus.test.TestXTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.