Examples of org.apache.hadoop.fs.Path

org.apache.hadoop.fs.Path
Names a file or directory in a {@link FileSystem}. Path strings use slash as the directory separator. A path string is absolute if it begins with a slash.

   * @throws IOException Thrown if failed update.
   */
  static Path updateHTableDescriptor(FileSystem fs, Path rootdir,
      HTableDescriptor hTableDescriptor)
  throws IOException {
    Path tableDir = FSUtils.getTablePath(rootdir, hTableDescriptor.getName());
    Path p = writeTableDescriptor(fs, hTableDescriptor, tableDir,
      getTableInfoPath(fs, tableDir));
    if (p == null) throw new IOException("Failed update");
    LOG.info("Updated tableinfo=" + p);
    return p;
  }

View Full Code Here

      final HTableDescriptor hTableDescriptor, final Path tableDir,
      final FileStatus status)
  throws IOException {
    // Get temporary dir into which we'll first write a file to avoid
    // half-written file phenomeon.
    Path tmpTableDir = new Path(tableDir, ".tmp");
    // What is current sequenceid?  We read the current sequenceid from
    // the current file.  After we read it, another thread could come in and
    // compete with us writing out next version of file.  The below retries
    // should help in this case some but its hard to do guarantees in face of
    // concurrent schema edits.
    int currentSequenceid =
      status == null? 0: getTableInfoSequenceid(status.getPath());
    int sequenceid = currentSequenceid;
    // Put arbitrary upperbound on how often we retry
    int retries = 10;
    int retrymax = currentSequenceid + retries;
    Path tableInfoPath = null;
    do {
      sequenceid += 1;
      Path p = getTableInfoFileName(tmpTableDir, sequenceid);
      if (fs.exists(p)) {
        LOG.debug(p + " exists; retrying up to " + retries + " times");
        continue;
      }
      try {

View Full Code Here

        return regionsToFlush;
      }
      // Do all the preparation outside of the updateLock to block
      // as less as possible the incoming writes
      long currentFilenum = this.filenum;
      Path oldPath = null;
      if (currentFilenum > 0) {
        oldPath = computeFilename(currentFilenum);
      }
      this.filenum = System.currentTimeMillis();
      Path newPath = computeFilename();


      // Tell our listeners that a new log is about to be created
      if (!this.listeners.isEmpty()) {
        for (WALActionsListener i : this.listeners) {
          i.preLogRoll(oldPath, newPath);
        }
      }
      HLog.Writer nextWriter = this.createWriterInstance(fs, newPath, conf);
      // Can we get at the dfsclient outputstream?  If an instance of
      // SFLW, it'll have done the necessary reflection to get at the
      // protected field name.
      FSDataOutputStream nextHdfsOut = null;
      if (nextWriter instanceof SequenceFileLogWriter) {
        nextHdfsOut = ((SequenceFileLogWriter)nextWriter).getWriterFSDataOutputStream();
      }


      synchronized (updateLock) {
        // Clean up current writer.
        Path oldFile = cleanupCurrentWriter(currentFilenum);
        this.writer = nextWriter;
        this.hdfs_out = nextHdfsOut;


        LOG.info((oldFile != null?
            "Roll " + FSUtils.getPath(oldFile) + ", entries=" +

View Full Code Here

   * @return True if we successfully created file.
   */
  public static boolean createTableDescriptor(FileSystem fs, Path rootdir,
      HTableDescriptor htableDescriptor, boolean forceCreation)
  throws IOException {
    Path tabledir = FSUtils.getTablePath(rootdir, htableDescriptor.getNameAsString());
    return createTableDescriptorForTableDirectory(fs, tabledir, htableDescriptor, forceCreation);
  }

View Full Code Here

   * Presumes we're operating inside an updateLock scope.
   * @return Path to current writer or null if none.
   * @throws IOException
   */
  Path cleanupCurrentWriter(final long currentfilenum) throws IOException {
    Path oldFile = null;
    if (this.writer != null) {
      // Close the current writer, get a new one.
      try {
        // Wait till all current transactions are written to the hlog.
        // No new transactions can occur because we have the updatelock.

View Full Code Here

          LOG.info("TableInfo already exists.. Skipping creation");
          return false;
        }
      }
    }
    Path p = writeTableDescriptor(fs, htableDescriptor, tabledir, status);
    return p != null;
  }

View Full Code Here

    }
    return oldFile;
  }


  private void archiveLogFile(final Path p, final Long seqno) throws IOException {
    Path newPath = getHLogArchivePath(this.oldLogDir, p);
    LOG.info("moving old hlog file " + FSUtils.getPath(p) +
      " whose highest sequenceid is " + seqno + " to " +
      FSUtils.getPath(newPath));


    // Tell our listeners that a log is going to be archived.

View Full Code Here

   */
  protected Path computeFilename(long filenum) {
    if (filenum < 0) {
      throw new RuntimeException("hlog file number can't be < 0");
    }
    return new Path(dir, prefix + "." + filenum);
  }

View Full Code Here

    close();
    if (!fs.exists(this.dir)) return;
    FileStatus[] files = fs.listStatus(this.dir);
    for(FileStatus file : files) {


      Path p = getHLogArchivePath(this.oldLogDir, file.getPath());
      // Tell our listeners that a log is going to be archived.
      if (!this.listeners.isEmpty()) {
        for (WALActionsListener i : this.listeners) {
          i.preLogArchive(file.getPath(), p);
        }

View Full Code Here

  public static boolean validateHLogFilename(String filename) {
    return pattern.matcher(filename).matches();
  }


  static Path getHLogArchivePath(Path oldLogDir, Path p) {
    return new Path(oldLogDir, p.getName());
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.fs.Path

co.nubetech.hiho.common.HihoTestCase

co.nubetech.hiho.dedup.TestDedupJob

co.nubetech.hiho.job.TestDBQueryInputJobWithCluster

co.nubetech.hiho.mapreduce.lib.output.AppendSequenceFileOutputFormat

co.nubetech.hiho.mapreduce.lib.output.AppendTextOutputFormat

co.nubetech.hiho.merge.TestMergeJob

com.alimama.mdrill.utils.HadoopBaseUtils

com.alimama.mdrill.utils.IndexUtils

com.alimama.mdrill.utils.ZipUtils

com.cloudera.circus.test.TestXTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.