Examples of org.apache.hadoop.fs.s3.S3Credentials

org.apache.hadoop.fs.s3.INode

Extracts AWS credentials from the filesystem URI or configuration.

  private synchronized void nextBlockOutputStream() throws IOException {
    long blockId = r.nextLong();
    while (store.blockExists(blockId)) {
      blockId = r.nextLong();
    }
    nextBlock = new Block(blockId, bytesWrittenToBlock);
    blocks.add(nextBlock);
    bytesWrittenToBlock = 0;
  }

View Full Code Here

        int numBlocks = dataIn.readInt();
        Block[] blocks = new Block[numBlocks];
        for (int i = 0; i < numBlocks; i++) {
          long id = dataIn.readLong();
          long length = dataIn.readLong();
          blocks[i] = new Block(id, length);
        }
        in.close();
        return new INode(fileType, blocks);
      default:
        throw new IllegalArgumentException("Cannot deserialize inode.");

View Full Code Here

    // TODO: Use passed in Progressable to report progress.
    nextBlockOutputStream();
    store.storeBlock(nextBlock, backupFile);
    Block[] arr = new Block[blocks.size()];
    arr = blocks.toArray(arr);
    store.storeINode(path, new INode(INode.FILE_TYPES[1], arr));


    //
    // Delete local backup, start new one
    //
    boolean b = backupFile.delete();

View Full Code Here

    this.workingDir =
        new Path("/user", System.getProperty("user.name")).makeQualified(this);
  }
  @Override
  public boolean isFile(Path path) throws IOException {
    INode inode = store.retrieveINode(makeAbsolute(path));
    if (inode == null) {
      return false;
    }
    return inode.isFile();
  }

View Full Code Here

    }
    return inode.isFile();
  }


  private INode checkFile(Path path) throws IOException {
    INode inode = store.retrieveINode(makeAbsolute(path));
    if (inode == null) {
      throw new IOException("No such file.");
    }
    if (inode.isDirectory()) {
      throw new IOException("Path " + path + " is a directory.");
    }
    return inode;
  }

View Full Code Here

  }


  @Override
  public FileStatus[] listStatus(Path f) throws IOException {
    Path absolutePath = makeAbsolute(f);
    INode inode = store.retrieveINode(absolutePath);
    if (inode == null) {
      throw new FileNotFoundException("File " + f + " does not exist.");
    }
    if (inode.isFile()) {
      return new FileStatus[] {
          new S3FileStatus(f.makeQualified(this), inode)
      };
    }
    ArrayList<FileStatus> ret = new ArrayList<FileStatus>();

View Full Code Here

  public FSDataOutputStream create(Path file, FsPermission permission,
                                   boolean overwrite, int bufferSize,
                                   short replication, long blockSize, Progressable progress)
      throws IOException {


    INode inode = store.retrieveINode(makeAbsolute(file));
    if (inode != null) {
      if (overwrite) {
        delete(file, true);
      } else {
        throw new IOException("File already exists: " + file);

View Full Code Here

  }


  @Override
  public boolean rename(Path src, Path dst) throws IOException {
    Path absoluteSrc = makeAbsolute(src);
    INode srcINode = store.retrieveINode(absoluteSrc);
    if (srcINode == null) {
      // src path doesn't exist
      return false;
    }
    Path absoluteDst = makeAbsolute(dst);
    INode dstINode = store.retrieveINode(absoluteDst);
    if (dstINode != null && dstINode.isDirectory()) {
      absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
      dstINode = store.retrieveINode(absoluteDst);
    }
    if (dstINode != null) {
      // dst path already exists - can't overwrite
      return false;
    }
    Path dstParent = absoluteDst.getParent();
    if (dstParent != null) {
      INode dstParentINode = store.retrieveINode(dstParent);
      if (dstParentINode == null || dstParentINode.isFile()) {
        // dst parent doesn't exist or is a file
        return false;
      }
    }
    return renameRecursive(absoluteSrc, absoluteDst);

View Full Code Here

    }
    return renameRecursive(absoluteSrc, absoluteDst);
  }


  private boolean renameRecursive(Path src, Path dst) throws IOException {
    INode srcINode = store.retrieveINode(src);
    store.storeINode(dst, srcINode);
    store.deleteINode(src);
    if (srcINode.isDirectory()) {
      for (Path oldSrc : store.listDeepSubPaths(src)) {
        INode inode = store.retrieveINode(oldSrc);
        if (inode == null) {
          return false;
        }
        String oldSrcPath = oldSrc.toUri().getPath();
        String srcPath = src.toUri().getPath();

View Full Code Here

  }


  @Override
  public boolean delete(Path path, boolean recursive) throws IOException {
    Path absolutePath = makeAbsolute(path);
    INode inode = store.retrieveINode(absolutePath);
    if (inode == null) {
      return false;
    }
    if (inode.isFile()) {
      store.deleteINode(absolutePath);
      for (Block block: inode.getBlocks()) {
        store.deleteBlock(block);
      }
    } else {
      FileStatus[] contents = null;
      try {

View Full Code Here

0 1 2 3 4 5 6

TOP

Related Classes of org.apache.hadoop.fs.s3.S3Credentials

com.facebook.presto.hive.PrestoS3FileSystem

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.FSDataInputStream

org.apache.hadoop.fs.FSDataOutputStream

org.apache.hadoop.fs.FSInputStream

org.apache.hadoop.fs.FSOutputStream

org.apache.hadoop.fs.Path

org.apache.hadoop.fs.s3native.Jets3tNativeFileSystemStore

org.apache.hadoop.io.retry.RetryPolicy

org.apache.tajo.storage.s3.INode

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.