Examples of HadoopShims


Examples of org.apache.hadoop.hive.shims.HadoopShims

      }
      boolean success = fs.mkdirs(f);
      if (!success) {
        return false;
      } else {
        HadoopShims shim = ShimLoader.getHadoopShims();
        HdfsFileStatus fullFileStatus = shim.getFullFileStatus(conf, fs, lastExistingParent);
        try {
          //set on the entire subtree
          shim.setFullFileStatus(conf, fullFileStatus, fs, firstNonExistentParent);
        } catch (Exception e) {
          LOG.warn("Error setting permissions of " + firstNonExistentParent, e);
        }
        return true;
      }
View Full Code Here

Examples of org.apache.hadoop.hive.shims.HadoopShims

    boolean overwrite,
    HiveConf conf) throws IOException {
    boolean copied = FileUtil.copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf);
    boolean inheritPerms = conf.getBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);
    if (copied && inheritPerms) {
      HadoopShims shims = ShimLoader.getHadoopShims();
      HdfsFileStatus fullFileStatus = shims.getFullFileStatus(conf, dstFS, dst);
      try {
        shims.setFullFileStatus(conf, fullFileStatus, dstFS, dst);
      } catch (Exception e) {
        LOG.warn("Error setting permissions or group of " + dst, e);
      }
    }
    return copied;
View Full Code Here

Examples of org.apache.hadoop.hive.shims.HadoopShims

   * @return true if move successful
   * @throws IOException
   */
  public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf) throws IOException {
    LOG.info("deleting  " + f);
    HadoopShims hadoopShim = ShimLoader.getHadoopShims();

    if (hadoopShim.moveToAppropriateTrash(fs, f, conf)) {
      LOG.info("Moved to trash: " + f);
      return true;
    }

    boolean result = fs.delete(f, true);
View Full Code Here

Examples of org.apache.hadoop.hive.shims.HadoopShims

      //just rename the directory
      return fs.rename(sourcePath, destPath);
    } else {
      //rename the directory
      if (fs.rename(sourcePath, destPath)) {
        HadoopShims shims = ShimLoader.getHadoopShims();
        HdfsFileStatus fullFileStatus = shims.getFullFileStatus(conf, fs, destPath.getParent());
        try {
          shims.setFullFileStatus(conf, fullFileStatus, fs, destPath);
        } catch (Exception e) {
          LOG.warn("Error setting permissions or group of " + destPath, e);
        }

        return true;
View Full Code Here

Examples of org.apache.hadoop.hive.shims.HadoopShims

      // no file/dir to be deleted
      return;
    }
    FileUtils.checkFileAccessWithImpersonation(fs, stat, FsAction.WRITE, user);

    HadoopShims shims = ShimLoader.getHadoopShims();
    if (!shims.supportStickyBit()) {
      // not supports sticky bit
      return;
    }

    // check if sticky bit is set on the parent dir
    FileStatus parStatus = fs.getFileStatus(path.getParent());
    if (!shims.hasStickyBit(parStatus.getPermission())) {
      // no sticky bit, so write permission on parent dir is sufficient
      // no further checks needed
      return;
    }
View Full Code Here

Examples of org.apache.hadoop.hive.shims.HadoopShims

  @Override
  public boolean deleteDir(FileSystem fs, Path f, boolean recursive,
      boolean ifPurge, Configuration conf) throws MetaException {
    LOG.info("deleting  " + f);
    HadoopShims hadoopShim = ShimLoader.getHadoopShims();

    try {
      if (ifPurge) {
        LOG.info("Not moving "+ f +" to trash");
      } else if (hadoopShim.moveToAppropriateTrash(fs, f, conf)) {
        LOG.info("Moved to trash: " + f);
        return true;
      }

      if (fs.delete(f, true)) {
View Full Code Here

Examples of org.apache.hadoop.hive.shims.HadoopShims

      // }

      //Set HADOOP_USER_NAME env variable for child process, so that
      // it also runs with hadoop permissions for the user the job is running as
      // This will be used by hadoop only in unsecure(/non kerberos) mode
      HadoopShims shim = ShimLoader.getHadoopShims();
      String endUserName = shim.getShortUserName(shim.getUGIForConf(job));
      LOG.debug("setting HADOOP_USER_NAME\t" + endUserName);
      variables.put("HADOOP_USER_NAME", endUserName);

      if (variables.containsKey(HADOOP_OPTS_KEY)) {
        variables.put(HADOOP_OPTS_KEY, variables.get(HADOOP_OPTS_KEY) + hadoopOpts);
View Full Code Here

Examples of org.apache.hadoop.hive.shims.HadoopShims

    // Create an archived version of the partition in a directory ending in
    // ARCHIVE_INTERMEDIATE_DIR_SUFFIX that's the same level as the partition,
    // if it does not already exist. If it does exist, we assume the dir is good
    // to use as the move operation that created it is atomic.
    HadoopShims shim = ShimLoader.getHadoopShims();
    if (!pathExists(intermediateArchivedDir) &&
        !pathExists(intermediateOriginalDir)) {

      // First create the archive in a tmp dir so that if the job fails, the
      // bad files don't pollute the filesystem
      Path tmpPath = new Path(driverContext.getCtx()
          .getExternalTmpPath(originalDir), "partlevel");

      console.printInfo("Creating " + archiveName +
          " for " + originalDir.toString());
      console.printInfo("in " + tmpPath);
      console.printInfo("Please wait... (this may take a while)");

      // Create the Hadoop archive
      int ret=0;
      try {
        int maxJobNameLen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
        String jobname = String.format("Archiving %s@%s",
            tbl.getTableName(), partSpecInfo.getName());
        jobname = Utilities.abbreviate(jobname, maxJobNameLen - 6);
        conf.setVar(HiveConf.ConfVars.HADOOPJOBNAME, jobname);
        ret = shim.createHadoopArchive(conf, originalDir, tmpPath, archiveName);
      } catch (Exception e) {
        throw new HiveException(e);
      }
      if (ret != 0) {
        throw new HiveException("Error while creating HAR");
View Full Code Here

Examples of org.apache.oozie.hadoop.utils.HadoopShims

            createFile(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_2.jar");
            createFile(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_3.jar");
            createFile(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_4.jar");

            HadoopShims fileSystem = new HadoopShims(fs);
            fileSystem.createSymlink(basePath, symlink, true);

            prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig", "/user/test/" + symlink.toString());
            createTestShareLibMetaFile(fs, prop);
            assertEquals(fileSystem.isSymlink(symlink), true);

            conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties");
            conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
            try {
                ShareLibService shareLibService = Services.get().get(ShareLibService.class);
                shareLibService.init(services);
                assertEquals(shareLibService.getShareLibJars("pig").size(), 2);
                new HadoopShims(fs).createSymlink(basePath1, symlink, true);
                assertEquals(shareLibService.getShareLibJars("pig").size(), 3);
            }
            finally {
                fs.delete(new Path("shareLibPath/"), true);
                fs.delete(symlink, true);
View Full Code Here

Examples of org.apache.oozie.hadoop.utils.HadoopShims

        if (!HadoopShims.isSymlinkSupported() || symlinkMapping.get(shareLibKey) == null
                || symlinkMapping.get(shareLibKey).isEmpty()) {
            return;
        }

        HadoopShims fileSystem = new HadoopShims(fs);
        for (Path path : symlinkMapping.get(shareLibKey).keySet()) {
            if (!symlinkMapping.get(shareLibKey).get(path).equals(fileSystem.getSymLinkTarget(path))) {
                synchronized (ShareLibService.class) {
                    Map<String, List<Path>> tempShareLibMap = new HashMap<String, List<Path>>(shareLibMap);
                    Map<String, Map<Path, Path>> tmpSymlinkMapping = new HashMap<String, Map<Path, Path>>(symlinkMapping);

                    LOG.info(MessageFormat.format("Symlink target for [{0}] has changed, was [{1}], now [{2}]", shareLibKey,
                            path, fileSystem.getSymLinkTarget(path)));
                    loadShareLibMetaFile(tempShareLibMap, tmpSymlinkMapping, sharelibMappingFile, shareLibKey);
                    shareLibMap = tempShareLibMap;
                    symlinkMapping = tmpSymlinkMapping;
                    return;
                }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.