Examples of org.apache.hadoop.tools.CopyListingFileStatus

org.apache.hadoop.tools.CopyListingFileStatus
CopyListingFileStatus is a specialized subclass of {@link FileStatus} forattaching additional data members useful to distcp. This class does not override {@link FileStatus#compareTo}, because the additional data members are not relevant to sort order.

    context.setStatus(description);


    LOG.info(description);


    try {
      CopyListingFileStatus sourceCurrStatus;
      FileSystem sourceFS;
      try {
        sourceFS = sourcePath.getFileSystem(conf);
        sourceCurrStatus = DistCpUtils.toCopyListingFileStatus(sourceFS,
          sourceFS.getFileStatus(sourcePath),
          fileAttributes.contains(FileAttribute.ACL), 
          fileAttributes.contains(FileAttribute.XATTR));
      } catch (FileNotFoundException e) {
        throw new IOException(new RetriableFileCopyCommand.CopyReadException(e));
      }


      FileStatus targetStatus = null;


      try {
        targetStatus = targetFS.getFileStatus(target);
      } catch (FileNotFoundException ignore) {
        if (LOG.isDebugEnabled())
          LOG.debug("Path could not be found: " + target, ignore);
      }


      if (targetStatus != null && (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) {
        throw new IOException("Can't replace " + target + ". Target is " +
            getFileType(targetStatus) + ", Source is " + getFileType(sourceCurrStatus));
      }


      if (sourceCurrStatus.isDirectory()) {
        createTargetDirsWithRetry(description, target, context);
        return;
      }


      FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target);
      if (action == FileAction.SKIP) {
        LOG.info("Skipping copy of " + sourceCurrStatus.getPath()
                 + " to " + target);
        updateSkipCounters(context, sourceCurrStatus);
        context.write(null, new Text("SKIP: " + sourceCurrStatus.getPath()));
      } else {
        copyFileWithRetry(description, sourceCurrStatus, target, context,
            action, fileAttributes);
      }

View Full Code Here

   * @throws IOException if there is an I/O error
   */
  public static CopyListingFileStatus toCopyListingFileStatus(
      FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls, 
      boolean preserveXAttrs) throws IOException {
    CopyListingFileStatus copyListingFileStatus =
      new CopyListingFileStatus(fileStatus);
    if (preserveAcls) {
      FsPermission perm = fileStatus.getPermission();
      if (perm.getAclBit()) {
        List<AclEntry> aclEntries = fileSystem.getAclStatus(
          fileStatus.getPath()).getEntries();
        copyListingFileStatus.setAclEntries(aclEntries);
      }
    }
    if (preserveXAttrs) {
      Map<String, byte[]> xAttrs = fileSystem.getXAttrs(fileStatus.getPath());
      copyListingFileStatus.setXAttrs(xAttrs);
    }
    return copyListingFileStatus;
  }

View Full Code Here

    List<DynamicInputChunk> openChunks
                  = new ArrayList<DynamicInputChunk>();
    
    List<DynamicInputChunk> chunksFinal = new ArrayList<DynamicInputChunk>();


    CopyListingFileStatus fileStatus = new CopyListingFileStatus();
    Text relPath = new Text();
    int recordCounter = 0;
    int chunkCount = 0;


    try {

View Full Code Here

            = new SequenceFile.Reader(cluster.getFileSystem().getConf(),
                    SequenceFile.Reader.file(listFile));


    try {
      reader.seek(lastEnd);
      CopyListingFileStatus srcFileStatus = new CopyListingFileStatus();
      Text srcRelPath = new Text();
      Assert.assertFalse(reader.next(srcRelPath, srcFileStatus));
    } finally {
      IOUtils.closeStream(reader);
    }

View Full Code Here

         = stubContext.getContext();
      
      recordReader.initialize(splits.get(0), taskAttemptContext);
      float previousProgressValue = 0f;
      while (recordReader.nextKeyValue()) {
        CopyListingFileStatus fileStatus = recordReader.getCurrentValue();
        String source = fileStatus.getPath().toString();
        System.out.println(source);
        Assert.assertTrue(expectedFilePaths.contains(source));
        final float progress = recordReader.getProgress();
        Assert.assertTrue(progress >= previousProgressValue);
        Assert.assertTrue(progress >= 0.0f);

View Full Code Here


      Path path = new Path("/tmp/abc");
      Path src = new Path("/tmp/src");
      fs.mkdirs(path);
      fs.mkdirs(src);
      CopyListingFileStatus srcStatus = new CopyListingFileStatus(
        fs.getFileStatus(src));


      FsPermission noPerm = new FsPermission((short) 0);
      fs.setPermission(path, noPerm);
      fs.setOwner(path, "nobody", "nobody");


      DistCpUtils.preserve(fs, path, srcStatus, attributes);
      FileStatus target = fs.getFileStatus(path);
      Assert.assertEquals(target.getPermission(), noPerm);
      Assert.assertEquals(target.getOwner(), "nobody");
      Assert.assertEquals(target.getGroup(), "nobody");


      attributes.add(FileAttribute.PERMISSION);
      DistCpUtils.preserve(fs, path, srcStatus, attributes);
      target = fs.getFileStatus(path);
      Assert.assertEquals(target.getPermission(), srcStatus.getPermission());
      Assert.assertEquals(target.getOwner(), "nobody");
      Assert.assertEquals(target.getGroup(), "nobody");


      attributes.add(FileAttribute.GROUP);
      attributes.add(FileAttribute.USER);
      DistCpUtils.preserve(fs, path, srcStatus, attributes);
      target = fs.getFileStatus(path);
      Assert.assertEquals(target.getPermission(), srcStatus.getPermission());
      Assert.assertEquals(target.getOwner(), srcStatus.getOwner());
      Assert.assertEquals(target.getGroup(), srcStatus.getGroup());


      fs.delete(path, true);
      fs.delete(src, true);
    } catch (IOException e) {
      LOG.error("Exception encountered ", e);

View Full Code Here

  private List<InputSplit> getSplits(Configuration configuration, int numSplits,
                                     long totalSizeBytes) throws IOException {
    List<InputSplit> splits = new ArrayList<InputSplit>(numSplits);
    long nBytesPerSplit = (long) Math.ceil(totalSizeBytes * 1.0 / numSplits);


    CopyListingFileStatus srcFileStatus = new CopyListingFileStatus();
    Text srcRelPath = new Text();
    long currentSplitSize = 0;
    long lastSplitStart = 0;
    long lastPosition = 0;


    final Path listingFilePath = getListingFilePath(configuration);


    if (LOG.isDebugEnabled()) {
      LOG.debug("Average bytes per map: " + nBytesPerSplit +
          ", Number of maps: " + numSplits + ", total size: " + totalSizeBytes);
    }
    SequenceFile.Reader reader=null;
    try {
      reader = getListingFileReader(configuration);
      while (reader.next(srcRelPath, srcFileStatus)) {
        // If adding the current file would cause the bytes per map to exceed
        // limit. Add the current file to new split
        if (currentSplitSize + srcFileStatus.getLen() > nBytesPerSplit && lastPosition != 0) {
          FileSplit split = new FileSplit(listingFilePath, lastSplitStart,
              lastPosition - lastSplitStart, null);
          if (LOG.isDebugEnabled()) {
            LOG.debug ("Creating split : " + split + ", bytes in split: " + currentSplitSize);
          }
          splits.add(split);
          lastSplitStart = lastPosition;
          currentSplitSize = 0;
        }
        currentSplitSize += srcFileStatus.getLen();
        lastPosition = reader.getPosition();
      }
      if (lastPosition > lastSplitStart) {
        FileSplit split = new FileSplit(listingFilePath, lastSplitStart,
            lastPosition - lastSplitStart, null);

View Full Code Here

    context.getConfiguration().setBoolean(
        DistCpOptionSwitch.APPEND.getConfigLabel(), true);
    copyMapper.setup(context);
    for (Path path: pathList) {
      copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)),
              new CopyListingFileStatus(cluster.getFileSystem().getFileStatus(
                  path)), context);
    }


    verifyCopy(fs, false);
    // verify that we only copied new appended data

View Full Code Here

    copyMapper.setup(context);


    for (Path path: pathList) {
      copyMapper.map(
          new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)),
          new CopyListingFileStatus(fs.getFileStatus(path)), context);
    }


    // Check that the maps worked.
    verifyCopy(fs, preserveChecksum);
    Assert.assertEquals(pathList.size(), stubContext.getReporter()

View Full Code Here

  private void testCopyingExistingFiles(FileSystem fs, CopyMapper copyMapper,
      Mapper<Text, CopyListingFileStatus, Text, Text>.Context context) {
    try {
      for (Path path : pathList) {
        copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)),
                new CopyListingFileStatus(fs.getFileStatus(path)), context);
      }


      Assert.assertEquals(nFiles,
              context.getCounter(CopyMapper.Counter.SKIP).getValue());
    }

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.hadoop.tools.CopyListingFileStatus

org.apache.hadoop.tools.mapred.CopyMapper

org.apache.hadoop.tools.mapred.lib.DynamicInputFormat

org.apache.hadoop.tools.mapred.lib.TestDynamicInputFormat

org.apache.hadoop.tools.mapred.TestCopyMapper

org.apache.hadoop.tools.mapred.TestUniformSizeInputFormat

org.apache.hadoop.tools.mapred.UniformSizeInputFormat

org.apache.hadoop.tools.util.DistCpUtils

org.apache.hadoop.tools.util.TestDistCpUtils

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.