Examples of org.apache.hadoop.hdfs.tools.FastCopy

org.apache.hadoop.hdfs.tools.FastCopy
There is a need to perform fast file copy on HDFS (primarily for the purpose of HBase Snapshot). The fast copy mechanism for a file works as follows : 1) Query metadata for all blocks of the source file. 2) For each block 'b' of the file, find out its datanode locations. 3) For each block of the file, add an empty block to the namesystem for the destination file. 4) For each location of the block, instruct the datanode to make a local copy of that block. 5) Once each datanode has copied over the its respective blocks, they report to the namenode about it. 6) Wait for all blocks to be copied and exit. This would speed up the copying process considerably by removing top of the rack data transfers.

  public void testFastCopyMultiple(boolean hardlink) throws Exception {
    // Create a source file.
    String src = "/testFastCopyMultipleSrc" + hardlink;
    generateRandomFile(fs, src, FILESIZE);
    String destination = "/testFastCopyMultipleDestination" + hardlink;
    FastCopy fastCopy = new FastCopy(conf);
    List<FastFileCopyRequest> requests = new ArrayList<FastFileCopyRequest>();
    for (int i = 0; i < COPIES; i++) {
      requests.add(new FastFileCopyRequest(src, destination + i, fs, fs));
    }
    NameNode namenode = cluster.getNameNode();
    try {
      fastCopy.copy(requests);
      for (FastFileCopyRequest r : requests) {
        assertTrue(verifyCopiedFile(r.getSrc(), r.getDestination(), namenode,
            namenode, fs, fs, hardlink));
        verifyFileStatus(r.getDestination(), namenode, fastCopy);
      }
    } catch (Exception e) {
      LOG.error("Fast Copy failed with exception : ", e);
      fail("Fast Copy failed");
    } finally {
      fastCopy.shutdown();
    }
    assertTrue(pass);
  }

View Full Code Here

    // Create a source file.
    setInjectionHandler();
    String src = "/testInterFileSystemFastCopySrc" + hardlink;
    generateRandomFile(fs, src, FILESIZE);
    String destination = "/testInterFileSystemFastCopyDst" + hardlink;
    FastCopy fastCopy = new FastCopy(conf);
    NameNode srcNameNode = cluster.getNameNode();
    NameNode dstNameNode = remoteCluster.getNameNode();
    try {
      for (int i = 0; i < COPIES; i++) {
        fastCopy.copy(src, destination + i, fs, remoteFs);
        assertTrue(verifyCopiedFile(src, destination + i, srcNameNode,
            dstNameNode, fs, remoteFs, hardlink));
        verifyFileStatus(destination + i, dstNameNode, fastCopy);
      }
    } catch (Exception e) {
      LOG.error("Fast Copy failed with exception : ", e);
      fail("Fast Copy failed");
    } finally {
      fastCopy.shutdown();
    }
    assertTrue(pass);
  }

View Full Code Here

    // Create a source file.
    String src = "/testInterFileSystemFastCopy MultipleSrc" + hardlink;
    generateRandomFile(fs, src, FILESIZE);
    String destination = "/testInterFileSystemFastCopy MultipleDestination"
        + hardlink;
    FastCopy fastCopy = new FastCopy(conf);
    List<FastFileCopyRequest> requests = new ArrayList<FastFileCopyRequest>();
    for (int i = 0; i < COPIES; i++) {
      requests.add(new FastFileCopyRequest(src, destination + i, fs, remoteFs));
    }
    NameNode srcNameNode = cluster.getNameNode();
    NameNode dstNameNode = remoteCluster.getNameNode();
    try {
      fastCopy.copy(requests);
      for (FastFileCopyRequest r : requests) {
        assertTrue(verifyCopiedFile(r.getSrc(), r.getDestination(),
            srcNameNode, dstNameNode, fs, remoteFs, hardlink));
        verifyFileStatus(r.getDestination(), dstNameNode, fastCopy);
      }
    } catch (Exception e) {
      LOG.error("Fast Copy failed with exception : ", e);
      fail("Fast Copy failed");
    } finally {
      fastCopy.shutdown();
    }
    assertTrue(pass);
  }

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.hadoop.hdfs.tools.FastCopy

org.apache.commons.cli.CommandLine

org.apache.commons.cli.CommandLineParser

org.apache.commons.cli.HelpFormatter

org.apache.commons.cli.PosixParser

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.FileStatus

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.Path

org.apache.hadoop.hdfs.DistributedFileSystem

org.apache.hadoop.hdfs.FastCopySetupUtil

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.