Package org.apache.hadoop.hdfs.tools

Examples of org.apache.hadoop.hdfs.tools.FastCopy$FastFileCopy


  @Test
  public void testFastCopy() throws Exception {
    DFSTestUtil util = new DFSTestUtil("testFastCopy", 1, 1, 1024 * 5);
    util.createFiles(fs, "/testFastCopy");
    FastCopy fcp = new FastCopy(conf);
    String[] files = util.getFileNames("/testFastCopy");
    assertEquals(1, files.length);
    fcp.copy(files[0], "/dst1", (DistributedFileSystem) fs,
        (DistributedFileSystem) fs);
    assertTrue(FastCopySetupUtil.compareFiles(files[0], fs, "/dst1", fs));

    cluster.failOver();

    fcp.copy(files[0], "/dst2", (DistributedFileSystem) fs,
        (DistributedFileSystem) fs);
    assertTrue(FastCopySetupUtil.compareFiles(files[0], fs, "/dst2", fs));
  }
View Full Code Here


    }

    public void run() {
      try {
        Random r = new Random();
        FastCopy fcp = new FastCopy(conf);
        while (running) {
          int suffix = r.nextInt();
          String dstFile = "/dst" + suffix;
          fcp.copy(src, dstFile, (DistributedFileSystem) fs,
              (DistributedFileSystem) fs);
          pass = FastCopySetupUtil.compareFiles(src, fs, dstFile, fs);
        }
      } catch (Exception e) {
        pass = false;
View Full Code Here

  @Test
  public void testDeadDatanodes() throws Exception {
    DFSTestUtil util = new DFSTestUtil("testDeadDatanodes", 1, 1, MAX_FILE_SIZE);
    String topDir = "/testDeadDatanodes";
    util.createFiles(fs, topDir);
    FastCopy fastCopy = new FastCopy(conf);

    // Find the locations for the last block of the file.
    String filename = util.getFileNames(topDir)[0];
    LocatedBlocks lbks = cluster.getNameNode().getBlockLocations(filename, 0,
        Long.MAX_VALUE);
    assertNotNull(lbks);

    int namespaceID = cluster.getNameNode().getNamespaceID();
    DataNode dn = cluster.getDataNodes().get(0);
    DatanodeID dnId = dn.getDNRegistrationForNS(namespaceID);
    List <Block> deleteList = new ArrayList <Block> ();
    for(LocatedBlock block : lbks.getLocatedBlocks()) {
      deleteList.add(block.getBlock());
    }

    assertEquals(lbks.locatedBlockCount(),
        dn.getFSDataset().getBlockReport(namespaceID).length);
    DatanodeDescriptor dnDs = cluster.getNameNode().namesystem.getDatanode(dnId);
    dnDs.addBlocksToBeInvalidated(deleteList);

    // Make sure all blocks are deleted.
    while(dn.getFSDataset().getBlockReport(namespaceID).length != 0) {
      Thread.sleep(1000);
    }

    // Now run FastCopy
    try {
      for (String fileName : util.getFileNames(topDir)) {
        fastCopy.copy(fileName, fileName + "dst", (DistributedFileSystem) fs,
            (DistributedFileSystem) fs);
      }
    } finally {
      fastCopy.shutdown();
    }

    // Make sure no errors are reported.
    Map<DatanodeInfo, Integer> dnErrors = fastCopy.getDatanodeErrors();
    assertEquals(0, dnErrors.size());
  }
View Full Code Here

  public void testCleanShutdown() throws Exception {
    String filename = "/testCleanShutdown";
    DFSTestUtil.createFile(fs, new Path(filename), MAX_FILE_SIZE, (short) 3,
        System.currentTimeMillis());

    FastCopy fastCopy = new FastCopy(conf);
    try {
      fastCopy.copy(filename, filename + "dst", (DistributedFileSystem) fs,
          (DistributedFileSystem) fs);
    } finally {
      fastCopy.shutdown();
    }
  }
View Full Code Here

  public void testDeadDatanodesSameFS() throws Exception {
    DFSTestUtil util = new DFSTestUtil("testDeadDatanodesSameFS", 1, 1, MAX_FILE_SIZE);
    String topDir = "/testDeadDatanodesSameFS";
    FileSystem fs = fileSystems[0];
    util.createFiles(fs, topDir);
    FastCopy fastCopy = new FastCopy(conf);

    // Find the locations for the last block of the file.
    String filename = util.getFileNames(topDir)[0];
    LocatedBlocks lbks = cluster.getNameNode(0).getBlockLocations(filename, 0,
        Long.MAX_VALUE);
    assertNotNull(lbks);
    LocatedBlock lastBlock = lbks.get(lbks.locatedBlockCount() - 1);

    // Shutdown all datanodes that have the last block of the file.
    for (DatanodeInfo dnInfo : lastBlock.getLocations()) {
      InetSocketAddress addr = new InetSocketAddress(dnInfo.getHost(),
          dnInfo.getPort());
      for (int i = 0; i < cluster.getDataNodes().size(); i++) {
        DataNode dn = cluster.getDataNodes().get(i);
        if (dn.getSelfAddr().equals(addr)) {
          cluster.shutdownDataNode(i, true);
        }
      }
    }

    // Now run FastCopy
    try {
      DistributedFileSystem dfs = (DistributedFileSystem) fs;
      for (String fileName : util.getFileNames(topDir)) {
        fastCopy.copy(fileName, fileName + "dst", dfs, dfs);
        assertTrue(fs.exists(new Path(fileName + "dst")));
        // assert the hard links.
        String[] hardlinks = dfs.getHardLinkedFiles(new Path(fileName));
        for (String hardLink : hardlinks) {
          assertEquals(hardLink, fileName + "dst");
        }
      }

    } catch (ExecutionException e) {
      System.out.println("Expected exception : " + e);
      assertEquals(IOException.class, e.getCause().getClass());
      return;
    } finally {
      fastCopy.shutdown();
    }
  }
View Full Code Here

    DFSTestUtil util = new DFSTestUtil("testDeadDatanodesCrossFS", 1, 1, MAX_FILE_SIZE);
    String topDir = "/testDeadDatanodesCrossFS";
    FileSystem fs1 = fileSystems[0];
    FileSystem fs2 = fileSystems[1];
    util.createFiles(fs1, topDir);
    FastCopy fastCopy = new FastCopy(conf);

    // Find the locations for the last block of the file.
    String filename = util.getFileNames(topDir)[0];
    LocatedBlocks lbks = cluster.getNameNode(0).getBlockLocations(filename, 0,
        Long.MAX_VALUE);
    assertNotNull(lbks);
    LocatedBlock lastBlock = lbks.get(lbks.locatedBlockCount() - 1);

    // Shutdown all datanodes that have the last block of the file.
    for (DatanodeInfo dnInfo : lastBlock.getLocations()) {
      InetSocketAddress addr = new InetSocketAddress(dnInfo.getHost(),
          dnInfo.getPort());
      for (int i = 0; i < cluster.getDataNodes().size(); i++) {
        DataNode dn = cluster.getDataNodes().get(i);
        if (dn.getSelfAddr().equals(addr)) {
          cluster.shutdownDataNode(i, true);
        }
      }
    }

    // Now run FastCopy
    try {
      for (String fileName : util.getFileNames(topDir)) {
        fastCopy.copy(fileName, fileName + "dst", (DistributedFileSystem) fs1,
            (DistributedFileSystem) fs2);
      }

    } catch (ExecutionException e) {
      System.out.println("Expected exception : " + e);
      assertEquals(IOException.class, e.getCause().getClass());
      return;
    } finally {
      fastCopy.shutdown();
    }
    fail("No exception thrown");
  }
View Full Code Here

  public void testBadDatanodeSameFS() throws Exception {
    DFSTestUtil util = new DFSTestUtil("testBadDatanodeSameFS", 3, 1, MAX_FILE_SIZE);
    String topDir = "/testBadDatanodeSameFS";
    FileSystem fs = fileSystems[0];
    util.createFiles(fs, topDir);
    FastCopy fastCopy = new FastCopy(conf);
    cluster.shutdownDataNode(0, true);
    try {
      DistributedFileSystem dfs = (DistributedFileSystem) fs;
      for (String fileName : util.getFileNames(topDir)) {
        fastCopy.copy(fileName, fileName + "dst", dfs, dfs);
        assertTrue(fs.exists(new Path(fileName + "dst")));
        // assert the hard links.
        String[] hardlinks = dfs.getHardLinkedFiles(new Path(fileName));
        for (String hardLink : hardlinks) {
          assertEquals(hardLink, fileName + "dst");
        }
      }
    } finally {
      fastCopy.shutdown();
    }
  }
View Full Code Here

    DFSTestUtil util = new DFSTestUtil("testBadDatanodeCrossFS", 3, 1, MAX_FILE_SIZE);
    String topDir = "/testBadDatanodeCrossFS";
    FileSystem fs1 = fileSystems[0];
    FileSystem fs2 = fileSystems[1];
    util.createFiles(fs1, topDir);
    FastCopy fastCopy = new FastCopy(conf);
    cluster.shutdownDataNode(0, true);
    try {
      for (String fileName : util.getFileNames(topDir)) {
        fastCopy.copy(fileName, fileName + "dst", (DistributedFileSystem) fs1,
            (DistributedFileSystem) fs2);
      }
      Map<DatanodeInfo, Integer> dnErrors = fastCopy.getDatanodeErrors();
      assertEquals(1, dnErrors.size());
      int errors = dnErrors.values().iterator().next();
      assertTrue(errors >= conf.getInt("dfs.fastcopy.max.datanode.errors", 3) + 1);
    } finally {
      fastCopy.shutdown();
    }
  }
View Full Code Here

  public void testFastCopy(boolean hardlink) throws Exception {
    // Create a source file.
    String src = "/testFastCopySrc" + hardlink;
    generateRandomFile(fs, src, FILESIZE);
    String destination = "/testFastCopyDestination" + hardlink;
    FastCopy fastCopy = new FastCopy(conf);
    NameNode namenode = cluster.getNameNode();
    try {
      for (int i = 0; i < COPIES; i++) {
        fastCopy.copy(src, destination + i, fs, fs);
        assertTrue(verifyCopiedFile(src, destination + i, namenode, namenode,
            fs, fs, hardlink));
        verifyFileStatus(destination + i, namenode, fastCopy);
      }
    } catch (Exception e) {
      LOG.error("Fast Copy failed with exception : ", e);
      fail("Fast Copy failed");
    } finally {
      fastCopy.shutdown();
    }
    assertTrue(pass);
  }
View Full Code Here

  public void testFastCopyOldAPI(boolean hardlink) throws Exception {
    // Create a source file.
    String src = "/testFastCopySrc" + hardlink;
    generateRandomFile(fs, src, FILESIZE);
    String destination = "/testFastCopyDestination" + hardlink;
    FastCopy fastCopy = new FastCopy(conf, fs, fs);
    NameNode namenode = cluster.getNameNode();
    try {
      for (int i = 0; i < COPIES; i++) {
        fastCopy.copy(src, destination + i);
        assertTrue(verifyCopiedFile(src, destination + i, namenode, namenode,
            fs, fs, hardlink));
        verifyFileStatus(destination + i, namenode, fastCopy);
      }
    } catch (Exception e) {
      LOG.error("Fast Copy failed with exception : ", e);
      fail("Fast Copy failed");
    } finally {
      fastCopy.shutdown();
    }
    assertTrue(pass);
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.tools.FastCopy$FastFileCopy

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.