Package org.apache.hadoop.util

Examples of org.apache.hadoop.util.CopyFiles$CopyFilesReducer


  }
 
  /** copy files from local file system to local file system */
  public void testCopyFromLocalToLocal() throws Exception {
    MyFile[] files = createFiles("local", TEST_ROOT_DIR+"/srcdat");
    new CopyFiles().doMain(new Configuration(),
                           new String[] {"file://"+TEST_ROOT_DIR+"/srcdat",
                                         "file://"+TEST_ROOT_DIR+"/destdat"});
    assertTrue("Source and destination directories do not match.",
               checkFiles("local", TEST_ROOT_DIR+"/destdat", files));
    deldir("local", TEST_ROOT_DIR+"/destdat");
View Full Code Here


      Configuration conf = new Configuration();
      cluster = new MiniDFSCluster(conf, 2, true, null);
      namenode = conf.get("fs.default.name", "local");
      if (!"local".equals(namenode)) {
        MyFile[] files = createFiles(namenode, "/srcdat");
        new CopyFiles().doMain(conf, new String[] {"hdfs://"+namenode+"/srcdat",
                                                   "hdfs://"+namenode+"/destdat"});
        assertTrue("Source and destination directories do not match.",
                   checkFiles(namenode, "/destdat", files));
        deldir(namenode, "/destdat");
        deldir(namenode, "/srcdat");
View Full Code Here

      Configuration conf = new Configuration();
      cluster = new MiniDFSCluster(conf, 1, true, null);
      namenode = conf.get("fs.default.name", "local");
      if (!"local".equals(namenode)) {
        MyFile[] files = createFiles("local", TEST_ROOT_DIR+"/srcdat");
        new CopyFiles().doMain(conf, new String[] {"file://"+TEST_ROOT_DIR+"/srcdat",
                                                   "hdfs://"+namenode+"/destdat"});
        assertTrue("Source and destination directories do not match.",
                   checkFiles(namenode, "/destdat", files));
        deldir(namenode, "/destdat");
        deldir("local", TEST_ROOT_DIR+"/srcdat");
View Full Code Here

      Configuration conf = new Configuration();
      cluster = new MiniDFSCluster(conf, 1, true, null);
      namenode = conf.get("fs.default.name", "local");
      if (!"local".equals(namenode)) {
        MyFile[] files = createFiles(namenode, "/srcdat");
        new CopyFiles().doMain(conf, new String[] {"hdfs://"+namenode+"/srcdat",
                                                   "file://"+TEST_ROOT_DIR+"/destdat"});
        assertTrue("Source and destination directories do not match.",
                   checkFiles("local", TEST_ROOT_DIR+"/destdat", files));
        deldir("local", TEST_ROOT_DIR+"/destdat");
        deldir(namenode, "/srcdat");
View Full Code Here

  }
 
  /** copy files from local file system to local file system */
  public void testCopyFromLocalToLocal() throws Exception {
    MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
    ToolRunner.run(new CopyFiles(new Configuration()),
                           new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                                         "file:///"+TEST_ROOT_DIR+"/destdat"});
    assertTrue("Source and destination directories do not match.",
               checkFiles("local", TEST_ROOT_DIR+"/destdat", files));
    deldir("local", TEST_ROOT_DIR+"/destdat");
View Full Code Here

      Configuration conf = new Configuration();
      cluster = new MiniDFSCluster(conf, 2, true, null);
      namenode = conf.get("fs.default.name", "local");
      if (!"local".equals(namenode)) {
        MyFile[] files = createFiles(URI.create("hdfs://"+namenode), "/srcdat");
        ToolRunner.run(new CopyFiles(conf), new String[] {
                                         "-log",
                                         "hdfs://"+namenode+"/logs",
                                         "hdfs://"+namenode+"/srcdat",
                                         "hdfs://"+namenode+"/destdat"});
        assertTrue("Source and destination directories do not match.",
View Full Code Here

      Configuration conf = new Configuration();
      cluster = new MiniDFSCluster(conf, 1, true, null);
      namenode = conf.get("fs.default.name", "local");
      if (!"local".equals(namenode)) {
        MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
        ToolRunner.run(new CopyFiles(conf), new String[] {
                                         "-log",
                                         "hdfs://"+namenode+"/logs",
                                         "file:///"+TEST_ROOT_DIR+"/srcdat",
                                         "hdfs://"+namenode+"/destdat"});
        assertTrue("Source and destination directories do not match.",
View Full Code Here

      Configuration conf = new Configuration();
      cluster = new MiniDFSCluster(conf, 1, true, null);
      namenode = conf.get("fs.default.name", "local");
      if (!"local".equals(namenode)) {
        MyFile[] files = createFiles(URI.create("hdfs://"+namenode), "/srcdat");
        ToolRunner.run(new CopyFiles(conf), new String[] {
                                         "-log",
                                         "/logs",
                                         "hdfs://"+namenode+"/srcdat",
                                         "file:///"+TEST_ROOT_DIR+"/destdat"});
        assertTrue("Source and destination directories do not match.",
View Full Code Here

      Configuration conf = new Configuration();
      cluster = new MiniDFSCluster(conf, 2, true, null);
      namenode = conf.get("fs.default.name", "local");
      if (!"local".equals(namenode)) {
        MyFile[] files = createFiles(URI.create("hdfs://"+namenode), "/srcdat");
        ToolRunner.run(new CopyFiles(conf), new String[] {
                                         "-p",
                                         "-log",
                                         "hdfs://"+namenode+"/logs",
                                         "hdfs://"+namenode+"/srcdat",
                                         "hdfs://"+namenode+"/destdat"});
        assertTrue("Source and destination directories do not match.",
                   checkFiles(namenode, "/destdat", files));
        FileSystem fs = FileSystem.get(URI.create("hdfs://"+namenode+"/logs"), conf);
        assertTrue("Log directory does not exist.",
                    fs.exists(new Path("hdfs://"+namenode+"/logs")));

        FileStatus[] dchkpoint = getFileStatus(namenode, "/destdat", files);
        final int nupdate = NFILES>>2;
        updateFiles(namenode, "/srcdat", files, nupdate);
        deldir(namenode, "/logs");

        ToolRunner.run(new CopyFiles(conf), new String[] {
                                         "-p",
                                         "-update",
                                         "-log",
                                         "hdfs://"+namenode+"/logs",
                                         "hdfs://"+namenode+"/srcdat",
                                         "hdfs://"+namenode+"/destdat"});
        assertTrue("Source and destination directories do not match.",
                   checkFiles(namenode, "/destdat", files));
        assertTrue("Update failed to replicate all changes in src",
                 checkUpdate(dchkpoint, namenode, "/destdat", files, nupdate));

        deldir(namenode, "/logs");
        ToolRunner.run(new CopyFiles(conf), new String[] {
                                         "-p",
                                         "-overwrite",
                                         "-log",
                                         "hdfs://"+namenode+"/logs",
                                         "hdfs://"+namenode+"/srcdat",
View Full Code Here

  }

  public void testCopyDuplication() throws Exception {
    try {   
      MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
      ToolRunner.run(new CopyFiles(new Configuration()),
          new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                        "file:///"+TEST_ROOT_DIR+"/src2/srcdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles("local", TEST_ROOT_DIR+"/src2/srcdat", files));
 
      assertEquals(CopyFiles.DuplicationException.ERROR_CODE,
          ToolRunner.run(new CopyFiles(new Configuration()),
          new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                        "file:///"+TEST_ROOT_DIR+"/src2/srcdat",
                        "file:///"+TEST_ROOT_DIR+"/destdat",}));
    }
    finally {
View Full Code Here

TOP

Related Classes of org.apache.hadoop.util.CopyFiles$CopyFilesReducer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.