* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* This class tests commands from DFSShell.

*/

  private static final Log LOG = LogFactory.getLog(TestDFSShell.class);

  private static final AtomicInteger counter = new AtomicInteger();

  private final int SUCCESS = 0;

  private final int ERROR = 1;

  static final String TEST_ROOT_DIR = PathUtils.getTestDirName(TestDFSShell.class);

  static Path writeFile(FileSystem fs, Path f) throws IOException {

    DataOutputStream out = fs.create(f);

    out.writeBytes("dhruba: " + f);

    out.close();

    assertTrue(fs.exists(f));

    return f;

  }

  static Path mkdir(FileSystem fs, Path p) throws IOException {

    assertTrue(fs.mkdirs(p));

    assertTrue(fs.exists(p));

    assertTrue(fs.getFileStatus(p).isDirectory());

    return p;

  }

  static File createLocalFile(File f) throws IOException {

    assertTrue(!f.exists());

    PrintWriter out = new PrintWriter(f);

    out.print("createLocalFile: " + f.getAbsolutePath());

    out.flush();

    out.close();

    assertTrue(f.exists());

    assertTrue(f.isFile());

    return f;

  }

  static File createLocalFileWithRandomData(int fileLength, File f)

      throws IOException {

    assertTrue(!f.exists());

    f.createNewFile();

    FileOutputStream out = new FileOutputStream(f.toString());

    byte[] buffer = new byte[fileLength];

    out.write(buffer);

    out.flush();

    out.close();

    return f;

  }

  static void show(String s) {

    System.out.println(Thread.currentThread().getStackTrace()[2] + " " + s);

  }

  @Test (timeout = 30000)

  public void testZeroSizeFile() throws IOException {

    Configuration conf = new HdfsConfiguration();

    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();

    FileSystem fs = cluster.getFileSystem();

    assertTrue("Not a HDFS: "+fs.getUri(),

               fs instanceof DistributedFileSystem);

    final DistributedFileSystem dfs = (DistributedFileSystem)fs;

    try {

      //create a zero size file

      final File f1 = new File(TEST_ROOT_DIR, "f1");

      assertTrue(!f1.exists());

      assertTrue(f1.createNewFile());

      assertTrue(f1.exists());

      assertTrue(f1.isFile());

      assertEquals(0L, f1.length());

     

      //copy to remote

      final Path root = mkdir(dfs, new Path("/test/zeroSizeFile"));

      final Path remotef = new Path(root, "dst");

      show("copy local " + f1 + " to remote " + remotef);

      dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), remotef);

     

      //getBlockSize() should not throw exception

      show("Block size = " + dfs.getFileStatus(remotef).getBlockSize());

      //copy back

      final File f2 = new File(TEST_ROOT_DIR, "f2");

      assertTrue(!f2.exists());

      dfs.copyToLocalFile(remotef, new Path(f2.getPath()));

      assertTrue(f2.exists());

      assertTrue(f2.isFile());

      assertEquals(0L, f2.length());

 

      f1.delete();

      f2.delete();

    } finally {

      try {dfs.close();} catch (Exception e) {}

      cluster.shutdown();

    }

  }

 

  @Test (timeout = 30000)

  public void testRecursiveRm() throws IOException {

    Configuration conf = new HdfsConfiguration();

    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();

    FileSystem fs = cluster.getFileSystem();

    assertTrue("Not a HDFS: " + fs.getUri(),

        fs instanceof DistributedFileSystem);

    try {

      fs.mkdirs(new Path(new Path("parent"), "child"));

      try {

        fs.delete(new Path("parent"), false);

        assert(false); // should never reach here.

      } catch(IOException e) {

         //should have thrown an exception

      }

      try {

        fs.delete(new Path("parent"), true);

      } catch(IOException e) {

        assert(false);

      }

    } finally { 

      try { fs.close();}catch(IOException e){};

      cluster.shutdown();

    }

  }

   

  @Test (timeout = 30000)

  public void testDu() throws IOException {

    Configuration conf = new HdfsConfiguration();

    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();

    DistributedFileSystem fs = cluster.getFileSystem();

    PrintStream psBackup = System.out;

    ByteArrayOutputStream out = new ByteArrayOutputStream();

    PrintStream psOut = new PrintStream(out);

    System.setOut(psOut);

    FsShell shell = new FsShell();

    shell.setConf(conf);

   

    try {

      Path myPath = new Path("/test/dir");

      assertTrue(fs.mkdirs(myPath));

      assertTrue(fs.exists(myPath));

      Path myFile = new Path("/test/dir/file");

      writeFile(fs, myFile);

      assertTrue(fs.exists(myFile));

      Path myFile2 = new Path("/test/dir/file2");

      writeFile(fs, myFile2);

      assertTrue(fs.exists(myFile2));

     

      String[] args = new String[2];

      args[0] = "-du";

      args[1] = "/test/dir";

      int val = -1;

      try {

        val = shell.run(args);

      } catch (Exception e) {

        System.err.println("Exception raised from DFSShell.run " +

                            e.getLocalizedMessage());

      }

      assertTrue(val == 0);

      String returnString = out.toString();

      out.reset();

      // Check if size matchs as expected

      assertTrue(returnString.contains("22"));

      assertTrue(returnString.contains("23"));

     

    } finally {

      System.setOut(psBackup);

      cluster.shutdown();

    }

                                 

  }

  @Test (timeout = 30000)

  public void testPut() throws IOException {

    Configuration conf = new HdfsConfiguration();

    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();

    FileSystem fs = cluster.getFileSystem();

    assertTrue("Not a HDFS: "+fs.getUri(),

               fs instanceof DistributedFileSystem);

    final DistributedFileSystem dfs = (DistributedFileSystem)fs;

    try {

      // remove left over crc files:

      new File(TEST_ROOT_DIR, ".f1.crc").delete();

      new File(TEST_ROOT_DIR, ".f2.crc").delete();   

      final File f1 = createLocalFile(new File(TEST_ROOT_DIR, "f1"));

      final File f2 = createLocalFile(new File(TEST_ROOT_DIR, "f2"));

 

      final Path root = mkdir(dfs, new Path("/test/put"));

      final Path dst = new Path(root, "dst");

 

      show("begin");

     

      final Thread copy2ndFileThread = new Thread() {

        @Override

        public void run() {

          try {

            show("copy local " + f2 + " to remote " + dst);

            dfs.copyFromLocalFile(false, false, new Path(f2.getPath()), dst);

          } catch (IOException ioe) {

            show("good " + StringUtils.stringifyException(ioe));

            return;

          }

          //should not be here, must got IOException

          assertTrue(false);

        }

      };

     

      //use SecurityManager to pause the copying of f1 and begin copying f2

      SecurityManager sm = System.getSecurityManager();

      System.out.println("SecurityManager = " + sm);

      System.setSecurityManager(new SecurityManager() {

        private boolean firstTime = true;

 

        @Override

        public void checkPermission(Permission perm) {

          if (firstTime) {

            Thread t = Thread.currentThread();

            if (!t.toString().contains("DataNode")) {

              String s = "" + Arrays.asList(t.getStackTrace());

              if (s.contains("FileUtil.copyContent")) {

                //pause at FileUtil.copyContent

 

                firstTime = false;

                copy2ndFileThread.start();

                try {Thread.sleep(5000);} catch (InterruptedException e) {}

              }

            }

          }

        }

      });

      show("copy local " + f1 + " to remote " + dst);

      dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), dst);

      show("done");

 

      try {copy2ndFileThread.join();} catch (InterruptedException e) { }

      System.setSecurityManager(sm);

      // copy multiple files to destination directory

      final Path destmultiple = mkdir(dfs, new Path("/test/putmultiple"));

      Path[] srcs = new Path[2];

      srcs[0] = new Path(f1.getPath());

      srcs[1] = new Path(f2.getPath());

      dfs.copyFromLocalFile(false, false, srcs, destmultiple);

      srcs[0] = new Path(destmultiple,"f1");

      srcs[1] = new Path(destmultiple,"f2");

      assertTrue(dfs.exists(srcs[0]));

      assertTrue(dfs.exists(srcs[1]));

      // move multiple files to destination directory

      final Path destmultiple2 = mkdir(dfs, new Path("/test/movemultiple"));

      srcs[0] = new Path(f1.getPath());

      srcs[1] = new Path(f2.getPath());

      dfs.moveFromLocalFile(srcs, destmultiple2);

      assertFalse(f1.exists());

      assertFalse(f2.exists());

      srcs[0] = new Path(destmultiple2, "f1");

      srcs[1] = new Path(destmultiple2, "f2");

      assertTrue(dfs.exists(srcs[0]));

      assertTrue(dfs.exists(srcs[1]));

      f1.delete();

      f2.delete();

    } finally {

      try {dfs.close();} catch (Exception e) {}

      cluster.shutdown();

    }

  }

  /** check command error outputs and exit statuses. */

  @Test (timeout = 30000)

  public void testErrOutPut() throws Exception {

    Configuration conf = new HdfsConfiguration();

    MiniDFSCluster cluster = null;

    PrintStream bak = null;

    try {

      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();

      FileSystem srcFs = cluster.getFileSystem();

      Path root = new Path("/nonexistentfile");

      bak = System.err;

      ByteArrayOutputStream out = new ByteArrayOutputStream();

      PrintStream tmp = new PrintStream(out);

      System.setErr(tmp);

      String[] argv = new String[2];

      argv[0] = "-cat";

      argv[1] = root.toUri().getPath();

      int ret = ToolRunner.run(new FsShell(), argv);

      assertEquals(" -cat returned 1 ", 1, ret);

      String returned = out.toString();

      assertTrue("cat does not print exceptions ",

          (returned.lastIndexOf("Exception") == -1));

      out.reset();

      argv[0] = "-rm";

      argv[1] = root.toString();

      FsShell shell = new FsShell();

      shell.setConf(conf);

      ret = ToolRunner.run(shell, argv);

      assertEquals(" -rm returned 1 ", 1, ret);

      returned = out.toString();

      out.reset();

      assertTrue("rm prints reasonable error ",

          (returned.lastIndexOf("No such file or directory") != -1));

      argv[0] = "-rmr";

      argv[1] = root.toString();

      ret = ToolRunner.run(shell, argv);

      assertEquals(" -rmr returned 1", 1, ret);

      returned = out.toString();

      assertTrue("rmr prints reasonable error ",

          (returned.lastIndexOf("No such file or directory") != -1));

      out.reset();

      argv[0] = "-du";

      argv[1] = "/nonexistentfile";

      ret = ToolRunner.run(shell, argv);

      returned = out.toString();

      assertTrue(" -du prints reasonable error ",

          (returned.lastIndexOf("No such file or directory") != -1));

      out.reset();

      argv[0] = "-dus";

      argv[1] = "/nonexistentfile";

      ret = ToolRunner.run(shell, argv);

      returned = out.toString();

      assertTrue(" -dus prints reasonable error",

          (returned.lastIndexOf("No such file or directory") != -1));

      out.reset();

      argv[0] = "-ls";

      argv[1] = "/nonexistenfile";

      ret = ToolRunner.run(shell, argv);

      returned = out.toString();

      assertTrue(" -ls does not return Found 0 items",

          (returned.lastIndexOf("Found 0") == -1));

      out.reset();

      argv[0] = "-ls";

      argv[1] = "/nonexistentfile";

      ret = ToolRunner.run(shell, argv);

      assertEquals(" -lsr should fail ", 1, ret);

      out.reset();

      srcFs.mkdirs(new Path("/testdir"));

      argv[0] = "-ls";

      argv[1] = "/testdir";

      ret = ToolRunner.run(shell, argv);

      returned = out.toString();

      assertTrue(" -ls does not print out anything ",

          (returned.lastIndexOf("Found 0") == -1));

      out.reset();

      argv[0] = "-ls";

      argv[1] = "/user/nonxistant/*";

      ret = ToolRunner.run(shell, argv);

      assertEquals(" -ls on nonexistent glob returns 1", 1, ret);

      out.reset();

      argv[0] = "-mkdir";

      argv[1] = "/testdir";

      ret = ToolRunner.run(shell, argv);

      returned = out.toString();

      assertEquals(" -mkdir returned 1 ", 1, ret);

      assertTrue(" -mkdir returned File exists",

          (returned.lastIndexOf("File exists") != -1));

      Path testFile = new Path("/testfile");

      OutputStream outtmp = srcFs.create(testFile);

      outtmp.write(testFile.toString().getBytes());

      outtmp.close();

      out.reset();

      argv[0] = "-mkdir";

      argv[1] = "/testfile";

      ret = ToolRunner.run(shell, argv);

      returned = out.toString();

      assertEquals(" -mkdir returned 1", 1, ret);

      assertTrue(" -mkdir returned this is a file ",

          (returned.lastIndexOf("not a directory") != -1));

      out.reset();

      argv = new String[3];

      argv[0] = "-mv";

      argv[1] = "/testfile";

      argv[2] = "file";

      ret = ToolRunner.run(shell, argv);

      assertEquals("mv failed to rename", 1,  ret);

      out.reset();

      argv = new String[3];

      argv[0] = "-mv";

      argv[1] = "/testfile";

      argv[2] = "/testfiletest";

      ret = ToolRunner.run(shell, argv);

      returned = out.toString();

      assertTrue("no output from rename",

          (returned.lastIndexOf("Renamed") == -1));

      out.reset();

      argv[0] = "-mv";

      argv[1] = "/testfile";

      argv[2] = "/testfiletmp";

      ret = ToolRunner.run(shell, argv);

      returned = out.toString();

      assertTrue(" unix like output",

          (returned.lastIndexOf("No such file or") != -1));

      out.reset();

      argv = new String[1];

      argv[0] = "-du";

      srcFs.mkdirs(srcFs.getHomeDirectory());

      ret = ToolRunner.run(shell, argv);

      returned = out.toString();

      assertEquals(" no error ", 0, ret);

      assertTrue("empty path specified",

          (returned.lastIndexOf("empty string") == -1));

      out.reset();

      argv = new String[3];

      argv[0] = "-test";

      argv[1] = "-d";

      argv[2] = "/no/such/dir";

      ret = ToolRunner.run(shell, argv);

      returned = out.toString();

      assertEquals(" -test -d wrong result ", 1, ret);

      assertTrue(returned.isEmpty());

    } finally {

      if (bak != null) {

        System.setErr(bak);

      }

      if (cluster != null) {

        cluster.shutdown();

      }

    }

  }

 

  @Test (timeout = 30000)

  public void testURIPaths() throws Exception {

    Configuration srcConf = new HdfsConfiguration();

    Configuration dstConf = new HdfsConfiguration();

    MiniDFSCluster srcCluster =  null;

    MiniDFSCluster dstCluster = null;

    File bak = new File(PathUtils.getTestDir(getClass()), "dfs_tmp_uri");