Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FsShell$TextRecordInputStream

1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
      final String localfcontent = DFSTestUtil.readFile(localf);
      final Path root = mkdir(dfs, new Path("/test/get"));
      final Path remotef = new Path(root, fname);
      dfs.copyFromLocalFile(false, false, new Path(localf.getPath()), remotef);

      final FsShell shell = new FsShell();
      shell.setConf(conf);
      TestGetRunner runner = new TestGetRunner() {
        private int count = 0;

        public String run(int exitcode, String... options) throws IOException {
          String dst = TEST_ROOT_DIR + "/" + fname+ ++count;
          String[] args = new String[options.length + 3];
          args[0] = "-get";
          args[args.length - 2] = remotef.toString();
          args[args.length - 1] = dst;
          for(int i = 0; i < options.length; i++) {
            args[i + 1] = options[i];
          }
          show("args=" + Arrays.asList(args));

          try {
            assertEquals(exitcode, shell.run(args));
          } catch (Exception e) {
            assertTrue(StringUtils.stringifyException(e), false);
          }
          return exitcode == 0? DFSTestUtil.readFile(new File(dst)): null;
        }
View Full Code Here
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
    try {
      final String root = createTree(dfs, "lsr");
      dfs.mkdirs(new Path(root, "zzz"));

      runLsr(new FsShell(conf), root, 0);

      final Path sub = new Path(root, "sub");
      dfs.setPermission(sub, new FsPermission((short)0));

      final UserGroupInformation ugi = UserGroupInformation.getCurrentUGI();
      final String tmpusername = ugi.getUserName() + "1";
      UnixUserGroupInformation tmpUGI = new UnixUserGroupInformation(
          tmpusername, new String[] {tmpusername});
      UnixUserGroupInformation.saveToConf(conf,
            UnixUserGroupInformation.UGI_PROPERTY_NAME, tmpUGI);
      String results = runLsr(new FsShell(conf), root, -1);
      assertTrue(results.contains("zzz"));
    } finally {
      cluster.shutdown();
    }
  }
View Full Code Here
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
      assertTrue(fs.mkdirs(directory));
      assertTrue(fs.exists(directory));
      writeFile(fs, tempFile);
      assertTrue(fs.exists(tempFile));

      FsShell shell = new FsShell();
      String argv[] = new String[3];
      argv[0] = "-rmdir";     
      argv[1] = "/dir";

      int ret = -100;
      try {
        ret = shell.run(argv);
        assertTrue(ret == -1);

        argv[1] = "-ignore-fail-on-non-empty";
        argv[2] = "/dir";
        ret = shell.run(argv);
        assertTrue(ret == 0);
        assertTrue(fs.exists(directory));

        assertTrue(fs.delete(tempFile, true));
        assertFalse(fs.exists(tempFile));

        argv[1] = "/dir";
        argv[2] = "";
        ret = shell.run(argv);
        assertTrue(ret == 0);
        assertFalse(fs.exists(directory));
      } catch (Exception e) {
        System.err.println("Exception raised from DFSShell.run " +
                            e.getLocalizedMessage());
View Full Code Here
262
263
264
265
266
267
268
269
270
271
272
273
274
      for (int i = 0; i < 32; i++) {
        filetmp.write(b);
      }
      filetmp.close();

      FsShell shell = new FsShell();
      String argv[]= new String[2];
      shell.setConf(conf);

      // head of large file, only contains AA
      argv[0] = "-head";
      argv[1] = fileNameLarge;
      ToolRunner.run(shell, argv);
View Full Code Here
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
    final DistributedFileSystem dfs = (DistributedFileSystem)fs;
    PrintStream psBackup = System.out;
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    PrintStream psOut = new PrintStream(out);
    System.setOut(psOut);
    FsShell shell = new FsShell();
    shell.setConf(conf);

    try {
      Path myPath = new Path("/test/dir");
      assertTrue(fs.mkdirs(myPath));
      assertTrue(fs.exists(myPath));
      Path myFile = new Path("/test/dir/file");
      writeFile(fs, myFile);
      assertTrue(fs.exists(myFile));
      Path myFile2 = new Path("/test/dir/file2");
      writeFile(fs, myFile2);
      assertTrue(fs.exists(myFile2));

      String[] args = new String[2];
      args[0] = "-du";
      args[1] = "/test/dir";
      int val = -1;
      try {
        val = shell.run(args);
      } catch (Exception e) {
        System.err.println("Exception raised from DFSShell.run " +
                            e.getLocalizedMessage());
      }
      assertTrue(val == 0);
View Full Code Here
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
      {
        Path path = new Path("test_file_1");
        writeFile(fs, path);
        assertTrue(fs.exists(path));

        FsShell shell = new FsShell();
        shell.setConf(conf);
        String argv[] = new String[2];

        argv[0] = "-rm";
        argv[1] = path.toString();
        assertTrue(shell.run(argv) == 0);
        assertTrue(!fs.exists(path));

        argv[0] = "-undelete";
        argv[1] = path.toString();
        assertTrue(shell.run(argv) == 0);
        assertTrue(fs.exists(path));
      }

      // test undelete of file that was deleted programatically
      {
        Path path = new Path("test_file_2");
        writeFile(fs, path);
        assertTrue(fs.exists(path));

        assertTrue(fs.delete(path, true));
        assertTrue(!fs.exists(path));

        FsShell shell = new FsShell();
        shell.setConf(conf);
        String argv[] = new String[2];
        argv[0] = "-undelete";
        argv[1] = path.toString();
        assertTrue(shell.run(argv) == 0);
        assertTrue(fs.exists(path));
      }

      // test undelete of directory that was deleted via shell
      {
        Path path = new Path("test_dir_1");
        assertTrue(fs.mkdirs(path));
        assertTrue(fs.exists(path));

        FsShell shell = new FsShell();
        shell.setConf(conf);
        String argv[] = new String[2];

        argv[0] = "-rmr";
        argv[1] = path.toString();
        assertTrue(shell.run(argv) == 0);
        assertTrue(!fs.exists(path));

        argv[0] = "-undelete";
        argv[1] = path.toString();
        assertTrue(shell.run(argv) == 0);
        assertTrue(fs.exists(path));
      }

      // test undelete of directory that was deleted programatically
      {
        Path path = new Path("test_dir_2");
        assertTrue(fs.mkdirs(path));
        assertTrue(fs.exists(path));

        assertTrue(fs.delete(path, true));
        assertTrue(!fs.exists(path));

        FsShell shell = new FsShell();
        shell.setConf(conf);
        String argv[] = new String[2];
        argv[0] = "-undelete";
        argv[1] = path.toString();
        assertTrue(shell.run(argv) == 0);
        assertTrue(fs.exists(path));
      }

      // test undelete of most recently deleted version
      {
        Path path = new Path("test_file_multiversion");

        writeFileContents(fs, path, "wrong version");
        assertTrue(fs.exists(path));
        assertTrue(fs.delete(path, true));
        assertTrue(!fs.exists(path));

        writeFileContents(fs, path, "right version");
        assertTrue(fs.exists(path));
        assertTrue(fs.delete(path, true));
        assertTrue(!fs.exists(path));

        FsShell shell = new FsShell();
        shell.setConf(conf);
        String argv[] = new String[2];
        argv[0] = "-undelete";
        argv[1] = path.toString();
        assertTrue(shell.run(argv) == 0);
        assertTrue(fs.exists(path));

        String verify = readFileContents(fs, path);
        System.err.println("verify=" + verify);
        assertTrue(verify.equals("right version"));
      }

      // test undelete of file from another user's trash
      {
        // make a fake trash for a different user
        Path path = new Path("my_file_in_joes_trash");
        Path joesTrashDir = new Path(
          "/user/joe/.Trash/Current/user/" + getUserName(fs));
        Path joesTrashFile = new Path(joesTrashDir, path);
        mkdir(fs, joesTrashDir);
        writeFileContents(fs, joesTrashFile, "some file contents");
        assertTrue(fs.exists(joesTrashFile));

        FsShell shell = new FsShell();
        shell.setConf(conf);
        String argv[] = new String[4];
        argv[0] = "-undelete";
        argv[1] = "-u";
        argv[2] = "joe";
        argv[3] = path.toString();
        assertTrue(shell.run(argv) == 0);
        assertTrue(fs.exists(path));
      }

      // test undelete of most recently deleted version across
      // checkpoint intervals
      {
        Path path = new Path("test_file_interval");

        writeFileContents(fs, path, "wrong version");
        assertTrue(fs.exists(path));
        assertTrue(fs.delete(path, true));
        assertTrue(!fs.exists(path));

        writeFileContents(fs, path, "right version");
        assertTrue(fs.exists(path));
        assertTrue(fs.delete(path, true));
        assertTrue(!fs.exists(path));

        // wait for the next interval before checking
        Thread.sleep(60500);

        FsShell shell = new FsShell();
        shell.setConf(conf);
        String argv[] = new String[2];
        argv[0] = "-undelete";
        argv[1] = path.toString();
        assertTrue(shell.run(argv) == 0);
        assertTrue(fs.exists(path));

        String verify = readFileContents(fs, path);
        System.err.println("verify=" + verify);
        assertTrue(verify.equals("right version"));
View Full Code Here
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
      PrintStream tmp = new PrintStream(out);
      System.setErr(tmp);
      String[] argv = new String[2];
      argv[0] = "-cat";
      argv[1] = root.toUri().getPath();
      int ret = ToolRunner.run(new FsShell(), argv);
      assertTrue(" -cat returned -1 ", 0>=ret);
      String returned = out.toString();
      assertTrue("cat does not print exceptions ",
          (returned.lastIndexOf("Exception") == -1));
      out.reset();
      argv[0] = "-rm";
      argv[1] = root.toString();
      FsShell shell = new FsShell();
      shell.setConf(conf);
      ret = ToolRunner.run(shell, argv);
      assertTrue(" -rm returned -1 ", 0>=ret);
      returned = out.toString();
      out.reset();
      assertTrue("rm prints reasonable error ",
View Full Code Here
788
789
790
791
792
793
794
795
796
797
798
799
      nameDir.mkdirs();
      System.setProperty("test.build.data", nameDir.toString());
      dstCluster = new MiniDFSCluster(dstConf, 2, true, null);
      FileSystem srcFs = srcCluster.getFileSystem();
      FileSystem dstFs = dstCluster.getFileSystem();
      FsShell shell = new FsShell();
      shell.setConf(srcConf);
      //check for ls
      String[] argv = new String[2];
      argv[0] = "-ls";
      argv[1] = dstFs.getUri().toString() + "/";
      int ret = ToolRunner.run(shell, argv);
View Full Code Here
893
894
895
896
897
898
899
900
901
902
903
      System.setOut(new PrintStream(out));

      String[] argv = new String[2];
      argv[0] = "-text";
      argv[1] = new Path(root, "file.gz").toUri().getPath();
      int ret = ToolRunner.run(new FsShell(), argv);
      assertTrue("-text returned -1", 0 >= ret);
      file.reset();
      out.reset();
      assertTrue("Output doesn't match input",
          Arrays.equals(file.toByteArray(), out.toByteArray()));
View Full Code Here
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
    MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
    FileSystem fs = cluster.getFileSystem();
    assertTrue("Not a HDFS: "+fs.getUri(),
               fs instanceof ChecksumDistributedFileSystem);
    ChecksumDistributedFileSystem dfs = (ChecksumDistributedFileSystem)fs;
    FsShell shell = new FsShell();
    shell.setConf(conf);

    try {
      String root = createTree(dfs, "copyToLocal");

      // Verify copying the tree
      {
        try {
          assertEquals(0,
              runCmd(shell, "-copyToLocal", root + "*", TEST_ROOT_DIR));
        } catch (Exception e) {
          System.err.println("Exception raised from DFSShell.run " +
                             e.getLocalizedMessage());
        }

        File localroot = new File(TEST_ROOT_DIR, "copyToLocal");
        File localroot2 = new File(TEST_ROOT_DIR, "copyToLocal2");

        File f1 = new File(localroot, "f1");
        assertTrue("Copying failed.", f1.isFile());

        File f2 = new File(localroot, "f2");
        assertTrue("Copying failed.", f2.isFile());

        File sub = new File(localroot, "sub");
        assertTrue("Copying failed.", sub.isDirectory());

        File f3 = new File(sub, "f3");
        assertTrue("Copying failed.", f3.isFile());

        File f4 = new File(sub, "f4");
        assertTrue("Copying failed.", f4.isFile());

        File f5 = new File(localroot2, "f1");
        assertTrue("Copying failed.", f5.isFile());

        f1.delete();
        f2.delete();
        f3.delete();
        f4.delete();
        f5.delete();
        sub.delete();
      }
      // Verify copying non existing sources do not create zero byte
      // destination files
      {
        String[] args = {"-copyToLocal", "nosuchfile", TEST_ROOT_DIR};
        try {
          assertEquals(-1, shell.run(args));
        } catch (Exception e) {
          System.err.println("Exception raised from DFSShell.run " +
                            e.getLocalizedMessage());
        }
        File f6 = new File(TEST_ROOT_DIR, "nosuchfile");
View Full Code Here
TOP

Related Classes of org.apache.hadoop.fs.FsShell$TextRecordInputStream

  • com.nexr.rhive.hadoop.FSUtils
  • org.apache.bigtop.itest.hadoop.hcfs.TestCLI
  • org.apache.bigtop.itest.hadoop.hdfs.TestCLI
  • org.apache.bigtop.itest.hadooptests.TestCLI
  • org.apache.falcon.resource.EntityManagerJerseyIT
  • org.apache.hadoop.ant.DfsTask
  • org.apache.hadoop.cli.util.CLITestCmd
  • org.apache.hadoop.cli.util.CmdFactory
  • org.apache.hadoop.cli.util.CommandExecutor
  • org.apache.hadoop.conf.Configuration
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.