Examples of org.apache.hadoop.tools.HadoopArchives

org.apache.hadoop.tools.HadoopArchives
a archive creation utility. This class provides methods that can be used to create hadoop archives. For understanding of Hadoop archives look at {@link HarFileSystem}.


  @Override
  public int createHadoopArchive(Configuration conf, Path sourceDir, Path destDir,
      String archiveName) throws Exception {


    HadoopArchives har = new HadoopArchives(conf);
    List<String> args = new ArrayList<String>();


    if (conf.get("hive.archive.har.parentdir.settable") == null) {
      throw new RuntimeException("hive.archive.har.parentdir.settable is not set");
    }

View Full Code Here


  @Override
  public int createHadoopArchive(Configuration conf, Path sourceDir, Path destDir,
      String archiveName) throws Exception {


    HadoopArchives har = new HadoopArchives(conf);
    List<String> args = new ArrayList<String>();


    if (conf.get("hive.archive.har.parentdir.settable") == null) {
      throw new RuntimeException("hive.archive.har.parentdir.settable is not set");
    }

View Full Code Here

          "-p",
          fs.getHomeDirectory().toString(),
          "test_files",
          fs.getHomeDirectory().toString()
      };
      int ret = ToolRunner.run(new HadoopArchives(getConf()), args);
      assert ret == 0;
      printMeasurements();
    }
    fs.delete(remotePath, true);
    fs.delete(archivePath, true);

View Full Code Here

    String[] args = {
        "-copyFromLocal",
        localDir.toString(),
        archivePath.toString()
    };
    int ret = ToolRunner.run(new HadoopArchives(getConf()), args);
    printMeasurements();
    assert ret == 0;
    fs.delete(archivePath, true);
  }

View Full Code Here

  // test archives with a -p option
  @Test
  public void testRelativeArchives() throws Exception {
    fs.delete(archivePath, true);
    Configuration conf = mapred.createJobConf();
    HadoopArchives har = new HadoopArchives(conf);


    {
      String[] args = new String[6];
      args[0] = "-archiveName";
      args[1] = "foo1.har";

View Full Code Here

    Configuration conf = mapred.createJobConf();
    
    boolean archivecaching = conf.getBoolean("fs.har.impl.disable.cache", false);
    assertTrue(archivecaching);
    fs.delete(archivePath, true);
    HadoopArchives har = new HadoopArchives(conf);
    String[] args = new String[4];
 
    //check for destination not specfied
    args[0] = "-archiveName";
    args[1] = "foo.har";

View Full Code Here


  @Test
  public void testSpaces() throws Exception {
     fs.delete(archivePath, true);
     Configuration conf = mapred.createJobConf();
     HadoopArchives har = new HadoopArchives(conf);
     String[] args = new String[6];
     args[0] = "-archiveName";
     args[1] = "foo bar.har";
     args[2] = "-p";
     args[3] = fs.getHomeDirectory().toString();

View Full Code Here

  
  @Test
  public void testLsOnTopArchiveDirectory() throws Exception {
    fs.delete(archivePath, true);
    Configuration conf = mapred.createJobConf();
    HadoopArchives har = new HadoopArchives(conf);
    String[] args = {
        "-archiveName",  
        "foobar.har",
        "-p",
        inputPath.toString(),

View Full Code Here

  public void testAppend() throws Exception {
    fs.delete(archivePath, true);
    Configuration conf = mapred.createJobConf();
    Path finalPath = new Path(archivePath, "foobar.har");
    {
      HadoopArchives har = new HadoopArchives(conf);
      // create archive
      String[] args = {
          "-archiveName",
          finalPath.getName(),
          "-p",
          inputPath.toString(),
          filea.getName(),
          fileb.getName(),
          archivePath.toString()
      };
      int ret = ToolRunner.run(har, args);
      assertTrue("failed test", ret == 0);
    }
    {
      HadoopArchives har = new HadoopArchives(conf);
      // append to archive
      String[] args = {
          "-append",
          finalPath.getName(),
          "-p",

View Full Code Here

    Path dirPath = new Path(inputPath, "dir");
    Path filee = new Path(dirPath, "e");
    fs.mkdirs(dirPath);
    CopyFilesBase.createFileWithContent(fs, filee, "e".getBytes());
    {
      HadoopArchives har = new HadoopArchives(conf);
      // create full archive
      String[] args = {
          "-archiveName",
          archive2.getName(),
          "-p", inputPath.toString(),
          filea.getName(),
          fileb.getName(),
          filec.getName(),
          filed.getName(),
          "dir/e",
          archivePath.toString()
      };
      int ret = ToolRunner.run(har, args);
      assertTrue("failed test", ret == 0);
    }
    {
      HadoopArchives har = new HadoopArchives(conf);
      // create archive without a, b, e
      String[] args = {
          "-archiveName",
          archive.getName(),
          "-p", inputPath.toString(),
          filec.getName(),
          filed.getName(),
          archivePath.toString()
      };
      int ret = ToolRunner.run(har, args);
      assertTrue("failed test", ret == 0);
    }
    {
      HadoopArchives har = new HadoopArchives(conf);
      // append to archive 
      String[] args = {
          "-appendFromArchive",
          archive2.toString(),
          "/a",

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.hadoop.tools.HadoopArchives

org.apache.commons.cli.CommandLine

org.apache.commons.cli.CommandLineParser

org.apache.commons.cli.Option

org.apache.commons.cli.Options

org.apache.commons.cli.PosixParser

org.apache.hadoop.cli.TestMRCLI$ArchiveCmdExecutor

org.apache.hadoop.fs.BenchmarkHar

org.apache.hadoop.fs.FileStatus

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.FSDataInputStream

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.