Examples of org.apache.hadoop.util.GenericOptionsParser

org.apache.hadoop.util.GenericOptionsParser

GenericOptionsParser is a utility to parse command line arguments generic to the Hadoop framework. GenericOptionsParser recognizes several standarad command line arguments, enabling applications to easily specify a namenode, a jobtracker, additional configuration resources etc.

Generic Options

The supported generic options are:

 -conf <configuration file>     specify a configuration file -D <property=value>            use value for given property -fs <local|namenode:port>      specify a namenode -jt <local|jobtracker:port>    specify a job tracker -files <comma separated list of files>    specify comma separated files to be copied to the map reduce cluster -libjars <comma separated list of jars>   specify comma separated jar files to include in the classpath. -archives <comma separated list of archives>    specify comma separated archives to be unarchived on the compute machines.

The general command line syntax is:

 bin/hadoop command [genericOptions] [commandOptions]

Generic command line arguments might modify Configuration objects, given to constructors.

The functionality is implemented using Commons CLI.

Examples:

 $ bin/hadoop dfs -fs darwin:8020 -ls /data list /data directory in dfs with namenode darwin:8020 $ bin/hadoop dfs -D fs.default.name=darwin:8020 -ls /data list /data directory in dfs with namenode darwin:8020 $ bin/hadoop dfs -conf hadoop-site.xml -ls /data list /data directory in dfs with conf specified in hadoop-site.xml $ bin/hadoop job -D mapred.job.tracker=darwin:50020 -submit job.xml submit a job to job tracker darwin:50020 $ bin/hadoop job -jt darwin:50020 -submit job.xml submit a job to job tracker darwin:50020 $ bin/hadoop job -jt local -submit job.xml submit a job to local runner $ bin/hadoop jar -libjars testlib.jar  -archives test.tgz -files file.txt inputjar args job submission with libjars, files and archives

@see Tool @see ToolRunner

        EXPORT_TABLE,
        OUTPUT_DIR,
        "1000"
    };


    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    Job job = Export.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());




    String IMPORT_TABLE = "importTableSimpleCase";
    t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
    args = new String[] {
        "-D" + Import.CF_RENAME_PROP + "="+FAMILYA_STRING+":"+FAMILYB_STRING,
        IMPORT_TABLE,
        OUTPUT_DIR
    };


    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
    conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    job = Import.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());

View Full Code Here

   */
  @Test
  public void testMetaExport() throws Exception {
    String EXPORT_TABLE = ".META.";
    String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1", "0", "0" };
    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
        cluster.getConfiguration()), args);
    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    Job job = Export.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());

View Full Code Here

        EXPORT_TABLE,
        OUTPUT_DIR,
        "1000"
    };


    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    Job job = Export.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());




    String IMPORT_TABLE = "importWithDeletes";
    desc = new HTableDescriptor(IMPORT_TABLE);
    desc.addFamily(new HColumnDescriptor(FAMILYA)
        .setMaxVersions(5)
        .setKeepDeletedCells(true)
    );
    UTIL.getHBaseAdmin().createTable(desc);
    t.close();
    t = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
    args = new String[] {
        IMPORT_TABLE,
        OUTPUT_DIR
    };


    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
    conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    job = Import.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());

View Full Code Here

    p.add(FAMILYA, QUAL, now + 4, QUAL);
    exportTable.put(p);


    String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1000" };


    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
        cluster.getConfiguration()), args);
    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    Job job = Export.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());


    String IMPORT_TABLE = "importWithFilter";
    desc = new HTableDescriptor(IMPORT_TABLE);
    desc.addFamily(new HColumnDescriptor(FAMILYA).setMaxVersions(5));
    UTIL.getHBaseAdmin().createTable(desc);


    HTable importTable = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
    args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + PrefixFilter.class.getName(),
        "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, OUTPUT_DIR,
        "1000" };


    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
    conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    job = Import.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());


    // get the count of the source table for that time range
    PrefixFilter filter = new PrefixFilter(ROW1);
    int count = getCount(exportTable, filter);


    Assert.assertEquals("Unexpected row count between export and import tables", count,
      getCount(importTable, null));


    // and then test that a broken command doesn't bork everything - easier here because we don't
    // need to re-run the export job


    args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + Filter.class.getName(),
        "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1) + "", EXPORT_TABLE,
        OUTPUT_DIR, "1000" };


    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
    conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    job = Import.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertFalse("Job succeeedd, but it had a non-instantiable filter!", job.isSuccessful());

View Full Code Here

    HBaseTestingUtility htu1 = new HBaseTestingUtility();


    htu1.startMiniCluster();
    htu1.startMiniMapReduceCluster();


    GenericOptionsParser opts = new GenericOptionsParser(htu1.getConfiguration(), args);
    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    try {
      FileSystem fs = FileSystem.get(conf);
      FSDataOutputStream op = fs.create(new Path(inputFile), true);
      if (data == null) {

View Full Code Here

  }


  @SuppressWarnings("static-access")
  private int parseArgs(String[] args) {
    Options opts = new Options();
    GenericOptionsParser parser =
      new GenericOptionsParser(this.getConf(), opts, args);
    String[] remainingArgs = parser.getRemainingArgs();
    if (remainingArgs.length != 1) {
      usage();
      return -1;
    }
    if (remainingArgs[0].compareTo("check") == 0) {

View Full Code Here

        EXPORT_TABLE,
        OUTPUT_DIR,
        "1000"
    };


    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    Job job = Export.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());




    String IMPORT_TABLE = "importTableSimpleCase";
    t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
    args = new String[] {
        "-D" + Import.CF_RENAME_PROP + "="+FAMILYA_STRING+":"+FAMILYB_STRING,
        IMPORT_TABLE,
        OUTPUT_DIR
    };


    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
    conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    job = Import.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());

View Full Code Here

        EXPORT_TABLE,
        OUTPUT_DIR,
        "1000"
    };


    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    Job job = Export.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());




    String IMPORT_TABLE = "importWithDeletes";
    desc = new HTableDescriptor(IMPORT_TABLE);
    desc.addFamily(new HColumnDescriptor(FAMILYA)
        .setMaxVersions(5)
        .setKeepDeletedCells(true)
    );
    UTIL.getHBaseAdmin().createTable(desc);
    t.close();
    t = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
    args = new String[] {
        IMPORT_TABLE,
        OUTPUT_DIR
    };


    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
    conf = opts.getConfiguration();
    args = opts.getRemainingArgs();


    job = Import.createSubmittableJob(conf, args);
    job.getConfiguration().set("mapreduce.framework.name", "yarn");
    job.waitForCompletion(false);
    assertTrue(job.isSuccessful());

View Full Code Here

        "Name of the delegation token renewer");
    fetcherOptions.addOption(CANCEL, false, "cancel the token");
    fetcherOptions.addOption(RENEW, false, "renew the token");
    fetcherOptions.addOption(PRINT, false, "print the token");
    fetcherOptions.addOption(HELP_SHORT, HELP, false, "print out help information");
    GenericOptionsParser parser = new GenericOptionsParser(conf,
        fetcherOptions, args);
    CommandLine cmd = parser.getCommandLine();
    
    // get options
    final String webUrl = cmd.hasOption(WEBSERVICE) ? cmd
        .getOptionValue(WEBSERVICE) : null;
    final String renewer = cmd.hasOption(RENEWER) ? 
        cmd.getOptionValue(RENEWER) : null;
    final boolean cancel = cmd.hasOption(CANCEL);
    final boolean renew = cmd.hasOption(RENEW);
    final boolean print = cmd.hasOption(PRINT);
    final boolean help = cmd.hasOption(HELP);
    String[] remaining = parser.getRemainingArgs();


    // check option validity
    if (help) {
      printUsage(System.out);
      System.exit(0);

View Full Code Here

    if (DFSUtil.parseHelpArgument(args, 
        ZKFailoverController.USAGE, System.out, true)) {
      System.exit(0);
    }
    
    GenericOptionsParser parser = new GenericOptionsParser(
        new HdfsConfiguration(), args);
    DFSZKFailoverController zkfc = DFSZKFailoverController.create(
        parser.getConfiguration());
    
    System.exit(zkfc.run(parser.getRemainingArgs()));
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.util.GenericOptionsParser

com.asakusafw.runtime.stage.launcher.LauncherOptionsParser

com.ebay.erl.mobius.core.MobiusJobRunner

com.sematext.hbase.hut.RollbackUpdatesMrJob

com.twitter.elephantbird.examples.DeprecatedWrapperWordCount

com.twitter.elephantbird.examples.ProtobufMRExample

com.twitter.elephantbird.examples.ThriftMRExample

org.apache.blur.mapreduce.lib.CsvBlurDriver

org.apache.commons.cli.CommandLineParser

org.apache.commons.cli.GnuParser

org.apache.commons.cli.HelpFormatter

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.