Examples of org.apache.hadoop.util.GenericOptionsParser

org.apache.hadoop.util.GenericOptionsParser

GenericOptionsParser is a utility to parse command line arguments generic to the Hadoop framework. GenericOptionsParser recognizes several standarad command line arguments, enabling applications to easily specify a namenode, a jobtracker, additional configuration resources etc.

Generic Options

The supported generic options are:

 -conf <configuration file>     specify a configuration file -D <property=value>            use value for given property -fs <local|namenode:port>      specify a namenode -jt <local|jobtracker:port>    specify a job tracker -files <comma separated list of files>    specify comma separated files to be copied to the map reduce cluster -libjars <comma separated list of jars>   specify comma separated jar files to include in the classpath. -archives <comma separated list of archives>    specify comma separated archives to be unarchived on the compute machines.

The general command line syntax is:

 bin/hadoop command [genericOptions] [commandOptions]

Generic command line arguments might modify Configuration objects, given to constructors.

The functionality is implemented using Commons CLI.

Examples:

 $ bin/hadoop dfs -fs darwin:8020 -ls /data list /data directory in dfs with namenode darwin:8020 $ bin/hadoop dfs -D fs.default.name=darwin:8020 -ls /data list /data directory in dfs with namenode darwin:8020 $ bin/hadoop dfs -conf hadoop-site.xml -ls /data list /data directory in dfs with conf specified in hadoop-site.xml $ bin/hadoop job -D mapred.job.tracker=darwin:50020 -submit job.xml submit a job to job tracker darwin:50020 $ bin/hadoop job -jt darwin:50020 -submit job.xml submit a job to job tracker darwin:50020 $ bin/hadoop job -jt local -submit job.xml submit a job to local runner $ bin/hadoop jar -libjars testlib.jar  -archives test.tgz -files file.txt inputjar args job submission with libjars, files and archives

@see Tool @see ToolRunner

  public static void main(String[] args)
                               throws ClassNotFoundException, DecoderException,
                                      IOException, InterruptedException {
    final Configuration conf = HBaseConfiguration.create();
    final String[] otherArgs =
      new GenericOptionsParser(conf, args).getRemainingArgs();


    if (otherArgs.length < 3) {
      System.err.println(
        "Usage: MRCoffeeJob <image_id> <outpath> <command>..."
      );

View Full Code Here

    Config.quiet_execution = true;


    String[] args = new String[] { "-local", "-spark" };


    conf = Evaluator.evaluator.new_configuration();
    GenericOptionsParser gop = new GenericOptionsParser(conf, args);
    conf = gop.getConfiguration();


    args = gop.getRemainingArgs();
    
    Config.hadoop_mode = true;
    Config.testing = true;    
    Config.parse_args(args, conf);

View Full Code Here

    Config.quiet_execution = true;


    String[] args = new String[] { "-local", "-spark" };


    conf = Evaluator.evaluator.new_configuration();
    GenericOptionsParser gop = new GenericOptionsParser(conf, args);
    conf = gop.getConfiguration();


    args = gop.getRemainingArgs();


    Config.hadoop_mode = true;
    Config.testing = true;
    Config.parse_args(args, conf);

View Full Code Here

    Config.quiet_execution = true;


    String[] args = new String[] { "-local", "-bsp" };


    conf = Evaluator.evaluator.new_configuration();
    GenericOptionsParser gop = new GenericOptionsParser(conf, args);
    conf = gop.getConfiguration();


    args = gop.getRemainingArgs();
  
    Config.hadoop_mode = true;
    Config.testing = true;
    Config.parse_args(args, conf);

View Full Code Here

    Config.quiet_execution = true;


    String[] args = new String[] { "-local", "-bsp" };


    conf = Evaluator.evaluator.new_configuration();
    GenericOptionsParser gop = new GenericOptionsParser(conf, args);
    conf = gop.getConfiguration();


    args = gop.getRemainingArgs();
    
    Config.hadoop_mode = true;
    Config.testing = true;
    Config.parse_args(args, conf);

View Full Code Here

    Config.quiet_execution = true;


    String[] args = new String[] { "-local" };


    conf = Evaluator.evaluator.new_configuration();
    GenericOptionsParser gop = new GenericOptionsParser(conf, args);
    conf = gop.getConfiguration();


    args = gop.getRemainingArgs();


    Config.hadoop_mode = true;
    Config.testing = true;
    Config.parse_args(args, conf);

View Full Code Here

    Config.quiet_execution = true;


    String[] args = new String[] { "-local" };


    conf = Evaluator.evaluator.new_configuration();
    GenericOptionsParser gop = new GenericOptionsParser(conf, args);
    conf = gop.getConfiguration();


    args = gop.getRemainingArgs();


    Config.hadoop_mode = true;
    Config.testing = true;
    Config.parse_args(args, conf);

View Full Code Here

   * @param region HRegion to add to <code>meta</code>
   *
   * @throws IOException
   */
  private int parseArgs(String[] args) throws IOException {
    GenericOptionsParser parser =
      new GenericOptionsParser(getConf(), args);


    String[] remainingArgs = parser.getRemainingArgs();
    if (remainingArgs.length != 3) {
      usage();
      return -1;
    }
    tableName = TableName.valueOf(remainingArgs[0]);

View Full Code Here

   * @param args  The command line parameters.
   * @throws Exception When running the job fails.
   */
  public static void main(String[] args) throws Exception {
    Configuration conf = HBaseConfiguration.create();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
      usage("Wrong number of arguments: " + otherArgs.length);
      System.exit(-1);
    }
    Job job = createSubmittableJob(conf, otherArgs);

View Full Code Here

   * @param args  The command line parameters.
   * @throws Exception When running the job fails.
   */
  public static void main(String[] args) throws Exception {
    Configuration conf = HBaseConfiguration.create();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
      usage("Wrong number of arguments: " + otherArgs.length);
      System.exit(-1);
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.util.GenericOptionsParser

com.asakusafw.runtime.stage.launcher.LauncherOptionsParser

com.ebay.erl.mobius.core.MobiusJobRunner

com.sematext.hbase.hut.RollbackUpdatesMrJob

com.twitter.elephantbird.examples.DeprecatedWrapperWordCount

com.twitter.elephantbird.examples.ProtobufMRExample

com.twitter.elephantbird.examples.ThriftMRExample

org.apache.blur.mapreduce.lib.CsvBlurDriver

org.apache.commons.cli.CommandLineParser

org.apache.commons.cli.GnuParser

org.apache.commons.cli.HelpFormatter

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.