Examples of org.apache.hadoop.util.Tool

org.apache.hadoop.util.Tool
e>Configuration processed by ToolRunner Configuration conf = getConf(); // Create a JobConf using the processed conf JobConf job = new JobConf(conf, MyApp.class); // Process custom command-line options Path in = new Path(args[1]); Path out = new Path(args[2]); // Specify various job-specific parameters job.setJobName("my-app"); job.setInputPath(in); job.setOutputPath(out); job.setMapperClass(MyMapper.class); job.setReducerClass(MyReducer.class); // Submit the job, then poll for progress until the job is complete JobClient.runJob(job); return 0; } public static void main(String[] args) throws Exception { // Let ToolRunner handle generic command-line options int res = ToolRunner.run(new Configuration(), new MyApp(), args); System.exit(res); } }
@see GenericOptionsParser @see ToolRunner

        return null;
    }


    protected void doExecute(List<Object> objects) throws Exception {
        org.apache.hadoop.conf.Configuration conf = getConfiguration();
        Tool run;
        try {
            Constructor cns = tool.getDeclaredConstructor();
            cns.setAccessible(true);
            run = (Tool) cns.newInstance();
            run.setConf(conf);
        } catch (NoSuchMethodException e) {
            Constructor cns = tool.getDeclaredConstructor(org.apache.hadoop.conf.Configuration.class);
            cns.setAccessible(true);
            run = (Tool) cns.newInstance(conf);
        }
        String[] args = new String[objects.size()];
        for (int i = 0; i < args.length; i++) {
            args[i] = objects.get(i) != null ? objects.get(i).toString() : null;
        }
        run.run(args);
    }

View Full Code Here

        if ( ( copyToLocal ) || ( runLocal ) ) {
          File path = new File(args[1]);
          path.mkdirs();
        }
    
        Tool first = new FirstDriver(configuration);
        int status = first.run(new String[] { args[0], args[1] + OUTPUT_PATH_POSTFIX_1 });
        if (status != 0){ return status ;}


        createOffsetsFile(fs, args[1] + OUTPUT_PATH_POSTFIX_1, args[1] + OUTPUT_PATH_POSTFIX_1);
        Path offsets = new Path(args[1] + OUTPUT_PATH_POSTFIX_1, Constants.OFFSETS_FILENAME);
        DistributedCache.addCacheFile(offsets.toUri(), configuration);
        
        Tool second = new SecondDriver(configuration);
        status = second.run(new String[] { args[0], args[1] + OUTPUT_PATH_POSTFIX_2 });
        if (status != 0){ return status ;}


        Tool third = new ThirdDriver(configuration);
        status = third.run(new String[] { args[1] + OUTPUT_PATH_POSTFIX_2, args[1] + OUTPUT_PATH_POSTFIX_3 });
        if (status != 0){ return status ;}


        Tool fourth = new FourthDriver(configuration);
        status = fourth.run(new String[] { args[1] + OUTPUT_PATH_POSTFIX_3, args[1] + OUTPUT_PATH_POSTFIX_4 });
        if (status != 0){ return status ;}


        if ( copyToLocal ) {
          Tool download = new download(configuration);
          download.run(new String[] { args[1] + OUTPUT_PATH_POSTFIX_2, args[1] + OUTPUT_PATH_POSTFIX_4, args[1] });
        }
        
        if ( verify ) {
            DatasetGraphTDB dsgMem = load(args[0]);
            Location location = new Location(args[1]);
            
            if (!copyToLocal) {
              // TODO: this is a sort of a cheat and it could go away (if it turns out to be too slow)!
              download.fixNodeTable2(location);
            }


            DatasetGraphTDB dsgDisk = SetupTDB.buildDataset(location) ;
            boolean isomorphic = isomorphic ( dsgMem, dsgDisk );
            System.out.println ("> " + isomorphic);

View Full Code Here

        FileSystem fs = FileSystem.get(new Path(args[1]).toUri(), configuration);
        if ( overrideOutput ) {
            fs.delete(new Path(args[1]), true);
        }
    
        Tool infer = new InferDriver(configuration);
        infer.run(new String[] { args[0], args[1] });


    return 0;
  }

View Full Code Here

        FileSystem fs = FileSystem.get(new Path(args[1]).toUri(), configuration);
        if ( overrideOutput ) {
            fs.delete(new Path(args[1]), true);
        }
    
        Tool stats = new StatsDriver(configuration);
        stats.run(new String[] { args[0], args[1] });


    return 0;
  }

View Full Code Here

  public int run(String[] args) throws Exception {
    //get the class, run with the conf
    if (args.length < 1) {
      return printUsage();
    }
    Tool tool = null;
    if (args[0].equals("Generator")) {
      tool = new Generator();
    } else if (args[0].equals("Verify")) {
      tool = new Verify();
    } else if (args[0].equals("Loop")) {

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of org.apache.hadoop.util.Tool

cmd.infer

cmd.stats

cmd.tdbloader4

com.asakusafw.directio.tools.DirectIoAbortTransactionTest

com.asakusafw.directio.tools.DirectIoApplyTransactionTest

com.asakusafw.directio.tools.DirectIoListTransactionTest

com.asakusafw.operation.tools.hadoop.fs.Clean

com.asakusafw.runtime.stage.launcher.ApplicationLauncher

hadoopbook.ch5.LocalDriverTest

io.fabric8.hadoop.commands.HadoopCommand

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.