Examples of org.apache.hadoop.util.Tool

org.apache.hadoop.util.Tool
e>Configuration processed by ToolRunner Configuration conf = getConf(); // Create a JobConf using the processed conf JobConf job = new JobConf(conf, MyApp.class); // Process custom command-line options Path in = new Path(args[1]); Path out = new Path(args[2]); // Specify various job-specific parameters job.setJobName("my-app"); job.setInputPath(in); job.setOutputPath(out); job.setMapperClass(MyMapper.class); job.setReducerClass(MyReducer.class); // Submit the job, then poll for progress until the job is complete JobClient.runJob(job); return 0; } public static void main(String[] args) throws Exception { // Let ToolRunner handle generic command-line options int res = ToolRunner.run(new Configuration(), new MyApp(), args); System.exit(res); } }
@see GenericOptionsParser @see ToolRunner

        table
    };


    // run the job, complete the load.
    util.createTable(table, cf);
    Tool t = TestImportTsv.doMROnTableTest(util, cf, simple_tsv, args);
    doLoadIncrementalHFiles(hfiles, table);


    // validate post-conditions
    validateDeletedPartitionsFile(t.getConf());


    // clean up after ourselves.
    util.deleteTable(table);
    util.cleanupDataTestDirOnTestFS(table);
    LOG.info("testGenerateAndLoad completed successfully.");

View Full Code Here

    }


    public static void main(final String[] args) throws Exception {
        System.out.println("Starting pig etl " + args.length);
        Configuration c = new Configuration();
        int res = ToolRunner.run(c, new Tool() {
                    Configuration conf;
                    @Override
                    public void setConf(Configuration conf) {
                        this.conf=conf;
                    }

View Full Code Here

    }


    // run the import
    List<String> argv = new ArrayList<String>(Arrays.asList(args));
    argv.add(inputPath.toString());
    Tool tool = new ImportTsv();
    LOG.debug("Running ImportTsv with arguments: " + argv);
    assertEquals(0, ToolRunner.run(conf, tool, argv.toArray(args)));


    // Perform basic validation. If the input args did not include
    // ImportTsv.BULK_OUTPUT_CONF_KEY then validate data in the table.

View Full Code Here

        optKey(DefaultOptionCreator.DISTANCE_MEASURE_OPTION), description.getDistanceMeasure(),
        optKey(DirichletDriver.MODEL_PROTOTYPE_CLASS_OPTION), description.getModelPrototype(),
        optKey(DefaultOptionCreator.NUM_CLUSTERS_OPTION), "20", optKey(DefaultOptionCreator.MAX_ITERATIONS_OPTION),
        maxIterations.toString(), optKey(DirichletDriver.ALPHA_OPTION), "1.0",
        optKey(DefaultOptionCreator.OVERWRITE_OPTION), optKey(DefaultOptionCreator.CLUSTERING_OPTION)};
    Tool dirichletDriver = new DirichletDriver();
    dirichletDriver.setConf(conf);
    ToolRunner.run(conf, dirichletDriver, args);
    // and inspect results
    printModels(getClusters(outputPath, maxIterations));
  }

View Full Code Here

    } finally {
      Closeables.closeQuietly(writer);
    }


    // run the job
    Tool job = new ConditionalEntropy();
    String[] args = { "-i", input.toString(), "-o", output.toString(),
        "--tempDir", getTestTempDirPath("tmp").toString() };
    ToolRunner.run(job, args);


    // check the output

View Full Code Here

    final Path topologyGoldFile =
        new Path(rootInputPath, "dispatch-topology-output.json.gz");
    final Path traceGoldFile =
        new Path(rootInputPath, "dispatch-trace-output.json.gz");


    Tool analyzer = new TraceBuilder();
    int result = ToolRunner.run(analyzer, args);
    assertEquals("Non-zero exit", 0, result);


    TestRumenJobTraces
        .<LoggedNetworkTopology> jsonFileMatchesGold(conf, topologyPath,

View Full Code Here

        new Path(rootInputPath, "counters-test-trace.json.gz");


    String[] args =
        { tracePath.toString(), topologyPath.toString(), inputPath.toString() };


    Tool analyzer = new TraceBuilder();
    int result = ToolRunner.run(analyzer, args);
    assertEquals("Non-zero exit", 0, result);


    TestRumenJobTraces.<LoggedJob> jsonFileMatchesGold(conf, tracePath,
        goldPath, LoggedJob.class, "trace");

View Full Code Here


    abstract String getInputPath();


    @Test
    public void runDriver() throws Exception {
        Tool driver = createDriver();
        driver.setConf(conf);


        int result = driver.run(new String[]{input.toString(), output.toString()});
        assertThat(result, is(0));


        verifyOutput();
    }

View Full Code Here

  private void runBalancerCli(Configuration conf,
      long totalUsedSpace, long totalCapacity) throws Exception {
    waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);


    final String[] args = { "-policy", "datanode" };
    final Tool tool = new Cli();    
    tool.setConf(conf);
    final int r = tool.run(args); // start rebalancing
    
    assertEquals("Tools should exit 0 on success", 0, r);
    waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
    LOG.info("Rebalancing with default ctor.");
    waitForBalancer(totalUsedSpace, totalCapacity, client, cluster);

View Full Code Here

  private void runBalancerCli(Configuration conf,
      long totalUsedSpace, long totalCapacity) throws Exception {
    waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);


    final String[] args = { "-policy", "datanode" };
    final Tool tool = new Cli();    
    tool.setConf(conf);
    final int r = tool.run(args); // start rebalancing
    
    assertEquals("Tools should exit 0 on success", 0, r);
    waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
    LOG.info("Rebalancing with default ctor.");
    waitForBalancer(totalUsedSpace, totalCapacity, client, cluster);

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of org.apache.hadoop.util.Tool

cmd.infer

cmd.stats

cmd.tdbloader4

com.asakusafw.directio.tools.DirectIoAbortTransactionTest

com.asakusafw.directio.tools.DirectIoApplyTransactionTest

com.asakusafw.directio.tools.DirectIoListTransactionTest

com.asakusafw.operation.tools.hadoop.fs.Clean

com.asakusafw.runtime.stage.launcher.ApplicationLauncher

hadoopbook.ch5.LocalDriverTest

io.fabric8.hadoop.commands.HadoopCommand

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.