Package org.apache.hadoop.util

Examples of org.apache.hadoop.util.ProgramDriver


* human-readable description.
*/
public class ExampleDriver {
 
  public static void main(String argv[]){
    ProgramDriver pgd = new ProgramDriver();
    try {
      pgd.addClass("wordcount", WordCount.class,
                   "A map/reduce program that counts the words in the input files.");
      pgd.addClass("aggregatewordcount", AggregateWordCount.class,
                   "An Aggregate based map/reduce program that counts the words in the input files.");
      pgd.addClass("grep", Grep.class,
                   "A map/reduce program that counts the matches of a regex in the input.");
      pgd.addClass("randomwriter", RandomWriter.class,
                   "A map/reduce program that writes 10GB of random data per node.");
      pgd.addClass("sort", Sort.class, "A map/reduce program that sorts the data written by the random writer.");
      pgd.addClass("pi", PiEstimator.class, "A map/reduce program that estimates Pi using monte-carlo method.");
      pgd.addClass("pentomino", DistributedPentomino.class,
      "A map/reduce tile laying program to find solutions to pentomino problems.");
      pgd.addClass("sudoku", Sudoku.class, "A sudoku solver.");
      pgd.driver(argv);
    }
    catch(Throwable e){
      e.printStackTrace();
    }
  }
View Full Code Here


* @author Owen O'Malley
*/
public class ExampleDriver {
 
    public static void main(String argv[]){
        ProgramDriver pgd = new ProgramDriver();
        try {
      pgd.addClass("wordcount", WordCount.class,
       "A map/reduce program that counts the words in the input files.");
      pgd.addClass("grep", Grep.class,
       "A map/reduce program that counts the matches of a regex in the input.");
      pgd.addClass("randomwriter", RandomWriter.class,
                        "A map/reduce program that writes 10GB of random data per node.");
            pgd.addClass("sort", Sort.class, "A map/reduce program that sorts the data written by the random writer.");
            pgd.addClass("pi", PiEstimator.class, "A map/reduce program that estimates Pi using monte-carlo method.");
            pgd.driver(argv);
  }
  catch(Throwable e){
      e.printStackTrace();
  }
    }
View Full Code Here

   * A description of the test program for running all the tests using jar file
   * @date April 2006
   */
   
    public static void main(String argv[]){
  ProgramDriver pgd = new ProgramDriver();
  try {
            pgd.addClass("nnbench", NNBench.class, "A benchmark that stresses the namenode.");
      pgd.addClass("mapredtest", TestMapRed.class, "A map/reduce test check.");
      pgd.addClass("clustertestdfs", ClusterTestDFS.class, "A pseudo distributed test for DFS.");
      pgd.addClass("testfilesystem", TestFileSystem.class, "A test for FileSystem read/write.");
      pgd.addClass("testsequencefile", TestSequenceFile.class, "A test for flat files of binary key value pairs.");
      pgd.addClass("testsetfile", TestSetFile.class, "A test for flat files of binary key/value pairs.");
      pgd.addClass("testarrayfile", TestArrayFile.class, "A test for flat files of binary key/value pairs.");
      pgd.addClass("testrpc", TestRPC.class, "A test for rpc.");
      pgd.addClass("testipc", TestIPC.class, "A test for ipc.");
      pgd.addClass("testsequencefileinputformat", TestSequenceFileInputFormat.class, "A test for sequence file input format.");
      pgd.addClass("testtextinputformat", TestTextInputFormat.class, "A test for text input format.");
      pgd.addClass("TestDFSIO", TestDFSIO.class, "Distributed i/o benchmark.");
      pgd.addClass("DFSCIOTest", DFSCIOTest.class, "Distributed i/o benchmark of libhdfs.");
      pgd.addClass("DistributedFSCheck", DistributedFSCheck.class, "Distributed checkup of the file system consistency.");
      pgd.driver(argv);
  }
  catch(Throwable e){
      e.printStackTrace();
  }
    }
View Full Code Here

  private static final DecimalFormat formatter = new DecimalFormat("###.##%");

  public static void main(String argv[]){
    int exitCode = -1;
    ProgramDriver pgd = new ProgramDriver();
    try {
      pgd.addClass("wordcount", WordCount.class,
          "A native Tez wordcount program that counts the words in the input files.");
      pgd.addClass("orderedwordcount", OrderedWordCount.class,
          "Word Count with words sorted on frequency");
      pgd.addClass("simplesessionexample", SimpleSessionExample.class,
          "Example to run multiple dags in a session");
      pgd.addClass("joinexample", JoinExample.class,
          "Identify all occurences of lines in file1 which also occur in file2");
      pgd.addClass("joindatagen", JoinDataGen.class,
          "Generate data to run the joinexample");
      pgd.addClass("joinvalidate", JoinValidate.class,
          "Validate data generated by joinexample and joindatagen");
      exitCode = pgd.run(argv);
    } catch(Throwable e){
      e.printStackTrace();
    }

    System.exit(exitCode);
View Full Code Here

public class TestDriver {

  public static void main(String argv[]){
    int exitCode = -1;
    ProgramDriver pgd = new ProgramDriver();
    try {
      pgd.addClass("FaultToleranceTestRunner", FaultToleranceTestRunner.class,
          "Run different DAGs for fault tolerance testing");
      exitCode = pgd.run(argv);
    }
    catch(Throwable e){
      e.printStackTrace();
    }
View Full Code Here

  private static final DecimalFormat formatter = new DecimalFormat("###.##%");

  public static void main(String argv[]){
    int exitCode = -1;
    ProgramDriver pgd = new ProgramDriver();
    try {
      pgd.addClass("rpcloadgen", RPCLoadGen.class, "Run a DAG to generate load for the task to AM RPC");
      pgd.addClass("wordcount", MapredWordCount.class,
          "A map/reduce program that counts the words in the input files.");
      pgd.addClass("mapredwordcount", MapredWordCount.class,
          "A map/reduce program that counts the words in the input files"
         + " using the mapred apis.");
      pgd.addClass("randomwriter", RandomWriter.class,
          "A map/reduce program that writes 10GB of random data per node.");
      pgd.addClass("randomtextwriter", RandomTextWriter.class,
      "A map/reduce program that writes 10GB of random textual data per node.");
      pgd.addClass("sort", Sort.class,
          "A map/reduce program that sorts the data written by the random"
          + " writer.");
      pgd.addClass("secondarysort", SecondarySort.class,
          "An example defining a secondary sort to the reduce.");
      pgd.addClass("join", Join.class,
          "A job that effects a join over sorted, equally partitioned"
          + " datasets");
      pgd.addClass("teragen", TeraGen.class,
          "Generate data for the terasort");
      pgd.addClass("terasort", TeraSort.class,
          "Run the terasort");
      pgd.addClass("teravalidate", TeraValidate.class,
          "Checking results of terasort");
      pgd.addClass("groupbyorderbymrrtest", GroupByOrderByMRRTest.class,
          "A map-reduce-reduce program that does groupby-order by. Takes input"
          + " containing employee_name department name per line of input"
          + " and generates count of employees per department and"
          + " sorted on employee count");
      pgd.addClass("mrrsleep", MRRSleepJob.class,
          "MRR Sleep Job");
      pgd.addClass("testorderedwordcount", TestOrderedWordCount.class,
          "Word Count with words sorted on frequency");
      pgd.addClass("unionexample", UnionExample.class,
          "Union example");
      pgd.addClass("broadcastAndOneToOneExample", BroadcastAndOneToOneExample.class,
          "BroadcastAndOneToOneExample example");
      pgd.addClass("filterLinesByWord", FilterLinesByWord.class,
          "Filters lines by the specified word using broadcast edge");
      pgd.addClass("filterLinesByWordOneToOne", FilterLinesByWordOneToOne.class,
          "Filters lines by the specified word using OneToOne edge");
      exitCode = pgd.run(argv);
    }
    catch(Throwable e){
      e.printStackTrace();
    }
View Full Code Here

import org.apache.hama.examples.graph.ShortestPaths;

public class ExampleDriver {

  public static void main(String[] args) {
    ProgramDriver pgd = new ProgramDriver();
    try {
      pgd.addClass("pi", PiEstimator.class, "Pi Estimator");
      pgd.addClass("bench", RandBench.class, "Random Communication Benchmark");
      pgd.addClass("test", SerializePrinting.class, "Serialize Printing Test");
      pgd.addClass("sssp", ShortestPaths.class, "Single Source Shortest Path");
      pgd.addClass("pagerank", PageRank.class, "PageRank");
     
      pgd.driver(args);
    } catch (Throwable e) {
      e.printStackTrace();
    }
  }
View Full Code Here

    if (mainClasses == null) {
      throw new IOException("Can't load any properties file?");
    }

    boolean foundShortName = false;
    ProgramDriver programDriver = new ProgramDriver();
    for (Object key :  mainClasses.keySet()) {
      String keyString = (String) key;
      if (args.length > 0 && shortName(mainClasses.getProperty(keyString)).equals(args[0])) {
        foundShortName = true;
      }
      if (args.length > 0 && keyString.equalsIgnoreCase(args[0]) && isDeprecated(mainClasses, keyString)) {
        log.error(desc(mainClasses.getProperty(keyString)));
        return;
      }
      if (isDeprecated(mainClasses, keyString)) {
        continue;
      }
      addClass(programDriver, keyString, mainClasses.getProperty(keyString));
    }

    if (args.length < 1 || args[0] == null || "-h".equals(args[0]) || "--help".equals(args[0])) {
      programDriver.driver(args);
    }

    String progName = args[0];
    if (!foundShortName) {
      addClass(programDriver, progName, progName);
    }
    shift(args);

    Properties mainProps = loadProperties(progName + ".props");
    if (mainProps == null) {
      log.warn("No {}.props found on classpath, will use command-line arguments only", progName);
      mainProps = new Properties();
    }

    Map<String,String[]> argMap = Maps.newHashMap();
    int i = 0;
    while (i < args.length && args[i] != null) {
      List<String> argValues = Lists.newArrayList();
      String arg = args[i];
      i++;
      if (arg.startsWith("-D")) { // '-Dkey=value' or '-Dkey=value1,value2,etc' case
        String[] argSplit = arg.split("=");
        arg = argSplit[0];
        if (argSplit.length == 2) {
          argValues.add(argSplit[1]);
        }
      } else {                                      // '-key [values]' or '--key [values]' case.
        while (i < args.length && args[i] != null) {
          if (args[i].startsWith("-")) {
            break;
          }
          argValues.add(args[i]);
          i++;
        }
      }
      argMap.put(arg, argValues.toArray(new String[argValues.size()]));
    }

    // Add properties from the .props file that are not overridden on the command line
    for (String key : mainProps.stringPropertyNames()) {
      String[] argNamePair = key.split("\\|");
      String shortArg = '-' + argNamePair[0].trim();
      String longArg = argNamePair.length < 2 ? null : "--" + argNamePair[1].trim();
      if (!argMap.containsKey(shortArg) && (longArg == null || !argMap.containsKey(longArg))) {
        argMap.put(longArg, new String[] {mainProps.getProperty(key)});
      }
    }

    // Now add command-line args
    List<String> argsList = Lists.newArrayList();
    argsList.add(progName);
    for (Map.Entry<String,String[]> entry : argMap.entrySet()) {
      String arg = entry.getKey();
      if (arg.startsWith("-D")) { // arg is -Dkey - if value for this !isEmpty(), then arg -> -Dkey + "=" + value
        String[] argValues = entry.getValue();
        if (argValues.length > 0 && !argValues[0].trim().isEmpty()) {
          arg += '=' + argValues[0].trim();
        }
        argsList.add(1, arg);
      } else {
        argsList.add(arg);
        for (String argValue : Arrays.asList(argMap.get(arg))) {
          if (!argValue.isEmpty()) {
            argsList.add(argValue);
          }
        }
      }
    }

    long start = System.currentTimeMillis();

    programDriver.driver(argsList.toArray(new String[argsList.size()]));

    if (log.isInfoEnabled()) {
      log.info("Program took {} ms (Minutes: {})", System.currentTimeMillis() - start,
          (System.currentTimeMillis() - start) / 60000.0);
    }
View Full Code Here

   * @author Owen O'Malley
   * @date april 2006
   */
   
    public static void main(String argv[]){
        ProgramDriver pgd = new ProgramDriver();
        try {
      pgd.addClass("wordcount", WordCount.class,
       "A map/reduce program that counts the words in the input files.");
      pgd.addClass("grep", Grep.class,
       "A map/reduce program that counts the matches of a regex in the input.");
      pgd.addClass("randomwriter", RandomWriter.class,
                        "A random writer benchmark that writes 10GB per node.");
            pgd.addClass("sort", Sort.class, "A sort benchmark that sorts the data written by the random writer.");
            pgd.addClass("pi", PiBenchmark.class, "A benchmark that estimates Pi using monte-carlo method.");
            pgd.addClass("nnbench", NNBench.class, "A benchmark that stresses the namenode");
            pgd.driver(argv);
  }
  catch(Throwable e){
      e.printStackTrace();
  }
    }
View Full Code Here

   * A description of the test program for running all the tests using jar file
   * @date April 2006
   */
   
    public static void main(String argv[]){
  ProgramDriver pgd = new ProgramDriver();
  try {
      pgd.addClass("mapredtest", TestMapRed.class, "A map/reduce test check.");
      pgd.addClass("clustertestdfs", ClusterTestDFS.class, "A pseudo distributed test for DFS.");
      pgd.addClass("testfilesystem", TestFileSystem.class, "A test for FileSystem read/write.");
      pgd.addClass("testsequencefile", TestSequenceFile.class, "A test for flat files of binary key value pairs.");
      pgd.addClass("testsetfile", TestSetFile.class, "A test for flat files of binary key/value pairs.");
      pgd.addClass("testarrayfile", TestArrayFile.class, "A test for flat files of binary key/value pairs.");
      pgd.addClass("testrpc", TestRPC.class, "A test for rpc.");
      pgd.addClass("testipc", TestIPC.class, "A test for ipc.");
      pgd.addClass("testsequencefileinputformat", TestSequenceFileInputFormat.class, "A test for sequence file input format.");
      pgd.addClass("testtextinputformat", TestTextInputFormat.class, "A test for text input format.");
      pgd.addClass("TestDFSIO", TestDFSIO.class, "Distributed i/o benchmark.");
      pgd.addClass("DFSCIOTest", DFSCIOTest.class, "Distributed i/o benchmark of libhdfs.");
      pgd.addClass("DistributedFSCheck", DistributedFSCheck.class, "Distributed checkup of the file system consistency.");
      pgd.driver(argv);
  }
  catch(Throwable e){
      e.printStackTrace();
  }
    }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.util.ProgramDriver

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.