Package org.apache.hadoop.util

Examples of org.apache.hadoop.util.ProgramDriver


import org.apache.hadoop.util.ProgramDriver;

public class ExampleDriver {
 
  public static void main(String[] args) {
    ProgramDriver pgd = new ProgramDriver();
    try {
      pgd.addClass("pi", PiEstimator.class, "Pi Estimator");
      pgd.addClass("bench", RandBench.class, "Random Communication Benchmark");
      pgd.addClass("test", SerializePrinting.class, "Serialize Printing Test");
     
      pgd.driver(args);
    } catch (Throwable e) {
      e.printStackTrace();
    }
  }
View Full Code Here


  private MahoutDriver() {
  }

  public static void main(String[] args) throws Throwable {
    ProgramDriver programDriver = new ProgramDriver();
    Properties mainClasses = new Properties();
    InputStream propsStream = Thread.currentThread()
                                    .getContextClassLoader()
                                    .getResourceAsStream("driver.classes.props");

    try {
      mainClasses.load(propsStream);
    } catch (Throwable e) {
      //try getting the default one
      propsStream = Thread.currentThread()
                                    .getContextClassLoader()
                                    .getResourceAsStream("driver.classes.default.props");
      mainClasses.load(propsStream);
    }

    boolean foundShortName = false;
    for (Object key :  mainClasses.keySet()) {
      String keyString = (String) key;
      if (args.length > 0 && shortName(mainClasses.getProperty(keyString)).equals(args[0])) {
        foundShortName = true;
      }
      addClass(programDriver, keyString, mainClasses.getProperty(keyString));
    }
    if (args.length < 1 || args[0] == null || args[0].equals("-h") || args[0].equals("--help")) {
      programDriver.driver(args);
    }
    String progName = args[0];
    if (!foundShortName) {
      addClass(programDriver, progName, progName);
    }
    shift(args);

    InputStream defaultsStream = Thread.currentThread()
                                       .getContextClassLoader()
                                       .getResourceAsStream(progName + ".props");

    Properties mainProps = new Properties();
    if (defaultsStream != null) { // can't find props file, use empty props.
      mainProps.load(defaultsStream);
    } else {
      log.warn("No " + progName + ".props found on classpath, will use command-line arguments only");
    }
    Map<String,String[]> argMap = new HashMap<String,String[]>();
    int i = 0;
    while (i < args.length && args[i] != null) {
      List<String> argValues = new ArrayList<String>();
      String arg = args[i];
      i++;
      if (arg.length() > 2 && arg.charAt(1) == 'D') { // '-Dkey=value' or '-Dkey=value1,value2,etc' case
        String[] argSplit = arg.split("=");
        arg = argSplit[0];
        if (argSplit.length == 2) {
          argValues.add(argSplit[1]);
        }
      } else {                                      // '-key [values]' or '--key [values]' case.
        while (i < args.length && args[i] != null) {
          if (args[i].length() > 0 && args[i].charAt(0) != '-') {
            argValues.add(args[i]);
            i++;
          } else {
            break;
          }
        }
      }
      argMap.put(arg, argValues.toArray(new String[argValues.size()]));
    }
    for (String key : mainProps.stringPropertyNames()) {
      String[] argNamePair = key.split("\\|");
      String shortArg = '-' + argNamePair[0].trim();
      String longArg = argNamePair.length < 2 ? null : "--" + argNamePair[1].trim();
      if (!argMap.containsKey(shortArg) && (longArg == null || !argMap.containsKey(longArg))) {
        argMap.put(longArg, new String[] {mainProps.getProperty(key)});
      }
    }
    List<String> argsList = new ArrayList<String>();
    argsList.add(progName);
    for (String arg : argMap.keySet()) {
      if (arg.startsWith("-D")) { // arg is -Dkey - if value for this !isEmpty(), then arg -> -Dkey + "=" + value
        if (argMap.get(arg).length > 0 && !argMap.get(arg)[0].trim().isEmpty()) {
          arg += '=' + argMap.get(arg)[0].trim();
        }
      }
      argsList.add(arg);
      if (!arg.startsWith("-D")) {
        argsList.addAll(Arrays.asList(argMap.get(arg)));
      }
    }
    long start = System.currentTimeMillis();
    programDriver.driver(argsList.toArray(new String[argsList.size()]));
    long finish = System.currentTimeMillis();
    if (log.isInfoEnabled()) {
      log.info("Program took " + (finish - start) + " ms");
    }
  }
View Full Code Here

  private MahoutDriver() {
  }

  public static void main(String[] args) throws Throwable {

    ProgramDriver programDriver = new ProgramDriver();

    Properties mainClasses = loadProperties("driver.classes.props");
    if (mainClasses == null) {
      mainClasses = loadProperties("driver.classes.default.props");
    }
    if (mainClasses == null) {
      throw new IOException("Can't load any properties file?");
    }

    boolean foundShortName = false;
    for (Object key :  mainClasses.keySet()) {
      String keyString = (String) key;
      if (args.length > 0 && shortName(mainClasses.getProperty(keyString)).equals(args[0])) {
        foundShortName = true;
      }
      addClass(programDriver, keyString, mainClasses.getProperty(keyString));
    }

    if (args.length < 1 || args[0] == null || "-h".equals(args[0]) || "--help".equals(args[0])) {
      programDriver.driver(args);
    }

    String progName = args[0];
    if (!foundShortName) {
      addClass(programDriver, progName, progName);
    }
    shift(args);

    Properties mainProps = loadProperties(progName + ".props");
    if (mainProps == null) {
      log.warn("No {}.props found on classpath, will use command-line arguments only", progName);
      mainProps = new Properties();
    }

    Map<String,String[]> argMap = Maps.newHashMap();
    int i = 0;
    while (i < args.length && args[i] != null) {
      List<String> argValues = Lists.newArrayList();
      String arg = args[i];
      i++;
      if (arg.startsWith("-D")) { // '-Dkey=value' or '-Dkey=value1,value2,etc' case
        String[] argSplit = arg.split("=");
        arg = argSplit[0];
        if (argSplit.length == 2) {
          argValues.add(argSplit[1]);
        }
      } else {                                      // '-key [values]' or '--key [values]' case.
        while (i < args.length && args[i] != null) {
          if (args[i].startsWith("-")) {
            break;
          }
          argValues.add(args[i]);
          i++;
        }
      }
      argMap.put(arg, argValues.toArray(new String[argValues.size()]));
    }

    // Add properties from the .props file that are not overridden on the command line
    for (String key : mainProps.stringPropertyNames()) {
      String[] argNamePair = key.split("\\|");
      String shortArg = '-' + argNamePair[0].trim();
      String longArg = argNamePair.length < 2 ? null : "--" + argNamePair[1].trim();
      if (!argMap.containsKey(shortArg) && (longArg == null || !argMap.containsKey(longArg))) {
        argMap.put(longArg, new String[] {mainProps.getProperty(key)});
      }
    }

    // Now add command-line args
    List<String> argsList = Lists.newArrayList();
    argsList.add(progName);
    for (Map.Entry<String,String[]> entry : argMap.entrySet()) {
      String arg = entry.getKey();
      if (arg.startsWith("-D")) { // arg is -Dkey - if value for this !isEmpty(), then arg -> -Dkey + "=" + value
        String[] argValues = entry.getValue();
        if (argValues.length > 0 && !argValues[0].trim().isEmpty()) {
          arg += '=' + argValues[0].trim();
        }
        argsList.add(1, arg);
      } else {
        argsList.add(arg);
        for (String argValue : Arrays.asList(argMap.get(arg))) {
          if (!argValue.isEmpty()) {
            argsList.add(argValue);
          }
        }
      }
    }

    long start = System.currentTimeMillis();

    programDriver.driver(argsList.toArray(new String[argsList.size()]));

    if (log.isInfoEnabled()) {
      log.info("Program took {} ms (Minutes: {})", System.currentTimeMillis() - start, (System.currentTimeMillis() - start)/60000.0);
    }
  }
View Full Code Here

 
  /**
   * A description of the test program for running all the tests using jar file
   */
  public static void main(String argv[]){
    ProgramDriver pgd = new ProgramDriver();
    try {
      pgd.addClass("mrbench", MRBench.class, "A map/reduce benchmark that can create many small jobs");
      pgd.addClass("nnbench", NNBench.class, "A benchmark that stresses the namenode.");
      pgd.addClass("mapredtest", TestMapRed.class, "A map/reduce test check.");
      pgd.addClass("clustertestdfs", ClusterTestDFS.class, "A pseudo distributed test for DFS.");
      pgd.addClass("testfilesystem", TestFileSystem.class, "A test for FileSystem read/write.");
      pgd.addClass("testsequencefile", TestSequenceFile.class, "A test for flat files of binary key value pairs.");
      pgd.addClass("testsetfile", TestSetFile.class, "A test for flat files of binary key/value pairs.");
      pgd.addClass("testarrayfile", TestArrayFile.class, "A test for flat files of binary key/value pairs.");
      pgd.addClass("testrpc", TestRPC.class, "A test for rpc.");
      pgd.addClass("testipc", TestIPC.class, "A test for ipc.");
      pgd.addClass("testsequencefileinputformat", TestSequenceFileInputFormat.class, "A test for sequence file input format.");
      pgd.addClass("testtextinputformat", TestTextInputFormat.class, "A test for text input format.");
      pgd.addClass("TestDFSIO", TestDFSIO.class, "Distributed i/o benchmark.");
      pgd.addClass("DFSCIOTest", DFSCIOTest.class, "Distributed i/o benchmark of libhdfs.");
      pgd.addClass("DistributedFSCheck", DistributedFSCheck.class, "Distributed checkup of the file system consistency.");
      pgd.addClass("testmapredsort", SortValidator.class,
      "A map/reduce program that validates the map-reduce framework's sort.");
      pgd.driver(argv);
    } catch(Throwable e) {
      e.printStackTrace();
    }
  }
View Full Code Here

  /**
   * @param args
   * @throws Throwable
   */
  public static void main(String[] args) throws Throwable {
    ProgramDriver pgd = new ProgramDriver();
    pgd.addClass(RowCounter.NAME, RowCounter.class,
      "Count rows in HBase table");
    ProgramDriver.class.getMethod("driver", new Class [] {String[].class}).
      invoke(pgd, new Object[]{args});
  }
View Full Code Here

*/
public class HCatTestDriver {

    public static void main(String argv[]) {
        int exitCode = -1;
        ProgramDriver pgd = new ProgramDriver();
        try {
            pgd.addClass("typedatacheck", TypeDataCheck.class,
                "A map/reduce program that checks the type of each field and" +
                    " outputs the entire table (to test hcat).");
            pgd.addClass("sumnumbers", SumNumbers.class,
                "A map/reduce program that performs a group by on the first column and a " +
                    "SUM operation on the other columns of the \"numbers\" table.");
            pgd.addClass("storenumbers", StoreNumbers.class, "A map/reduce program that " +
                "reads from the \"numbers\" table and adds 10 to each fields and writes " +
                "to the \"numbers_partitioned\" table into the datestamp=20100101 " +
                "partition OR the \"numbers_empty_initially\" table based on a " +
                "cmdline arg");
            pgd.addClass("storecomplex", StoreComplex.class, "A map/reduce program that " +
                "reads from the \"complex\" table and stores as-is into the " +
                "\"complex_empty_initially\" table.");
            pgd.addClass("storedemo", StoreDemo.class, "demo prog.");
            pgd.driver(argv);

            // Success
            exitCode = 0;
        } catch (Throwable e) {
            e.printStackTrace();
View Full Code Here

  /**
   * @param args
   * @throws Throwable
   */
  public static void main(String[] args) throws Throwable {
    ProgramDriver pgd = new ProgramDriver();
    pgd.addClass(RowCounter.NAME, RowCounter.class,
      "Count rows in HBase table");
    ProgramDriver.class.getMethod("driver", new Class [] {String[].class}).
      invoke(pgd, new Object[]{args});
  }
View Full Code Here

  /**
   * @param args
   * @throws Throwable
   */
  public static void main(String[] args) throws Throwable {
    ProgramDriver pgd = new ProgramDriver();
    pgd.addClass(RowCounter.NAME, RowCounter.class,
      "Count rows in HBase table");
    pgd.addClass(CellCounter.NAME, CellCounter.class,
      "Count cells in HBase table");
    pgd.addClass(Export.NAME, Export.class, "Write table data to HDFS.");
    pgd.addClass(Import.NAME, Import.class, "Import data written by Export.");
    pgd.addClass(ImportTsv.NAME, ImportTsv.class, "Import data in TSV format.");
    pgd.addClass(LoadIncrementalHFiles.NAME, LoadIncrementalHFiles.class,
                 "Complete a bulk data load.");
    pgd.addClass(CopyTable.NAME, CopyTable.class,
        "Export a table from local cluster to peer cluster");
    pgd.addClass(VerifyReplication.NAME, VerifyReplication.class, "Compare" +
        " the data from tables in two different clusters. WARNING: It" +
        " doesn't work for incrementColumnValues'd cells since the" +
        " timestamp is changed after being appended to the log.");
    ProgramDriver.class.getMethod("driver", new Class [] {String[].class}).
      invoke(pgd, new Object[]{args});
View Full Code Here

  /**
   * @param args
   * @throws Throwable
   */
  public static void main(String[] args) throws Throwable {
    ProgramDriver pgd = new ProgramDriver();
    pgd.addClass(RowCounter.NAME, RowCounter.class,
      "Count rows in HBase table");
    ProgramDriver.class.getMethod("driver", new Class [] {String[].class}).
      invoke(pgd, new Object[]{args});
  }
View Full Code Here

  /**
   * @param args
   * @throws Throwable
   */
  public static void main(String[] args) throws Throwable {
    ProgramDriver pgd = new ProgramDriver();
    pgd.addClass(RowCounter.NAME, RowCounter.class,
      "Count rows in HBase table");
    pgd.addClass(Export.NAME, Export.class, "Write table data to HDFS.");
    pgd.addClass(Import.NAME, Import.class, "Import data written by Export.");
    pgd.addClass(ImportTsv.NAME, ImportTsv.class, "Import data in TSV format.");
    pgd.addClass(LoadIncrementalHFiles.NAME, LoadIncrementalHFiles.class,
                 "Complete a bulk data load.");
    pgd.addClass(CopyTable.NAME, CopyTable.class,
        "Export a table from local cluster to peer cluster");
    pgd.driver(args);
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.util.ProgramDriver

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.