Package org.apache.hadoop.streaming

Examples of org.apache.hadoop.streaming.StreamJob


    job.setJobName(name);
  }

  private Configuration createStreamJob(Configuration cfg, String[] args) {
    // ugly reflection to add an extra method to #createJob
    StreamJob job = new StreamJob();
    job.setConf(cfg);
    Field argv = ReflectionUtils.findField(job.getClass(), "argv_");
    // job.argv_ = args
    ReflectionUtils.makeAccessible(argv);
    ReflectionUtils.setField(argv, job, args);

    // job.init();
    invokeMethod(job, "init");
    // job.preProcessArgs();
    invokeMethod(job, "preProcessArgs");
    // job.parseArgv();
    invokeMethod(job, "parseArgv");
    // job.postProcessArgs();
    invokeMethod(job, "postProcessArgs");
    // job.setJobConf();
    invokeMethod(job, "setJobConf");
    // return job.jobConf_;
    Field jobConf = ReflectionUtils.findField(job.getClass(), "jobConf_");
    ReflectionUtils.makeAccessible(jobConf);
    return (Configuration) ReflectionUtils.getField(jobConf, job);
  }
View Full Code Here


  }
 
  private JobID getJobIdOfRunningStreamJob(String [] runtimeArgs)
      throws IOException {
    JobID jobId = null;
    StreamJob streamJob = new StreamJob();
    int counter = 0;
    jtClient = cluster.getJTClient();
    client = jtClient.getClient();
    int totalJobs = client.getAllJobs().length;
    String [] streamingArgs = generateArgs(runtimeArgs);
View Full Code Here

      LOG.info("user name changed is :" + conf.get("user.name"));
    }

    if (conf.get("mapred.task.tracker.task-controller").
        equals("org.apache.hadoop.mapred.LinuxTaskController")) {
      StreamJob streamJob = new StreamJob();
      String shellFile = System.getProperty("user.dir") +
        "/src/test/system/scripts/StreamMapper.sh";
      String runtimeArgs [] = {
        "-D", "mapred.job.name=Streaming job",
        "-D", "mapred.map.tasks=1",
View Full Code Here

  }
 
  private JobID getJobIdOfRunningStreamJob(String [] runtimeArgs)
      throws IOException {
    JobID jobId = null;
    StreamJob streamJob = new StreamJob();
    int counter = 0;
    jtClient = cluster.getJTClient();
    client = jtClient.getClient();
    int totalJobs = client.getAllJobs().length;
    String [] streamingArgs = generateArgs(runtimeArgs);
View Full Code Here

 
  private JobID getJobId(String [] runtimeArgs, String [] otherArgs)
      throws IOException {
    JobID jobId = null;
    final RunStreamJob runSJ;
    StreamJob streamJob = new StreamJob();
    int counter = 0;
    JTClient jtClient = cluster.getJTClient();
    JobClient jobClient = jtClient.getClient();
    int totalJobs = jobClient.getAllJobs().length;
    String [] args = buildArgs(runtimeArgs, otherArgs);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.streaming.StreamJob

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.