Examples of com.amazonaws.services.elasticmapreduce.model.HadoopJarStepConfig

Package com.amazonaws.services.elasticmapreduce.model

Examples of com.amazonaws.services.elasticmapreduce.model.HadoopJarStepConfig

com.amazonaws.services.elasticmapreduce.model.HadoopJarStepConfig

A job flow step consisting of a JAR file whose main function will be executed. The main function submits a job for Hadoop to execute and waits for the job to finish or fail.

            StepConfig c=steps.get(j++);
            assertEquals(debugStep,c);
        }
        {
            StepConfig c=steps.get(j++);
            HadoopJarStepConfig that=c.getHadoopJarStep();
            assertEquals(
                    Arrays.asList(
                            "run"
                            , "freebaseRDFPrefilter"
                            , "s3n://freebase-dumps/freebase-rdf-1942-12-07-00-00/"
                            , "/preprocessed/1942-12-07-00-00/")
                    ,that.getArgs());
        }


        {
            StepConfig c=steps.get(j++);
            HadoopJarStepConfig that=c.getHadoopJarStep();
            assertEquals(
                    Arrays.asList(
                            "run"
                            ,"pse3"
                            ,"-r"
                            ,"210"
                            ,"/preprocessed/1942-12-07-00-00/"
                            ,"s3n://basekb-now/1942-12-07-00-00/")
                    ,that.getArgs());
        }


        {
            StepConfig c=steps.get(j++);
            HadoopJarStepConfig that=c.getHadoopJarStep();
            assertEquals(
                    Arrays.asList(
                            "run"
                            ,"fs"
                            ,"-rmr"
                            ,"/preprocessed/1942-12-07-00-00/")
                    ,that.getArgs());
        }
    }

View Full Code Here

    req.setName(jobName);


    JobFlowInstancesConfig instances = getInstances();
    req.setInstances(instances);


    HadoopJarStepConfig hadoopJarStepConfig = new HadoopJarStepConfig()
        .withArgs(args).withMainClass(mainClass).withJar(path);


    req.getSteps().add(new StepConfig("custom-jar-exec", hadoopJarStepConfig));


    return req;

View Full Code Here

    if (onFailure != null) {
      args.add("--on-failure");
      args.add(onFailure.toString());
    }


    return new HadoopJarStepConfig()
      .withJar("s3://" + bucket + "/libs/resize-job-flow/0.1/resize-job-flow.jar")
      .withArgs(args);
  }

View Full Code Here

        List<String> argsList = new ArrayList<String>();
        argsList.add(script);
        for (String arg : args) {
            argsList.add(arg);
        }
        return new HadoopJarStepConfig()
            .withJar("s3://" + bucket + "/libs/script-runner/script-runner.jar")
            .withArgs(argsList);
    }

View Full Code Here

    if (onFailure != null) {
      args.add("--on-failure");
      args.add(onFailure.toString());
    }


    return new HadoopJarStepConfig()
      .withJar("s3://" + bucket + "/libs/resize-job-flow/0.1/resize-job-flow.jar")
      .withArgs(args);
  }

View Full Code Here

        List<String> argsList = new ArrayList<String>();
        argsList.add(script);
        for (String arg : args) {
            argsList.add(arg);
        }
        return new HadoopJarStepConfig()
            .withJar("s3://" + bucket + "/libs/script-runner/script-runner.jar")
            .withArgs(argsList);
    }

View Full Code Here

    if (reducer != null) {
      args.add("-reducer");
      args.add(reducer);
    }


    return new HadoopJarStepConfig()
      .withJar("/home/hadoop/contrib/streaming/hadoop-streaming.jar")
      .withArgs(args);
  }

View Full Code Here

    if (onFailure != null) {
      args.add("--on-failure");
      args.add(onFailure.toString());
    }


    return new HadoopJarStepConfig()
      .withJar("s3://" + bucket + "/libs/resize-job-flow/0.1/resize-job-flow.jar")
      .withArgs(args);
  }

View Full Code Here

        List<String> argsList = new ArrayList<String>();
        argsList.add(script);
        for (String arg : args) {
            argsList.add(arg);
        }
        return new HadoopJarStepConfig()
            .withJar("s3://" + bucket + "/libs/script-runner/script-runner.jar")
            .withArgs(argsList);
    }

View Full Code Here

    if (reducer != null) {
      args.add("-reducer");
      args.add(reducer);
    }


    return new HadoopJarStepConfig()
      .withJar("/home/hadoop/contrib/streaming/hadoop-streaming.jar")
      .withArgs(args);
  }

View Full Code Here

0 1 2

TOP

Related Classes of com.amazonaws.services.elasticmapreduce.model.HadoopJarStepConfig

br.com.ingenieux.mojo.mapreduce.RunJobFlowsMojo

com.amazonaws.services.elasticmapreduce.util.ResizeJobFlowStep

com.amazonaws.services.elasticmapreduce.util.StepFactory

com.amazonaws.services.elasticmapreduce.util.StreamingStep

com.ontology2.haruhi.TestApplicationConfigurationLoader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.