Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.Job

93
94
95
96
97
98
99
100
101
102
103
    static List<StageInputSplit> computeSplits(JobContext context) throws IOException, InterruptedException {
        assert context != null;
        Map<FormatAndMapper, List<StageInput>> paths = getPaths(context);
        Map<Class<? extends InputFormat<?, ?>>, InputFormat<?, ?>> formats =
            instantiateFormats(context, paths.keySet());
        Job temporaryJob = JobCompatibility.newJob(context.getConfiguration());
        List<StageInputSplit> results = new ArrayList<StageInputSplit>();
        for (Map.Entry<FormatAndMapper, List<StageInput>> entry : paths.entrySet()) {
            FormatAndMapper formatAndMapper = entry.getKey();
            List<StageInput> current = entry.getValue();
            InputFormat<?, ?> format = formats.get(formatAndMapper.formatClass);
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
            TestContext context) throws IOException {
        LOG.info("エクスポート結果を取得します: {}", description);
        VariableTable variables = createVariables(context);
        checkType(definition, description);
        Configuration conf = configurations.newInstance();
        Job job = JobCompatibility.newJob(conf);
        String resolved = variables.parse(description.getPathPrefix(), false);
        FileInputFormat.setInputPaths(job, new Path(resolved));
        TaskAttemptContext taskContext = JobCompatibility.newTaskAttemptContext(
                job.getConfiguration(),
                JobCompatibility.newTaskAttemptId(JobCompatibility.newTaskId(JobCompatibility.newJobId())));
        FileInputFormat<?, V> format = getOpposite(conf, description.getOutputFormat());
        FileInputFormatDriver<V> result = new FileInputFormatDriver<V>(definition, taskContext, format);
        return result;
    }
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
            final String destination,
            FileOutputFormat<? super NullWritable, ? super V> output) throws IOException {
        assert destination != null;
        assert output != null;
        LOG.debug("Opening {} using {}", destination, output.getClass().getName());
        Job job = JobCompatibility.newJob(configuration);
        job.setOutputKeyClass(NullWritable.class);
        job.setOutputValueClass(definition.getModelClass());
        final File temporaryDir = File.createTempFile("asakusa", ".tempdir");
        if (temporaryDir.delete() == false || temporaryDir.mkdirs() == false) {
            throw new IOException("Failed to create temporary directory");
        }
        LOG.debug("Using staging deploy target: {}", temporaryDir);
        URI uri = temporaryDir.toURI();
        FileOutputFormat.setOutputPath(job, new Path(uri));
        TaskAttemptContext context = JobCompatibility.newTaskAttemptContext(
                job.getConfiguration(),
                JobCompatibility.newTaskAttemptId(JobCompatibility.newTaskId(JobCompatibility.newJobId())));
        FileOutputFormatDriver<V> result = new FileOutputFormatDriver<V>(context, output, NullWritable.get()) {
            @Override
            public void close() throws IOException {
                super.close();
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
   */
  private JobControl createDependencies(Configuration conf, Job job1)
      throws Exception {
    List<ControlledJob> dependingJobs = null;
    cjob1 = new ControlledJob(job1, dependingJobs);
    Job job2 = MapReduceTestUtil.createCopyJob(conf, outdir_2, indir);
    cjob2 = new ControlledJob(job2, dependingJobs);

    Job job3 = MapReduceTestUtil.createCopyJob(conf, outdir_3,
                                     outdir_1, outdir_2);
    dependingJobs = new ArrayList<ControlledJob>();
    dependingJobs.add(cjob1);
    dependingJobs.add(cjob2);
    cjob3 = new ControlledJob(job3, dependingJobs);

    Job job4 = MapReduceTestUtil.createCopyJob(conf, outdir_4, outdir_3);
    dependingJobs = new ArrayList<ControlledJob>();
    dependingJobs.add(cjob3);
    cjob4 = new ControlledJob(job4, dependingJobs);

    JobControl theControl = new JobControl("Test");
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
    Configuration conf = createJobConf();

    cleanupData(conf);
   
    // create a Fail job
    Job job1 = MapReduceTestUtil.createFailJob(conf, outdir_1, indir);
   
    // create job dependencies
    JobControl theControl = createDependencies(conf, job1);
   
    // wait till all the jobs complete
View Full Code Here
147
148
149
150
151
152
153
154
155
156
157
  public void testJobControlWithKillJob() throws Exception {
    LOG.info("Starting testJobControlWithKillJob");

    Configuration conf = createJobConf();
    cleanupData(conf);
    Job job1 = MapReduceTestUtil.createKillJob(conf, outdir_1, indir);
    JobControl theControl = createDependencies(conf, job1);

    while (cjob1.getJobState() != ControlledJob.State.RUNNING) {
      try {
        Thread.sleep(100);
View Full Code Here
187
188
189
190
191
192
193
194
195
196
197
    Configuration conf = createJobConf();

    cleanupData(conf);
   
    Job job1 = MapReduceTestUtil.createCopyJob(conf, outdir_1, indir);
   
    JobControl theControl = createDependencies(conf, job1);
   
    // wait till all the jobs complete
    waitTillAllFinished(theControl);
View Full Code Here
205
206
207
208
209
210
211
212
213
214
215
  public void testControlledJob() throws Exception {
    LOG.info("Starting testControlledJob");

    Configuration conf = createJobConf();
    cleanupData(conf);
    Job job1 = MapReduceTestUtil.createCopyJob(conf, outdir_1, indir);
    JobControl theControl = createDependencies(conf, job1);
    while (cjob1.getJobState() != ControlledJob.State.RUNNING) {
      try {
        Thread.sleep(100);
      } catch (InterruptedException e) {
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
  @Test
  public void testErrorWhileSubmitting() throws Exception {
    JobControl jobControl = new JobControl("Test");
   
    Job mockJob = mock(Job.class);
   
    ControlledJob job1 = new ControlledJob(mockJob, null);
    when(mockJob.getConfiguration()).thenReturn(new Configuration());
    doThrow(new IncompatibleClassChangeError("This is a test")).when(mockJob).submit();
   
    jobControl.addJob(job1);
   
    runJobControl(jobControl);
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
  }
 
  private Job createJob(boolean complete, boolean successful)
    throws IOException, InterruptedException {
    // Create a stub Job that responds in a controlled way
    Job mockJob = mock(Job.class);
    when(mockJob.getConfiguration()).thenReturn(new Configuration());
    when(mockJob.isComplete()).thenReturn(complete);
    when(mockJob.isSuccessful()).thenReturn(successful);
    return mockJob;
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.Job

  • bulkimport.BulkImportJobExample
  • co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService
  • com.alimama.quanjingmonitor.kmeans.KMeansDriver
  • com.asakusafw.runtime.stage.AbstractStageClient
  • com.asakusafw.thundergate.runtime.cache.mapreduce.CacheBuildClient
  • com.datasalt.pangool.tuplemr.mapred.lib.output.PangoolMultipleOutputs
  • com.datasalt.pangool.tuplemr.mapred.lib.output.TestTupleTextInputOutputFormat
  • com.datasalt.pangool.tuplemr.mapred.TestRollup
  • com.datasalt.pangool.tuplemr.mapred.TestTupleMRJob
  • com.hadoop.compression.lzo.DistributedLzoIndexer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.