Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.JobContext

812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
                                              new Path [] {submitJobDir},
                                              jobCopy);

          Path submitJobFile = JobSubmissionFiles.getJobConfPath(submitJobDir);
          int reduces = jobCopy.getNumReduceTasks();
          JobContext context = new JobContext(jobCopy, jobId);

          jobCopy = (JobConf)context.getConfiguration();

          // Check the output specification
          if (reduces == 0 ? jobCopy.getUseNewMapper() :
            jobCopy.getUseNewReducer()) {
            org.apache.hadoop.mapreduce.OutputFormat<?,?> output =
              ReflectionUtils.newInstance(context.getOutputFormatClass(),
                  jobCopy);
            output.checkOutputSpecs(context);
          } else {
            jobCopy.getOutputFormat().checkOutputSpecs(fs, jobCopy);
          }
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
    checkExpectedRangeBins(10, new String[] {"C", "A", "B"}, new int[] {29, 9, 19});
    checkExpectedRangeBins(10, new String[] {"", "AA", "BB", "CC"}, new int[] {9, 19, 29, 39});
  }
 
  private RangePartitioner prepPartitioner(int numSubBins) {
    JobContext job = ContextFactory.createJobContext();
    RangePartitioner.setNumSubBins(job, numSubBins);
    RangePartitioner rp = new RangePartitioner();
    rp.setConf(job.getConfiguration());
    return rp;
  }
View Full Code Here
232
233
234
235
236
237
238
239
240
241
242
   
    CommitterEventHandler handler = new CommitterEventHandler(mockContext,
        mockCommitter, new TestingRMHeartbeatHandler());
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    JobContext mockJobContext = mock(JobContext.class);
    ApplicationAttemptId attemptid =
      ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
    JobId jobId =  TypeConverter.toYarn(
        TypeConverter.fromYarn(attemptid.getApplicationId()));
   
View Full Code Here
280
281
282
283
284
285
286
287
288
289
290
   
    CommitterEventHandler handler = new CommitterEventHandler(mockContext,
        mockCommitter, new TestingRMHeartbeatHandler());
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    JobContext mockJobContext = mock(JobContext.class);
    ApplicationAttemptId attemptid =
      ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
    JobId jobId =  TypeConverter.toYarn(
        TypeConverter.fromYarn(attemptid.getApplicationId()));
   
View Full Code Here
232
233
234
235
236
237
238
239
240
241
242
   
    CommitterEventHandler handler = new CommitterEventHandler(mockContext,
        mockCommitter, new TestingRMHeartbeatHandler());
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    JobContext mockJobContext = mock(JobContext.class);
    ApplicationAttemptId attemptid =
      ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
    JobId jobId =  TypeConverter.toYarn(
        TypeConverter.fromYarn(attemptid.getApplicationId()));
   
View Full Code Here
280
281
282
283
284
285
286
287
288
289
290
   
    CommitterEventHandler handler = new CommitterEventHandler(mockContext,
        mockCommitter, new TestingRMHeartbeatHandler());
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    JobContext mockJobContext = mock(JobContext.class);
    ApplicationAttemptId attemptid =
      ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
    JobId jobId =  TypeConverter.toYarn(
        TypeConverter.fromYarn(attemptid.getApplicationId()));
   
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
  job.setInputFormatClass(FileInputFormat.class);

  Configuration conf = job.getConfiguration();
  FileInputFormat.setInputPaths(job, new Path(baseDir.getAbsolutePath()));

  JobContext jobContext = new JobContext(conf, job.getJobID());

  partitionHelper.setPartitionFilterExpression("year < '2010'",
    PigStorage.class, "1");
  partitionHelper.setPartitionKeys(baseDir.getAbsolutePath(), conf,
    PigStorage.class, "1");
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
  job.setInputFormatClass(FileInputFormat.class);

  Configuration conf = job.getConfiguration();
  FileInputFormat.setInputPaths(job, new Path(baseDir.getAbsolutePath()));

  JobContext jobContext = new JobContext(conf, job.getJobID());

  partitionHelper.setPartitionFilterExpression(
    "year<='2010' and month=='01' and day>='01'", PigStorage.class, "2");
  partitionHelper.setPartitionKeys(baseDir.getAbsolutePath(), conf,
    PigStorage.class, "2");
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
  job.setInputFormatClass(FileInputFormat.class);

  Configuration conf = job.getConfiguration();
  FileInputFormat.setInputPaths(job, new Path(baseDir.getAbsolutePath()));

  JobContext jobContext = new JobContext(conf, job.getJobID());

  partitionHelper.setPartitionKeys(baseDir.getAbsolutePath(), conf,
    PigStorage.class, "3");

  List<FileStatus> files = partitionHelper.listStatus(jobContext,
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
   
    public void store(DataBag data, FuncSpec storeFuncSpec, PigContext pigContext) throws IOException {
        Configuration conf = ConfigurationUtil.toConfiguration(pigContext.getProperties());
        // create a simulated JobContext
        JobContext jc = new JobContext(conf, new JobID());
        StoreFuncInterface sfunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(
                storeFuncSpec);
        OutputFormat<?,?> of = sfunc.getOutputFormat();
       
        POStore store = new POStore(new OperatorKey());
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.JobContext

  • co.cask.cdap.hive.datasets.DatasetInputFormat
  • co.gridport.kafka.hadoop.FakeHadoopJob
  • com.atlantbh.hadoop.s3.io.S3InputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.output.PangoolMultipleOutputs
  • com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat$ProxyOutputCommitter
  • com.facebook.giraph.hive.impl.input.benchmark.InputBenchmark
  • com.facebook.hiveio.benchmark.InputBenchmark
  • com.facebook.hiveio.input.HiveInput
  • com.facebook.hiveio.output.CheckOutputSpecsTest
  • com.facebook.hiveio.output.FaultyCheckOutputSpecsTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.