Examples of org.apache.hadoop.mapred.JobConf

org.apache.hadoop.mapred.JobConf
te a new JobConf JobConf job = new JobConf(new Configuration(), MyJob.class); // Specify various job-specific parameters job.setJobName("myjob"); job.setInputPath(new Path("in")); job.setOutputPath(new Path("out")); job.setMapperClass(MyJob.MyMapper.class); job.setCombinerClass(MyJob.MyReducer.class); job.setReducerClass(MyJob.MyReducer.class); job.setInputFormat(SequenceFileInputFormat.class); job.setOutputFormat(SequenceFileOutputFormat.class);
@see JobClient @see ClusterStatus @see Tool @see DistributedCache

    System.setProperty("hadoop.log.dir", logDir);
    c.set("mapred.output.dir", tmpDir);
    mrCluster = new MiniMRCluster(servers,
      FileSystem.get(conf).getUri().toString(), 1);
    LOG.info("Mini mapreduce cluster started");
    JobConf mrClusterJobConf = mrCluster.createJobConf();
    c.set("mapred.job.tracker", mrClusterJobConf.get("mapred.job.tracker"));
    /* this for mrv2 support */
    conf.set("mapreduce.framework.name", "yarn");
    String rmAdress = mrClusterJobConf.get("yarn.resourcemanager.address");
    if (rmAdress != null) {
      conf.set("yarn.resourcemanager.address", rmAdress);
    }
    String schedulerAdress =
      mrClusterJobConf.get("yarn.resourcemanager.scheduler.address");
    if (schedulerAdress != null) {
      conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress);
    }
  }

View Full Code Here

    }


    testMerge_ = (-1 != userJobConfProps_.toString().indexOf("stream.testmerge"));


    // general MapRed job properties
    jobConf_ = new JobConf(config_);
    
    // All streaming jobs have, by default, no time-out for tasks
    jobConf_.setLong("mapred.task.timeout", 0);


    setUserJobConfProps(true);

View Full Code Here


  /** Uses default mapper with no reduces for a map-only identity job. */
  @Test
  @SuppressWarnings("deprecation")
  public void testMapOnly() throws Exception {
    JobConf job = new JobConf();
    String inDir = System.getProperty("share.dir",".")+"/test/data";
    Path input = new Path(inDir+"/weather.avro");
    Path output = new Path(System.getProperty("test.dir",".")+"/weather-ident");
    
    output.getFileSystem(job).delete(output);
    
    job.setJobName("identity map weather");
    
    AvroJob.setInputSchema(job, Weather.SCHEMA$);
    AvroJob.setMapOutputSchema(job, Weather.SCHEMA$);


    FileInputFormat.setInputPaths(job, input);
    FileOutputFormat.setOutputPath(job, output);
    FileOutputFormat.setCompressOutput(job, true);
    
    job.setNumReduceTasks(0);                     // map-only
    
    JobClient.runJob(job);


    // check output is correct
    DatumReader<Weather> reader = new SpecificDatumReader<Weather>();

View Full Code Here

  }    


  @Test
  @SuppressWarnings("deprecation")
  public void testSort() throws Exception {
    JobConf job = new JobConf();
    String inDir = System.getProperty("share.dir",".")+"/test/data";
    Path input = new Path(inDir+"/weather.avro");
    Path output = new Path(System.getProperty("test.dir",".")+"/weather-sort");
    
    output.getFileSystem(job).delete(output);
    
    job.setJobName("sort weather");
    
    AvroJob.setInputSchema(job, Weather.SCHEMA$);
    AvroJob.setMapOutputSchema
      (job, Pair.getPairSchema(Weather.SCHEMA$, Schema.create(Type.NULL)));
    AvroJob.setOutputSchema(job, Weather.SCHEMA$);

View Full Code Here

    reader.close();
  }


  @Test
  public void testSequenceFileInputFormat() throws Exception {
    JobConf job = new JobConf();
    Path output = new Path(System.getProperty("test.dir",".")+"/seq-out");


    output.getFileSystem(job).delete(output);
    
    Schema schema = Pair.getPairSchema(Schema.create(Schema.Type.LONG),

View Full Code Here


  public void setConf(Configuration conf) {
    if (conf instanceof JobConf) {
      this.conf = (JobConf) conf;
    } else {
      this.conf = new JobConf(conf);
    }
  }

View Full Code Here

      EnumSet<Options> flags) throws IOException {
    LOG.info("srcPaths=" + srcPaths);
    LOG.info("destPath=" + destPath);
    checkSrcPath(conf, srcPaths);


    JobConf job = createJobConf(conf);
    //Initialize the mapper
    try {
      setup(conf, job, srcPaths, destPath, logPath, flags);
      JobClient.runJob(job);
    } finally {
      //delete tmp
      fullyDelete(job.get(TMP_DIR_LABEL), job);
      //delete jobDirectory
      fullyDelete(job.get(JOB_DIR_LABEL), job);
    }
  }

View Full Code Here

    }
    return 0;
  }


  public static void main(String[] args) throws Exception {
    JobConf job = new JobConf(CopyFiles.class);
    CopyFiles distcp = new CopyFiles(job);
    int res = ToolRunner.run(distcp, args);
    System.exit(res);
  }

View Full Code Here

    }
  }


  //Job configuration
  private static JobConf createJobConf(Configuration conf) {
    JobConf jobconf = new JobConf(conf, CopyFiles.class);
    jobconf.setJobName(NAME);


    // turn off speculative execution, because DFS doesn't handle
    // multiple writers to the same file.
    jobconf.setMapSpeculativeExecution(false);


    jobconf.setInputFormat(CopyInputFormat.class);
    jobconf.setOutputKeyClass(Text.class);
    jobconf.setOutputValueClass(Text.class);


    jobconf.setMapperClass(FSCopyFilesMapper.class);
    jobconf.setNumReduceTasks(0);
    return jobconf;
  }

View Full Code Here

    // new API - just delegates to the Old API
    //
    @Override
    public List<InputSplit> getSplits(JobContext context) throws IOException {
        org.elasticsearch.hadoop.mr.compat.JobContext compatJobContext = CompatHandler.jobContext(context);
        JobConf conf = HadoopCfgUtils.asJobConf(compatJobContext.getConfiguration());
        // NOTE: this method expects a ShardInputSplit to be returned (which implements both the old and the new API).
        return Arrays.asList((InputSplit[]) getSplits(conf, conf.getNumMapTasks()));
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapred.JobConf

bixo.config.BixoPlatform

com.cloudera.circus.test.TestXTest

com.cloudera.circus.test.XTest$TestMethodListener

com.cloudera.lib.service.hadoop.HadoopService

com.cloudera.lib.service.hadoop.TestHadoopService

com.facebook.hive.orc.TestInputOutputFormat

com.facebook.hiveio.output.HiveApiOutputFormat

com.facebook.hiveio.output.PerThread

com.facebook.LinkBench.LinkBenchDriverMR

com.facebook.presto.hive.HiveRecordSet

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.