Examples of org.apache.hadoop.mapred.JobConf

org.apache.hadoop.mapred.JobConf
te a new JobConf JobConf job = new JobConf(new Configuration(), MyJob.class); // Specify various job-specific parameters job.setJobName("myjob"); job.setInputPath(new Path("in")); job.setOutputPath(new Path("out")); job.setMapperClass(MyJob.MyMapper.class); job.setCombinerClass(MyJob.MyReducer.class); job.setReducerClass(MyJob.MyReducer.class); job.setInputFormat(SequenceFileInputFormat.class); job.setOutputFormat(SequenceFileOutputFormat.class);
@see JobClient @see ClusterStatus @see Tool @see DistributedCache

  }


  @Test
  @TestHadoop
  public void testHadoopMapReduce() throws Exception {
    JobConf conf = getHadoopConf();
    FileSystem fs = FileSystem.get(conf);
    JobClient jobClient = new JobClient(conf);
    try {
      Path inputDir = new Path(getHadoopTestDir(), "input");
      Path outputDir = new Path(getHadoopTestDir(), "output");


      fs.mkdirs(inputDir);
      Writer writer = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
      writer.write("a\n");
      writer.write("b\n");
      writer.write("c\n");
      writer.close();


      JobConf jobConf = getHadoopConf();
      jobConf.setInt("mapred.map.tasks", 1);
      jobConf.setInt("mapred.map.max.attempts", 1);
      jobConf.setInt("mapred.reduce.max.attempts", 1);
      jobConf.set("mapred.input.dir", inputDir.toString());
      jobConf.set("mapred.output.dir", outputDir.toString());
      final RunningJob runningJob = jobClient.submitJob(jobConf);
      waitFor(60 * 1000, true, new Predicate() {
        @Override
        public boolean evaluate() throws Exception {
          return runningJob.isComplete();

View Full Code Here

        super(applicationJarClass);
        if (platform == Platform.Local) {
            _platform = new LocalPlatform(applicationJarClass);
            setJobPollingInterval(CASCADING_LOCAL_JOB_POLLING_INTERVAL);
        } else {
            configureHadoopPlatform(applicationJarClass, new JobConf());
        }
    }

View Full Code Here

            configureHadoopPlatform(applicationJarClass, new JobConf());
        }
    }


    public BixoPlatform(Class applicationJarClass, Configuration conf) throws Exception {
        this(applicationJarClass, new JobConf(conf));
    }

View Full Code Here

    protected abstract void setupJob(Job job) throws Exception;


    protected JobConf getConfiguration(String[] args, io.lumify.core.config.Configuration lumifyConfig) {
        Configuration hadoopConfig = lumifyConfig.toHadoopConfiguration(getConf());
        hadoopConfig.set(ElementMapper.GRAPH_CONFIG_PREFIX, "graph.");
        JobConf result = new JobConf(hadoopConfig, this.getClass());
        parseArgs(result, args);
        setConf(result);
        LOGGER.info("Using config:\n" + result);
        return result;
    }

View Full Code Here

    private Timer periodicCounterOutputTimer;


    @Override
    public int run(String[] args) throws Exception {
        io.lumify.core.config.Configuration lumifyConfig = ConfigurationLoader.load();
        JobConf conf = getConfiguration(args, lumifyConfig);
        AccumuloGraphConfiguration accumuloGraphConfiguration = new AccumuloGraphConfiguration(conf, "graph.");
        InjectHelper.inject(this, LumifyBootstrap.bootstrapModuleMaker(lumifyConfig));


        Job job = new Job(conf, getJobName());


        instanceName = accumuloGraphConfiguration.getAccumuloInstanceName();
        zooKeepers = accumuloGraphConfiguration.getZookeeperServers();
        principal = accumuloGraphConfiguration.getAccumuloUsername();
        authorizationToken = accumuloGraphConfiguration.getAuthenticationToken();
        AccumuloElementOutputFormat.setOutputInfo(job, instanceName, zooKeepers, principal, authorizationToken);


        boolean periodicCounterOutput = conf.getBoolean("lumify.periodic.counter.output.enabled", false);


        if (job.getConfiguration().get("mapred.job.tracker").equals("local")) {
            LOGGER.warn("!!!!!! Running in local mode !!!!!!");
            local = true;
            periodicCounterOutput = true;

View Full Code Here

    final Configuration conf, int pollingInterval, CountDownLatch startFlag)
    throws IOException, InterruptedException {
    UserGroupInformation ugi = UserGroupInformation.getLoginUser();
      this.cluster = ugi.doAs(new PrivilegedExceptionAction<JobClient>(){
        public JobClient run() throws IOException {
          return new JobClient(new JobConf(conf));
        }
      });


    this.jtPollingInterval = pollingInterval;
    maxJobCompletedInInterval = conf.getInt(

View Full Code Here

    }
    return pid;
  }


  static JobTracker getJobTracker() {
    JobConf conf = new JobConf();
    conf.set("mapred.job.tracker", "localhost:0");
    conf.set("mapred.job.tracker.http.address", "0.0.0.0:0");
    JobTracker jt;
    try {
      jt = new JobTracker(conf);
      return jt;
    } catch (Exception e) {

View Full Code Here


  public void setConf(Configuration conf) {
    if (conf instanceof JobConf) {
      this.conf = (JobConf) conf;
    } else {
      this.conf = new JobConf(conf, HadoopArchives.class);
    }
  }

View Full Code Here

    return 0;
  }


  /** the main functions **/
  public static void main(String[] args) {
    JobConf job = new JobConf(HadoopArchives.class);
    HadoopArchives harchives = new HadoopArchives(job);
    int ret = 0;


    try{
      ret = ToolRunner.run(harchives, args);

View Full Code Here

  @Override
  public synchronized void connect() throws IOException {
    if (isConnected()) {
      return;
    }
    client = new JobClient(new JobConf(getConf()));
    setConnected(true);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapred.JobConf

bixo.config.BixoPlatform

com.cloudera.circus.test.TestXTest

com.cloudera.circus.test.XTest$TestMethodListener

com.cloudera.lib.service.hadoop.HadoopService

com.cloudera.lib.service.hadoop.TestHadoopService

com.facebook.hive.orc.TestInputOutputFormat

com.facebook.hiveio.output.HiveApiOutputFormat

com.facebook.hiveio.output.PerThread

com.facebook.LinkBench.LinkBenchDriverMR

com.facebook.presto.hive.HiveRecordSet

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.