Examples of org.apache.hadoop.mapred.JobConf

org.apache.hadoop.mapred.JobConf
te a new JobConf JobConf job = new JobConf(new Configuration(), MyJob.class); // Specify various job-specific parameters job.setJobName("myjob"); job.setInputPath(new Path("in")); job.setOutputPath(new Path("out")); job.setMapperClass(MyJob.MyMapper.class); job.setCombinerClass(MyJob.MyReducer.class); job.setReducerClass(MyJob.MyReducer.class); job.setInputFormat(SequenceFileInputFormat.class); job.setOutputFormat(SequenceFileOutputFormat.class);
@see JobClient @see ClusterStatus @see Tool @see DistributedCache

      validateJobtracker(new URI(conf.get(JOB_TRACKER_PROPERTY)).getAuthority());
      validateNamenode(new URI(conf.get(NAME_NODE_PROPERTY)).getAuthority());
      UserGroupInformation ugi = getUGI(user);
      return ugi.doAs(new PrivilegedExceptionAction<T>() {
        public T run() throws Exception {
          JobConf jobtrackerConf = createJobTrackerConf(conf);
          Configuration namenodeConf = createNameNodeConf(conf);
          JobClient jobClient = createJobClient(jobtrackerConf);
          try {
            checkJobTrackerHealth(jobClient);
            FileSystem fs = createFileSystem(namenodeConf);

View Full Code Here

      
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        long start = System.currentTimeMillis();
        LOG.info("ElasticsearchIndexer: starting at " + sdf.format(start));


        final JobConf job = new NutchJob(getConf());
        job.setJobName("index-elasticsearch " + elasticsearchUrl);


        IndexerMapReduce.initMRJob(crawlDb, linkDb, segments, job);


        job.set(ElasticsearchConstants.SERVER_URL, elasticsearchUrl);
        job.set(ElasticsearchConstants.SERVER_PORT, elasticsearchPort);


        NutchIndexWriterFactory.addClassToConf(job, ElasticsearchWriter.class);


        job.setReduceSpeculativeExecution(false);


        final Path tmp = new Path("tmp_" + System.currentTimeMillis() + "-" +
                             new Random().nextInt());


        FileOutputFormat.setOutputPath(job, tmp);

View Full Code Here

   *
   * @return the Hadoop <code>JobConf</code> preconfigured with the Hadoop cluster
   * settings for testing
   */
  protected JobConf getHadoopConf() {
    JobConf jobConf = TEST_HADOOP_TL.get();
    if (jobConf == null) {
      throw new IllegalStateException("This test does not use @TestHadoop");
    }
    return new JobConf(jobConf);
  }

View Full Code Here


      TestHadoop th = iInvokedMethod.getTestMethod().getMethod().getAnnotation(TestHadoop.class);
      if (th != null) {
        synchronized (HADOOP_LOCK) {
          if (HADOOP_CONF == null) {
            JobConf conf = new JobConf();
            for (String name : System.getProperties().stringPropertyNames()) {
              conf.set(name, System.getProperty(name));
            }
            if (Boolean.parseBoolean(System.getProperty(HADOOP_MINICLUSTER, "true"))) {
              try {
                HADOOP_CONF = setUpEmbeddedHadoop(conf);
              }

View Full Code Here

        System.setProperty("test.build.data", new File(TEST_DIR_ROOT, "hadoop-data").getAbsolutePath());
      }


      int taskTrackers = 2;
      int dataNodes = 2;
      conf = new JobConf(conf);
      conf.set("fs.hdfs.impl.disable.cache", "true");
      conf.set("dfs.block.access.token.enable", "false");
      conf.set("dfs.permissions", "true");
      conf.set("hadoop.security.authentication", "simple");
      conf.set("hadoop.proxyuser." + getHadoopProxyUser() + ".hosts", getHadoopProxyUserHosts());

View Full Code Here

    conf.set("server.services", services);
    Server server = new Server("server", dir, dir, dir, dir, conf);
    server.init();
    Hadoop hadoop = server.get(Hadoop.class);


    JobConf jobConf = getHadoopConf();
    jobConf.set("fs.default.name", "");
    hadoop.execute("u", jobConf, new Hadoop.FileSystemExecutor<Void>() {
      @Override
      public Void execute(FileSystem fs) throws IOException {
        return null;
      }

View Full Code Here

    conf.set("server.services", services);
    Server server = new Server("server", dir, dir, dir, dir, conf);
    server.init();
    Hadoop hadoop = server.get(Hadoop.class);


    JobConf jobConf = getHadoopConf();
    jobConf.set("fs.default.name", "");
    hadoop.execute("u", jobConf, new Hadoop.JobClientExecutor<Void>() {
      @Override
      public Void execute(JobClient jobClient, FileSystem fs) throws IOException {
        return null;
      }

View Full Code Here

    conf.set("server.services", services);
    Server server = new Server("server", dir, dir, dir, dir, conf);
    server.init();
    Hadoop hadoop = server.get(Hadoop.class);


    JobConf jobConf = getHadoopConf();
    jobConf.set("mapred.job.tracker", "");
    hadoop.execute("u", jobConf, new Hadoop.JobClientExecutor<Void>() {
      @Override
      public Void execute(JobClient jobClient, FileSystem fs) throws IOException {
        return null;
      }

View Full Code Here

  }


  @Test
  @TestHadoop
  public void testHadoopMinicluster() throws Exception {
    JobConf conf = getHadoopConf();
    Assert.assertNotNull(conf);
    FileSystem fs = FileSystem.get(conf);
    Assert.assertNotNull(fs);
    Assert.assertEquals(fs.getUri().getScheme(), "hdfs");
    Assert.assertTrue(fs.exists(getHadoopTestDir()));

View Full Code Here

  }


  @Test
  @TestHadoop
  public void testHadoopFileSystem() throws Exception {
    JobConf conf = getHadoopConf();
    FileSystem fs = FileSystem.get(conf);
    try {
      OutputStream os = fs.create(new Path(getHadoopTestDir(), "foo"));
      os.write(new byte[]{1});
      os.close();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapred.JobConf

bixo.config.BixoPlatform

com.cloudera.circus.test.TestXTest

com.cloudera.circus.test.XTest$TestMethodListener

com.cloudera.lib.service.hadoop.HadoopService

com.cloudera.lib.service.hadoop.TestHadoopService

com.facebook.hive.orc.TestInputOutputFormat

com.facebook.hiveio.output.HiveApiOutputFormat

com.facebook.hiveio.output.PerThread

com.facebook.LinkBench.LinkBenchDriverMR

com.facebook.presto.hive.HiveRecordSet

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.