Examples of org.apache.hadoop.mapred.JobConf

org.apache.hadoop.mapred.JobConf
te a new JobConf JobConf job = new JobConf(new Configuration(), MyJob.class); // Specify various job-specific parameters job.setJobName("myjob"); job.setInputPath(new Path("in")); job.setOutputPath(new Path("out")); job.setMapperClass(MyJob.MyMapper.class); job.setCombinerClass(MyJob.MyReducer.class); job.setReducerClass(MyJob.MyReducer.class); job.setInputFormat(SequenceFileInputFormat.class); job.setOutputFormat(SequenceFileOutputFormat.class);
@see JobClient @see ClusterStatus @see Tool @see DistributedCache

        }
    }


    @Parameters
    public static Collection<Object[]> configs() {
        JobConf conf = HdpBootstrap.hadoopConfig();


        conf.setInputFormat(SplittableTextInputFormat.class);
        conf.setOutputFormat(EsOutputFormat.class);
        conf.setReducerClass(IdentityReducer.class);
        HadoopCfgUtils.setGenericOptions(conf);
        conf.setNumMapTasks(2);
        conf.setInt("actual.splits", 2);
        conf.setNumReduceTasks(0);




        JobConf standard = new JobConf(conf);
        standard.setMapperClass(TabMapper.class);
        standard.setMapOutputValueClass(LinkedMapWritable.class);
        standard.set(ConfigurationOptions.ES_INPUT_JSON, "false");
        FileInputFormat.setInputPaths(standard, new Path(TestUtils.sampleArtistsDat(conf)));


        JobConf json = new JobConf(conf);
        json.setMapperClass(IdentityMapper.class);
        json.setMapOutputValueClass(Text.class);
        json.set(ConfigurationOptions.ES_INPUT_JSON, "true");
        FileInputFormat.setInputPaths(json, new Path(TestUtils.sampleArtistsJson(conf)));


        return Arrays.asList(new Object[][] {
                { standard, "" },
                { json, "json-" }

View Full Code Here

        copyFromLocal(localPath, localPath);
    }


    public static void copyFromLocal(String localPath, String destination) {
        try {
            JobConf hadoopConfig = HdpBootstrap.hadoopConfig();
            FileSystem fs = FileSystem.get(hadoopConfig);
            if (!(fs instanceof LocalFileSystem)) {
                Path src = new Path(localPath);
                Path dst = new Path(destination);
                fs.copyFromLocalFile(false, true, src, dst);

View Full Code Here

        this.config = config;
    }


  @Test
  public void testBasicMultiSave() throws Exception {
    JobConf conf = createJobConf();
    conf.set(ConfigurationOptions.ES_RESOURCE, "oldapi/multi-save");


    MultiOutputFormat.addOutputFormat(conf, EsOutputFormat.class);
    MultiOutputFormat.addOutputFormat(conf, PrintStreamOutputFormat.class);
    //MultiOutputFormat.addOutputFormat(conf, TextOutputFormat.class);


    PrintStreamOutputFormat.stream(conf, Stream.OUT);
    //conf.set("mapred.output.dir", "foo/bar");
    //FileOutputFormat.setOutputPath(conf, new Path("foo/bar"));


    conf.setClass("mapred.output.format.class", MultiOutputFormat.class, OutputFormat.class);
    runJob(conf);
  }

View Full Code Here

  }




    @Test
    public void testNoInput() throws Exception {
        JobConf conf = createJobConf();


        // use only when dealing with constant input
        assumeFalse(conf.get(ConfigurationOptions.ES_INPUT_JSON).equals("true"));
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/constant");
        conf.setMapperClass(ConstantMapper.class);


        runJob(conf);
    }

View Full Code Here

        runJob(conf);
    }


    @Test
    public void testBasicIndex() throws Exception {
        JobConf conf = createJobConf();
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/save");


        runJob(conf);
    }

View Full Code Here

        runJob(conf);
    }


    @Test
    public void testBasicIndexWithId() throws Exception {
        JobConf conf = createJobConf();
        conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/savewithid");


        runJob(conf);
    }

View Full Code Here

        runJob(conf);
    }


    @Test
    public void testCreateWithId() throws Exception {
        JobConf conf = createJobConf();
        conf.set(ConfigurationOptions.ES_WRITE_OPERATION, "create");
        conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/createwithid");


        runJob(conf);
    }

View Full Code Here

        testCreateWithId();
    }


    @Test(expected = IOException.class)
    public void testUpdateWithoutId() throws Exception {
        JobConf conf = createJobConf();
        conf.set(ConfigurationOptions.ES_WRITE_OPERATION, "upsert");
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/update");


        runJob(conf);
    }

View Full Code Here

        runJob(conf);
    }


    @Test
    public void testUpsertWithId() throws Exception {
        JobConf conf = createJobConf();
        conf.set(ConfigurationOptions.ES_WRITE_OPERATION, "upsert");
        conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/update");


        runJob(conf);
    }

View Full Code Here

        runJob(conf);
    }


    @Test(expected = IOException.class)
    public void testUpdateWithoutUpsert() throws Exception {
        JobConf conf = createJobConf();
        conf.set(ConfigurationOptions.ES_WRITE_OPERATION, "update");
        conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/updatewoupsert");


        runJob(conf);
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapred.JobConf

bixo.config.BixoPlatform

com.cloudera.circus.test.TestXTest

com.cloudera.circus.test.XTest$TestMethodListener

com.cloudera.lib.service.hadoop.HadoopService

com.cloudera.lib.service.hadoop.TestHadoopService

com.facebook.hive.orc.TestInputOutputFormat

com.facebook.hiveio.output.HiveApiOutputFormat

com.facebook.hiveio.output.PerThread

com.facebook.LinkBench.LinkBenchDriverMR

com.facebook.presto.hive.HiveRecordSet

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.