Examples of JobConf

cgl.imr.base.impl.JobConf
com.dp.nebula.wormhole.common.config.JobConf
org.apache.hadoop.mapred.JobConf
te a new JobConf JobConf job = new JobConf(new Configuration(), MyJob.class); // Specify various job-specific parameters job.setJobName("myjob"); job.setInputPath(new Path("in")); job.setOutputPath(new Path("out")); job.setMapperClass(MyJob.MyMapper.class); job.setCombinerClass(MyJob.MyReducer.class); job.setReducerClass(MyJob.MyReducer.class); job.setInputFormat(SequenceFileInputFormat.class); job.setOutputFormat(SequenceFileOutputFormat.class);
@see JobClient @see ClusterStatus @see Tool @see DistributedCache
org.apache.nutch.mapReduce.JobConf
A map/reduce job configuration. This names the {@link Mapper}, combiner (if any), {@link Partitioner}, {@link Reducer}, {@link InputFormat}, and {@link OutputFormat} implementations to be used. It also indicates the setof input files, and where the output files should be written.

Examples of org.apache.hadoop.mapred.JobConf

        runJob(conf);
    }


    @Test
    public void testUpsertWithId() throws Exception {
        JobConf conf = createJobConf();
        conf.set(ConfigurationOptions.ES_WRITE_OPERATION, "upsert");
        conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/update");


        runJob(conf);
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

        runJob(conf);
    }


    @Test(expected = IOException.class)
    public void testUpdateWithoutUpsert() throws Exception {
        JobConf conf = createJobConf();
        conf.set(ConfigurationOptions.ES_WRITE_OPERATION, "update");
        conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/updatewoupsert");


        runJob(conf);
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

        runJob(conf);
    }


    @Test
    public void testUpdateOnlyScript() throws Exception {
        JobConf conf = createJobConf();
        // use an existing id to allow the update to succeed
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/createwithid");
        conf.set(ConfigurationOptions.ES_WRITE_OPERATION, "update");
        conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");


        conf.set(ConfigurationOptions.ES_INDEX_AUTO_CREATE, "yes");
        conf.set(ConfigurationOptions.ES_UPDATE_RETRY_ON_CONFLICT, "3");
        conf.set(ConfigurationOptions.ES_UPDATE_SCRIPT, "counter = 3");
        conf.set(ConfigurationOptions.ES_UPDATE_SCRIPT_LANG, "groovy");


        runJob(conf);
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

        runJob(conf);
    }


    @Test
    public void testUpdateOnlyParamScript() throws Exception {
        JobConf conf = createJobConf();
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/createwithid");
        conf.set(ConfigurationOptions.ES_INDEX_AUTO_CREATE, "yes");


        conf.set(ConfigurationOptions.ES_WRITE_OPERATION, "update");
        conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
        conf.set(ConfigurationOptions.ES_UPDATE_SCRIPT, "counter = param1; anothercounter = param2");
        conf.set(ConfigurationOptions.ES_UPDATE_SCRIPT_LANG, "groovy");
        conf.set(ConfigurationOptions.ES_UPDATE_SCRIPT_PARAMS, " param1:<1>,   param2:number ");


        runJob(conf);
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

        runJob(conf);
    }


    @Test
    public void testUpdateOnlyParamJsonScript() throws Exception {
        JobConf conf = createJobConf();
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/createwithid");
        conf.set(ConfigurationOptions.ES_INDEX_AUTO_CREATE, "yes");


        conf.set(ConfigurationOptions.ES_WRITE_OPERATION, "update");
        conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
        conf.set(ConfigurationOptions.ES_UPDATE_SCRIPT, "counter = param1; anothercounter = param2");
        conf.set(ConfigurationOptions.ES_UPDATE_SCRIPT_LANG, "groovy");
        conf.set(ConfigurationOptions.ES_UPDATE_SCRIPT_PARAMS_JSON, "{ \"param1\":1, \"param2\":2}");


        runJob(conf);
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

        runJob(conf);
    }


    @Test
    public void testUpdateOnlyParamJsonScriptWithArray() throws Exception {
        JobConf conf = createJobConf();
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/createwithid");
        conf.set(ConfigurationOptions.ES_INDEX_AUTO_CREATE, "yes");


        conf.set(ConfigurationOptions.ES_WRITE_OPERATION, "update");
        conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
        conf.set(ConfigurationOptions.ES_UPDATE_SCRIPT, "list = new HashSet(); list.add(ctx._source.list); list.add(some_list); ctx._source.list= list.toArray()");
        conf.set(ConfigurationOptions.ES_UPDATE_SCRIPT_LANG, "groovy");
        conf.set(ConfigurationOptions.ES_UPDATE_SCRIPT_PARAMS_JSON, "{ \"some_list\": [\"one\", \"two\"]}");


        runJob(conf);


//        conf = createJobConf();
//        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/createwithid");

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

            }
        }
    }


    public static void run1(Map<String, String> path) throws IOException {
        JobConf conf = Recommend.config();


        String input = path.get("Step3Input1");
        String output = path.get("Step3Output1");


        HdfsDAO hdfs = new HdfsDAO(Recommend.HDFS, conf);
        hdfs.rmr(output);


        conf.setOutputKeyClass(IntWritable.class);
        conf.setOutputValueClass(Text.class);


        conf.setMapperClass(Step31_UserVectorSplitterMapper.class);


        conf.setInputFormat(TextInputFormat.class);
        conf.setOutputFormat(TextOutputFormat.class);


        FileInputFormat.setInputPaths(conf, new Path(input));
        FileOutputFormat.setOutputPath(conf, new Path(output));


        RunningJob job = JobClient.runJob(conf);

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

            output.collect(k, v);
        }
    }


    public static void run2(Map<String, String> path) throws IOException {
        JobConf conf = Recommend.config();


        String input = path.get("Step3Input2");
        String output = path.get("Step3Output2");


        HdfsDAO hdfs = new HdfsDAO(Recommend.HDFS, conf);
        hdfs.rmr(output);


        conf.setOutputKeyClass(Text.class);
        conf.setOutputValueClass(IntWritable.class);


        conf.setMapperClass(Step32_CooccurrenceColumnWrapperMapper.class);


        conf.setInputFormat(TextInputFormat.class);
        conf.setOutputFormat(TextOutputFormat.class);


        FileInputFormat.setInputPaths(conf, new Path(input));
        FileOutputFormat.setOutputPath(conf, new Path(output));


        RunningJob job = JobClient.runJob(conf);

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

        }


    }


    public static void run(Map<String, String> path) throws IOException, InterruptedException, ClassNotFoundException {
        JobConf conf = config();
        String local_data = path.get("sell");
        String input = path.get("input");
        String output = path.get("output");


        // 初始化sell

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf


        job.waitForCompletion(true);
    }


    public static JobConf config() {// Hadoop集群的远程配置信息
        JobConf conf = new JobConf(Purchase.class);
        conf.setJobName("purchase");
        conf.addResource("classpath:/hadoop/core-site.xml");
        conf.addResource("classpath:/hadoop/hdfs-site.xml");
        conf.addResource("classpath:/hadoop/mapred-site.xml");
        return conf;
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.