package com.commons;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import com.mapreduce.DataGettingMapper;
import com.mapreduce.DataGettingReducer;
import com.mapreduce.UrlGettingMapper;
import com.mapreduce.UrlGettingReducer;
import com.utils.LoggerUtils;
public class Main {
public static void main(String[] args) throws Exception {
startJob1();
startJob2();
}
private static void startJob2() throws Exception {
FileSystem fs = FileSystem.get(new Configuration());
Job job = Job.getInstance();
JobConf config = ((JobConf) job.getConfiguration());
config.setJar("jingdong.jar");
job.setJarByClass(Main.class);
job.setMapOutputValueClass(Text.class);
job.setInputFormatClass(KeyValueTextInputFormat.class);
job.setMapperClass(DataGettingMapper.class);
job.setReducerClass(DataGettingReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
Path output = new Path("/price/jingdong/output/data");
if (fs.exists(output)) {
fs.delete(output, true);
}
Path input = new Path("/price/jingdong/output/url");
FileInputFormat.addInputPath(config, input);
FileOutputFormat.setOutputPath(config, output);
job.waitForCompletion(true);
}
private static void startJob1() throws Exception {
FileSystem fs = FileSystem.get(new Configuration());
Job job = Job.getInstance();
JobConf config = ((JobConf) job.getConfiguration());
config.setJar("jingdong.jar");
job.setJarByClass(Main.class);
job.setMapOutputValueClass(Text.class);
job.setInputFormatClass(TextInputFormat.class);
job.setMapperClass(UrlGettingMapper.class);
job.setReducerClass(UrlGettingReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
Path input = new Path("/price/jingdong/input/url");
fs.mkdirs(input);
LoggerUtils.log(Main.class.getName(), "create dir:" + input.toString());
Path inputFile = new Path(input, "jingdong");
if (!fs.exists(inputFile)) {
FSDataOutputStream file = fs.create(inputFile);
file.writeUTF("http://www.jd.com");
file.flush();
file.close();
}
Path output = new Path("/price/jingdong/output/url");
if (fs.exists(output)) {
fs.delete(output, true);
}
FileInputFormat.addInputPath(config, input);
FileOutputFormat.setOutputPath(config, output);
job.waitForCompletion(true);
LoggerUtils.log(Main.class.getName(), "finish job1!");
}
}