| 1 | '''WordCount.java''' |
| 2 | |
| 3 | {{{ |
| 4 | #!java |
| 5 | |
| 6 | import java.io.IOException; |
| 7 | import java.util.*; |
| 8 | |
| 9 | import org.apache.hadoop.fs.Path; |
| 10 | import org.apache.hadoop.conf.*; |
| 11 | import org.apache.hadoop.io.*; |
| 12 | import org.apache.hadoop.mapred.*; |
| 13 | import org.apache.hadoop.util.*; |
| 14 | |
| 15 | public class WordCount { |
| 16 | |
| 17 | public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> { |
| 18 | private final static IntWritable one = new IntWritable(1); |
| 19 | private Text word = new Text(); |
| 20 | |
| 21 | public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException { |
| 22 | String line = value.toString(); |
| 23 | StringTokenizer tokenizer = new StringTokenizer(line); |
| 24 | while (tokenizer.hasMoreTokens()) { |
| 25 | word.set(tokenizer.nextToken()); |
| 26 | output.collect(word, one); |
| 27 | } |
| 28 | } |
| 29 | } |
| 30 | |
| 31 | public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> { |
| 32 | |
| 33 | public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException { |
| 34 | int sum = 0; |
| 35 | while (values.hasNext()) { |
| 36 | sum += values.next().get(); |
| 37 | } |
| 38 | output.collect(key, new IntWritable(sum)); |
| 39 | } |
| 40 | } |
| 41 | |
| 42 | public static void main(String[] args) throws Exception { |
| 43 | JobConf conf = new JobConf(WordCount.class); |
| 44 | conf.setJobName("wordcount"); |
| 45 | |
| 46 | conf.setOutputKeyClass(Text.class); |
| 47 | conf.setOutputValueClass(IntWritable.class); |
| 48 | |
| 49 | conf.setMapperClass(Map.class); |
| 50 | conf.setCombinerClass(Reduce.class); |
| 51 | conf.setReducerClass(Reduce.class); |
| 52 | |
| 53 | conf.setInputFormat(TextInputFormat.class); |
| 54 | conf.setOutputFormat(TextOutputFormat.class); |
| 55 | |
| 56 | FileInputFormat.setInputPaths(conf, new Path(args[0])); |
| 57 | FileOutputFormat.setOutputPath(conf, new Path(args[1])); |
| 58 | |
| 59 | JobClient.runJob(conf); |
| 60 | } |
| 61 | |
| 62 | } |
| 63 | }}} |