Changes between Initial Version and Version 1 of waue/2009/0715

Timestamp:

Jul 15, 2009, 3:54:28 PM (15 years ago)

Author:

waue

Comment:

--

waue/2009/0715

@@ +1 @@
+{{{
+                public void map(LongWritable key, Text value,
+                                OutputCollector<Text, IntWritable> output, Reporter reporter)
+                                throws IOException {
+                        String line = (caseSensitive) ? value.toString() : value.toString()
+                                        .toLowerCase();
+
+                        for (String pattern : patternsToSkip) {
+                                line = line.replaceAll(pattern, "");
+                        }
+
+                        StringTokenizer tokenizer = new StringTokenizer(line);
+                        while (tokenizer.hasMoreTokens()) {
+                                word.set(tokenizer.nextToken());
+                                output.collect(word, one);
+                                reporter.incrCounter(Counters.INPUT_WORDS, 1);
+                        }
+
+                        if ((++numRecords % 100) == 0) {
+                                reporter.setStatus("Finished processing " + numRecords
+                                                + " records " + "from the input file: " + inputFile);
+                        }
+                }
+        }
+
+        public static class Reduce extends MapReduceBase implements
+                        Reducer<Text, IntWritable, Text, IntWritable> {
+                public void reduce(Text key, Iterator<IntWritable> values,
+                                OutputCollector<Text, IntWritable> output, Reporter reporter)
+                                throws IOException {
+                        int sum = 0;
+                        while (values.hasNext()) {
+                                sum += values.next().get();
+                        }
+                        output.collect(key, new IntWritable(sum));
+                }
+        }
+}}}