| 1 | |
| 2 | {{{ |
| 3 | #!html |
| 4 | <div style="text-align: center; color:#151B8D"><big style="font-weight: bold;"><big><big> |
| 5 | Hadoop 進階課程 |
| 6 | </big></big></big></div> <div style="text-align: center; color:#7E2217"><big style="font-weight: bold;"><big> |
| 7 | 範例練習 |
| 8 | </big></big></div> |
| 9 | }}} |
| 10 | |
| 11 | = [wiki:NCHCCloudCourse100928_4_EXM 上一關 < ] 第二關 [wiki:NCHCCloudCourse100928_4_EXM3 > 下一關] = |
| 12 | |
| 13 | |
| 14 | * HelloHadoopV2 |
| 15 | {{{ |
| 16 | #!java |
| 17 | package org.nchc.hadoop; |
| 18 | import java.io.IOException; |
| 19 | |
| 20 | import org.apache.hadoop.conf.Configuration; |
| 21 | import org.apache.hadoop.fs.Path; |
| 22 | import org.apache.hadoop.io.Text; |
| 23 | import org.apache.hadoop.mapreduce.Job; |
| 24 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; |
| 25 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; |
| 26 | |
| 27 | // HelloHadoopV2 |
| 28 | // 說明: |
| 29 | // 此程式碼比HelloHadoop 增加 |
| 30 | // * 檢查輸出資料夾是否存在並刪除 |
| 31 | // * input 資料夾內的資料若大於兩個,則資料不會被覆蓋 |
| 32 | // * map 與 reduce 拆開以利程式再利用 |
| 33 | // |
| 34 | // 測試方法: |
| 35 | // 將此程式運作在hadoop 0.20 平台上,執行: |
| 36 | // --------------------------- |
| 37 | // hadoop jar HelloHadoopV2.jar |
| 38 | // --------------------------- |
| 39 | // |
| 40 | // 注意: |
| 41 | // 1. 在hdfs 上來源檔案的路徑為 "/user/$YOUR_NAME/input" |
| 42 | // 請注意必須先放資料到此hdfs上的資料夾內,且此資料夾內只能放檔案,不可再放資料夾 |
| 43 | // 2. 運算完後,程式將執行結果放在hdfs 的輸出路徑為 "/user/$YOUR_NAME/output-hh2" |
| 44 | // |
| 45 | |
| 46 | public class HelloHadoopV2 { |
| 47 | |
| 48 | |
| 49 | public static void main(String[] args) throws IOException, |
| 50 | InterruptedException, ClassNotFoundException { |
| 51 | |
| 52 | Configuration conf = new Configuration(); |
| 53 | Job job = new Job(conf, "Hadoop Hello World 2"); |
| 54 | job.setJarByClass(HelloHadoopV2.class); |
| 55 | // 設定 map and reduce 以及 Combiner class |
| 56 | job.setMapperClass(HelloMapperV2.class); |
| 57 | job.setCombinerClass(HelloReducerV2.class); |
| 58 | job.setReducerClass(HelloReducerV2.class); |
| 59 | |
| 60 | // 設定map的輸出型態 |
| 61 | job.setMapOutputKeyClass(Text.class); |
| 62 | job.setMapOutputValueClass(Text.class); |
| 63 | // 設定reduce的輸出型態 |
| 64 | job.setOutputKeyClass(Text.class); |
| 65 | job.setOutputValueClass(Text.class); |
| 66 | |
| 67 | FileInputFormat.addInputPath(job, new Path("/user/hadooper/input")); |
| 68 | |
| 69 | FileOutputFormat.setOutputPath(job, new Path("/user/hadooper/output-hh2")); |
| 70 | |
| 71 | // 呼叫checkAndDelete函式,檢查是否存在該資料夾,若有則刪除之 |
| 72 | CheckAndDelete.checkAndDelete("/user/hadooper/output-hh2", conf); |
| 73 | |
| 74 | boolean status = job.waitForCompletion(true); |
| 75 | |
| 76 | if (status) { |
| 77 | System.err.println("Integrate Alert Job Finished !"); |
| 78 | |
| 79 | } else { |
| 80 | System.err.println("Integrate Alert Job Failed !"); |
| 81 | System.exit(1); |
| 82 | } |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | }}} |
| 87 | |
| 88 | * HelloMapperV2 |
| 89 | |
| 90 | {{{ |
| 91 | #!java |
| 92 | package org.nchc.hadoop; |
| 93 | import java.io.IOException; |
| 94 | |
| 95 | import org.apache.hadoop.io.LongWritable; |
| 96 | import org.apache.hadoop.io.Text; |
| 97 | import org.apache.hadoop.mapreduce.Mapper; |
| 98 | |
| 99 | public class HelloMapperV2 extends Mapper<LongWritable, Text, Text, Text> { |
| 100 | |
| 101 | public void map(LongWritable key, Text value, Context context) |
| 102 | throws IOException, InterruptedException { |
| 103 | context.write(new Text(key.toString()), value); |
| 104 | } |
| 105 | |
| 106 | } |
| 107 | |
| 108 | }}} |
| 109 | |
| 110 | * HelloReducerV2 |
| 111 | |
| 112 | {{{ |
| 113 | #!java |
| 114 | package org.nchc.hadoop; |
| 115 | import java.io.IOException; |
| 116 | |
| 117 | import org.apache.hadoop.io.Text; |
| 118 | import org.apache.hadoop.mapreduce.Reducer; |
| 119 | |
| 120 | public class HelloReducerV2 extends Reducer<Text, Text, Text, Text> { |
| 121 | public void reduce(Text key, Iterable<Text> values, Context context) |
| 122 | throws IOException, InterruptedException { |
| 123 | |
| 124 | String str = new String(""); |
| 125 | Text final_key = new Text(); |
| 126 | Text final_value = new Text(); |
| 127 | // 將key值相同的values,透過 && 符號分隔之 |
| 128 | for (Text tmp : values) { |
| 129 | str += tmp.toString() + " &&"; |
| 130 | } |
| 131 | |
| 132 | final_key.set(key); |
| 133 | final_value.set(str); |
| 134 | |
| 135 | context.write(final_key, final_value); |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | }}} |