Index: sample/hadoop-0.16/tw/org/nchc/code/HBaseRecordPro.java =================================================================== --- sample/hadoop-0.16/tw/org/nchc/code/HBaseRecordPro.java (revision 23) +++ sample/hadoop-0.16/tw/org/nchc/code/HBaseRecordPro.java (revision 25) @@ -228,7 +228,7 @@ conf.setNumMapTasks(mapTasks); conf.setNumReduceTasks(reduceTasks); - // 0.16 -// conf.setInputPath(text_path); - Convert.setInputPath(conf, text_path); + + conf.setInputPath(text_path); + conf.setMapperClass(IdentityMapper.class); @@ -240,6 +240,5 @@ // delete tmp file // 0.16 -// FileSystem.get(conf).delete(text_path); - FileSystem.get(conf).delete(text_path,true); + FileSystem.get(conf).delete(text_path); setup.deleteFile(conf_tmp); Index: sample/hadoop-0.16/tw/org/nchc/code/WordCount.java =================================================================== --- sample/hadoop-0.16/tw/org/nchc/code/WordCount.java (revision 23) +++ sample/hadoop-0.16/tw/org/nchc/code/WordCount.java (revision 25) @@ -110,12 +110,11 @@ conf.setNumMapTasks(mapTasks); conf.setNumReduceTasks(reduceTasks); - // 0.16 - // conf.setInputPath(new Path(wc.filepath)); - Convert.setInputPath(conf, new Path(wc.filepath)); + + conf.setInputPath(new Path(wc.filepath)); + conf.setOutputKeyClass(Text.class); conf.setOutputValueClass(IntWritable.class); - // 0.16 - // conf.setOutputPath(new Path(wc.outputPath)); - Convert.setOutputPath(conf, new Path(wc.outputPath)); + + conf.setOutputPath(new Path(wc.outputPath)); conf.setMapperClass(MapClass.class); @@ -125,7 +124,5 @@ // Delete the output directory if it exists already Path outputDir = new Path(wc.outputPath); - // 0.16 - FileSystem.get(conf).delete(outputDir,true); - + FileSystem.get(conf).delete(outputDir); JobClient.runJob(conf); } Index: sample/hadoop-0.16/tw/org/nchc/code/WordCountFromHBase.java =================================================================== --- sample/hadoop-0.16/tw/org/nchc/code/WordCountFromHBase.java (revision 23) +++ sample/hadoop-0.16/tw/org/nchc/code/WordCountFromHBase.java (revision 25) @@ -168,9 +168,7 @@ // input is Hbase format => TableInputFormat conf.setInputFormat(TableInputFormat.class); - // 0.16 -// conf.setOutputPath(new Path(outputPath)); - Convert.setOutputPath(conf, new Path(outputPath)); + conf.setOutputPath(new Path(outputPath)); // delete the old path with the same name - FileSystem.get(conf).delete(new Path(outputPath),true); + FileSystem.get(conf).delete(new Path(outputPath)); JobClient.runJob(conf); } Index: sample/hadoop-0.16/tw/org/nchc/demo/DemoWordCount.java =================================================================== --- sample/hadoop-0.16/tw/org/nchc/demo/DemoWordCount.java (revision 23) +++ sample/hadoop-0.16/tw/org/nchc/demo/DemoWordCount.java (revision 25) @@ -28,6 +28,4 @@ import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reporter; - -import tw.org.nchc.code.Convert; /** @@ -107,12 +105,11 @@ conf.setNumMapTasks(mapTasks); conf.setNumReduceTasks(reduceTasks); - //0.16 -// conf.setInputPath(new Path(filename)); - Convert.setInputPath(conf, new Path(filename)); + + conf.setInputPath(new Path(filename)); + conf.setOutputKeyClass(Text.class); conf.setOutputValueClass(IntWritable.class); - // 0.16 -// conf.setOutputPath(new Path(outputPath)); - Convert.setInputPath(conf, new Path(outputPath)); + + conf.setOutputPath(new Path(outputPath)); conf.setMapperClass(MapClass.class); conf.setCombinerClass(ReduceClass.class); @@ -121,7 +118,5 @@ // Delete the output directory if it exists already Path outputDir = new Path(outputPath); - // 0.16 -// FileSystem.get(conf).delete(outputDir); - FileSystem.get(conf).delete(outputDir,true); + FileSystem.get(conf).delete(outputDir); JobClient.runJob(conf); } Index: sample/hadoop-0.16/tw/org/nchc/demo/LogFetcher.java =================================================================== --- sample/hadoop-0.16/tw/org/nchc/demo/LogFetcher.java (revision 23) +++ sample/hadoop-0.16/tw/org/nchc/demo/LogFetcher.java (revision 25) @@ -23,4 +23,5 @@ import java.text.ParseException; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -40,6 +41,4 @@ import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reporter; - -import tw.org.nchc.code.Convert; /** @@ -96,5 +95,15 @@ } } - + static public Path[] listPaths(FileSystem fsm,Path path) throws IOException + { + FileStatus[] fss = fsm.listStatus(path); + int length = fss.length; + Path[] pi = new Path[length]; + for (int i=0 ; i< length; i++) + { + pi[i] = fss[i].getPath(); + } + return pi; + } public static void runMapReduce(String table, String dir) throws IOException { @@ -106,23 +115,17 @@ jobConf.set(TABLE, table); // my convert function from 0.16 to 0.17 - Path[] in = Convert.listPaths(fs, InputDir); + Path[] in = listPaths(fs, InputDir); if (fs.isFile(InputDir)) { - // 0.16 -// jobConf.setInputPath(InputDir); - Convert.setInputPath(jobConf, InputDir); + jobConf.setInputPath(InputDir); } else { for (int i = 0; i < in.length; i++) { if (fs.isFile(in[i])) { - // 0.16 -// jobConf.addInputPath(in[i]); - Convert.addInputPath(jobConf,in[i]); + jobConf.addInputPath(in[i]); } else { // my convert function from 0.16 to 0.17 - Path[] sub = Convert.listPaths(fs, in[i]); + Path[] sub = listPaths(fs, in[i]); for (int j = 0; j < sub.length; j++) { if (fs.isFile(sub[j])) { - // 0.16 -// jobConf.addInputPath(sub[j]); - Convert.addInputPath(jobConf, sub[j]); + jobConf.addInputPath(sub[j]); } } @@ -130,7 +133,5 @@ } } - // 0.16 -// jobConf.setOutputPath(tempDir); - Convert.setOutputPath(jobConf, tempDir); + jobConf.setOutputPath(tempDir); jobConf.setMapperClass(MapClass.class); @@ -142,8 +143,6 @@ JobClient.runJob(jobConf); - // 0.16 -// fs.delete(tempDir); - fs.delete(tempDir,true); - + + fs.delete(tempDir); fs.close(); } Index: sample/hadoop-0.16/tw/org/nchc/util/SequenceFileProcessor.java =================================================================== --- sample/hadoop-0.16/tw/org/nchc/util/SequenceFileProcessor.java (revision 23) +++ sample/hadoop-0.16/tw/org/nchc/util/SequenceFileProcessor.java (revision 25) @@ -1,11 +1,17 @@ -/** - * Program: BuildHTable.java - * Editor: Waue Chen - * From : NCHC. Taiwn - * Last Update Date: 07/02/2008 - * Upgrade to 0.17 - * Re-code from : Cloud9: A MapReduce Library for Hadoop +/* + * Cloud9: A MapReduce Library for Hadoop + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. */ - package tw.org.nchc.util; @@ -13,5 +19,4 @@ import java.io.IOException; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -22,5 +27,4 @@ /** - * Upgrade from hadoop 0.16 to 0.17 *

* Harness for processing one or more {@link SequenceFile}s within a single @@ -115,10 +119,5 @@ private void run() throws IOException { if (!FileSystem.get(conf).isFile(mPath)) { - Path[] pa = new Path[] { mPath }; - Path p; - // hadoop 0.17 -> listStatus(); - FileStatus[] fi = FileSystem.get(conf).listStatus(pa); - for (int i =0 ; i