Changeset 25 for sample/hadoop-0.16/tw/org

Timestamp:

Jul 3, 2008, 4:45:54 PM (16 years ago)

Author:

waue

Message:

downgrade from 0.17 to 0.16
test for work -> not yet

Location:

sample/hadoop-0.16/tw/org/nchc

Files:

• code/HBaseRecordPro.java (modified) (2 diffs)
• code/WordCount.java (modified) (2 diffs)
• code/WordCountFromHBase.java (modified) (1 diff)
• demo/DemoHBaseSink$ReduceClass.class (modified) (previous)
• demo/DemoHBaseSource$MapClass.class (modified) (previous)
• demo/DemoHBaseSource$ReduceClass.class (modified) (previous)
• demo/DemoWordCondProb$MapClass.class (modified) (previous)
• demo/DemoWordCondProb$MyPartitioner.class (modified) (previous)
• demo/DemoWordCondProb$ReduceClass.class (modified) (previous)
• demo/DemoWordCount$MapClass.class (modified) (previous)
• demo/DemoWordCount$ReduceClass.class (modified) (previous)
• demo/DemoWordCount.class (modified) (previous)
• demo/DemoWordCount.java (modified) (3 diffs)
• demo/DemoWordCountTuple$MapClass.class (modified) (previous)
• demo/DemoWordCountTuple$ReduceClass.class (modified) (previous)
• demo/DemoWordCountTuple2$MapClass.class (modified) (previous)
• demo/DemoWordCountTuple2$ReduceClass.class (modified) (previous)
• demo/LogFetcher$MapClass.class (modified) (previous)
• demo/LogFetcher.class (modified) (previous)
• demo/LogFetcher.java (modified) (6 diffs)
• util/SequenceFileProcessor.class (modified) (previous)
• util/SequenceFileProcessor.java (modified) (4 diffs)

sample/hadoop-0.16/tw/org/nchc/code/HBaseRecordPro.java

@@ -228 +228 @@
     conf.setNumMapTasks(mapTasks);
     conf.setNumReduceTasks(reduceTasks);
-    // 0.16
-//    conf.setInputPath(text_path);
-    Convert.setInputPath(conf, text_path);
+    
+    conf.setInputPath(text_path);
+
     
     conf.setMapperClass(IdentityMapper.class);
@@ -240 +240 @@
     // delete tmp file
     // 0.16
-//    FileSystem.get(conf).delete(text_path);
-    FileSystem.get(conf).delete(text_path,true);
+    FileSystem.get(conf).delete(text_path);
     
     setup.deleteFile(conf_tmp);

sample/hadoop-0.16/tw/org/nchc/code/WordCount.java

@@ -110 +110 @@
     conf.setNumMapTasks(mapTasks);
     conf.setNumReduceTasks(reduceTasks);
-    // 0.16
-    // conf.setInputPath(new Path(wc.filepath));
-    Convert.setInputPath(conf, new Path(wc.filepath));
+    
+    conf.setInputPath(new Path(wc.filepath));
+
     conf.setOutputKeyClass(Text.class);
     conf.setOutputValueClass(IntWritable.class);
-    // 0.16
-    // conf.setOutputPath(new Path(wc.outputPath));
-    Convert.setOutputPath(conf, new Path(wc.outputPath));
+
+    conf.setOutputPath(new Path(wc.outputPath));
 
     conf.setMapperClass(MapClass.class);
@@ -125 +124 @@
     // Delete the output directory if it exists already
     Path outputDir = new Path(wc.outputPath);
-    // 0.16
-    FileSystem.get(conf).delete(outputDir,true);
-
+    FileSystem.get(conf).delete(outputDir);
     JobClient.runJob(conf);
   }

sample/hadoop-0.16/tw/org/nchc/code/WordCountFromHBase.java

@@ -168 +168 @@
     // input is Hbase format => TableInputFormat
     conf.setInputFormat(TableInputFormat.class);
-    // 0.16
-//    conf.setOutputPath(new Path(outputPath));
-    Convert.setOutputPath(conf, new Path(outputPath));
+    conf.setOutputPath(new Path(outputPath));
 //     delete the old path with the same name 
-    FileSystem.get(conf).delete(new Path(outputPath),true);
+    FileSystem.get(conf).delete(new Path(outputPath));
     JobClient.runJob(conf);
   }

sample/hadoop-0.16/tw/org/nchc/demo/DemoWordCount.java

@@ -28 +28 @@
 import org.apache.hadoop.mapred.Reducer;
 import org.apache.hadoop.mapred.Reporter;
-
-import tw.org.nchc.code.Convert;
 
 /**
@@ -107 +105 @@
     conf.setNumMapTasks(mapTasks);
     conf.setNumReduceTasks(reduceTasks);
-    //0.16
-//    conf.setInputPath(new Path(filename));
-    Convert.setInputPath(conf, new Path(filename));
+
+    conf.setInputPath(new Path(filename));
+
     conf.setOutputKeyClass(Text.class);
     conf.setOutputValueClass(IntWritable.class);
-    // 0.16
-//    conf.setOutputPath(new Path(outputPath));
-    Convert.setInputPath(conf, new Path(outputPath));
+
+    conf.setOutputPath(new Path(outputPath));
     conf.setMapperClass(MapClass.class);
     conf.setCombinerClass(ReduceClass.class);
@@ -121 +118 @@
     // Delete the output directory if it exists already
     Path outputDir = new Path(outputPath);
-    // 0.16
-//    FileSystem.get(conf).delete(outputDir);
-    FileSystem.get(conf).delete(outputDir,true);
+    FileSystem.get(conf).delete(outputDir);
     JobClient.runJob(conf);
   }

sample/hadoop-0.16/tw/org/nchc/demo/LogFetcher.java

@@ -23 +23 @@
 import java.text.ParseException;
 
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -40 +41 @@
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
-
-import tw.org.nchc.code.Convert;
 
 /**
@@ -96 +95 @@
     }
   }
-
+  static public Path[] listPaths(FileSystem fsm,Path path) throws IOException
+  {
+    FileStatus[] fss = fsm.listStatus(path);
+    int length = fss.length;
+    Path[] pi = new Path[length];
+    for (int i=0 ; i< length; i++)
+    {
+      pi[i] = fss[i].getPath();
+    }
+    return pi;
+  } 
   public static void runMapReduce(String table, String dir)
       throws IOException {
@@ -106 +115 @@
     jobConf.set(TABLE, table);
     // my convert function from 0.16 to 0.17
-    Path[] in = Convert.listPaths(fs, InputDir);
+    Path[] in = listPaths(fs, InputDir);
     if (fs.isFile(InputDir)) {
-      // 0.16
-//      jobConf.setInputPath(InputDir);
-      Convert.setInputPath(jobConf, InputDir);
+      jobConf.setInputPath(InputDir);
     } else {
       for (int i = 0; i < in.length; i++) {
         if (fs.isFile(in[i])) {
-          // 0.16
-//          jobConf.addInputPath(in[i]);
-          Convert.addInputPath(jobConf,in[i]);
+          jobConf.addInputPath(in[i]);
         } else {
           // my convert function from 0.16 to 0.17
-          Path[] sub = Convert.listPaths(fs, in[i]);
+          Path[] sub = listPaths(fs, in[i]);
           for (int j = 0; j < sub.length; j++) {
             if (fs.isFile(sub[j])) {
-              // 0.16
-//              jobConf.addInputPath(sub[j]);
-              Convert.addInputPath(jobConf, sub[j]);
+              jobConf.addInputPath(sub[j]);
             }
           }
@@ -130 +133 @@
       }
     }
-    // 0.16
-//    jobConf.setOutputPath(tempDir);
-    Convert.setOutputPath(jobConf, tempDir);
+    jobConf.setOutputPath(tempDir);
     
     jobConf.setMapperClass(MapClass.class);
@@ -142 +143 @@
 
     JobClient.runJob(jobConf);
-    // 0.16
-//    fs.delete(tempDir);
-    fs.delete(tempDir,true);
-    
+
+    fs.delete(tempDir);   
     fs.close();
   }

sample/hadoop-0.16/tw/org/nchc/util/SequenceFileProcessor.java

@@ -1 @@
-/**
- * Program: BuildHTable.java
- * Editor: Waue Chen 
- * From :  NCHC. Taiwn
- * Last Update Date: 07/02/2008
- * Upgrade to 0.17
- * Re-code from : Cloud9: A MapReduce Library for Hadoop
+/*
+ * Cloud9: A MapReduce Library for Hadoop
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0 
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
  */
-
 
 package tw.org.nchc.util;
@@ -13 +19 @@
 import java.io.IOException;
 
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -22 +27 @@
 
 /**
- * Upgrade from hadoop 0.16 to 0.17 
  * <p>
  * Harness for processing one or more {@link SequenceFile}s within a single
@@ -115 +119 @@
   private void run() throws IOException {
     if (!FileSystem.get(conf).isFile(mPath)) {
-      Path[] pa = new Path[] { mPath };
-      Path p;
-      // hadoop 0.17 -> listStatus();
-      FileStatus[] fi = FileSystem.get(conf).listStatus(pa);
-      for (int i =0 ; i<fi.length ; i++) {
-        p = fi[i].getPath();
+      for (Path p : FileSystem.get(conf).listPaths(new Path[] { mPath })) {
         // System.out.println("Applying to " + p);
         applyToFile(p);