Changeset 25 for sample/hadoop-0.16/tw/org/nchc/demo

Timestamp:

Jul 3, 2008, 4:45:54 PM (16 years ago)

Author:

waue

Message:

downgrade from 0.17 to 0.16
test for work -> not yet

Location:

sample/hadoop-0.16/tw/org/nchc/demo

Files:

• DemoHBaseSink$ReduceClass.class (modified) (previous)
• DemoHBaseSource$MapClass.class (modified) (previous)
• DemoHBaseSource$ReduceClass.class (modified) (previous)
• DemoWordCondProb$MapClass.class (modified) (previous)
• DemoWordCondProb$MyPartitioner.class (modified) (previous)
• DemoWordCondProb$ReduceClass.class (modified) (previous)
• DemoWordCount$MapClass.class (modified) (previous)
• DemoWordCount$ReduceClass.class (modified) (previous)
• DemoWordCount.class (modified) (previous)
• DemoWordCount.java (modified) (3 diffs)
• DemoWordCountTuple$MapClass.class (modified) (previous)
• DemoWordCountTuple$ReduceClass.class (modified) (previous)
• DemoWordCountTuple2$MapClass.class (modified) (previous)
• DemoWordCountTuple2$ReduceClass.class (modified) (previous)
• LogFetcher$MapClass.class (modified) (previous)
• LogFetcher.class (modified) (previous)
• LogFetcher.java (modified) (6 diffs)

sample/hadoop-0.16/tw/org/nchc/demo/DemoWordCount.java

@@ -28 +28 @@
 import org.apache.hadoop.mapred.Reducer;
 import org.apache.hadoop.mapred.Reporter;
-
-import tw.org.nchc.code.Convert;
 
 /**
@@ -107 +105 @@
     conf.setNumMapTasks(mapTasks);
     conf.setNumReduceTasks(reduceTasks);
-    //0.16
-//    conf.setInputPath(new Path(filename));
-    Convert.setInputPath(conf, new Path(filename));
+
+    conf.setInputPath(new Path(filename));
+
     conf.setOutputKeyClass(Text.class);
     conf.setOutputValueClass(IntWritable.class);
-    // 0.16
-//    conf.setOutputPath(new Path(outputPath));
-    Convert.setInputPath(conf, new Path(outputPath));
+
+    conf.setOutputPath(new Path(outputPath));
     conf.setMapperClass(MapClass.class);
     conf.setCombinerClass(ReduceClass.class);
@@ -121 +118 @@
     // Delete the output directory if it exists already
     Path outputDir = new Path(outputPath);
-    // 0.16
-//    FileSystem.get(conf).delete(outputDir);
-    FileSystem.get(conf).delete(outputDir,true);
+    FileSystem.get(conf).delete(outputDir);
     JobClient.runJob(conf);
   }

sample/hadoop-0.16/tw/org/nchc/demo/LogFetcher.java

@@ -23 +23 @@
 import java.text.ParseException;
 
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -40 +41 @@
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
-
-import tw.org.nchc.code.Convert;
 
 /**
@@ -96 +95 @@
     }
   }
-
+  static public Path[] listPaths(FileSystem fsm,Path path) throws IOException
+  {
+    FileStatus[] fss = fsm.listStatus(path);
+    int length = fss.length;
+    Path[] pi = new Path[length];
+    for (int i=0 ; i< length; i++)
+    {
+      pi[i] = fss[i].getPath();
+    }
+    return pi;
+  } 
   public static void runMapReduce(String table, String dir)
       throws IOException {
@@ -106 +115 @@
     jobConf.set(TABLE, table);
     // my convert function from 0.16 to 0.17
-    Path[] in = Convert.listPaths(fs, InputDir);
+    Path[] in = listPaths(fs, InputDir);
     if (fs.isFile(InputDir)) {
-      // 0.16
-//      jobConf.setInputPath(InputDir);
-      Convert.setInputPath(jobConf, InputDir);
+      jobConf.setInputPath(InputDir);
     } else {
       for (int i = 0; i < in.length; i++) {
         if (fs.isFile(in[i])) {
-          // 0.16
-//          jobConf.addInputPath(in[i]);
-          Convert.addInputPath(jobConf,in[i]);
+          jobConf.addInputPath(in[i]);
         } else {
           // my convert function from 0.16 to 0.17
-          Path[] sub = Convert.listPaths(fs, in[i]);
+          Path[] sub = listPaths(fs, in[i]);
           for (int j = 0; j < sub.length; j++) {
             if (fs.isFile(sub[j])) {
-              // 0.16
-//              jobConf.addInputPath(sub[j]);
-              Convert.addInputPath(jobConf, sub[j]);
+              jobConf.addInputPath(sub[j]);
             }
           }
@@ -130 +133 @@
       }
     }
-    // 0.16
-//    jobConf.setOutputPath(tempDir);
-    Convert.setOutputPath(jobConf, tempDir);
+    jobConf.setOutputPath(tempDir);
     
     jobConf.setMapperClass(MapClass.class);
@@ -142 +143 @@
 
     JobClient.runJob(jobConf);
-    // 0.16
-//    fs.delete(tempDir);
-    fs.delete(tempDir,true);
-    
+
+    fs.delete(tempDir);   
     fs.close();
   }