Changeset 25 for sample/hadoop-0.16/tw/org/nchc/demo/LogFetcher.java

Timestamp:

Jul 3, 2008, 4:45:54 PM (16 years ago)

Author:

waue

Message:

downgrade from 0.17 to 0.16
test for work -> not yet

File:

• sample/hadoop-0.16/tw/org/nchc/demo/LogFetcher.java (modified) (6 diffs)

sample/hadoop-0.16/tw/org/nchc/demo/LogFetcher.java

@@ -23 +23 @@
 import java.text.ParseException;
 
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -40 +41 @@
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
-
-import tw.org.nchc.code.Convert;
 
 /**
@@ -96 +95 @@
     }
   }
-
+  static public Path[] listPaths(FileSystem fsm,Path path) throws IOException
+  {
+    FileStatus[] fss = fsm.listStatus(path);
+    int length = fss.length;
+    Path[] pi = new Path[length];
+    for (int i=0 ; i< length; i++)
+    {
+      pi[i] = fss[i].getPath();
+    }
+    return pi;
+  } 
   public static void runMapReduce(String table, String dir)
       throws IOException {
@@ -106 +115 @@
     jobConf.set(TABLE, table);
     // my convert function from 0.16 to 0.17
-    Path[] in = Convert.listPaths(fs, InputDir);
+    Path[] in = listPaths(fs, InputDir);
     if (fs.isFile(InputDir)) {
-      // 0.16
-//      jobConf.setInputPath(InputDir);
-      Convert.setInputPath(jobConf, InputDir);
+      jobConf.setInputPath(InputDir);
     } else {
       for (int i = 0; i < in.length; i++) {
         if (fs.isFile(in[i])) {
-          // 0.16
-//          jobConf.addInputPath(in[i]);
-          Convert.addInputPath(jobConf,in[i]);
+          jobConf.addInputPath(in[i]);
         } else {
           // my convert function from 0.16 to 0.17
-          Path[] sub = Convert.listPaths(fs, in[i]);
+          Path[] sub = listPaths(fs, in[i]);
           for (int j = 0; j < sub.length; j++) {
             if (fs.isFile(sub[j])) {
-              // 0.16
-//              jobConf.addInputPath(sub[j]);
-              Convert.addInputPath(jobConf, sub[j]);
+              jobConf.addInputPath(sub[j]);
             }
           }
@@ -130 +133 @@
       }
     }
-    // 0.16
-//    jobConf.setOutputPath(tempDir);
-    Convert.setOutputPath(jobConf, tempDir);
+    jobConf.setOutputPath(tempDir);
     
     jobConf.setMapperClass(MapClass.class);
@@ -142 +143 @@
 
     JobClient.runJob(jobConf);
-    // 0.16
-//    fs.delete(tempDir);
-    fs.delete(tempDir,true);
-    
+
+    fs.delete(tempDir);   
     fs.close();
   }