Changeset 31 for sample/hadoop-0.16/tw/org

Timestamp:

Jul 17, 2008, 3:35:18 PM (16 years ago)

Author:

waue

Message:

update some new ..

Location:

sample/hadoop-0.16/tw/org/nchc

Files:

• code/LogParser.java (modified) (3 diffs)
• code/LogParserGo.java (modified) (10 diffs)
• code/WordCount$MapClass.class (modified) (previous)
• code/WordCount$ReduceClass.class (modified) (previous)
• code/WordCount.class (modified) (previous)
• code/WordCount.java (modified) (4 diffs)
• code/WordCountIntoHBase.class (modified) (previous)
• code/WordCountIntoHBase.java (modified) (1 diff)
• demo/LogFetcher$MapClass.class (modified) (previous)
• demo/LogFetcher.class (modified) (previous)
• demo/LogFetcher.java (modified) (1 diff)
• util/SequenceFileProcessor.class (modified) (previous)

sample/hadoop-0.16/tw/org/nchc/code/LogParser.java

@@ +1 @@
+/**
+ * Program: LogParser.java
+ * Editor: Waue Chen 
+ * From :  NCHC. Taiwn
+ * Last Update Date: 07/02/2008
+ */
+
 package tw.org.nchc.code;
 
@@ -10 +17 @@
 
 
-public class AccessLogParser {
+public class LogParser {
   private String ip;
   private String protocol;
@@ -27 +34 @@
   
   
-  public AccessLogParser(String line) throws ParseException, Exception{
+  public LogParser(String line) throws ParseException, Exception{
    
    Matcher matcher = p.matcher(line);

sample/hadoop-0.16/tw/org/nchc/code/LogParserGo.java

@@ -1 +1 @@
 /**
- * Program: LogFetcher.java
+ * Program: LogParserGo.java
  * Editor: Waue Chen 
  * From :  NCHC. Taiwn
@@ -6 +6 @@
  */
 /**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Purpose : 
+ *  This program will parse your apache log and store it into Hbase.
+ * 
+ * HowToUse : 
+ *  Make sure two thing :
+ *  1. Upload apache logs ( /var/log/apache2/access.log* ) to \ 
+ *    hdfs (default: /user/waue/apache-log) \
+ *   $ bin/hadoop dfs -put /var/log/apache2/ apache-log
+ *  2. parameter "dir" in main contains the logs.
+ *  3. you should filter the exception contents manually, \ 
+ *    ex:  ::1 - - [29/Jun/2008:07:35:15 +0800] "GET / HTTP/1.0" 200 729 "...
+ *  
+ * Check Result:
+ *  Go to hbase console, type : 
+ *    hql > select * from apache-log;
+
++-------------------------+-------------------------+-------------------------+
+| Row                     | Column                  | Cell                    |
++-------------------------+-------------------------+-------------------------+
+| 118.170.101.250         | http:agent              | Mozilla/4.0 (compatible;|
+|                         |                         |  MSIE 4.01; Windows 95) |
++-------------------------+-------------------------+-------------------------+
+| 118.170.101.250         | http:bytesize           | 318                     |
++-------------------------+-------------------------+-------------------------+
+..........(skip)........
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | http:method             | OPTIONS                 |
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | http:protocol           | HTTP/1.1                |
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | referrer:-              | *                       |
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | url:*                   | -                       |
++-------------------------+-------------------------+-------------------------+
+31 row(s) in set. (0.58 sec)
+
+
+
  */
 package tw.org.nchc.code;
 
 import java.io.IOException;
-import java.text.ParseException;
 
 import org.apache.hadoop.fs.FileStatus;
@@ -47 +67 @@
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
+
 // import AccessLogParser
 /**
@@ -52 +73 @@
  * W3CExtended, IISw3cExtended)
  */
-public class LogFetcher {
+public class LogParserGo {
   static HBaseConfiguration conf = new HBaseConfiguration();
 
@@ -60 +81 @@
 
   static HTable table = null;
-
-  static boolean eclipseRun = false;
-
+  
+  static void print(String str){
+    System.out.println("STR  = "+str);
+  }
   public static class MapClass extends MapReduceBase implements
       Mapper<WritableComparable, Text, Text, Writable> {
 
     @Override
+    // MapReduceBase.configure(JobConf job) 
+    // Default implementation that does nothing.
     public void configure(JobConf job) {
+      // String get(String name,String defaultValue) 
+      // Get the value of the name property. If no such property exists,\
+      //  then defaultValue is returned.
       tableName = job.get(TABLE, "");
     }
@@ -74 +101 @@
         OutputCollector<Text, Writable> output, Reporter reporter)
         throws IOException {
+      
       try {
+        /*
+        print(value.toString());
+        FileWriter out = new FileWriter(new File(
+        "/home/waue/mr-result.txt"));
+        out.write(value.toString());
+        out.flush();
+        out.close();
+        */
+        LogParser log = new LogParser(value.toString());
         
-        AccessLogParser log = new AccessLogParser(value.toString());
         if (table == null)
           table = new HTable(conf, new Text(tableName));
@@ -95 +131 @@
             log.getUrl().getBytes());
         table.commit(lockId, log.getTimestamp());
-      } catch (ParseException e) {
-        e.printStackTrace();
+        
       } catch (Exception e) {
         e.printStackTrace();
       }
-    }
-  }
-//   do it to resolve warning : FileSystem.listPaths 
-  static public Path[] listPaths(FileSystem fsm,Path path) throws IOException
-  {
+      
+    }
+  }
+
+  // do it to resolve warning : FileSystem.listPaths
+  static public Path[] listPaths(FileSystem fsm, Path path)
+      throws IOException {
     FileStatus[] fss = fsm.listStatus(path);
     int length = fss.length;
     Path[] pi = new Path[length];
-    for (int i=0 ; i< length; i++)
-    {
+    for (int i = 0; i < length; i++) {
       pi[i] = fss[i].getPath();
     }
     return pi;
-  } 
+  }
+
   public static void runMapReduce(String table, String dir)
       throws IOException {
-    Path tempDir = new Path("log/temp");
+    Path tempDir = new Path("/tmp/Mylog/");
     Path InputDir = new Path(dir);
     FileSystem fs = FileSystem.get(conf);
-    JobConf jobConf = new JobConf(conf, LogFetcher.class);
+    JobConf jobConf = new JobConf(conf, LogParserGo.class);
     jobConf.setJobName("apache log fetcher");
     jobConf.set(TABLE, table);
@@ -140 +177 @@
     }
     jobConf.setOutputPath(tempDir);
-    
+
     jobConf.setMapperClass(MapClass.class);
 
@@ -150 +187 @@
     JobClient.runJob(jobConf);
 
-    fs.delete(tempDir);   
+    fs.delete(tempDir);
     fs.close();
   }
@@ -171 +208 @@
 
   public static void main(String[] args) throws IOException {
-    String table_name = "log";
-    String dir = "apache-log";
-
-    if (eclipseRun) {
-      table_name = "log";
-      dir = "apache-log";
-    } else if (args.length < 2) {
-      System.out
-          .println("Usage: logfetcher <access_log file or directory> <table_name>");
-      System.exit(1);
-    } else {
-      table_name = args[1];
-      dir = args[0];
-    }
+    String table_name = "apache-log2";
+    String dir = "/user/waue/apache-log";
+    
+    // if (eclipseRun) {
+    // table_name = "log";
+    // dir = "apache-log";
+    // } else if (args.length < 2) {
+    // System.out
+    // .println("Usage: logfetcher <access_log file or directory>
+    // <table_name>");
+    // System.exit(1);
+    // } else {
+    // table_name = args[1];
+    // dir = args[0];
+    // }
+
     creatTable(table_name);
     runMapReduce(table_name, dir);

sample/hadoop-0.16/tw/org/nchc/code/WordCount.java

@@ -56 +56 @@
 
   // mapper: emits (token, 1) for every word occurrence
-  private static class MapClass extends MapReduceBase implements
-      Mapper<LongWritable, Text, Text, IntWritable> {
+  private static class MapClass extends MapReduceBase 
+  implements Mapper<LongWritable, Text, Text, IntWritable> 
+  {
 
     // reuse objects to save overhead of object creation
@@ -77 +78 @@
 
   // reducer: sums up all the counts
-  private static class ReduceClass extends MapReduceBase implements
-      Reducer<Text, IntWritable, Text, IntWritable> {
+  private static class ReduceClass extends MapReduceBase 
+  implements Reducer<Text, IntWritable, Text, IntWritable> 
+  {
 
     // reuse objects
@@ -105 +107 @@
     int reduceTasks = 1;
     JobConf conf = new JobConf(WordCount.class);
-    conf.setJobName("wordcount");
+//    conf.setJobName("wordcount");
 
     conf.setNumMapTasks(mapTasks);
@@ -118 +120 @@
 
     conf.setMapperClass(MapClass.class);
-    conf.setCombinerClass(ReduceClass.class);
+//    conf.setCombinerClass(ReduceClass.class);
     conf.setReducerClass(ReduceClass.class);
 

sample/hadoop-0.16/tw/org/nchc/code/WordCountIntoHBase.java

@@ -45 +45 @@
   // $Input_Path. Please make sure the path is correct and contains input
   // files
-  static final String Input_Path = "/user/waue/simple";
+  static final String Input_Path = "/user/waue/input";
 
   // Hbase table name, the program will create it

sample/hadoop-0.16/tw/org/nchc/demo/LogFetcher.java

@@ -6 +6 @@
  */
 /**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Purpose : 
+ *  This program will parse your apache log and store it into Hbase.
+ * 
+ * HowToUse : 
+ *  Make sure two thing :
+ *  1. Upload apache logs ( /var/log/apache2/access.log* ) to \ 
+ *    hdfs (default: /user/waue/apache-log) \
+ *   $ bin/hadoop dfs -put /var/log/apache2/ apache-log
+ *  2. parameter "dir" in main contains the logs.
+ *  3. you should filter the exception contents manually, \ 
+ *    ex:  ::1 - - [29/Jun/2008:07:35:15 +0800] "GET / HTTP/1.0" 200 729 "...
+ *  
+ * Check Result:
+ *  Go to hbase console, type : 
+ *    hql > select * from apache-log;
+
++-------------------------+-------------------------+-------------------------+
+| Row                     | Column                  | Cell                    |
++-------------------------+-------------------------+-------------------------+
+| 118.170.101.250         | http:agent              | Mozilla/4.0 (compatible;|
+|                         |                         |  MSIE 4.01; Windows 95) |
++-------------------------+-------------------------+-------------------------+
+| 118.170.101.250         | http:bytesize           | 318                     |
++-------------------------+-------------------------+-------------------------+
+..........(skip)........
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | http:method             | OPTIONS                 |
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | http:protocol           | HTTP/1.1                |
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | referrer:-              | *                       |
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | url:*                   | -                       |
++-------------------------+-------------------------+-------------------------+
+31 row(s) in set. (0.58 sec)
+
  */
+
+
 package tw.org.nchc.demo;