Changeset 31

Timestamp:

Jul 17, 2008, 3:35:18 PM (16 years ago)

Author:

waue

Message:

update some new ..

Location:

sample/hadoop-0.16

Files:

• test.class (modified) (previous)
• test.java (modified) (7 diffs)
• testb.class (modified) (previous)
• tw/org/nchc/code/LogParser.java (modified) (3 diffs)
• tw/org/nchc/code/LogParserGo.java (modified) (10 diffs)
• tw/org/nchc/code/WordCount$MapClass.class (modified) (previous)
• tw/org/nchc/code/WordCount$ReduceClass.class (modified) (previous)
• tw/org/nchc/code/WordCount.class (modified) (previous)
• tw/org/nchc/code/WordCount.java (modified) (4 diffs)
• tw/org/nchc/code/WordCountIntoHBase.class (modified) (previous)
• tw/org/nchc/code/WordCountIntoHBase.java (modified) (1 diff)
• tw/org/nchc/demo/LogFetcher$MapClass.class (modified) (previous)
• tw/org/nchc/demo/LogFetcher.class (modified) (previous)
• tw/org/nchc/demo/LogFetcher.java (modified) (1 diff)
• tw/org/nchc/util/SequenceFileProcessor.class (modified) (previous)

sample/hadoop-0.16/test.java

@@ -8 +8 @@
 import java.io.RandomAccessFile;
 import java.util.StringTokenizer;
+
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseAdmin;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.JobConf;
 
@@ -27 +23 @@
     System.out.println(test.sna);
   }
-
 }
 
 public class test {
   int na;
-
   static int sna;
-
   static String str;
-
   public test() {
-
   }
 
@@ -49 +40 @@
     sna = b;
     str = st;
-
   }
 
@@ -108 +98 @@
     } while (true);
     fw.close();
-
     return first_line;
   }
@@ -114 +103 @@
   boolean deleteFile(String str) throws IOException {
     File ttt = new File(str);
-
     if (ttt.exists()) {
       if (!ttt.delete()) {
@@ -122 +110 @@
       System.out.println("file not exit!");
     }
-
     return true;
   }
@@ -143 +130 @@
     // "/home/waue/out.tmp.txt"));
     // ttt();
+    
+    
+    animal a = new animal();
+    bird b = new bird();
+    dog d = new dog(b);
+    animal aaa = new bird();
+//    dog ddd = new animal();  -> error 
+    
+    animal aa = (animal) b;
+    System.out.println(aa.hand);
+    System.out.println(aa.mind);    
+//    bird bb = (bird) a; -> error
+//    d.setbird(aa); -> error
+//    System.out.println(aa.feather); -> error
 
-
-    // setup Table name
-    String table_name = "test_create_table2";
-    Text text_table_name = new Text(table_name);
-    // setup Column Family
-    String[] Column_Family = { "http:", "url:", "referrer:" };
-    HBaseConfiguration conf = new HBaseConfiguration();
-
-    HBaseAdmin admin = new HBaseAdmin(conf);
-    boolean b = admin.tableExists(text_table_name);
-    System.out.println(b);
-
-    if (!admin.tableExists(text_table_name)) {
-
-      System.out.println("HTable : " + table_name
-          + "  creating ... please wait");
-      HTableDescriptor tableDesc = new HTableDescriptor(table_name);
-      for (int i = 0; i < Column_Family.length; i++) {
-        String st = Column_Family[i];
-        // check name format "string:"
-        if (!st.endsWith(":")) {
-          Column_Family[i] = st + ":";
-          System.out.println("normize :" + st + "->"
-              + Column_Family[i]);
-        }
-        // add column family
-        tableDesc.addFamily(new HColumnDescriptor(Column_Family[i]));
-      }
-      admin.createTable(tableDesc);
-    } else {
-      System.out.println("table exist!");
-    }
-
-
+    // Fu <T>
+    Fu<? extends String> ak;
+    ak = new Fu<String>();
+    ak.getFu();
+    Fu<?> in = ak;
+//    in.setFu("ss"); -> error
+    in.getFu();
   }
 }
+class Fu <T>{
+  T go;
+  void setFu(T go){
+    this.go = go;
+  }
+  T getFu(){
+    return go;
+  }
+}
+
+class animal{
+  String head = "head";
+  String feet = "feet";
+  String body = "body";
+  String hand = "head";
+  String mind = "mind";
+  int id = 1234567;
+}
+class bird extends animal{
+  String feather = "feather";
+  bird(){
+    hand = "wing";
+    id = 1234568;
+    System.out.println(head + hand + body + feet + "id="+id);
+    System.out.println("super = " + super.hand +"this = " + this.hand);
+  }
+}
+class dog extends animal{
+  dog(animal a){
+    System.out.println(a.head + hand + body + feet + "id="+id);
+  }
+  void setbird(bird b){
+    System.out.println(b.head + hand + body + feet + "id="+id);
+  }
+}

sample/hadoop-0.16/tw/org/nchc/code/LogParser.java

@@ +1 @@
+/**
+ * Program: LogParser.java
+ * Editor: Waue Chen 
+ * From :  NCHC. Taiwn
+ * Last Update Date: 07/02/2008
+ */
+
 package tw.org.nchc.code;
 
@@ -10 +17 @@
 
 
-public class AccessLogParser {
+public class LogParser {
   private String ip;
   private String protocol;
@@ -27 +34 @@
   
   
-  public AccessLogParser(String line) throws ParseException, Exception{
+  public LogParser(String line) throws ParseException, Exception{
    
    Matcher matcher = p.matcher(line);

sample/hadoop-0.16/tw/org/nchc/code/LogParserGo.java

@@ -1 +1 @@
 /**
- * Program: LogFetcher.java
+ * Program: LogParserGo.java
  * Editor: Waue Chen 
  * From :  NCHC. Taiwn
@@ -6 +6 @@
  */
 /**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Purpose : 
+ *  This program will parse your apache log and store it into Hbase.
+ * 
+ * HowToUse : 
+ *  Make sure two thing :
+ *  1. Upload apache logs ( /var/log/apache2/access.log* ) to \ 
+ *    hdfs (default: /user/waue/apache-log) \
+ *   $ bin/hadoop dfs -put /var/log/apache2/ apache-log
+ *  2. parameter "dir" in main contains the logs.
+ *  3. you should filter the exception contents manually, \ 
+ *    ex:  ::1 - - [29/Jun/2008:07:35:15 +0800] "GET / HTTP/1.0" 200 729 "...
+ *  
+ * Check Result:
+ *  Go to hbase console, type : 
+ *    hql > select * from apache-log;
+
++-------------------------+-------------------------+-------------------------+
+| Row                     | Column                  | Cell                    |
++-------------------------+-------------------------+-------------------------+
+| 118.170.101.250         | http:agent              | Mozilla/4.0 (compatible;|
+|                         |                         |  MSIE 4.01; Windows 95) |
++-------------------------+-------------------------+-------------------------+
+| 118.170.101.250         | http:bytesize           | 318                     |
++-------------------------+-------------------------+-------------------------+
+..........(skip)........
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | http:method             | OPTIONS                 |
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | http:protocol           | HTTP/1.1                |
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | referrer:-              | *                       |
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | url:*                   | -                       |
++-------------------------+-------------------------+-------------------------+
+31 row(s) in set. (0.58 sec)
+
+
+
  */
 package tw.org.nchc.code;
 
 import java.io.IOException;
-import java.text.ParseException;
 
 import org.apache.hadoop.fs.FileStatus;
@@ -47 +67 @@
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
+
 // import AccessLogParser
 /**
@@ -52 +73 @@
  * W3CExtended, IISw3cExtended)
  */
-public class LogFetcher {
+public class LogParserGo {
   static HBaseConfiguration conf = new HBaseConfiguration();
 
@@ -60 +81 @@
 
   static HTable table = null;
-
-  static boolean eclipseRun = false;
-
+  
+  static void print(String str){
+    System.out.println("STR  = "+str);
+  }
   public static class MapClass extends MapReduceBase implements
       Mapper<WritableComparable, Text, Text, Writable> {
 
     @Override
+    // MapReduceBase.configure(JobConf job) 
+    // Default implementation that does nothing.
     public void configure(JobConf job) {
+      // String get(String name,String defaultValue) 
+      // Get the value of the name property. If no such property exists,\
+      //  then defaultValue is returned.
       tableName = job.get(TABLE, "");
     }
@@ -74 +101 @@
         OutputCollector<Text, Writable> output, Reporter reporter)
         throws IOException {
+      
       try {
+        /*
+        print(value.toString());
+        FileWriter out = new FileWriter(new File(
+        "/home/waue/mr-result.txt"));
+        out.write(value.toString());
+        out.flush();
+        out.close();
+        */
+        LogParser log = new LogParser(value.toString());
         
-        AccessLogParser log = new AccessLogParser(value.toString());
         if (table == null)
           table = new HTable(conf, new Text(tableName));
@@ -95 +131 @@
             log.getUrl().getBytes());
         table.commit(lockId, log.getTimestamp());
-      } catch (ParseException e) {
-        e.printStackTrace();
+        
       } catch (Exception e) {
         e.printStackTrace();
       }
-    }
-  }
-//   do it to resolve warning : FileSystem.listPaths 
-  static public Path[] listPaths(FileSystem fsm,Path path) throws IOException
-  {
+      
+    }
+  }
+
+  // do it to resolve warning : FileSystem.listPaths
+  static public Path[] listPaths(FileSystem fsm, Path path)
+      throws IOException {
     FileStatus[] fss = fsm.listStatus(path);
     int length = fss.length;
     Path[] pi = new Path[length];
-    for (int i=0 ; i< length; i++)
-    {
+    for (int i = 0; i < length; i++) {
       pi[i] = fss[i].getPath();
     }
     return pi;
-  } 
+  }
+
   public static void runMapReduce(String table, String dir)
       throws IOException {
-    Path tempDir = new Path("log/temp");
+    Path tempDir = new Path("/tmp/Mylog/");
     Path InputDir = new Path(dir);
     FileSystem fs = FileSystem.get(conf);
-    JobConf jobConf = new JobConf(conf, LogFetcher.class);
+    JobConf jobConf = new JobConf(conf, LogParserGo.class);
     jobConf.setJobName("apache log fetcher");
     jobConf.set(TABLE, table);
@@ -140 +177 @@
     }
     jobConf.setOutputPath(tempDir);
-    
+
     jobConf.setMapperClass(MapClass.class);
 
@@ -150 +187 @@
     JobClient.runJob(jobConf);
 
-    fs.delete(tempDir);   
+    fs.delete(tempDir);
     fs.close();
   }
@@ -171 +208 @@
 
   public static void main(String[] args) throws IOException {
-    String table_name = "log";
-    String dir = "apache-log";
-
-    if (eclipseRun) {
-      table_name = "log";
-      dir = "apache-log";
-    } else if (args.length < 2) {
-      System.out
-          .println("Usage: logfetcher <access_log file or directory> <table_name>");
-      System.exit(1);
-    } else {
-      table_name = args[1];
-      dir = args[0];
-    }
+    String table_name = "apache-log2";
+    String dir = "/user/waue/apache-log";
+    
+    // if (eclipseRun) {
+    // table_name = "log";
+    // dir = "apache-log";
+    // } else if (args.length < 2) {
+    // System.out
+    // .println("Usage: logfetcher <access_log file or directory>
+    // <table_name>");
+    // System.exit(1);
+    // } else {
+    // table_name = args[1];
+    // dir = args[0];
+    // }
+
     creatTable(table_name);
     runMapReduce(table_name, dir);

sample/hadoop-0.16/tw/org/nchc/code/WordCount.java

@@ -56 +56 @@
 
   // mapper: emits (token, 1) for every word occurrence
-  private static class MapClass extends MapReduceBase implements
-      Mapper<LongWritable, Text, Text, IntWritable> {
+  private static class MapClass extends MapReduceBase 
+  implements Mapper<LongWritable, Text, Text, IntWritable> 
+  {
 
     // reuse objects to save overhead of object creation
@@ -77 +78 @@
 
   // reducer: sums up all the counts
-  private static class ReduceClass extends MapReduceBase implements
-      Reducer<Text, IntWritable, Text, IntWritable> {
+  private static class ReduceClass extends MapReduceBase 
+  implements Reducer<Text, IntWritable, Text, IntWritable> 
+  {
 
     // reuse objects
@@ -105 +107 @@
     int reduceTasks = 1;
     JobConf conf = new JobConf(WordCount.class);
-    conf.setJobName("wordcount");
+//    conf.setJobName("wordcount");
 
     conf.setNumMapTasks(mapTasks);
@@ -118 +120 @@
 
     conf.setMapperClass(MapClass.class);
-    conf.setCombinerClass(ReduceClass.class);
+//    conf.setCombinerClass(ReduceClass.class);
     conf.setReducerClass(ReduceClass.class);
 

sample/hadoop-0.16/tw/org/nchc/code/WordCountIntoHBase.java

@@ -45 +45 @@
   // $Input_Path. Please make sure the path is correct and contains input
   // files
-  static final String Input_Path = "/user/waue/simple";
+  static final String Input_Path = "/user/waue/input";
 
   // Hbase table name, the program will create it

sample/hadoop-0.16/tw/org/nchc/demo/LogFetcher.java

@@ -6 +6 @@
  */
 /**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Purpose : 
+ *  This program will parse your apache log and store it into Hbase.
+ * 
+ * HowToUse : 
+ *  Make sure two thing :
+ *  1. Upload apache logs ( /var/log/apache2/access.log* ) to \ 
+ *    hdfs (default: /user/waue/apache-log) \
+ *   $ bin/hadoop dfs -put /var/log/apache2/ apache-log
+ *  2. parameter "dir" in main contains the logs.
+ *  3. you should filter the exception contents manually, \ 
+ *    ex:  ::1 - - [29/Jun/2008:07:35:15 +0800] "GET / HTTP/1.0" 200 729 "...
+ *  
+ * Check Result:
+ *  Go to hbase console, type : 
+ *    hql > select * from apache-log;
+
++-------------------------+-------------------------+-------------------------+
+| Row                     | Column                  | Cell                    |
++-------------------------+-------------------------+-------------------------+
+| 118.170.101.250         | http:agent              | Mozilla/4.0 (compatible;|
+|                         |                         |  MSIE 4.01; Windows 95) |
++-------------------------+-------------------------+-------------------------+
+| 118.170.101.250         | http:bytesize           | 318                     |
++-------------------------+-------------------------+-------------------------+
+..........(skip)........
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | http:method             | OPTIONS                 |
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | http:protocol           | HTTP/1.1                |
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | referrer:-              | *                       |
++-------------------------+-------------------------+-------------------------+
+| 87.65.93.58             | url:*                   | -                       |
++-------------------------+-------------------------+-------------------------+
+31 row(s) in set. (0.58 sec)
+
  */
+
+
 package tw.org.nchc.demo;