Index: /sample/hadoop-0.16/test.java =================================================================== --- /sample/hadoop-0.16/test.java (revision 44) +++ /sample/hadoop-0.16/test.java (revision 45) @@ -7,5 +7,7 @@ import java.io.IOException; import java.io.RandomAccessFile; -import java.io.Writer; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Locale; import java.util.StringTokenizer; import java.util.regex.Matcher; @@ -242,5 +244,20 @@ } - public static void main(String[] args) throws IOException { + void enumTest() { + for (EnumTest t : EnumTest.values()) { + + System.out.println(t); + } + + } + + void Date()throws Exception{ + String da = "20/Jul/2008:08:26:55"; + SimpleDateFormat sdf = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss",Locale.US); + Date timestamp = sdf.parse(da); + System.out.println(timestamp.toString()); + } + + public static void main(String[] args) throws Exception { test t = new test(); @@ -259,6 +276,12 @@ // 測試 正規表示法 // t.regular(); - - } - -} + // 測試 enum 列舉 + // t.enumTest(); + t.Date(); + } + +} + +enum EnumTest { + R, L, GO +} Index: /sample/hadoop-0.16/tw/org/nchc/code/LogParserGo.java =================================================================== --- /sample/hadoop-0.16/tw/org/nchc/code/LogParserGo.java (revision 44) +++ /sample/hadoop-0.16/tw/org/nchc/code/LogParserGo.java (revision 45) @@ -47,4 +47,6 @@ package tw.org.nchc.code; +import java.io.File; +import java.io.FileWriter; import java.io.IOException; @@ -103,14 +105,12 @@ try { - /* + LogParser log = new LogParser(value.toString()); print(value.toString()); FileWriter out = new FileWriter(new File( - "/home/waue/mr-result.txt")); + "/home/waue/Desktop/mr-result.txt")); out.write(value.toString()); out.flush(); out.close(); - */ - LogParser log = new LogParser(value.toString()); - + if (table == null) table = new HTable(conf, new Text(tableName)); Index: /sample/hadoop-0.16/tw/org/nchc/code/SnortBase.java =================================================================== --- /sample/hadoop-0.16/tw/org/nchc/code/SnortBase.java (revision 44) +++ /sample/hadoop-0.16/tw/org/nchc/code/SnortBase.java (revision 45) @@ -22,23 +22,23 @@ * hql > select * from apache-log; -+-------------------------+-------------------------+-------------------------+ -| Row | Column | Cell | -+-------------------------+-------------------------+-------------------------+ -| 118.170.101.250 | http:agent | Mozilla/4.0 (compatible;| -| | | MSIE 4.01; Windows 95) | -+-------------------------+-------------------------+-------------------------+ -| 118.170.101.250 | http:bytesize | 318 | -+-------------------------+-------------------------+-------------------------+ -..........(skip)........ -+-------------------------+-------------------------+-------------------------+ -| 87.65.93.58 | http:method | OPTIONS | -+-------------------------+-------------------------+-------------------------+ -| 87.65.93.58 | http:protocol | HTTP/1.1 | -+-------------------------+-------------------------+-------------------------+ -| 87.65.93.58 | referrer:- | * | -+-------------------------+-------------------------+-------------------------+ -| 87.65.93.58 | url:* | - | -+-------------------------+-------------------------+-------------------------+ -31 row(s) in set. (0.58 sec) + +-------------------------+-------------------------+-------------------------+ + | Row | Column | Cell | + +-------------------------+-------------------------+-------------------------+ + | 118.170.101.250 | http:agent | Mozilla/4.0 (compatible;| + | | | MSIE 4.01; Windows 95) | + +-------------------------+-------------------------+-------------------------+ + | 118.170.101.250 | http:bytesize | 318 | + +-------------------------+-------------------------+-------------------------+ + ..........(skip)........ + +-------------------------+-------------------------+-------------------------+ + | 87.65.93.58 | http:method | OPTIONS | + +-------------------------+-------------------------+-------------------------+ + | 87.65.93.58 | http:protocol | HTTP/1.1 | + +-------------------------+-------------------------+-------------------------+ + | 87.65.93.58 | referrer:- | * | + +-------------------------+-------------------------+-------------------------+ + | 87.65.93.58 | url:* | - | + +-------------------------+-------------------------+-------------------------+ + 31 row(s) in set. (0.58 sec) @@ -47,5 +47,9 @@ package tw.org.nchc.code; +import java.io.File; +import java.io.FileWriter; import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Locale; import org.apache.hadoop.fs.FileStatus; @@ -68,9 +72,48 @@ import org.apache.hadoop.mapred.Reporter; +class Log { + + public Log(String data) throws Exception { + + String[] arr = data.split(";"); + + this.gid = arr[0]; + this.sid = arr[1]; + this.version = arr[2]; + this.alert_name = arr[3]; + this.class_type = arr[4]; + this.priority = arr[5]; + // this.timestamp = "2008" + arr[6] + arr[7] + arr[8] + arr[9] + + // arr[10]; + this.timestamp = getTime(arr[7] + "/" + arr[6] + "/2008:" + arr[8] + + ":" + arr[9] + ":" + arr[10]); + this.source = arr[11]; + this.destination = arr[12]; + this.type = arr[13]; + this.ttl = arr[14]; + this.tos = arr[15]; + this.id = arr[16]; + this.iplen = arr[17]; + this.dgmlen = arr[18]; + + } + + long timestamp; + + String gid, sid, version; + + String alert_name, class_type, priority; + + String source, destination, type, ttl, tos, id, iplen, dgmlen; + + long getTime(String str) throws Exception { + SimpleDateFormat sdf = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss", + Locale.TAIWAN); + Long timestamp = sdf.parse(str).getTime(); + return timestamp; + } +} + // import AccessLogParser -/** - * Access_log fetcher. TODO: FgnStatLog, Error_log, Access_log (Default, - * W3CExtended, IISw3cExtended) - */ public class SnortBase { static HBaseConfiguration conf = new HBaseConfiguration(); @@ -78,22 +121,19 @@ public static final String TABLE = "table.name"; - static String tableName; + static String tableName = "mySnort"; static HTable table = null; - - static void print(String str){ - System.out.println("STR = "+str); - } + + public static class MapClass extends MapReduceBase implements Mapper { @Override - // MapReduceBase.configure(JobConf job) + // MapReduceBase.configure(JobConf job) // Default implementation that does nothing. public void configure(JobConf job) { - // String get(String name,String defaultValue) + // String get(String name,String defaultValue) // Get the value of the name property. If no such property exists,\ - // then defaultValue is returned. - tableName = job.get(TABLE, ""); + // then defaultValue is returned. } @@ -101,41 +141,48 @@ OutputCollector output, Reporter reporter) throws IOException { - + try { - /* - print(value.toString()); + + Log log = new Log(value.toString()); + + // 查看value的值 FileWriter out = new FileWriter(new File( - "/home/waue/mr-result.txt")); - out.write(value.toString()); + "/home/waue/Desktop/snort-result.txt")); + out.write(value.toString() + "_time=" + log.timestamp + "\n"); out.flush(); out.close(); - */ -// SnortParser log = new SnortParser(value.toString(),0); - + if (table == null) table = new HTable(conf, new Text(tableName)); - /* - long lockId = table.startUpdate(new Text(log.getIp())); - table.put(lockId, new Text("http:protocol"), log.getProtocol() - .getBytes()); - table.put(lockId, new Text("http:method"), log.getMethod() - .getBytes()); - table.put(lockId, new Text("http:code"), log.getCode() - .getBytes()); - table.put(lockId, new Text("http:bytesize"), log.getByteSize() - .getBytes()); - table.put(lockId, new Text("http:agent"), log.getAgent() - .getBytes()); - table.put(lockId, new Text("url:" + log.getUrl()), log - .getReferrer().getBytes()); - table.put(lockId, new Text("referrer:" + log.getReferrer()), - log.getUrl().getBytes()); - table.commit(lockId, log.getTimestamp()); - */ - + + long lockId = table.startUpdate(new Text(log.destination)); + table.put(lockId, new Text("id:gid"), log.gid.getBytes()); + table.put(lockId, new Text("id:sid"), log.sid.getBytes()); + table.put(lockId, new Text("id:version"), log.version + .getBytes()); + table.put(lockId, new Text("name:name"), log.alert_name + .getBytes()); + table.put(lockId, new Text("name:class"), log.class_type + .getBytes()); + table.put(lockId, new Text("index:priority"), log.priority + .getBytes()); + table.put(lockId, new Text("index:soure"), log.source + .getBytes()); + table + .put(lockId, new Text("payload:type"), log.type + .getBytes()); + table.put(lockId, new Text("payload:ttl"), log.ttl.getBytes()); + table.put(lockId, new Text("payload:tos"), log.tos.getBytes()); + table.put(lockId, new Text("payload:id"), log.id.getBytes()); + table.put(lockId, new Text("payload:iplen"), log.iplen + .getBytes()); + table.put(lockId, new Text("payload:dgmlen"), log.dgmlen + .getBytes()); + table.commit(lockId, log.timestamp); + } catch (Exception e) { e.printStackTrace(); } - + } } @@ -153,29 +200,22 @@ } - public static void runMapReduce(String table, String dir) + public static void runMapReduce(String table, String inpath) throws IOException { Path tempDir = new Path("/tmp/Mylog/"); - Path InputDir = new Path(dir); + Path InputPath = new Path(inpath); FileSystem fs = FileSystem.get(conf); JobConf jobConf = new JobConf(conf, SnortBase.class); - jobConf.setJobName("apache log fetcher"); + jobConf.setJobName("Snort Parse"); jobConf.set(TABLE, table); - Path[] in = listPaths(fs, InputDir); - if (fs.isFile(InputDir)) { - jobConf.setInputPath(InputDir); - } else { - for (int i = 0; i < in.length; i++) { - if (fs.isFile(in[i])) { - jobConf.addInputPath(in[i]); - } else { - Path[] sub = listPaths(fs, in[i]); - for (int j = 0; j < sub.length; j++) { - if (fs.isFile(sub[j])) { - jobConf.addInputPath(sub[j]); - } - } - } - } - } + // 先省略 自動搜尋目錄的功能 + /* + * Path[] in = listPaths(fs, InputDir); if (fs.isFile(InputDir)) { + * jobConf.setInputPath(InputDir); } else { for (int i = 0; i < + * in.length; i++) { if (fs.isFile(in[i])) { + * jobConf.addInputPath(in[i]); } else { Path[] sub = listPaths(fs, + * in[i]); for (int j = 0; j < sub.length; j++) { if (fs.isFile(sub[j])) { + * jobConf.addInputPath(sub[j]); } } } } } + */ + jobConf.setInputPath(InputPath); jobConf.setOutputPath(tempDir); jobConf.setMapperClass(MapClass.class); @@ -195,7 +235,9 @@ + " table creating ... please wait"); HTableDescriptor tableDesc = new HTableDescriptor(table); - tableDesc.addFamily(new HColumnDescriptor("http:")); - tableDesc.addFamily(new HColumnDescriptor("url:")); - tableDesc.addFamily(new HColumnDescriptor("referrer:")); + tableDesc.addFamily(new HColumnDescriptor("id:")); + tableDesc.addFamily(new HColumnDescriptor("name:")); + tableDesc.addFamily(new HColumnDescriptor("index:")); + tableDesc.addFamily(new HColumnDescriptor("payload:")); + tableDesc.addFamily(new HColumnDescriptor("priority:")); admin.createTable(tableDesc); } else { @@ -205,23 +247,16 @@ } - public static void main(String[] args) throws IOException { + public static void main(String[] args) throws IOException, Exception { String table_name = "snort"; - String dir = "/user/waue/apache-log"; - - // if (eclipseRun) { - // table_name = "log"; - // dir = "apache-log"; - // } else if (args.length < 2) { - // System.out - // .println("Usage: logfetcher - // "); - // System.exit(1); - // } else { - // table_name = args[1]; - // dir = args[0]; - // } - + String path = "/user/waue/alert_meta"; + + // 先省略掉 parse完後自動上傳部份 + /* + * SnortParser sp = new + * SnortParser("/tmp/alert","/tmp/alert_SnortBase"); sp.parseToLine(); + */ creatTable(table_name); - runMapReduce(table_name, dir); + + runMapReduce(table_name, path); } Index: /sample/hadoop-0.16/tw/org/nchc/code/SnortParser.java =================================================================== --- /sample/hadoop-0.16/tw/org/nchc/code/SnortParser.java (revision 44) +++ /sample/hadoop-0.16/tw/org/nchc/code/SnortParser.java (revision 45) @@ -102,5 +102,5 @@ public static void main(String[] args) throws ParseException, Exception { String in = new String("/home/waue/Desktop/alert_m"); - String ou = new String("/tmp/alert_my"); + String ou = new String("/tmp/alert_SnortBase"); SnortParser a = new SnortParser(in, ou); a.parseToLine();