| 1 | {{{ |
| 2 | package tw.org.nchc.code; |
| 3 | import java.io.IOException; |
| 4 | import java.text.ParsePosition; |
| 5 | import java.text.SimpleDateFormat; |
| 6 | import java.util.Date; |
| 7 | import java.util.Locale; |
| 8 | import org.apache.hadoop.fs.FileStatus; |
| 9 | import org.apache.hadoop.fs.FileSystem; |
| 10 | import org.apache.hadoop.fs.Path; |
| 11 | import org.apache.hadoop.hbase.HBaseAdmin; |
| 12 | import org.apache.hadoop.hbase.HBaseConfiguration; |
| 13 | import org.apache.hadoop.hbase.HColumnDescriptor; |
| 14 | import org.apache.hadoop.hbase.HTable; |
| 15 | import org.apache.hadoop.hbase.HTableDescriptor; |
| 16 | import org.apache.hadoop.io.Text; |
| 17 | import org.apache.hadoop.io.Writable; |
| 18 | import org.apache.hadoop.io.WritableComparable; |
| 19 | import org.apache.hadoop.mapred.ClusterStatus; |
| 20 | import org.apache.hadoop.mapred.JobClient; |
| 21 | import org.apache.hadoop.mapred.JobConf; |
| 22 | import org.apache.hadoop.mapred.MapReduceBase; |
| 23 | import org.apache.hadoop.mapred.Mapper; |
| 24 | import org.apache.hadoop.mapred.OutputCollector; |
| 25 | import org.apache.hadoop.mapred.Reporter; |
| 26 | class Log { |
| 27 | String gid, sid, version; |
| 28 | String alert_name, class_type, priority; |
| 29 | String source, destination, type; |
| 30 | String srcport, dstport,tmp; |
| 31 | public Log(String data) { |
| 32 | String[] arr = data.split(";"); |
| 33 | this.gid = arr[0]; |
| 34 | this.sid = arr[1]; |
| 35 | this.version = arr[2]; |
| 36 | this.alert_name = arr[3]; |
| 37 | this.class_type = arr[4]; |
| 38 | this.priority = arr[5]; |
| 39 | this.timestamp = getTime(arr[7] + "/" + arr[6] + ":" + arr[8] + ":" |
| 40 | + arr[9] + ":" + arr[10]); |
| 41 | this.source = getIP(arr[11]); |
| 42 | this.srcport = this.tmp; |
| 43 | this.destination = getIP(arr[12]); |
| 44 | this.dstport = this.tmp; |
| 45 | this.type = arr[13]; |
| 46 | } |
| 47 | long timestamp; |
| 48 | String getIP(String str){ |
| 49 | String res; |
| 50 | int n = str.indexOf(":"); |
| 51 | if (n == -1) { |
| 52 | res = str; |
| 53 | this.tmp = "0"; |
| 54 | } else { |
| 55 | String[] vec = str.split(":"); |
| 56 | res = vec[0]; |
| 57 | this.tmp = vec[1]; |
| 58 | } |
| 59 | return res; |
| 60 | } |
| 61 | long getTime(String str) { |
| 62 | SimpleDateFormat sdf = new SimpleDateFormat("dd/MM:HH:mm:ss", |
| 63 | Locale.TAIWAN); |
| 64 | Long timestamp = sdf.parse(str, new ParsePosition(0)).getTime(); |
| 65 | return timestamp; |
| 66 | } |
| 67 | } |
| 68 | // import AccessLogParser |
| 69 | public class SnortBase { |
| 70 | static HBaseConfiguration conf = new HBaseConfiguration(); |
| 71 | public static final String TABLE = "table.name"; |
| 72 | static String tableName = "NewSnort"; |
| 73 | static HTable table = null; |
| 74 | public static class MapClass extends MapReduceBase implements |
| 75 | Mapper<WritableComparable, Text, Text, Writable> { |
| 76 | public void configure(JobConf job) { |
| 77 | } |
| 78 | public void map(WritableComparable key, Text value, |
| 79 | OutputCollector<Text, Writable> output, Reporter reporter) |
| 80 | throws IOException { |
| 81 | Log log = new Log(value.toString()); |
| 82 | String property_name = |
| 83 | "priority="+log.priority+ |
| 84 | ";class="+log.class_type+ |
| 85 | ";snort_id="+log.sid; |
| 86 | String property_source = |
| 87 | log.source+":"+log.srcport+" => " |
| 88 | +log.destination+":"+log.dstport; |
| 89 | String property_payload = log.type; |
| 90 | if (table == null) |
| 91 | table = new HTable(conf, new Text(tableName)); |
| 92 | long lockId = table.startUpdate(new Text(log.destination+":"+log.sid)); |
| 93 | table.put(lockId, new Text("name:"+log.alert_name), property_name.getBytes()); |
| 94 | table.put(lockId, new Text("from:"+log.source), property_source.getBytes()); |
| 95 | table.put(lockId, new Text("payload:"+log.type), property_payload.getBytes()); |
| 96 | table.commit(lockId, log.timestamp); |
| 97 | } |
| 98 | } |
| 99 | |
| 100 | // do it to resolve warning : FileSystem.listPaths |
| 101 | static public Path[] listPaths(FileSystem fsm, Path path) |
| 102 | throws IOException { |
| 103 | FileStatus[] fss = fsm.listStatus(path); |
| 104 | int length = fss.length; |
| 105 | Path[] pi = new Path[length]; |
| 106 | for (int i = 0; i < length; i++) { |
| 107 | pi[i] = fss[i].getPath(); |
| 108 | } |
| 109 | return pi; |
| 110 | } |
| 111 | |
| 112 | public static void runMapReduce(String tableName, String inpath) |
| 113 | throws IOException { |
| 114 | Path tempDir = new Path("/tmp/Mylog/"); |
| 115 | Path InputPath = new Path(inpath); |
| 116 | FileSystem fs = FileSystem.get(conf); |
| 117 | JobConf jobConf = new JobConf(conf, SnortBase.class); |
| 118 | jobConf.setJobName("Snort Parse"); |
| 119 | jobConf.set(TABLE, tableName); |
| 120 | jobConf.setInputPath(InputPath); |
| 121 | jobConf.setOutputPath(tempDir); |
| 122 | jobConf.setMapperClass(MapClass.class); |
| 123 | JobClient client = new JobClient(jobConf); |
| 124 | ClusterStatus cluster = client.getClusterStatus(); |
| 125 | jobConf.setNumMapTasks(cluster.getMapTasks()); |
| 126 | jobConf.setNumReduceTasks(0); |
| 127 | fs.delete(tempDir); |
| 128 | JobClient.runJob(jobConf); |
| 129 | fs.delete(tempDir); |
| 130 | fs.close(); |
| 131 | } |
| 132 | |
| 133 | public static void creatTable(String table) throws IOException { |
| 134 | HBaseAdmin admin = new HBaseAdmin(conf); |
| 135 | if (!admin.tableExists(new Text(table))) { |
| 136 | System.out.println("1. " + table |
| 137 | + " table creating ... please wait"); |
| 138 | HTableDescriptor tableDesc = new HTableDescriptor(table); |
| 139 | tableDesc.addFamily(new HColumnDescriptor("name:")); |
| 140 | tableDesc.addFamily(new HColumnDescriptor("from:")); |
| 141 | tableDesc.addFamily(new HColumnDescriptor("payload:")); |
| 142 | admin.createTable(tableDesc); |
| 143 | } else { |
| 144 | System.out.println("1. " + table + " table already exists."); |
| 145 | } |
| 146 | System.out.println("2. access_log files fetching using map/reduce"); |
| 147 | } |
| 148 | public static void main(String[] args) throws IOException, Exception { |
| 149 | String path = "/user/waue/snort-log/alert_flex_parsed.txt"; |
| 150 | creatTable(tableName); |
| 151 | runMapReduce(tableName, path); |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | }}} |
| 156 | |
| 157 | -------- |
| 158 | |