Changes between Version 1 and Version 2 of ExperimentLog1

Timestamp:

Aug 11, 2008, 3:10:28 PM (17 years ago)

Author:

waue

Comment:

--

ExperimentLog1

@@ +1 @@
+{{{
+package tw.org.nchc.code;
+import java.io.IOException;
+import java.text.ParsePosition;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Locale;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseAdmin;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTable;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapred.ClusterStatus;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+class Log {
+        String gid, sid, version;
+        String alert_name, class_type, priority;
+        String source, destination, type;
+        String srcport, dstport,tmp;
+        public Log(String data) {
+                String[] arr = data.split(";");
+                this.gid = arr[0];
+                this.sid = arr[1];
+                this.version = arr[2];
+                this.alert_name = arr[3];
+                this.class_type = arr[4];
+                this.priority = arr[5];
+                this.timestamp = getTime(arr[7] + "/" + arr[6] + ":" + arr[8] + ":"
+                                + arr[9] + ":" + arr[10]);
+                this.source = getIP(arr[11]);
+                this.srcport = this.tmp;
+                this.destination = getIP(arr[12]);
+                this.dstport = this.tmp;
+                this.type = arr[13];
+        }
+        long timestamp;
+        String getIP(String str){
+                String res;
+                int n = str.indexOf(":");
+                if (n == -1) {
+                        res = str;
+                        this.tmp = "0";
+                } else {
+                        String[] vec = str.split(":");
+                        res = vec[0];
+                        this.tmp = vec[1];
+                }
+                return res;
+        }
+        long getTime(String str) {
+                SimpleDateFormat sdf = new SimpleDateFormat("dd/MM:HH:mm:ss",
+                                Locale.TAIWAN);
+                Long timestamp = sdf.parse(str, new ParsePosition(0)).getTime();
+                return timestamp;
+        }
+}
+// import AccessLogParser
+public class SnortBase {
+        static HBaseConfiguration conf = new HBaseConfiguration();
+        public static final String TABLE = "table.name";
+        static String tableName = "NewSnort";
+        static HTable table = null;
+        public static class MapClass extends MapReduceBase implements
+                        Mapper<WritableComparable, Text, Text, Writable> {
+                public void configure(JobConf job) {
+                }
+                public void map(WritableComparable key, Text value,
+                                OutputCollector<Text, Writable> output, Reporter reporter)
+                                throws IOException {
+                        Log log = new Log(value.toString());
+                        String property_name = 
+                                        "priority="+log.priority+
+                                        ";class="+log.class_type+
+                                        ";snort_id="+log.sid;
+                        String property_source = 
+                                log.source+":"+log.srcport+" => "
+                                +log.destination+":"+log.dstport;
+                        String property_payload = log.type;
+                        if (table == null)
+                                table = new HTable(conf, new Text(tableName));
+                        long lockId = table.startUpdate(new Text(log.destination+":"+log.sid));
+                        table.put(lockId, new Text("name:"+log.alert_name), property_name.getBytes());                  
+                        table.put(lockId, new Text("from:"+log.source), property_source.getBytes());
+                        table.put(lockId, new Text("payload:"+log.type), property_payload.getBytes());
+                        table.commit(lockId, log.timestamp);
+                }
+        }
+
+        // do it to resolve warning : FileSystem.listPaths
+        static public Path[] listPaths(FileSystem fsm, Path path)
+                        throws IOException {
+                FileStatus[] fss = fsm.listStatus(path);
+                int length = fss.length;
+                Path[] pi = new Path[length];
+                for (int i = 0; i < length; i++) {
+                        pi[i] = fss[i].getPath();
+                }
+                return pi;
+        }
+
+        public static void runMapReduce(String tableName, String inpath)
+                        throws IOException {
+                Path tempDir = new Path("/tmp/Mylog/");
+                Path InputPath = new Path(inpath);
+                FileSystem fs = FileSystem.get(conf);
+                JobConf jobConf = new JobConf(conf, SnortBase.class);
+                jobConf.setJobName("Snort Parse");
+                jobConf.set(TABLE, tableName);
+                jobConf.setInputPath(InputPath);
+                jobConf.setOutputPath(tempDir);
+                jobConf.setMapperClass(MapClass.class);
+                JobClient client = new JobClient(jobConf);
+                ClusterStatus cluster = client.getClusterStatus();
+                jobConf.setNumMapTasks(cluster.getMapTasks());
+                jobConf.setNumReduceTasks(0);
+                fs.delete(tempDir);
+                JobClient.runJob(jobConf);
+                fs.delete(tempDir);
+                fs.close();
+        }
+
+        public static void creatTable(String table) throws IOException {
+                HBaseAdmin admin = new HBaseAdmin(conf);
+                if (!admin.tableExists(new Text(table))) {
+                        System.out.println("1. " + table
+                                        + " table creating ... please wait");
+                        HTableDescriptor tableDesc = new HTableDescriptor(table);
+                        tableDesc.addFamily(new HColumnDescriptor("name:"));
+                        tableDesc.addFamily(new HColumnDescriptor("from:"));
+                        tableDesc.addFamily(new HColumnDescriptor("payload:"));
+                        admin.createTable(tableDesc);
+                } else {
+                        System.out.println("1. " + table + " table already exists.");
+                }
+                System.out.println("2. access_log files fetching using map/reduce");
+        }
+        public static void main(String[] args) throws IOException, Exception {
+                String path = "/user/waue/snort-log/alert_flex_parsed.txt";
+                creatTable(tableName);
+                runMapReduce(tableName, path);
+        }
+}
+
+}}}
+
+--------
+
 {{{
 #!html
 
-<p>Enter 'help;' -- thats 'help' plus a semi-colon -- for the list of <em>HQL</em> commands.
- Data Definition, SHELL, INSERTS, DELETES, and UPDATE commands are disabled in this interface
- </p>
- 
- 
- <hr>
  <table>