Changes between Initial Version and Version 1 of waue/2009/1020

Timestamp:

Oct 22, 2009, 10:32:44 AM (15 years ago)

Author:

waue

Comment:

--

waue/2009/1020

@@ +1 @@
+
+
+ = rawDataCheck =
+
+ * uploadHDFS 待實做
+ * 其他單元已測試完成
+{{{
+#!java
+/**
+ * Program: LogParser.java
+ * Editor: Waue Chen 
+ * From :  GTD. NCHC. Taiwn
+ * Last Update Date: 10/20/2009
+ * support version : java 6 upper
+ * 
+ * How to Use :
+ * see as SnortProduce.java
+ */
+
+package tw.org.nchc.icas;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+
+public class RawDataCheck {
+        // TODO improve sourcepath include Directory
+        String sourcepath;
+        String out;
+        String tmp = "/home/waue/tmp";
+        String bak = "/home/waue/bak";
+
+        File[] src_files;
+
+        RawDataCheck(String in, String out) {
+                this.sourcepath = in;
+                this.out = out;
+
+        }
+
+        public String now() {
+
+                Calendar cal = Calendar.getInstance();
+                SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
+                return sdf.format(cal.getTime());
+
+        }
+
+        boolean checkData() {
+                File srcpath = new File(sourcepath);
+                File outpath = new File(out);
+                File tmppath = new File(tmp);
+                File bakpath = new File(bak);
+                // check path is ready
+                if (!srcpath.exists()) {
+                        System.err
+                                        .print("Error(RawDataCheck.checkData): sourcepath is not exists.");
+                        return false;
+                }
+                if (!outpath.exists()) {
+                        System.err
+                                        .print("Error(RawDataCheck.checkData): outpathpath is not exists.");
+                        return false;
+                }
+                if (!tmppath.exists()) {
+                        System.err
+                                        .print("Error(RawDataCheck.checkData): tmp path is not exists.");
+                        return false;
+                }
+                if (!bakpath.exists()) {
+                        System.err
+                                        .print("Error(RawDataCheck.checkData): bak path is not exists.");
+                        return false;
+                }
+                // check src path is a director
+                if (srcpath.isDirectory()) {
+                        // point src path files
+                        src_files = srcpath.listFiles();
+
+                        if (src_files.length != 0) {
+                                // list src path files
+                                for (File fi : src_files) {
+                                        System.err
+                                                        .println("Message(RawDataCheck.checkData): detect "
+                                                                        + fi);
+                                }
+                        } else {
+                                System.err
+                                                .println("Message(RawDataCheck.checkData): There is no data in "
+                                                                + sourcepath + "!");
+                                return false;
+                        }
+                } else {
+                        System.err
+                                        .println("Error(RawDataCheck.checkData): Input path is not a directory.");
+                        return false;
+                }
+
+                return true;
+        }
+
+        boolean regular() throws IOException, ParseException, Exception {
+
+                for (File src_fi : src_files) {
+                        File tmppath = new File(tmp);
+                        String filename = now() + "_" + src_fi.getName() + ".log";
+                        File tmpfile = new File(tmppath + "/" + filename);
+                        SnortRegular snr = new SnortRegular(src_fi, tmpfile);
+                        if (snr.parseToLine()) {
+
+                                System.err.println("Message(RawDataCheck.regular): "
+                                                + src_fi.toString() + " -> " + tmpfile.toString()
+                                                + " done .");
+                        } else {
+                                System.err
+                                                .println("Error(RawDataCheck.regular): parseToLine() error.");
+                                return false;
+                        }
+
+                }
+                return true;
+        }
+
+        boolean uploadHDFS() {
+                return true;
+        }
+
+        boolean purgeData() {
+
+                // mv tmp data to bak path
+                File[] tmp_files = (new File(tmp)).listFiles();
+                for (File old_file : tmp_files) {
+                        // mkdir
+                        File dir = new File(bak + "/parsed/");
+                        if (!dir.exists())
+                                dir.mkdir();
+                        File new_file = new File(bak + "/parsed/" + old_file.getName());
+                        if (old_file.renameTo(new_file)) {
+                                System.err.println("Message(RawDataCheck.purgeData): mv "
+                                                + old_file.toString() + " to " + new_file.toString()
+                                                + " .");
+                        } else {
+                                System.err
+                                                .println("Error(RawDataCheck.purgeData): mv file error.");
+                        }
+
+                }
+                // mv src data to bak path
+
+                for (File old_file : src_files) {
+                        // mkdir
+                        File dir = new File(bak + "/raw/");
+                        if (!dir.exists())
+                                dir.mkdir();
+                        // mv file
+                        File new_file = new File(bak + "/raw/" + old_file.getName());
+                        if (old_file.renameTo(new_file)) {
+                                System.err.println("Message(RawDataCheck.purgeData): mv "
+                                                + old_file.toString() + " to " + new_file.toString()
+                                                + " .");
+                        } else {
+                                System.err
+                                                .println("Error(RawDataCheck.purgeData): mv file error.");
+                        }
+
+                }
+
+                return true;
+        }
+
+        public static void main(String[] args) throws ParseException, Exception {
+                long Start_Time = Util.getTime();
+                RawDataCheck rdc = new RawDataCheck("/home/waue/in", "/home/waue/out");
+                if (!rdc.checkData()) {
+                        System.err.println("Error(RawDataCheck.Main): checkData() error.");
+                        return;
+                }
+                if (!rdc.regular()) {
+                        System.err.println("Error(RawDataCheck.Main): regular() error .");
+                        return;
+                }
+                // if (! rdc.uploadHDFS()){
+                // System.err.println("Error(RawDataCheck.Main): uploadHDFS() error .");
+                // return;
+                // }
+                if (!rdc.purgeData()) {
+                        System.err.println("Error(RawDataCheck.Main): purgeData() error .");
+                        return;
+                }
+                Util.calcuTime("RawDataCheck", Start_Time);
+
+        }
+}
+}}}