wiki:waue/1020

Version 1 (modified by waue, 15 years ago) (diff)

--

rawDataCheck

  • uploadHDFS 待實做
  • 其他單元已測試完成
    /**
     * Program: LogParser.java
     * Editor: Waue Chen 
     * From :  GTD. NCHC. Taiwn
     * Last Update Date: 07/23/2008
     * support version : java 6 upper
     * 
     * How to Use :
     * see as SnortProduce.java
     */
    
    package tw.org.nchc.icas;
    
    import java.io.File;
    import java.io.IOException;
    import java.text.ParseException;
    import java.text.SimpleDateFormat;
    import java.util.Calendar;
    
    public class RawDataCheck {
      // TODO improve sourcepath include Directory
      String sourcepath;
      String out;
      String tmp = "/home/waue/tmp";
      String bak = "/home/waue/bak";
    
      File[] src_files;
    
      RawDataCheck(String in, String out) {
        this.sourcepath = in;
        this.out = out;
    
      }
    
      public String now() {
    
        Calendar cal = Calendar.getInstance();
        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
        return sdf.format(cal.getTime());
    
      }
    
      boolean checkData() {
        File srcpath = new File(sourcepath);
        File outpath = new File(out);
        File tmppath = new File(tmp);
        File bakpath = new File(bak);
        // check path is ready
        if (!srcpath.exists()) {
          System.err
              .print("Error(RawDataCheck.checkData): sourcepath is not exists.");
          return false;
        }
        if (!outpath.exists()) {
          System.err
              .print("Error(RawDataCheck.checkData): outpathpath is not exists.");
          return false;
        }
        if (!tmppath.exists()) {
          System.err
              .print("Error(RawDataCheck.checkData): tmp path is not exists.");
          return false;
        }
        if (!bakpath.exists()) {
          System.err
              .print("Error(RawDataCheck.checkData): bak path is not exists.");
          return false;
        }
        // check src path is a director
        if (srcpath.isDirectory()) {
          // point src path files
          src_files = srcpath.listFiles();
    
          if (src_files.length != 0) {
            // list src path files
            for (File fi : src_files) {
              System.err
                  .println("Message(RawDataCheck.checkData): detect "
                      + fi);
            }
          } else {
            System.err
                .println("Message(RawDataCheck.checkData): There is no data in "
                    + sourcepath + "!");
            return false;
          }
        } else {
          System.err
              .println("Error(RawDataCheck.checkData): Input path is not a directory.");
          return false;
        }
    
        return true;
      }
    
      boolean regular() throws IOException, ParseException, Exception {
    
        for (File src_fi : src_files) {
          File tmppath = new File(tmp);
          String filename = now() + "_" + src_fi.getName() + ".log";
          File tmpfile = new File(tmppath + "/" + filename);
          SnortRegular snr = new SnortRegular(src_fi, tmpfile);
          if (snr.parseToLine()) {
    
            System.err.println("Message(RawDataCheck.regular): "
                + src_fi.toString() + " -> " + tmpfile.toString()
                + " done .");
          } else {
            System.err
                .println("Error(RawDataCheck.regular): parseToLine() error.");
            return false;
          }
    
        }
        return true;
      }
    
      boolean uploadHDFS() {
        return true;
      }
    
      boolean purgeData() {
    
        // mv tmp data to bak path
        File[] tmp_files = (new File(tmp)).listFiles();
        for (File old_file : tmp_files) {
          // mkdir
          File dir = new File(bak + "/parsed/");
          if (!dir.exists())
            dir.mkdir();
          File new_file = new File(bak + "/parsed/" + old_file.getName());
          if (old_file.renameTo(new_file)) {
            System.err.println("Message(RawDataCheck.purgeData): mv "
                + old_file.toString() + " to " + new_file.toString()
                + " .");
          } else {
            System.err
                .println("Error(RawDataCheck.purgeData): mv file error.");
          }
    
        }
        // mv src data to bak path
    
        for (File old_file : src_files) {
          // mkdir
          File dir = new File(bak + "/raw/");
          if (!dir.exists())
            dir.mkdir();
          // mv file
          File new_file = new File(bak + "/raw/" + old_file.getName());
          if (old_file.renameTo(new_file)) {
            System.err.println("Message(RawDataCheck.purgeData): mv "
                + old_file.toString() + " to " + new_file.toString()
                + " .");
          } else {
            System.err
                .println("Error(RawDataCheck.purgeData): mv file error.");
          }
    
        }
    
        return true;
      }
    
      public static void main(String[] args) throws ParseException, Exception {
        long Start_Time = Util.getTime();
        RawDataCheck rdc = new RawDataCheck("/home/waue/in", "/home/waue/out");
        if (!rdc.checkData()) {
          System.err.println("Error(RawDataCheck.Main): checkData() error.");
          return;
        }
        if (!rdc.regular()) {
          System.err.println("Error(RawDataCheck.Main): regular() error .");
          return;
        }
        // if (! rdc.uploadHDFS()){
        // System.err.println("Error(RawDataCheck.Main): uploadHDFS() error .");
        // return;
        // }
        if (!rdc.purgeData()) {
          System.err.println("Error(RawDataCheck.Main): purgeData() error .");
          return;
        }
        Util.calcuTime("RawDataCheck", Start_Time);
    
      }
    }