Changeset 94

Timestamp:

Jul 8, 2009, 2:20:41 PM (15 years ago)

Author:

waue

Message:

0.1-2 fix some bug

Location:

nutchez-0.1

Files:

• Makefile (modified) (1 diff)
• README (modified) (2 diffs)
• bin/nutchez (modified) (5 diffs)
• bin/nutchez-func.sh (modified) (9 diffs)
• debian/changelog (modified) (1 diff)
• debian/nutchez.install (modified) (1 diff)
• debian/nutchez.postinst (modified) (1 diff)
• debian/nutchez.postrm (modified) (1 diff)
• debian/rules (modified) (1 diff)

nutchez-0.1/Makefile

@@ -13 +13 @@
   @echo "make deb     - Build Debian Package."
   @echo "make clean   - Clean up Debian Package temparate files."
-  @echo "make source  - download source tarball from hadoop mirror site."
   @echo "make help    - show Makefile options."
   @echo " "

nutchez-0.1/README

@@ -3 +3 @@
 ************************************************
 
-NutchEz is developed by NCHC in Taiwan
+NutchEz is developed by National Center for High-Performance Computing (NCHC) in Taiwan
 
- * How to install :
-You should download the deb file and type the instruction at the same dir.
-============================  
-sudo dpkg -i nutchez-*.deb
-============================
+ * How to lunch nutchez  (Type following Instruction)
+========================================
+nutchez
+========================================
 
-
- * How to lunch nutchez
-============
-nutchez
-============
-
- * How to remove nutchez
-=====================
+ * How to remove nutchez  (Type following Instruction)
+========================================
 sudo dpkg -P nutchez
-=====================
+========================================
 
  * Where are the program and configuration files
@@ -27 +20 @@
 2. Nutch conf dir is set on ~/.nutchez/conf
 3. Tomcat home is installed on ~/.nutchez/tomcat
-
-
+4. The final crawl results is located on ~/.nutchez/search
+5. The nutchez log will be recorded on ~/.nutchez/log
+6. The full path of this README is /etc/nutch/README

nutchez-0.1/bin/nutchez

@@ -12 +12 @@
   export JAVA_HOME=/usr/lib/jvm/java-6-sun
   export NUTCH_HOME=/opt/nutch
-#  export HADOOP_HOME=/opt/nutch
   export NUTCH_CONF_DIR=~/.nutchez/conf
-#  export HADOOP_CONF_DIR=~/.nutchez/conf
   export NUTCH_LOG_DIR=~/.nutchez/log
 
+. /opt/nutch/bin/nutchez-func.sh
+
+init_nutchez
 
 
+tfile=`tempfile 2>/dev/null` || tfile=/tmp/test$$
+trap "rm -f $tfile" 0 1 2 5 15
 
-. /opt/nutch/bin/nutchez-func.sh
-#. ~/.nutchez/hadoop-env.sh
+$DIALOG --backtitle "Developed By NCHC" --clear --item-help --title "NutchEz 雛型版" \
+        --menu " 你好，歡迎使用NutchEz！ \n\
+這套軟體是用來打造專屬於你的搜尋引擎 \n\
+你有網頁不希望被公開的搜尋引擎找到， \n\
+卻又希望能有個搜尋介面的困擾嗎？ \n\
+用NutchEz就對了！因為他操作簡單， \n\
+除了基本的網頁以外，還支援多種格式（ppt,doc,txt...） \n\
+並且是開源碼軟體，完全免費，安全無虞\n\
+趕快來使用看看吧!\n\n\
+          選擇你要的模式:" 20 60 4 \
+        "1"  "開始建構搜尋內容"    "透過NutchEz來建構專屬於你自己所需的內容的搜尋引擎" \
+        "2"  "開啟或關閉NutchEz的網頁伺服器"  "若您之前已經執行完1之後才需做網頁伺服器的管理" 2> $tfile
 
+rev=$?
 
+MAIN_CHOISE=`cat $tfile`
 
-init_nutchez
+case $rev in
+  0)
+    echo_dialog_v "OK '$MAIN_CHOISE' chosen.";;
+  1)
+    echo_dialog_v "Cancel pressed."
+    exit 0 ;;
+  2)
+    echo_dialog_v "HELP '$MAIN_CHOISE' chosen.";;
+  255)
+    echo_dialog_v "ESC pressed.";;
+  *)
+    echo_dialog_v "Unexpected code $MAIN_CHOISE";;
+esac
+
+if [ $MAIN_CHOISE -eq 2 ];then
+
+  pid_tc=$(ps axw -eo pid,command |\
+  grep "catalina" | grep "java" |\
+  grep "start" | awk '{print $1}')
+  if [ -z "$pid_tc" ]; then
+      echo_dialog_v "0. pid = $pid_tc ! no another toddmcat is running"
+
+      $DIALOG --title "你的NutchEz網頁伺服器沒打開.." --clear \
+        --yesno "\n 要開啟NutchEz的網頁伺服器嗎？ \n" 15 61
+      if [ $? -eq 0 ];then
+          ~/.nutchez/tomcat/bin/startup.sh
+    PORT=`cat ~/.nutchez/sav/n.tomcat.txt`
+          $DIALOG --msgbox "已經試著開啟瀏覽伺服器，你可以瀏覽這個網址看看: \n  http://localhost:$PORT" 0 0
+      else 
+    $DIALOG --msgbox "你選擇不要打開瀏覽伺服器！" 0 0
+      fi
+      
+  else
+      echo_dialog_v "0. tomcat had been started and the pid is $pid_tc"
+
+      $DIALOG --title "偵測到NutchEz的網頁伺服器正在運作.." --clear \
+        --yesno "\n 你要關閉他嗎？ \n" 15 61
+      if [ $? -eq 0 ];then
+          ~/.nutchez/tomcat/bin/shutdown.sh
+    kill -9 $pid_tc
+          $DIALOG --msgbox "已經試著關閉NutchEz網頁伺服器" 0 0
+      else 
+          $DIALOG --msgbox "你選擇不要關閉瀏覽伺服器！" 0 0
+      fi
+
+  fi
+  exit 0
+
+elif [ $MAIN_CHOISE -eq 1 ];then
 
 CHECK=0
@@ -35 +98 @@
     show_urls
     URL=$?
-    echo_vb "2.2 show_urls return $URL"
+    echo_dialog_v "2.2 show_urls return $URL"
     
 # add or delete url:  ok , exit
@@ -63 +126 @@
   setup_tomcat
 
+  # continue last search
+  if [ -e ~/.nutchez/search ];then
+    continue_previous
+  fi
   # show the final checklist
   final_confirm
   FC=$?
-  echo_vb "6.3 final confirm return = $FC "
+  echo_dialog_v "6.3 final confirm return = $FC "
   #   START , back, cancel
   # start =0 , back =1 
@@ -90 +157 @@
       ;;
     255)
-      echo_vb "ESC pressed !!"
+      echo_dialog_v "ESC pressed !!"
       ;;
   esac
@@ -108 +175 @@
 # Done 
 
+
+
+
+else
+
+  echo_dialog_v "Main Choise is $MAIN_CHOISE ! Error !!"
+
+fi

nutchez-0.1/bin/nutchez-func.sh

@@ -6 +6 @@
 
 
-: ${DIALOG=dialog}
+#DIALOG='dialog --backtitle "     NutchEz Setup Menu           -- powered by NCHC "'
+DIALOG=dialog
 
 # set 1 to display more for debug, 
@@ -39 +40 @@
 
 check_if_root() {
-   if [ ! "$UID" = "0" ]; then
+   if [ ! "$UID" -eq "0" ]; then
      echo_dialog_v "Hi [$LOGNAME] !! "
      echo_dialog_v "You need to run this script \"`basename $0`\" as root."
@@ -71 +72 @@
 
   cp ~/.nutchez/sav/n.urls.txt ~/.nutchez/urls/urls.txt
+
+  if [ $NOCONTINUE -eq 1 ]; then
+    echo_dialog_v " delete the ~/.nutchez/search/*"
+    DATE=`date +%y%m%d%H%M%S`
+    mv ~/.nutchez/search ~/.nutchez/search-$DATE
+    $DIALOG --msgbox "上次搜尋的結果改放到 ~/.nutchez/search-$DATE " 0 0
+    rm -f /tmp/search
+#    rm -rf ~/.nutchez/search/*
+  fi
 
   if [ -e ~/.nutchez/conf/nutch-site.xml ] ; then
@@ -100 +110 @@
   echo_dialog_v "2.1 test_file ~/.nutchez/sav return : $?"
   # dialog begin
-  dialog --title "The URLS that you want" --editbox ~/.nutchez/sav/n.urls.txt 16 51 2>/tmp/n.urls.tmp
+  dialog --title "請輸入你要抓取的網址（一行一個網址）" --editbox ~/.nutchez/sav/n.urls.txt 16 51 2>/tmp/n.urls.tmp
   RET=$?
   echo_dialog_v "2.1 cat url: `cat /tmp/n.urls.tmp`"
@@ -110 +120 @@
   echo_dialog_v "3. setup_robot"
   # dialog
-  dialog --nocancel --inputbox " This agent name \n" 16 51 "`cat ~/.nutchez/sav/n.robot.txt`" 2>/tmp/n.robot.tmp
+  dialog --title "設定機器人名稱" --nocancel --inputbox " 這個爬網機器人，你要將他取名為：\n\n (ps: 這個設定只是禮貌性宣告，並不會對結果造成影響) \n" 16 55 "`cat ~/.nutchez/sav/n.robot.txt`" 2>/tmp/n.robot.tmp
   echo_dialog_v "3.1 cat robot : `cat /tmp/n.robot.tmp`"
 }
@@ -117 +127 @@
   echo_dialog_v "4. setup_crawler"
   test_file ~/.nutchez/sav/n.crawler.txt
-  dialog --nocancel --inputbox " Depth  \n " 16 51 "`cat ~/.nutchez/sav/n.crawler.txt`" 2>/tmp/n.crawler.tmp
+  dialog --title "設定抓取深度" --nocancel --inputbox " 對於每個網址，你需要NutchEz爬多深呢？\n\n (ps: 初次體驗建議將深度設為1來感受需要多久)  \n " 16 51 "`cat ~/.nutchez/sav/n.crawler.txt`" 2>/tmp/n.crawler.tmp
   echo_dialog_v "4.1 cat robot : `cat /tmp/n.robot.tmp`"
 }
@@ -124 +134 @@
   echo_dialog_v "5. setup_tomcat"
   test_file ~/.nutchez/sav/n.tomcat.txt
-  dialog --nocancel --inputbox " explorer port \n " 16 51 "`cat ~/.nutchez/sav/n.tomcat.txt`" 2>/tmp/n.tomcat.tmp
+  dialog --title "設定網頁伺服器" --nocancel --inputbox " 你希望NutchEz將網頁伺服器開在哪個port \n\n (ps: 請選擇一個沒用到的port以免造成衝突 \n 也請盡量不要設成80以免造成你誤以為是apache的混淆) \n " 16 51 "`cat ~/.nutchez/sav/n.tomcat.txt`" 2>/tmp/n.tomcat.tmp
   echo_dialog_v "5.1 cat tomcat : `cat /tmp/n.tomcat.tmp`"
 }
 
+continue_previous () {
+  echo_dialog_v "6. setup_tomcat"
+$DIALOG --title "清除上次搜尋" --clear \
+        --yesno "你是否要清除上一次爬網所得的結果，\n否則將加入到URL列裡增加搜尋負擔 \n\n ps: 選no的話，會跑相當相當久，\n 請慎重考慮之\n" 16 51
+
+case $? in
+  0)
+    NOCONTINUE=1;;
+  1)
+    NOCONTINUE=0;;
+  255)
+    echo "ESC pressed.";;
+esac
+echo_dialog_v " 6continue = $CONTINUE"
+}
+
 final_confirm () {
-  echo_dialog_v "6. final_confirm : start =0 , back =1 "
+  echo_dialog_v "7. final_confirm : start =0 , back =1 "
   tempfile=/tmp/n.finalcheck.tmp
 
-  echo " \n 1. The url list is : \n " > $tempfile
+  echo " \n 1. 你所選擇要爬取的網址為 : \n " > $tempfile
   cat /tmp/n.urls.tmp >> $tempfile
-  echo " \n 2. The robot name is : \n" >> $tempfile
+  echo " \n\n 2. 對於這個爬網機器人，你取名為 : \n" >> $tempfile
   cat /tmp/n.robot.tmp >> $tempfile
-  echo " \n 3. The crawled depth is : \n " >> $tempfile
+  echo " \n\n 3. 爬網的深度，你設定為 : \n " >> $tempfile
   cat /tmp/n.crawler.tmp >> $tempfile
-  echo " \n 4. The explorer port is : \n " >> $tempfile
+  echo " \n\n 4. NutchEz將會把你的搜尋結果呈現在這個Port : \n " >> $tempfile
   cat /tmp/n.tomcat.tmp >> $tempfile
-
+  if [ $NOCONTINUE -eq 0 ];then
+    echo " \n\n 5. 是否要清除上一次的收尋結果 : \n " >> $tempfile
+    echo_dialog_v " 7continue = $CONTINUE"
+    echo "NO" >> $tempfile
+  elif [ $NOCONTINUE -eq 1 ];then
+    echo " \n\n 5. 是否要清除上一次的收尋結果繼續搜尋 : \n " >> $tempfile
+    echo_dialog_v " 7continue = $CONTINUE"
+    echo "YES" >> $tempfile
+  else
+    echo_dialog_v " 無資料可匯入 "
+  fi
   MSG=`cat $tempfile`
-  echo_dialog_v "6.1 final message :\n $MSG"
+  echo_dialog_v "7.1 final message :\n $MSG"
   #read READ
-  $DIALOG --title "Check It !!" --clear \
+  $DIALOG --title "請檢查你的選擇 ! \n\n 若所有的設定都是正確的，你可以按 \"ok\",\n 若你按了 \"reset\" 則會重頭開始設定, \n 若你選擇 \"exit\" 則會跳出NutchEz的設定選單 \n ps: reset 與 exit都不會把資料記成預設值，請放心使用 " --clear \
         --extra-button --extra-label "reset" --ok-label "ok" --cancel-label "exit" \
         --yesno "$MSG" 26 51
@@ -179 +215 @@
 
   pid_tc=$(ps axw -eo pid,command |\
- grep "catalina.startup.Bootstrap" |\
- grep "start" | awk '{print $1}')
+  grep "catalina" | grep "java" |\
+  grep "start" | awk '{print $1}')
   if [ -z "$pid_tc" ]; then
-      echo "starting tomcat"
+      echo_dialog_v "no another tomcat is running"
   else
-      echo "tomcat had been started and the pid is $pid_tc"
-      echo "stop it first"
+      echo_dialog_v "tomcat had been started and the pid is $pid_tc"
+      echo_dialog_v "stop it first"
       kill -9 $pid_tc
       if [ -z $? ];then
-    echo " tomcat ($pid_tc) is  killed ..."
+    echo_dialog_v " tomcat ($pid_tc) is  killed ..."
       else
-    echo "kill error ..." 
+    echo_dialog_v "kill error ..." 
       fi
   fi
+  echo "Starting Tomcat ...."
   ~/.nutchez/tomcat/bin/startup.sh
   sleep 3
@@ -201 +238 @@
   FIREFOX=`which firefox`
   RET=$?
-  if [ $RET == 0 ];then 
+  if [ $RET -eq 0 ];then 
     $FIREFOX -D 0.0 http://localhost:$PORT
     RET=$?
   fi
-  if ! [ $RET == 0 ];then
-    $DIALOG --msgbox "Congratulations! \n you can explore the url: \n  http://localhost:$PORT" 0 0 
-  fi
-}
+  if ! [ $RET -eq 0 ];then
+    $DIALOG --msgbox "恭喜你已經完成了! \n 你可以用瀏覽器瀏覽: \n  http://host_ip:$PORT" 0 0 
+  fi
+}

nutchez-0.1/debian/changelog

@@ -1 @@
-nutchez (0.1-1) unstable; urgency=low
+nutchez (0.1-2) unstable; urgency=low
 
   * Initial release (Closes: #nnnn)  <nnnn is the bug number of your ITP>
+  * 
 
- -- Wei-Yu Chen <waue0920@gmail.com>  Tue, 12 May 2009 11:15:51 +0800
-
+ -- Wei-Yu Chen <waue0920@gmail.com>  Tue, 07 Jul 2009 11:22:46 +0800

nutchez-0.1/debian/nutchez.install

@@ -3 +3 @@
 README*   etc/nutch
 bin   opt/nutch
-bin/nutchez*  usr/local/bin
 lib   opt/nutch
-webapps   opt/nutch
 tomcat    opt/nutch
 plugins   opt/nutch

nutchez-0.1/debian/nutchez.postinst

@@ -8 +8 @@
 fi
 
-ln -sf /opt/nutch/bin/nutchez /usr/local/sbin/
-ln -sf /opt/nutch/bin/nutchez-func.sh /usr/local/sbin/
+ln -sf /opt/nutch/bin/nutchez /usr/sbin/
+ln -sf /opt/nutch/bin/nutchez-func.sh /usr/sbin/
 
-setup_hdfsadm_user() {
-  if ! getent passwd hdfsadm >/dev/null; then
-    useradd hdfsadm
-    mkdir -p /home/hdfsadm/.ssh
-    mkdir -p /var/log/nutch
-    ssh-keygen -t rsa -q -f /home/hdfsadm/.ssh/id_rsa -N ""
-    cp /home/hdfsadm/.ssh/id_rsa.pub /home/hdfsadm/.ssh/authorized_keys
-    chown hdfsadm:hdfsadm /var/log/nutch
-    chown -R hdfsadm:hdfsadm /opt/nutch
-    chown -R hdfsadm:hdfsadm /home/hdfsadm
-  fi
+show_message(){
+  cat /etc/nutch/README
 }
-
-check_root() {
-  if ! test -e /root/.ssh/id_rsa ; then
-    ssh-keygen -t rsa -q -f /root/.ssh/id_rsa -N ""
-  fi
-  if test -e /root/.ssh/id_rsa.pub ; then
-    if ! test -e /root/.ssh/authorized_keys ; then
-      cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
-    fi
-  else
-    ssh-keygen -t rsa -q -f /root/.ssh/id_rsa -N ""
-    cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
-  fi
-
-}
-
-start_hadoop() {
-  if getent passwd hdfsadm >/dev/null; then
-    su -c "/opt/nutch/bin/hadoop namenode -format" hdfsadm -
-    su -c "/opt/nutch/bin/start-all.sh" hdfsadm -
-  else
-    /opt/nutch/conf/hadoop-env.sh
-    /opt/nutch/bin/hadoop namenode -format
-    /opt/nutch/bin/start-all.sh
-  fi
-}
-show_message() {
+show_old_message() {
   echo "You can use the instruction : \" nutchez\" to easyly use nutch"
   echo "Enjoy"
 }
-show_old_message() {
-  echo "You can quickly start by following ways [in /opt/nutch/ with root privilege]:"
-  echo "(1) Modify the urls/urls.txt file with indicate urls, one site one line."
-  echo "(2) Use this instruction \"bin/nutch crawl urls -dir search -depth 4 -topN 50\" to crawl web"
-  echo "(3) Type \" tomcat/bin/startup.sh \" and use browser to check the result in http://localhost:8080/"
-  echo "Enjoy !"
-}
-#setup_hdfsadm_user
-#check_root
-#start_hadoop
 show_message

nutchez-0.1/debian/nutchez.postrm

@@ -18 +18 @@
 done
 
-rm -f /usr/local/sbin/nutchez*
+rm -f /usr/sbin/nutchez*
 
 rm -rf /tmp/search

nutchez-0.1/debian/rules

@@ -13 +13 @@
   dh_compress
   dh_fixperms
+  find ./debian/nutchez/ -name ".svn" -type d > svnfolders
+  rm -rf `cat svnfolders`
   dh_installdeb
   dh_link