Index: /nutchez-0.1/Makefile =================================================================== --- /nutchez-0.1/Makefile (revision 93) +++ /nutchez-0.1/Makefile (revision 94) @@ -13,5 +13,4 @@ @echo "make deb - Build Debian Package." @echo "make clean - Clean up Debian Package temparate files." - @echo "make source - download source tarball from hadoop mirror site." @echo "make help - show Makefile options." @echo " " Index: /nutchez-0.1/README =================================================================== --- /nutchez-0.1/README (revision 93) +++ /nutchez-0.1/README (revision 94) @@ -3,22 +3,15 @@ ************************************************ -NutchEz is developed by NCHC in Taiwan +NutchEz is developed by National Center for High-Performance Computing (NCHC) in Taiwan - * How to install : -You should download the deb file and type the instruction at the same dir. -============================ -sudo dpkg -i nutchez-*.deb -============================ + * How to lunch nutchez (Type following Instruction) +======================================== +nutchez +======================================== - - * How to lunch nutchez -============ -nutchez -============ - - * How to remove nutchez -===================== + * How to remove nutchez (Type following Instruction) +======================================== sudo dpkg -P nutchez -===================== +======================================== * Where are the program and configuration files @@ -27,4 +20,5 @@ 2. Nutch conf dir is set on ~/.nutchez/conf 3. Tomcat home is installed on ~/.nutchez/tomcat - - +4. The final crawl results is located on ~/.nutchez/search +5. The nutchez log will be recorded on ~/.nutchez/log +6. The full path of this README is /etc/nutch/README Index: /nutchez-0.1/bin/nutchez =================================================================== --- /nutchez-0.1/bin/nutchez (revision 93) +++ /nutchez-0.1/bin/nutchez (revision 94) @@ -12,18 +12,81 @@ export JAVA_HOME=/usr/lib/jvm/java-6-sun export NUTCH_HOME=/opt/nutch -# export HADOOP_HOME=/opt/nutch export NUTCH_CONF_DIR=~/.nutchez/conf -# export HADOOP_CONF_DIR=~/.nutchez/conf export NUTCH_LOG_DIR=~/.nutchez/log +. /opt/nutch/bin/nutchez-func.sh + +init_nutchez +tfile=`tempfile 2>/dev/null` || tfile=/tmp/test$$ +trap "rm -f $tfile" 0 1 2 5 15 -. /opt/nutch/bin/nutchez-func.sh -#. ~/.nutchez/hadoop-env.sh +$DIALOG --backtitle "Developed By NCHC" --clear --item-help --title "NutchEz 雛型版" \ + --menu " 你好，歡迎使用NutchEz！ \n\ +這套軟體是用來打造專屬於你的搜尋引擎 \n\ +你有網頁不希望被公開的搜尋引擎找到， \n\ +卻又希望能有個搜尋介面的困擾嗎？ \n\ +用NutchEz就對了！因為他操作簡單， \n\ +除了基本的網頁以外，還支援多種格式（ppt,doc,txt...） \n\ +並且是開源碼軟體，完全免費，安全無虞\n\ +趕快來使用看看吧!\n\n\ + 選擇你要的模式:" 20 60 4 \ + "1" "開始建構搜尋內容" "透過NutchEz來建構專屬於你自己所需的內容的搜尋引擎" \ + "2" "開啟或關閉NutchEz的網頁伺服器" "若您之前已經執行完1之後才需做網頁伺服器的管理" 2> $tfile +rev=$? +MAIN_CHOISE=`cat $tfile` -init_nutchez +case $rev in + 0) + echo_dialog_v "OK '$MAIN_CHOISE' chosen.";; + 1) + echo_dialog_v "Cancel pressed." + exit 0 ;; + 2) + echo_dialog_v "HELP '$MAIN_CHOISE' chosen.";; + 255) + echo_dialog_v "ESC pressed.";; + *) + echo_dialog_v "Unexpected code $MAIN_CHOISE";; +esac + +if [ $MAIN_CHOISE -eq 2 ];then + + pid_tc=$(ps axw -eo pid,command |\ + grep "catalina" | grep "java" |\ + grep "start" | awk '{print $1}') + if [ -z "$pid_tc" ]; then + echo_dialog_v "0. pid = $pid_tc ! no another toddmcat is running" + + $DIALOG --title "你的NutchEz網頁伺服器沒打開.." --clear \ + --yesno "\n 要開啟NutchEz的網頁伺服器嗎？ \n" 15 61 + if [ $? -eq 0 ];then + ~/.nutchez/tomcat/bin/startup.sh + PORT=`cat ~/.nutchez/sav/n.tomcat.txt` + $DIALOG --msgbox "已經試著開啟瀏覽伺服器，你可以瀏覽這個網址看看: \n http://localhost:$PORT" 0 0 + else + $DIALOG --msgbox "你選擇不要打開瀏覽伺服器！" 0 0 + fi + + else + echo_dialog_v "0. tomcat had been started and the pid is $pid_tc" + + $DIALOG --title "偵測到NutchEz的網頁伺服器正在運作.." --clear \ + --yesno "\n 你要關閉他嗎？ \n" 15 61 + if [ $? -eq 0 ];then + ~/.nutchez/tomcat/bin/shutdown.sh + kill -9 $pid_tc + $DIALOG --msgbox "已經試著關閉NutchEz網頁伺服器" 0 0 + else + $DIALOG --msgbox "你選擇不要關閉瀏覽伺服器！" 0 0 + fi + + fi + exit 0 + +elif [ $MAIN_CHOISE -eq 1 ];then CHECK=0 @@ -35,5 +98,5 @@ show_urls URL=$? - echo_vb "2.2 show_urls return $URL" + echo_dialog_v "2.2 show_urls return $URL" # add or delete url: ok , exit @@ -63,8 +126,12 @@ setup_tomcat + # continue last search + if [ -e ~/.nutchez/search ];then + continue_previous + fi # show the final checklist final_confirm FC=$? - echo_vb "6.3 final confirm return = $FC " + echo_dialog_v "6.3 final confirm return = $FC " # START , back, cancel # start =0 , back =1 @@ -90,5 +157,5 @@ ;; 255) - echo_vb "ESC pressed !!" + echo_dialog_v "ESC pressed !!" ;; esac @@ -108,2 +175,10 @@ # Done + + + +else + + echo_dialog_v "Main Choise is $MAIN_CHOISE ! Error !!" + +fi Index: /nutchez-0.1/bin/nutchez-func.sh =================================================================== --- /nutchez-0.1/bin/nutchez-func.sh (revision 93) +++ /nutchez-0.1/bin/nutchez-func.sh (revision 94) @@ -6,5 +6,6 @@ -: ${DIALOG=dialog} +#DIALOG='dialog --backtitle " NutchEz Setup Menu -- powered by NCHC "' +DIALOG=dialog # set 1 to display more for debug, @@ -39,5 +40,5 @@ check_if_root() { - if [ ! "$UID" = "0" ]; then + if [ ! "$UID" -eq "0" ]; then echo_dialog_v "Hi [$LOGNAME] !! " echo_dialog_v "You need to run this script \"`basename $0`\" as root." @@ -71,4 +72,13 @@ cp ~/.nutchez/sav/n.urls.txt ~/.nutchez/urls/urls.txt + + if [ $NOCONTINUE -eq 1 ]; then + echo_dialog_v " delete the ~/.nutchez/search/*" + DATE=`date +%y%m%d%H%M%S` + mv ~/.nutchez/search ~/.nutchez/search-$DATE + $DIALOG --msgbox "上次搜尋的結果改放到 ~/.nutchez/search-$DATE " 0 0 + rm -f /tmp/search +# rm -rf ~/.nutchez/search/* + fi if [ -e ~/.nutchez/conf/nutch-site.xml ] ; then @@ -100,5 +110,5 @@ echo_dialog_v "2.1 test_file ~/.nutchez/sav return : $?" # dialog begin - dialog --title "The URLS that you want" --editbox ~/.nutchez/sav/n.urls.txt 16 51 2>/tmp/n.urls.tmp + dialog --title "請輸入你要抓取的網址（一行一個網址）" --editbox ~/.nutchez/sav/n.urls.txt 16 51 2>/tmp/n.urls.tmp RET=$? echo_dialog_v "2.1 cat url: `cat /tmp/n.urls.tmp`" @@ -110,5 +120,5 @@ echo_dialog_v "3. setup_robot" # dialog - dialog --nocancel --inputbox " This agent name \n" 16 51 "`cat ~/.nutchez/sav/n.robot.txt`" 2>/tmp/n.robot.tmp + dialog --title "設定機器人名稱" --nocancel --inputbox " 這個爬網機器人，你要將他取名為：\n\n (ps: 這個設定只是禮貌性宣告，並不會對結果造成影響) \n" 16 55 "`cat ~/.nutchez/sav/n.robot.txt`" 2>/tmp/n.robot.tmp echo_dialog_v "3.1 cat robot : `cat /tmp/n.robot.tmp`" } @@ -117,5 +127,5 @@ echo_dialog_v "4. setup_crawler" test_file ~/.nutchez/sav/n.crawler.txt - dialog --nocancel --inputbox " Depth \n " 16 51 "`cat ~/.nutchez/sav/n.crawler.txt`" 2>/tmp/n.crawler.tmp + dialog --title "設定抓取深度" --nocancel --inputbox " 對於每個網址，你需要NutchEz爬多深呢？\n\n (ps: 初次體驗建議將深度設為1來感受需要多久) \n " 16 51 "`cat ~/.nutchez/sav/n.crawler.txt`" 2>/tmp/n.crawler.tmp echo_dialog_v "4.1 cat robot : `cat /tmp/n.robot.tmp`" } @@ -124,25 +134,51 @@ echo_dialog_v "5. setup_tomcat" test_file ~/.nutchez/sav/n.tomcat.txt - dialog --nocancel --inputbox " explorer port \n " 16 51 "`cat ~/.nutchez/sav/n.tomcat.txt`" 2>/tmp/n.tomcat.tmp + dialog --title "設定網頁伺服器" --nocancel --inputbox " 你希望NutchEz將網頁伺服器開在哪個port \n\n (ps: 請選擇一個沒用到的port以免造成衝突 \n 也請盡量不要設成80以免造成你誤以為是apache的混淆) \n " 16 51 "`cat ~/.nutchez/sav/n.tomcat.txt`" 2>/tmp/n.tomcat.tmp echo_dialog_v "5.1 cat tomcat : `cat /tmp/n.tomcat.tmp`" } +continue_previous () { + echo_dialog_v "6. setup_tomcat" +$DIALOG --title "清除上次搜尋" --clear \ + --yesno "你是否要清除上一次爬網所得的結果，\n否則將加入到URL列裡增加搜尋負擔 \n\n ps: 選no的話，會跑相當相當久，\n 請慎重考慮之\n" 16 51 + +case $? in + 0) + NOCONTINUE=1;; + 1) + NOCONTINUE=0;; + 255) + echo "ESC pressed.";; +esac +echo_dialog_v " 6continue = $CONTINUE" +} + final_confirm () { - echo_dialog_v "6. final_confirm : start =0 , back =1 " + echo_dialog_v "7. final_confirm : start =0 , back =1 " tempfile=/tmp/n.finalcheck.tmp - echo " \n 1. The url list is : \n " > $tempfile + echo " \n 1. 你所選擇要爬取的網址為 : \n " > $tempfile cat /tmp/n.urls.tmp >> $tempfile - echo " \n 2. The robot name is : \n" >> $tempfile + echo " \n\n 2. 對於這個爬網機器人，你取名為 : \n" >> $tempfile cat /tmp/n.robot.tmp >> $tempfile - echo " \n 3. The crawled depth is : \n " >> $tempfile + echo " \n\n 3. 爬網的深度，你設定為 : \n " >> $tempfile cat /tmp/n.crawler.tmp >> $tempfile - echo " \n 4. The explorer port is : \n " >> $tempfile + echo " \n\n 4. NutchEz將會把你的搜尋結果呈現在這個Port : \n " >> $tempfile cat /tmp/n.tomcat.tmp >> $tempfile - + if [ $NOCONTINUE -eq 0 ];then + echo " \n\n 5. 是否要清除上一次的收尋結果 : \n " >> $tempfile + echo_dialog_v " 7continue = $CONTINUE" + echo "NO" >> $tempfile + elif [ $NOCONTINUE -eq 1 ];then + echo " \n\n 5. 是否要清除上一次的收尋結果繼續搜尋 : \n " >> $tempfile + echo_dialog_v " 7continue = $CONTINUE" + echo "YES" >> $tempfile + else + echo_dialog_v " 無資料可匯入 " + fi MSG=`cat $tempfile` - echo_dialog_v "6.1 final message :\n $MSG" + echo_dialog_v "7.1 final message :\n $MSG" #read READ - $DIALOG --title "Check It !!" --clear \ + $DIALOG --title "請檢查你的選擇 ! \n\n 若所有的設定都是正確的，你可以按 \"ok\",\n 若你按了 \"reset\" 則會重頭開始設定, \n 若你選擇 \"exit\" 則會跳出NutchEz的設定選單 \n ps: reset 與 exit都不會把資料記成預設值，請放心使用 " --clear \ --extra-button --extra-label "reset" --ok-label "ok" --cancel-label "exit" \ --yesno "$MSG" 26 51 @@ -179,18 +215,19 @@ pid_tc=$(ps axw -eo pid,command |\ - grep "catalina.startup.Bootstrap" |\ - grep "start" | awk '{print $1}') + grep "catalina" | grep "java" |\ + grep "start" | awk '{print $1}') if [ -z "$pid_tc" ]; then - echo "starting tomcat" + echo_dialog_v "no another tomcat is running" else - echo "tomcat had been started and the pid is $pid_tc" - echo "stop it first" + echo_dialog_v "tomcat had been started and the pid is $pid_tc" + echo_dialog_v "stop it first" kill -9 $pid_tc if [ -z $? ];then - echo " tomcat ($pid_tc) is killed ..." + echo_dialog_v " tomcat ($pid_tc) is killed ..." else - echo "kill error ..." + echo_dialog_v "kill error ..." fi fi + echo "Starting Tomcat ...." ~/.nutchez/tomcat/bin/startup.sh sleep 3 @@ -201,10 +238,10 @@ FIREFOX=`which firefox` RET=$? - if [ $RET == 0 ];then + if [ $RET -eq 0 ];then $FIREFOX -D 0.0 http://localhost:$PORT RET=$? fi - if ! [ $RET == 0 ];then - $DIALOG --msgbox "Congratulations! \n you can explore the url: \n http://localhost:$PORT" 0 0 - fi -} + if ! [ $RET -eq 0 ];then + $DIALOG --msgbox "恭喜你已經完成了! \n 你可以用瀏覽器瀏覽: \n http://host_ip:$PORT" 0 0 + fi +} Index: /nutchez-0.1/debian/changelog =================================================================== --- /nutchez-0.1/debian/changelog (revision 93) +++ /nutchez-0.1/debian/changelog (revision 94) @@ -1,6 +1,6 @@ -nutchez (0.1-1) unstable; urgency=low +nutchez (0.1-2) unstable; urgency=low * Initial release (Closes: #nnnn) + * - -- Wei-Yu Chen Tue, 12 May 2009 11:15:51 +0800 - + -- Wei-Yu Chen Tue, 07 Jul 2009 11:22:46 +0800 Index: /nutchez-0.1/debian/nutchez.install =================================================================== --- /nutchez-0.1/debian/nutchez.install (revision 93) +++ /nutchez-0.1/debian/nutchez.install (revision 94) @@ -3,7 +3,5 @@ README* etc/nutch bin opt/nutch -bin/nutchez* usr/local/bin lib opt/nutch -webapps opt/nutch tomcat opt/nutch plugins opt/nutch Index: /nutchez-0.1/debian/nutchez.postinst =================================================================== --- /nutchez-0.1/debian/nutchez.postinst (revision 93) +++ /nutchez-0.1/debian/nutchez.postinst (revision 94) @@ -8,58 +8,13 @@ fi -ln -sf /opt/nutch/bin/nutchez /usr/local/sbin/ -ln -sf /opt/nutch/bin/nutchez-func.sh /usr/local/sbin/ +ln -sf /opt/nutch/bin/nutchez /usr/sbin/ +ln -sf /opt/nutch/bin/nutchez-func.sh /usr/sbin/ -setup_hdfsadm_user() { - if ! getent passwd hdfsadm >/dev/null; then - useradd hdfsadm - mkdir -p /home/hdfsadm/.ssh - mkdir -p /var/log/nutch - ssh-keygen -t rsa -q -f /home/hdfsadm/.ssh/id_rsa -N "" - cp /home/hdfsadm/.ssh/id_rsa.pub /home/hdfsadm/.ssh/authorized_keys - chown hdfsadm:hdfsadm /var/log/nutch - chown -R hdfsadm:hdfsadm /opt/nutch - chown -R hdfsadm:hdfsadm /home/hdfsadm - fi +show_message(){ + cat /etc/nutch/README } - -check_root() { - if ! test -e /root/.ssh/id_rsa ; then - ssh-keygen -t rsa -q -f /root/.ssh/id_rsa -N "" - fi - if test -e /root/.ssh/id_rsa.pub ; then - if ! test -e /root/.ssh/authorized_keys ; then - cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys - fi - else - ssh-keygen -t rsa -q -f /root/.ssh/id_rsa -N "" - cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys - fi - -} - -start_hadoop() { - if getent passwd hdfsadm >/dev/null; then - su -c "/opt/nutch/bin/hadoop namenode -format" hdfsadm - - su -c "/opt/nutch/bin/start-all.sh" hdfsadm - - else - /opt/nutch/conf/hadoop-env.sh - /opt/nutch/bin/hadoop namenode -format - /opt/nutch/bin/start-all.sh - fi -} -show_message() { +show_old_message() { echo "You can use the instruction : \" nutchez\" to easyly use nutch" echo "Enjoy" } -show_old_message() { - echo "You can quickly start by following ways [in /opt/nutch/ with root privilege]:" - echo "(1) Modify the urls/urls.txt file with indicate urls, one site one line." - echo "(2) Use this instruction \"bin/nutch crawl urls -dir search -depth 4 -topN 50\" to crawl web" - echo "(3) Type \" tomcat/bin/startup.sh \" and use browser to check the result in http://localhost:8080/" - echo "Enjoy !" -} -#setup_hdfsadm_user -#check_root -#start_hadoop show_message Index: /nutchez-0.1/debian/nutchez.postrm =================================================================== --- /nutchez-0.1/debian/nutchez.postrm (revision 93) +++ /nutchez-0.1/debian/nutchez.postrm (revision 94) @@ -18,5 +18,5 @@ done -rm -f /usr/local/sbin/nutchez* +rm -f /usr/sbin/nutchez* rm -rf /tmp/search Index: /nutchez-0.1/debian/rules =================================================================== --- /nutchez-0.1/debian/rules (revision 93) +++ /nutchez-0.1/debian/rules (revision 94) @@ -13,4 +13,6 @@ dh_compress dh_fixperms + find ./debian/nutchez/ -name ".svn" -type d > svnfolders + rm -rf `cat svnfolders` dh_installdeb dh_link