source: nutchez-0.1/bin/nutchez-func.sh @ 108

Last change on this file since 108 was 108, checked in by waue, 15 years ago

years and log

  • Property svn:executable set to *
File size: 7.9 KB
Line 
1#!/bin/bash
2# Author: WeiYu Chen <waue _at_ nchc org tw>
3# License: GPL
4# Description: Eazily use for Nutch
5# .
6
7
8#DIALOG='dialog --backtitle "     NutchEz Setup Menu           -- powered by NCHC "'
9DIALOG=dialog
10
11# set 1 to display more for debug,
12VERB=0
13
14init_nutchez () {
15  if ! [ -e ~/.nutchez ] ;then
16    # copy from /etc/nutch
17    mkdir ~/.nutchez
18    cp -rf /etc/nutch/conf ~/.nutchez/
19    mkdir ~/.nutchez/log
20    cp -rf /etc/nutch/sav ~/.nutchez/
21    chown -R $LOGNAME:$LOGNAME ~/.nutchez
22  fi
23 
24}
25
26echo_dialog_v () {
27  if [[ $VERB -eq 1 ]]; then
28    $DIALOG --msgbox "$1" 16 51
29  fi
30}
31
32test_file () {
33  if ! test -e "$1" ; then
34    echo_dialog_v "test_file: \n can not find $1"
35    echo "" >  $1
36  else
37     echo_dialog_v "test_file: \n Touch  $1 ! \n Its content is \n `cat $1`"
38  fi
39}
40
41check_if_root() {
42   if [[ ! "$UID" -eq "0" ]]; then
43     echo_dialog_v "Hi [$LOGNAME] !! "
44     echo_dialog_v "You need to run this script \"`basename $0`\" as root."
45     exit 1
46   fi
47}
48
49promote_tempfile () {
50  echo_dialog_v "7. chang tmp as txt"
51  rm -f ~/.nutchez/sav/n.*.txt
52  mv /tmp/n.urls.tmp ~/.nutchez/sav/n.urls.txt
53  mv /tmp/n.robot.tmp ~/.nutchez/sav/n.robot.txt
54  mv /tmp/n.crawler.tmp ~/.nutchez/sav/n.crawler.txt
55  mv /tmp/n.tomcat.tmp ~/.nutchez/sav/n.tomcat.txt
56  rm -f /tmp/n.*.tmp
57}
58
59clean_tempfile () {
60  echo_dialog_v "7. delete tmp"
61  rm -f /tmp/n.*.tmp
62}
63
64setup_nutchez () {   
65  if ! [ -e ~/.nutchez/urls ] ; then
66    # make url list dir
67    mkdir ~/.nutchez/urls
68  fi
69
70  if  [ -e ~/.nutchez/urls/urls.txt ] ; then
71    rm ~/.nutchez/urls/urls.txt
72  fi
73
74  cp ~/.nutchez/sav/n.urls.txt ~/.nutchez/urls/urls.txt
75  if ! [ -z $NOCONTINUE ]; then
76    if [[ $NOCONTINUE -eq 1 ]]; then
77      echo_dialog_v " delete the ~/.nutchez/search/*"
78      DATE=`date +%Y%m%d%H%M%S`
79      mv ~/.nutchez/search ~/.nutchez/search-$DATE
80      $DIALOG --msgbox "上次搜尋的結果改放到 ~/.nutchez/search-$DATE " 0 0
81  #    rm -f /tmp/search
82  #    rm -rf ~/.nutchez/search/*
83    fi
84  fi
85
86  if [ -e ~/.nutchez/conf/nutch-site.xml ] ; then
87    # set nutch-site.xml
88    sed -i -e "4s/<value>[a-zA-Z0-9]*</<value>$ROBOT</" ~/.nutchez/conf/nutch-site.xml
89  fi
90}
91
92install_tomcat (){
93  if ! [ -e ~/.nutchez/tomcat ] ;then
94    # isntall tomcat to home
95    cp -rf /opt/nutchez/tomcat ~/.nutchez/
96    chown -R $LOGNAME:$LOGNAME ~/.nutchez/tomcat/
97    # make search dir
98    if ! [ -e ~/.nutchez/search ] ;then
99      mkdir ~/.nutchez/search
100    fi
101
102   # setup search dir
103    cd ~
104    HOMEDIR=`pwd`/.nutchez/search
105    sed -i -e "8s|<value>[a-zA-Z0-9/]*<|<value>$HOMEDIR<|" ~/.nutchez/tomcat/webapps/ROOT/WEB-INF/classes/nutch-site.xml
106
107  fi
108  # change explorer port
109  sed -i -e "67s/<Connector port=\"[0-9]*\"/<Connector port=\"$PORT\"/" ~/.nutchez/tomcat/conf/server.xml
110}
111
112
113show_urls (){
114  # show urls : ok =0 ,cancel = 1
115  echo_dialog_v "2. show_urls !"
116  test_file  ~/.nutchez/sav/n.urls.txt
117  echo_dialog_v "2.1 test_file ~/.nutchez/sav return : $?"
118  # dialog begin
119  dialog --title "請輸入你要抓取的網址(一行一個網址)" --editbox ~/.nutchez/sav/n.urls.txt 16 51 2>/tmp/n.urls.tmp
120  RET=$?
121  echo_dialog_v "2.1 cat url: `cat /tmp/n.urls.tmp`"
122  return $RET
123}
124
125setup_robot () {
126  test_file ~/.nutchez/sav/n.robot.txt
127  echo_dialog_v "3. setup_robot"
128  # dialog
129  dialog --title "設定機器人名稱" --nocancel --inputbox " 這個爬網機器人,你要將他取名為:\n\n (ps: 這個設定只是禮貌性宣告,並不會對結果造成影響) \n" 16 55 "`cat ~/.nutchez/sav/n.robot.txt`" 2>/tmp/n.robot.tmp
130  echo_dialog_v "3.1 cat robot : `cat /tmp/n.robot.tmp`"
131}
132
133setup_crawler () {
134  echo_dialog_v "4. setup_crawler"
135  test_file ~/.nutchez/sav/n.crawler.txt
136  dialog --title "設定抓取深度" --nocancel --inputbox " 對於每個網址,你需要NutchEz爬多深呢?\n\n (ps: 初次體驗建議將深度設為1來感受需要多久)  \n " 16 51 "`cat ~/.nutchez/sav/n.crawler.txt`" 2>/tmp/n.crawler.tmp
137  echo_dialog_v "4.1 cat robot : `cat /tmp/n.robot.tmp`"
138}
139
140setup_tomcat () {
141  echo_dialog_v "5. setup_tomcat"
142  test_file ~/.nutchez/sav/n.tomcat.txt
143  dialog --title "設定網頁伺服器" --nocancel --inputbox " 你希望NutchEz將網頁伺服器開在哪個port \n\n (ps: 請選擇一個沒用到的port以免造成衝突 \n 也請盡量不要設成80以免造成你誤以為是apache的混淆) \n " 16 51 "`cat ~/.nutchez/sav/n.tomcat.txt`" 2>/tmp/n.tomcat.tmp
144  echo_dialog_v "5.1 cat tomcat : `cat /tmp/n.tomcat.tmp`"
145}
146
147continue_previous () {
148  echo_dialog_v "6. setup_tomcat"
149$DIALOG --title "清除上次搜尋" --clear \
150        --yesno "你是否要清除上一次爬網所得的結果,\n否則將加入到URL列裡增加搜尋負擔 \n\n ps: 選no的話,會跑相當相當久,\n 請慎重考慮之\n" 16 51
151
152case $? in
153  0)
154    NOCONTINUE=1;;
155  1)
156    NOCONTINUE=0;;
157  255)
158    echo "ESC pressed.";;
159esac
160echo_dialog_v " 6continue = $CONTINUE"
161}
162
163final_confirm () {
164  echo_dialog_v "7. final_confirm : start =0 , back =1 "
165  tempfile=/tmp/n.finalcheck.tmp
166
167  echo " \n 1. 你所選擇要爬取的網址為 : \n " > $tempfile
168  cat /tmp/n.urls.tmp >> $tempfile
169  echo " \n\n 2. 對於這個爬網機器人,你取名為 : \n" >> $tempfile
170  cat /tmp/n.robot.tmp >> $tempfile
171  echo " \n\n 3. 爬網的深度,你設定為 : \n " >> $tempfile
172  cat /tmp/n.crawler.tmp >> $tempfile
173  echo " \n\n 4. NutchEz將會把你的搜尋結果呈現在這個Port : \n " >> $tempfile
174  cat /tmp/n.tomcat.tmp >> $tempfile
175  if [[ $NOCONTINUE -eq 0 ]];then
176    echo " \n\n 5. 是否要清除上一次的收尋結果 : \n " >> $tempfile
177    echo_dialog_v " 7continue = $CONTINUE"
178    echo "NO" >> $tempfile
179  elif [[ $NOCONTINUE -eq 1 ]];then
180    echo " \n\n 5. 是否要清除上一次的收尋結果繼續搜尋 : \n " >> $tempfile
181    echo_dialog_v " 7continue = $CONTINUE"
182    echo "YES" >> $tempfile
183  else
184    echo_dialog_v " 無資料可匯入 "
185  fi
186  MSG=`cat $tempfile`
187  echo_dialog_v "7.1 final message :\n $MSG"
188  #read READ
189  $DIALOG --title "請檢查你的選擇 ! \n\n 若所有的設定都是正確的,你可以按 \"ok\",\n 若你按了 \"reset\" 則會重頭開始設定, \n 若你選擇 \"exit\" 則會跳出NutchEz的設定選單 \n ps: reset 與 exit都不會把資料記成預設值,請放心使用 " --clear \
190        --extra-button --extra-label "reset" --ok-label "ok" --cancel-label "exit" \
191        --yesno "$MSG" 26 51
192  RET=$?
193  echo_dialog_v "final return = $RET" 
194  return $RET
195}
196
197# define paramaters
198
199set_nutchez_p () {
200  ROBOT=`cat ~/.nutchez/sav/n.robot.txt`
201  URLS=`cat ~/.nutchez/sav/n.urls.txt`
202  DEPTH=`cat ~/.nutchez/sav/n.crawler.txt`
203  PORT=`cat ~/.nutchez/sav/n.tomcat.txt`
204}
205start_crawl () {
206
207  echo_dialog_v "7. start_crawl"
208  setup_nutchez
209  install_tomcat
210  echo_dialog_v "/opt/nutchez/nutch/bin/nutch crawl ~/.nutchez/urls -dir ~/.nutchez/search -depth $DEPTH"
211  echo_dialog_v "nutch conf dir = $NUTCH_CONF_DIR"
212  /opt/nutchez/nutch/bin/nutch crawl ~/.nutchez/urls -dir ~/.nutchez/search -depth $DEPTH -topN 5000 -threads 1000
213}
214
215start_tomcat () {
216  echo_dialog_v "8. start_tomcat "
217  echo_dialog_v "/opt/nutchez/nutch/tomcat/bin/startup.sh"
218  #if [ -e /tmp/search ];then
219  #  rm -rf /tmp/search
220  #fi
221  #ln -sf ~/.nutchez/search/ /tmp/
222
223  pid_tc=$(ps axw -eo pid,command |\
224  grep "catalina" | grep "java" |\
225  grep "start" | awk '{print $1}')
226  if [ -z "$pid_tc" ]; then
227      echo_dialog_v "no another tomcat is running"
228  else
229      echo_dialog_v "tomcat had been started and the pid is $pid_tc"
230      echo_dialog_v "stop it first"
231      kill -9 $pid_tc
232      if [ -z $? ];then
233    echo_dialog_v " tomcat ($pid_tc) is  killed ..."
234      else
235    echo_dialog_v "kill error ..." 
236      fi
237  fi
238  echo "Starting Tomcat ...."
239  ~/.nutchez/tomcat/bin/startup.sh
240  sleep 3
241}
242
243show_report () {
244  echo_dialog_v "9. show_report "
245  FIREFOX=`which firefox`
246  RET=$?
247  if [[ $RET -eq 0 ]];then 
248    $FIREFOX -D 0.0 http://localhost:$PORT
249    RET=$?
250  fi
251  if ! [[ $RET -eq 0 ]];then
252    $DIALOG --msgbox "恭喜你已經完成了! \n 你可以用瀏覽器瀏覽: \n  http://host_ip:$PORT" 0 0 
253  fi
254}
Note: See TracBrowser for help on using the repository browser.