Changeset 68 for nutchez-0.1/bin/nutchez-func.sh

Timestamp:

May 22, 2009, 6:36:48 PM (15 years ago)

Author:

waue

Message:

big modification

File:

• nutchez-0.1/bin/nutchez-func.sh (modified) (8 diffs)

nutchez-0.1/bin/nutchez-func.sh

@@ -9 +9 @@
 : ${DIALOG=dialog}
 
+# display more for debug
 VERB=0
-
 
 echo_vb () {
@@ -37 +37 @@
 promote_tempfile () {
   echo_vb "7. chang tmp as txt"
-  rm /tmp/n.*.txt
-  mv /tmp/n.url.tmp /tmp/n.url.txt
-  mv /tmp/n.robot.tmp /tmp/n.robot.txt
-  mv /tmp/n.crawler.tmp /tmp/n.crawler.txt
-  mv /tmp/n.tomcat.tmp /tmp/n.tomcat.txt
+  rm ~/.nutchez/sav/n.*.txt
+  mv /tmp/n.url.tmp ~/.nutchez/sav/
+  mv /tmp/n.robot.tmp ~/.nutchez/sav/
+  mv /tmp/n.crawler.tmp ~/.nutchez/sav/
+  mv /tmp/n.tomcat.tmp ~/.nutchez/sav/
 }
 
@@ -49 +49 @@
 }
 
-setup_nutchez() {
+init_nutchez () {
   if ! [ -e ~/.nutchez ] ;then
-    cp -rf /etc/nutch/.nutchez ~/
+    # copy from /etc/nutch
+    cp -rf /etc/nutch/* ~/.nutchez
     chown -R $LOGNAME:$LOGNAME ~/.nutchez
-    if [ -e /tmp/nutch ] ;then
-      rm -rf /tmp/nutch
-    fi
-    mkdir /tmp/nutch
-    ln -sf ~/.nutchez/search /tmp/nutch/
-
   fi
 }
+
+setup_nutchez () {    
+  if ! [ -e ~/.nutchez/urls ] ; then
+    # make url list dir
+    mkdir ~/.nutchez/urls
+  fi
+
+  if  [ -e ~/.nutchez/urls/urls.txt ] ; then
+    rm ~/.nutchez/urls/urls.txt
+  fi
+
+  cp ~/.nutchez/sav/n.urls.txt ~/.nutchez/urls/urls.txt
+
+  if [ -e ~/.nutchez/nutch-site.xml ] ; then
+    # set nutch-site.xml
+    sed -ie "s/>user</>$ROBOT</" ~/.nutchez/nutch-site.xml
+  fi
+
+}
+
+install_tomcat (){
+  if ! [ -e ~/.nutchez/tomcat ] ;then
+    # isntall tomcat to home
+    cp -rf /opt/nutch/tomcat ~/.nutchez/
+    chown -R $LOGNAME:$LOGNAME ~/.nutchez/tomcat/
+    # make search dir
+    if ! [ -e ~/.nutchez/search ] ;then
+      mkdir ~/.nutchez/search
+    fi
+    # change explorer port
+    sed -ie "s/8080/>$PORT</" ~/.nutchez/tomcat/conf/server.xml
+}
+
 
 show_urls (){
   # show urls : ok =0 ,cancel = 1
   echo_vb "2. show_urls !"
-  test_file  /tmp/n.url.txt
-  echo_vb "2.1 test_file /tmp/n.url.txt return : $?"
+  test_file  ~/.nutchez/sav/n.url.txt
+  echo_vb "2.1 test_file ~/.nutchez/sav return : $?"
   # dialog begin
-  dialog  --editbox /tmp/n.url.txt 16 51 2>/tmp/n.url.tmp
+  dialog  --editbox ~/.nutchez/sav/n.url.txt 16 51 2>/tmp/n.url.tmp
   RET=$?
   echo_vb "2.1 cat url: `cat /tmp/n.url.tmp`"
@@ -75 +103 @@
 
 setup_robot () {
-  test_file /tmp/n.robot.txt
+  test_file ~/.nutchez/sav/n.robot.txt
   echo_vb "3. setup_robot"
   # dialog
-  dialog --nocancel --inputbox " This agent name \n" 16 51 "`cat /tmp/n.robot.txt`" 2>/tmp/n.robot.tmp
+  dialog --nocancel --inputbox " This agent name \n" 16 51 "`cat ~/.nutchez/sav/n.robot.txt`" 2>/tmp/n.robot.tmp
   echo_vb "3.1 cat robot : `cat /tmp/n.robot.tmp`"
 }
@@ -84 +112 @@
 setup_crawler () {
   echo_vb "4. setup_crawler"
-  test_file /tmp/n.crawler.txt
-  dialog --nocancel --inputbox " Depth  \n " 16 51 "`cat /tmp/n.crawler.txt`" 2>/tmp/n.crawler.tmp
+  test_file ~/.nutchez/sav/n.crawler.txt
+  dialog --nocancel --inputbox " Depth  \n " 16 51 "`cat ~/.nutchez/sav/n.crawler.txt`" 2>/tmp/n.crawler.tmp
   echo_vb "4.1 cat robot : `cat /tmp/n.robot.tmp`"
 }
@@ -91 +119 @@
 setup_tomcat () {
   echo_vb "5. setup_tomcat"
-  test_file /tmp/n.tomcat.txt
-  dialog --nocancel --inputbox " explorer port \n " 16 51 "`cat /tmp/n.tomcat.txt`" 2>/tmp/n.tomcat.tmp
+  test_file ~/.nutchez/sav/n.tomcat.txt
+  dialog --nocancel --inputbox " explorer port \n " 16 51 "`cat ~/.nutchez/sav/n.tomcat.txt`" 2>/tmp/n.tomcat.tmp
   echo_vb "5.1 cat tomcat : `cat /tmp/n.tomcat.tmp`"
 }
@@ -121 +149 @@
 
 # define paramaters
-ROBOT=`cat /tmp/n.url.txt`
-URLS=`cat /tmp/n.url.txt`
-DEPTH=`cat /tmp/n.url.txt`
-PORT=`cat /tmp/n.url.txt`
 
 start_crawl () {
+
+  ROBOT=`cat ~/.nutchez/sav`
+  URLS=`cat ~/.nutchez/sav`
+  DEPTH=`cat ~/.nutchez/sav`
+  PORT=`cat ~/.nutchez/sav`
+
   echo_vb "7. start_crawl"
   setup_nutch
-  /opt/nutch/bin/nutch crawl ~/.nutch/urls -dir ~/.nutch/search -threads 2 -depth $DEPTH 
+  install_tomcat
+  # /opt/nutch/bin/nutch crawl ~/.nutchez/urls -dir ~/.nutchez/search -depth $DEPTH
+  echo "/opt/nutch/bin/nutch crawl ~/.nutchez/urls -dir ~/.nutchez/search -depth $DEPTH"
 }
 
@@ -139 +171 @@
 show_report () {
   echo_vb "9. show_report "
+  FIREFOX=`which firefox`
+  RET=$?
+  if [ RET == 0 ];then 
+    $FIREFOX -D 0.0 http://localhost:$PORT
+  else
+    $DIALOG --msgbox "Congratulations! \n you can explore the url: \n  http://localhost:8080" 0 0 
+  fi
 }