Changeset 68

Timestamp:

May 22, 2009, 6:36:48 PM (15 years ago)

Author:

waue

Message:

big modification

Location:

nutchez-0.1

Files:

• bin/nutchez (modified) (1 diff)
• bin/nutchez-func.sh (modified) (8 diffs)
• conf/sav (moved) (moved from nutchez-0.1/conf/.nutchez)
• debian/nutchez.postinst (modified) (2 diffs)
• debian/nutchez.postrm (modified) (1 diff)
• tomcat/webapps/ROOT/WEB-INF/classes/nutch-site.xml (modified) (1 diff)
• urls (deleted)

nutchez-0.1/bin/nutchez

@@ -4 +4 @@
 # Description: Eazily use for Nutch
 # .
-
+export NUTCH_CONF_DIR=~/.nutchez
 NUTCHEZ_SCRIPT_PATH="${NUTCHEZ_SCRIPT_PATH:-/opt/nutch}"
 
 . ./nutchez-func.sh
+
 
 # root ?

nutchez-0.1/bin/nutchez-func.sh

@@ -9 +9 @@
 : ${DIALOG=dialog}
 
+# display more for debug
 VERB=0
-
 
 echo_vb () {
@@ -37 +37 @@
 promote_tempfile () {
   echo_vb "7. chang tmp as txt"
-  rm /tmp/n.*.txt
-  mv /tmp/n.url.tmp /tmp/n.url.txt
-  mv /tmp/n.robot.tmp /tmp/n.robot.txt
-  mv /tmp/n.crawler.tmp /tmp/n.crawler.txt
-  mv /tmp/n.tomcat.tmp /tmp/n.tomcat.txt
+  rm ~/.nutchez/sav/n.*.txt
+  mv /tmp/n.url.tmp ~/.nutchez/sav/
+  mv /tmp/n.robot.tmp ~/.nutchez/sav/
+  mv /tmp/n.crawler.tmp ~/.nutchez/sav/
+  mv /tmp/n.tomcat.tmp ~/.nutchez/sav/
 }
 
@@ -49 +49 @@
 }
 
-setup_nutchez() {
+init_nutchez () {
   if ! [ -e ~/.nutchez ] ;then
-    cp -rf /etc/nutch/.nutchez ~/
+    # copy from /etc/nutch
+    cp -rf /etc/nutch/* ~/.nutchez
     chown -R $LOGNAME:$LOGNAME ~/.nutchez
-    if [ -e /tmp/nutch ] ;then
-      rm -rf /tmp/nutch
-    fi
-    mkdir /tmp/nutch
-    ln -sf ~/.nutchez/search /tmp/nutch/
-
   fi
 }
+
+setup_nutchez () {    
+  if ! [ -e ~/.nutchez/urls ] ; then
+    # make url list dir
+    mkdir ~/.nutchez/urls
+  fi
+
+  if  [ -e ~/.nutchez/urls/urls.txt ] ; then
+    rm ~/.nutchez/urls/urls.txt
+  fi
+
+  cp ~/.nutchez/sav/n.urls.txt ~/.nutchez/urls/urls.txt
+
+  if [ -e ~/.nutchez/nutch-site.xml ] ; then
+    # set nutch-site.xml
+    sed -ie "s/>user</>$ROBOT</" ~/.nutchez/nutch-site.xml
+  fi
+
+}
+
+install_tomcat (){
+  if ! [ -e ~/.nutchez/tomcat ] ;then
+    # isntall tomcat to home
+    cp -rf /opt/nutch/tomcat ~/.nutchez/
+    chown -R $LOGNAME:$LOGNAME ~/.nutchez/tomcat/
+    # make search dir
+    if ! [ -e ~/.nutchez/search ] ;then
+      mkdir ~/.nutchez/search
+    fi
+    # change explorer port
+    sed -ie "s/8080/>$PORT</" ~/.nutchez/tomcat/conf/server.xml
+}
+
 
 show_urls (){
   # show urls : ok =0 ,cancel = 1
   echo_vb "2. show_urls !"
-  test_file  /tmp/n.url.txt
-  echo_vb "2.1 test_file /tmp/n.url.txt return : $?"
+  test_file  ~/.nutchez/sav/n.url.txt
+  echo_vb "2.1 test_file ~/.nutchez/sav return : $?"
   # dialog begin
-  dialog  --editbox /tmp/n.url.txt 16 51 2>/tmp/n.url.tmp
+  dialog  --editbox ~/.nutchez/sav/n.url.txt 16 51 2>/tmp/n.url.tmp
   RET=$?
   echo_vb "2.1 cat url: `cat /tmp/n.url.tmp`"
@@ -75 +103 @@
 
 setup_robot () {
-  test_file /tmp/n.robot.txt
+  test_file ~/.nutchez/sav/n.robot.txt
   echo_vb "3. setup_robot"
   # dialog
-  dialog --nocancel --inputbox " This agent name \n" 16 51 "`cat /tmp/n.robot.txt`" 2>/tmp/n.robot.tmp
+  dialog --nocancel --inputbox " This agent name \n" 16 51 "`cat ~/.nutchez/sav/n.robot.txt`" 2>/tmp/n.robot.tmp
   echo_vb "3.1 cat robot : `cat /tmp/n.robot.tmp`"
 }
@@ -84 +112 @@
 setup_crawler () {
   echo_vb "4. setup_crawler"
-  test_file /tmp/n.crawler.txt
-  dialog --nocancel --inputbox " Depth  \n " 16 51 "`cat /tmp/n.crawler.txt`" 2>/tmp/n.crawler.tmp
+  test_file ~/.nutchez/sav/n.crawler.txt
+  dialog --nocancel --inputbox " Depth  \n " 16 51 "`cat ~/.nutchez/sav/n.crawler.txt`" 2>/tmp/n.crawler.tmp
   echo_vb "4.1 cat robot : `cat /tmp/n.robot.tmp`"
 }
@@ -91 +119 @@
 setup_tomcat () {
   echo_vb "5. setup_tomcat"
-  test_file /tmp/n.tomcat.txt
-  dialog --nocancel --inputbox " explorer port \n " 16 51 "`cat /tmp/n.tomcat.txt`" 2>/tmp/n.tomcat.tmp
+  test_file ~/.nutchez/sav/n.tomcat.txt
+  dialog --nocancel --inputbox " explorer port \n " 16 51 "`cat ~/.nutchez/sav/n.tomcat.txt`" 2>/tmp/n.tomcat.tmp
   echo_vb "5.1 cat tomcat : `cat /tmp/n.tomcat.tmp`"
 }
@@ -121 +149 @@
 
 # define paramaters
-ROBOT=`cat /tmp/n.url.txt`
-URLS=`cat /tmp/n.url.txt`
-DEPTH=`cat /tmp/n.url.txt`
-PORT=`cat /tmp/n.url.txt`
 
 start_crawl () {
+
+  ROBOT=`cat ~/.nutchez/sav`
+  URLS=`cat ~/.nutchez/sav`
+  DEPTH=`cat ~/.nutchez/sav`
+  PORT=`cat ~/.nutchez/sav`
+
   echo_vb "7. start_crawl"
   setup_nutch
-  /opt/nutch/bin/nutch crawl ~/.nutch/urls -dir ~/.nutch/search -threads 2 -depth $DEPTH 
+  install_tomcat
+  # /opt/nutch/bin/nutch crawl ~/.nutchez/urls -dir ~/.nutchez/search -depth $DEPTH
+  echo "/opt/nutch/bin/nutch crawl ~/.nutchez/urls -dir ~/.nutchez/search -depth $DEPTH"
 }
 
@@ -139 +171 @@
 show_report () {
   echo_vb "9. show_report "
+  FIREFOX=`which firefox`
+  RET=$?
+  if [ RET == 0 ];then 
+    $FIREFOX -D 0.0 http://localhost:$PORT
+  else
+    $DIALOG --msgbox "Congratulations! \n you can explore the url: \n  http://localhost:8080" 0 0 
+  fi
 }

nutchez-0.1/debian/nutchez.postinst

@@ -7 +7 @@
   exit 0
 fi
+
+ln -sf /opt/nutch/bin/nutchez /usr/local/sbin/
 
 setup_hdfsadm_user() {
@@ -47 +49 @@
 }
 show_message() {
+  echo "You can use the instruction : \" nutchez\" to easyly use nutch"
+  echo "Enjoy"
+}
+show_old_message() {
   echo "You can quickly start by following ways [in /opt/nutch/ with root privilege]:"
   echo "(1) Modify the urls/urls.txt file with indicate urls, one site one line."

nutchez-0.1/debian/nutchez.postrm

@@ -8 +8 @@
 fi
 
-setup_hdfsadm_user() {
-  if ! getent passwd hdfsadm >/dev/null; then
-    echo "no account found: 'hdfsadm'."
-  else
-    userdel hdfsadm
-    rm -rf /home/hdfsadm
-    rm -rf /opt/nutch
-    rm -rf /tmp/hadoop*
-    rm -rf /tmp/hsperfdata*
-    rm /usr/local/sbin/nutchez*
-  fi
-}
+rm -rf /tmp/hsperfdata*
+if [ -e HOME/.nutchez ]; then
+  rm -rf $HOME/.nutchez
+fi
+rm /usr/local/sbin/nutchez*
 
-setup_hdfsadm_user

nutchez-0.1/tomcat/webapps/ROOT/WEB-INF/classes/nutch-site.xml

@@ -6 +6 @@
     <property>
         <name>searcher.dir</name>
-        <value>/tmp/nutch/search</value>
+        <value>../../../../../search</value>
     </property>
 </configuration>