Changeset 125


Ignore:
Timestamp:
May 24, 2010, 12:25:09 PM (14 years ago)
Author:
shunfa
Message:

shunfa ci install.sh and install_func.sh

Location:
nutchez-0.2/src/test
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • nutchez-0.2/src/test/install

    r119 r125  
    1414  # check_dialog
    1515  # check_php
     16
     17  #set install mode
    1618  set_install_information
    1719  show_info
    … …  
    2224    # make_ssh_key
    2325
    24     Hadoop_install
    25     Nutch_install
    26 
    27     # tomcat_install
     26    Install_Nutch
     27    Install_Tomcat
     28   
    2829    # make_client_install
    2930   
    3031    format_HDFS
    3132    start_up_NutchEZ
    32     set_tomcat
     33    install_tomcat
    3334    start_up_tomcat
     35
     36    # 安裝流程結束，並進入網頁管理頁面設定爬網網址...等 
     37    echo "Install Successfully!!"
     38    echo "Visit http://$MasterIP_Address:portNO"
    3439  elif [ $confirm -eq 2 ]; then
    3540    main
    3641  fi
    37   echo "Install Successfully!!"
    38   echo "Visit http://$MasterIP_Address:portNO"
    3942}
    4043

  • nutchez-0.2/src/test/install_func.sh

    r117 r125  
    1616####### fafa code here ###########
    1717
     18# 參數假設
     19# /home/nutchuser/NutchEZ_source下有3個檔案
     20# install.sh, nutch-1.0.tar.gz, apache-tomcat-6.0.18.tar.gz
     21# 安裝路徑為/opt/nutchEZ
     22
     23Nutch_install_folder=/opt/nutchEZ
     24Install_source=/home/nutchuser/NutchEZ_source
     25NutchEZ_HOME=/opt/nutchEZ
     26MasterIP=`/sbin/ifconfig eth0 | grep 'inet addr' |  sed 's/^.*addr://g' | sed 's/Bcast.*$//g'`
     27
     28
    1829set_install_information () {
    19   read -p "Please select install mode, 1:Single mode 2:Cluster mode: " install_mode
    2030  read -p "Please enter administrator's e-mail address:  " admin_email
    21   if [ $install_mode -eq 1 ]; then
    22     select_mode="Single"
    23     read -p "Please enter the ip address:  " MasterIP_Address
    24 
    25   elif [ $install_mode -eq 2 ]; then
    26     select_mode="Cluster"
    27     read -p "Please enter the master ip address:  " MasterIP_Address
    28     read -p "Please enter the slaves number:  " Slaves_NO
    29     for (( i=1; i<=$Slaves_NO; i=i+1 ))
    30     do
    31       read -p "Please enter the slaves of $i's ip address:  " SlavesAddress$i
    32     done
    33   fi
     31  read -p "Please enter the Master DNS:  " MasterDNS
    3432}
    3533
    3634show_info () {
    37   echo "Your choose is $select_mode install mode."
    38   echo "Administrator's e-mail address is $admin_email."
    39   echo "The master IP address is: $MasterIP_Address"
    40   if [ $install_mode -eq 2 ]; then
    41     echo "You have $Slaves_NO slaves, the list of slave as following: "
    42     for (( i=1; i<=$Slaves_NO; i=i+1))
    43     do
    44       echo "The slaves $i: $((SlavesAddress$i))"
    45     done
    46   fi
     35  echo "Administrator's e-mail address is $Admin_email."
     36  echo "The master DNS is: $MasterDNS"
    4737}
    4838
    … …  
    5141}
    5242
    53 # Hadoop install
    54 Hadoop_install () {
    55   set_hadoop-env
    56   set_hadoop-site
    57   set_masters_slaves
    58 
    59   if [ $install_mode -eq 2 ]; then
    60     echo "Hadoop Cluster install version($Slaves_NO slaves...)"
    61   fi
    62 
     43Install_Nutch () {
     44  tar zxvf $Install_source/nutch-1.0.tar.gz /opt/
     45  mv nutch-1.0 nutchEZ
     46  chown -R nutchuser:nutchuser /opt/nutchEZ
     47  set_Nutch_conf
    6348}
    6449
    65 # set $Hadoop_HOME/conf/hadoop-env.sh
    66 set_hadoop-env () {
    67   echo "set $Hadoop_HOME/conf/hadoop-env.sh"
     50set_Nutch_conf () {
     51  set_hadoop-env
     52  set_haoop-site
     53  set_nutch-site
     54  set_crawl-urlfilter
    6855}
    6956
    70 # set $Hadoop_HOME/conf/hadoop-site.xml
    71 set_hadoop-site () {
    72   echo "set $Hadoop_HOME/conf/hadoop-site.xml"
    73 }
    74 # set $Hadoop_HOME/conf/masters and slaves
    75 set_masters_slaves () {
    76   echo "set $Hadoop_HOME/conf/masters and slaves"
     57# set $NutchEZ_HOME/conf/hadoop-env.sh
     58set_hadoop-env () {
     59  echo "set $NutchEZ_HOME/conf/hadoop-env.sh"
     60  cat >> hadoop-env.sh << EOF
     61export JAVA_HOME=/usr/lib/jvm/java-6-sun
     62export HADOOP_HOME=/opt/nutch
     63export HADOOP_LOG_DIR=/tmp/nutch/logs
     64export HADOOP_SLAVES=/opt/nutch/conf/slaves
     65EOF
    7766}
    7867
    79 # Nutch install
    80 Nutch_install () {
    81   echo "install Nutch..."
    82   echo "cp -rf $Hadoop_HOME/* $Nutch_HOME/*"
    83   echo "cp -rf $Nutch_HOME/*.jar lib/"
    84   set_nutch-site
    85   set_crawl-urlfilter
    86   if [ $install_mode -eq 2 ]; then
    87     echo "Nutch Cluster install version($Slaves_NO slaves...)"
    88     nutch_cluster_install
    89   fi
     68# set $NutchEZ_HOME/conf/hadoop-site.xml
     69set_hadoop-site () {
     70  echo "set $NutchEZ_HOME/conf/hadoop-site.xml"
     71  cat > $NutchEZ_HOME/conf/hadoop-site.xml << EOF
     72<configuration>
     73<property>
     74    <name>fs.default.name</name>
     75    <value>$MasterDNS:9000</value>
     76    <description> The name of the default file system. Either the literal string "local" or a host:port for NDFS. </description>
     77</property>
     78<property>
     79    <name>mapred.job.tracker</name>
     80    <value>$MasterDNS:9001</value>
     81    <description> The host and port that the MapReduce job tracker runs at. If "local", then jobs are run in-process as a single map and reduce task. </description>
     82</property>
     83</configuration>
     84EOF
     85}
    9086
     87set_nutch-site () {
     88  echo "set $NutchEZ_HOME/conf/nutch-site.xml"
     89  cat > $NutchEZ_HOME/conf/nutch-site.xml << EOF
     90<configuration>
     91<property>
     92  <name>http.agent.name</name>
     93  <value>nutchuser</value>
     94  <description>HTTP 'User-Agent' request header. </description>
     95</property>
     96<property>
     97  <name>http.agent.description</name>
     98  <value>MyTest</value>
     99  <description>Further description</description>
     100</property>
     101<property>
     102  <name>http.agent.url</name>
     103  <value>$MasterDNS</value>
     104  <description>A URL to advertise in the User-Agent header. </description>
     105</property>
     106<property>
     107  <name>$MasterDNS</name>
     108  <value>$Admin_email</value>
     109  <description>An email address
     110  </description>
     111</property>
     112</configuration>
     113EOF
    91114}
    92 # cp -rf $Hadoop_HOME/* $Nutch_HOME/*
    93 # cp -rf $Nutch_HOME/*.jar lib/
    94115
    95 #********these steps will merge with hadoop********
    96 # set $Nutch_HOME/conf/hadoop-env.sh              *
    97 # source ./$Nutch_HOME/conf/hadoop-env.sh         *
    98 #**************************************************
     116###################
     117# undo... sed or cp
     118set_crawl-urlfilter () {
     119  echo "set NutchEZ_HOME/conf/set_crawl-urlfilter.txt"
     120}
    99121
    100 # set $Nutch_HOME/conf/nutch-site.xml
    101 set_nutch-site () {
    102   echo "set $Nutch_HOME/conf/nutch-site.xml"
    103 }
    104 # set $Nutch_HOME/conf/crawl-urlfilter.txt       
    105 set_crawl-urlfilter () {
    106   echo "set $Nutch_HOME/conf/crawl-urlfilter.txt"
     122format_HDFS () {
     123  echo "format HDFS..."
     124  $NutchEZ_HOME/bin/hadoop namenode -format
    107125}
    108126
    109127
    110 #******Cluster version to do*********
    111 # scp nutch_dir to the other slaves *
    112 #************************************
    113 nutch_cluster_install () {
    114   for (( i=1; i<=$Slaves_NO; i=i+1))
    115   do
    116     echo "ssh $((SlavesAddress$i)) chown"
    117     echo "scp nutch_dir to slave $((SlavesAddress$i))."
    118   done
     128start_up_NutchEZ (){
     129  echo "start up NutchEZ..."
     130  $NutchEZ_HOME/bin/start-all.sh
     131}
     132
     133# install tomcat
     134Install_Tomcat () {
     135  tar zxvf $Install_source/apache-tomcat-6.0.18.tar.gz $NutchEZ_HOME
     136  mv $NutchEZ_HOME/apache-tomcat-6.0.18 $NutchEZ_HOME/tomcat
     137  chown -R nutchuser:nutchuser $NutchEZ_HOME
     138  mkdir $NutchEZ_HOME/web
     139  jar -xvf $NutchEZ_HOME/nutch-1.0.war $NutchEZ_HOME/web
     140  mv $NutchEZ_HOME/tomcat/webapps/ROOT $NutchEZ_HOME/tomcat/webapps/ROOT-ori
     141  mv $NutchEZ_HOME/web $NutchEZ_HOME/tomcat/webapps/ROOT
     142  mkdir $NutchEZ_HOME/search
     143  set_server
     144  set_nutch-site
     145}
     146
     147####################
     148# undo...sed or cp
     149set_server () {
     150  echo "$NutchEZ_HOME/tomcat/conf/server.xml"
     151}
     152
     153####################
     154# undo...sed or cp
     155set_nutch-site () {
     156  echo "$NutchEZ_HOME/tomcat/webapps/ROOT/WEB-INF/classes/nutch-site.xml"
    119157}
    120158
    121159
    122 # format HDFS
    123 format_HDFS () {
    124   echo "format HDFS"
     160start_up_tomcat () {
     161  echo "start up tomcat..."
     162  $NutchEZ_HOME/tomcat/bin/startup.sh
    125163}
    126 
    127 # start up NutchEZ
    128 # bin/start-all.sh
    129 start_up_NutchEZ (){
    130   echo "start up NutchEZ"
    131 }
    132 
    133 # Crawl(Maybe in administrator page to do this)
    134 # edit url list(mkdir urls and urls.txt)
    135 # upload to HDFS
    136 # bin/hadoop dfs -put urls urls
    137 # bin/nutch crawl urls -dir search -threads 2 -depth 3 -topN 100000
    138 # or save as a log file
    139 # bin/nutch crawl urls -dir search -threads 2 -depth 3 -topN 100000 >& nutch.log
    140 
    141 
    142 #****only master do this****
    143 # Tomcat install function  *
    144 #***************************
    145 
    146 # install tomcat
    147 set_tomcat () {
    148   echo "install tomcat.."
    149   echo "wget Tomcat"
    150   edit_server
    151   download_crawl_result
    152 }
    153 
    154 # edit $Tomcat_HOME/conf/server.xml
    155 edit_server () {
    156   echo "edit $Tomcat_HOME/conf/server.xml"
    157 }
    158 
    159 # download crawl result
    160 download_crawl_result () {
    161   echo "download crawl result"
    162   echo "bin/hadoop dfs -get search /opt/search"
    163   echo "set search engine page to tomcat"
    164   echo "set search engine page source page"
    165 }
    166 
    167 start_up_tomcat () {
    168   echo "start up tomcat"
    169 }
Note: See TracChangeset for help on using the changeset viewer.