[116] | 1 | #!/bin/bash |
---|
| 2 | source install_lang |
---|
[117] | 3 | ####### garbage here ############# |
---|
[116] | 4 | function mainFunction ( ) |
---|
| 5 | { |
---|
| 6 | echo "$Good" |
---|
| 7 | } |
---|
| 8 | function braBraBra ( ) |
---|
| 9 | { |
---|
| 10 | echo "$Bra_Bra_Bra" |
---|
| 11 | } |
---|
[117] | 12 | ####### garbage end ############### |
---|
| 13 | |
---|
| 14 | |
---|
| 15 | |
---|
| 16 | ####### fafa code here ########### |
---|
| 17 | |
---|
| 18 | set_install_information () { |
---|
| 19 | read -p "Please select install mode, 1:Single mode 2:Cluster mode: " install_mode |
---|
| 20 | read -p "Please enter administrator's e-mail address: " admin_email |
---|
| 21 | if [ $install_mode -eq 1 ]; then |
---|
| 22 | select_mode="Single" |
---|
| 23 | read -p "Please enter the ip address: " MasterIP_Address |
---|
| 24 | |
---|
| 25 | elif [ $install_mode -eq 2 ]; then |
---|
| 26 | select_mode="Cluster" |
---|
| 27 | read -p "Please enter the master ip address: " MasterIP_Address |
---|
| 28 | read -p "Please enter the slaves number: " Slaves_NO |
---|
| 29 | for (( i=1; i<=$Slaves_NO; i=i+1 )) |
---|
| 30 | do |
---|
| 31 | read -p "Please enter the slaves of $i's ip address: " SlavesAddress$i |
---|
| 32 | done |
---|
| 33 | fi |
---|
| 34 | } |
---|
| 35 | |
---|
| 36 | show_info () { |
---|
| 37 | echo "Your choose is $select_mode install mode." |
---|
| 38 | echo "Administrator's e-mail address is $admin_email." |
---|
| 39 | echo "The master IP address is: $MasterIP_Address" |
---|
| 40 | if [ $install_mode -eq 2 ]; then |
---|
| 41 | echo "You have $Slaves_NO slaves, the list of slave as following: " |
---|
| 42 | for (( i=1; i<=$Slaves_NO; i=i+1)) |
---|
| 43 | do |
---|
| 44 | echo "The slaves $i: $((SlavesAddress$i))" |
---|
| 45 | done |
---|
| 46 | fi |
---|
| 47 | } |
---|
| 48 | |
---|
| 49 | confirm_install_information () { |
---|
| 50 | read -p "Please confirm your install infomation: 1.Yes 2.No " confirm |
---|
| 51 | } |
---|
| 52 | |
---|
| 53 | # Hadoop install |
---|
| 54 | Hadoop_install () { |
---|
| 55 | set_hadoop-env |
---|
| 56 | set_hadoop-site |
---|
| 57 | set_masters_slaves |
---|
| 58 | |
---|
| 59 | if [ $install_mode -eq 2 ]; then |
---|
| 60 | echo "Hadoop Cluster install version($Slaves_NO slaves...)" |
---|
| 61 | fi |
---|
| 62 | |
---|
| 63 | } |
---|
| 64 | |
---|
| 65 | # set $Hadoop_HOME/conf/hadoop-env.sh |
---|
| 66 | set_hadoop-env () { |
---|
| 67 | echo "set $Hadoop_HOME/conf/hadoop-env.sh" |
---|
| 68 | } |
---|
| 69 | |
---|
| 70 | # set $Hadoop_HOME/conf/hadoop-site.xml |
---|
| 71 | set_hadoop-site () { |
---|
| 72 | echo "set $Hadoop_HOME/conf/hadoop-site.xml" |
---|
| 73 | } |
---|
| 74 | # set $Hadoop_HOME/conf/masters and slaves |
---|
| 75 | set_masters_slaves () { |
---|
| 76 | echo "set $Hadoop_HOME/conf/masters and slaves" |
---|
| 77 | } |
---|
| 78 | |
---|
| 79 | # Nutch install |
---|
| 80 | Nutch_install () { |
---|
| 81 | echo "install Nutch..." |
---|
| 82 | echo "cp -rf $Hadoop_HOME/* $Nutch_HOME/*" |
---|
| 83 | echo "cp -rf $Nutch_HOME/*.jar lib/" |
---|
| 84 | set_nutch-site |
---|
| 85 | set_crawl-urlfilter |
---|
| 86 | if [ $install_mode -eq 2 ]; then |
---|
| 87 | echo "Nutch Cluster install version($Slaves_NO slaves...)" |
---|
| 88 | nutch_cluster_install |
---|
| 89 | fi |
---|
| 90 | |
---|
| 91 | } |
---|
| 92 | # cp -rf $Hadoop_HOME/* $Nutch_HOME/* |
---|
| 93 | # cp -rf $Nutch_HOME/*.jar lib/ |
---|
| 94 | |
---|
| 95 | #********these steps will merge with hadoop******** |
---|
| 96 | # set $Nutch_HOME/conf/hadoop-env.sh * |
---|
| 97 | # source ./$Nutch_HOME/conf/hadoop-env.sh * |
---|
| 98 | #************************************************** |
---|
| 99 | |
---|
| 100 | # set $Nutch_HOME/conf/nutch-site.xml |
---|
| 101 | set_nutch-site () { |
---|
| 102 | echo "set $Nutch_HOME/conf/nutch-site.xml" |
---|
| 103 | } |
---|
| 104 | # set $Nutch_HOME/conf/crawl-urlfilter.txt |
---|
| 105 | set_crawl-urlfilter () { |
---|
| 106 | echo "set $Nutch_HOME/conf/crawl-urlfilter.txt" |
---|
| 107 | } |
---|
| 108 | |
---|
| 109 | |
---|
| 110 | #******Cluster version to do********* |
---|
| 111 | # scp nutch_dir to the other slaves * |
---|
| 112 | #************************************ |
---|
| 113 | nutch_cluster_install () { |
---|
| 114 | for (( i=1; i<=$Slaves_NO; i=i+1)) |
---|
| 115 | do |
---|
| 116 | echo "ssh $((SlavesAddress$i)) chown" |
---|
| 117 | echo "scp nutch_dir to slave $((SlavesAddress$i))." |
---|
| 118 | done |
---|
| 119 | } |
---|
| 120 | |
---|
| 121 | |
---|
| 122 | # format HDFS |
---|
| 123 | format_HDFS () { |
---|
| 124 | echo "format HDFS" |
---|
| 125 | } |
---|
| 126 | |
---|
| 127 | # start up NutchEZ |
---|
| 128 | # bin/start-all.sh |
---|
| 129 | start_up_NutchEZ (){ |
---|
| 130 | echo "start up NutchEZ" |
---|
| 131 | } |
---|
| 132 | |
---|
| 133 | # Crawl(Maybe in administrator page to do this) |
---|
| 134 | # edit url list(mkdir urls and urls.txt) |
---|
| 135 | # upload to HDFS |
---|
| 136 | # bin/hadoop dfs -put urls urls |
---|
| 137 | # bin/nutch crawl urls -dir search -threads 2 -depth 3 -topN 100000 |
---|
| 138 | # or save as a log file |
---|
| 139 | # bin/nutch crawl urls -dir search -threads 2 -depth 3 -topN 100000 >& nutch.log |
---|
| 140 | |
---|
| 141 | |
---|
| 142 | #****only master do this**** |
---|
| 143 | # Tomcat install function * |
---|
| 144 | #*************************** |
---|
| 145 | |
---|
| 146 | # install tomcat |
---|
| 147 | set_tomcat () { |
---|
| 148 | echo "install tomcat.." |
---|
| 149 | echo "wget Tomcat" |
---|
| 150 | edit_server |
---|
| 151 | download_crawl_result |
---|
| 152 | } |
---|
| 153 | |
---|
| 154 | # edit $Tomcat_HOME/conf/server.xml |
---|
| 155 | edit_server () { |
---|
| 156 | echo "edit $Tomcat_HOME/conf/server.xml" |
---|
| 157 | } |
---|
| 158 | |
---|
| 159 | # download crawl result |
---|
| 160 | download_crawl_result () { |
---|
| 161 | echo "download crawl result" |
---|
| 162 | echo "bin/hadoop dfs -get search /opt/search" |
---|
| 163 | echo "set search engine page to tomcat" |
---|
| 164 | echo "set search engine page source page" |
---|
| 165 | } |
---|
| 166 | |
---|
| 167 | start_up_tomcat () { |
---|
| 168 | echo "start up tomcat" |
---|
| 169 | } |
---|