Changes between Version 2 and Version 3 of waue/Hadoop_DRBL

Timestamp:

Feb 12, 2009, 5:04:21 PM (15 years ago)

Author:

waue

Comment:

--

waue/Hadoop_DRBL

@@ -25 +25 @@
 }}}
 
+
+ * download Hadoop 0.18.3 
+{{{
+$ cd /opt
+$ wget http://ftp.twaren.net/Unix/Web/apache/hadoop/core/hadoop-0.18.3/hadoop-0.18.3.tar.gz
+$ tar zxvf hadoop-0.18.3.tar.gz
+hadoop:/opt# ln -sf hadoop-0.18.3 hadoop
+}}}
+ * setup JAVA_HOME environment variable
+{{{
+$ echo "export JAVA_HOME=/usr/lib/jvm/java-6-sun" >> ~/.bash_profile
+$ source ~/.bash_profile
+}}}
+ * edit hadoop-0.18.3/conf/hadoop-env.sh
+{{{
+#!diff
+--- hadoop-0.18.3/conf/hadoop-env.sh.org
++++ hadoop-0.18.3/conf/hadoop-env.sh
+@@ -6,7 +6,9 @@
+ # remote nodes.
+
+ # The java implementation to use.  Required.
+-# export JAVA_HOME=/usr/lib/j2sdk1.5-sun
++export JAVA_HOME=/usr/lib/jvm/java-6-sun
++export HADOOP_HOME=/opt/hadoop-0.18.3
++export HADOOP_CONF_DIR=$HADOOP_HOME/conf
+
+ # Extra Java CLASSPATH elements.  Optional.
+ # export HADOOP_CLASSPATH=
+}}}
+
+ * edit hadoop-0.18.3/conf/hadoop-site.xml
+{{{
+#!diff
+--- hadoop-0.18.3/conf/hadoop-site.xml.org
++++ hadoop-0.18.3/conf/hadoop-site.xml
+@@ -4,5 +4,31 @@
+ <!-- Put site-specific property overrides in this file. -->
+
+ <configuration>
+-
++  <property>
++    <name>fs.default.name</name>
++    <value>hdfs://192.168.1.254:9000/</value>
++    <description>
++      The name of the default file system. Either the literal string
++      "local" or a host:port for NDFS.
++    </description>
++  </property>
++  <property>
++    <name>mapred.job.tracker</name>
++    <value>hdfs://192.168.1.254:9001</value>
++    <description>
++      The host and port that the MapReduce job tracker runs at. If
++      "local", then jobs are run in-process as a single map and
++      reduce task.
++    </description>
++  </property>
+ </configuration>
+}}}
+
+ * here is current DRBL setup
+{{{
+******************************************************
+          NIC    NIC IP                    Clients
++------------------------------+
+|         DRBL SERVER          |
+|                              |
+|    +-- [eth0] X.X.X.X        +- to WAN
+|                              |
+|    +-- [eth1] 192.168.1.254 +- to clients group 1 [ 16 clients, their IP
+|                              |             from 192.168.1.1 - 192.168.1.16]
++------------------------------+
+******************************************************
+Total clients: 16
+******************************************************
+}}}
+ * Hadoop will use ssh connections for internal connection, thus we have to do SSH key exchange.
+{{{
+$ ssh-keygen
+$ cp .ssh/id_rsa.pub .ssh/authorized_keys
+$ sudo apt-get install dsh
+$ mkdir -p .dsh
+$ for ((i=1;i<=16;i++)); do echo "192.168.1.$i" >> .dsh/machines.list; done
+}}}
+
+ * edit /etc/rc.local for DRBL Server as Hadoop namenode
+{{{
+#!diff
+--- /etc/rc.local.org   2008-11-07 18:09:10.000000000 +0800
++++ /etc/rc.local       2008-11-07 17:58:14.000000000 +0800
+@@ -11,4 +11,7 @@
+ #
+ # By default this script does nothing.
+
++echo 3 > /proc/sys/vm/drop_caches
++/opt/hadoop-0.18.3/bin/hadoop namenode -format
++/opt/hadoop-0.18.3/bin/hadoop-daemon.sh start namenode
++/opt/hadoop-0.18.3/bin/hadoop-daemon.sh start jobtracker
++/opt/hadoop-0.18.3/bin/hadoop-daemon.sh start tasktracker
+ exit 0
+}}}
+ * edit hadoop_datanode for DRBL client as datanode
+{{{
+$ cat > hadoop_datanode << EOF
+}}}
+{{{
+#! /bin/sh
+set -e
+
+# /etc/init.d/hadoop_datanode: start and stop Hadoop DFS datanode for DRBL Client
+
+export PATH="${PATH:+$PATH:}/usr/sbin:/sbin"
+
+case "\$1" in
+  start)
+        echo -n "starting datanode:"
+        /opt/hadoop-0.18.3/bin/hadoop-daemon.sh start datanode
+        echo "[OK]"
+        ;;
+  stop)
+        echo -n "stoping datanode:"
+        /opt/hadoop-0.18.3/bin/hadoop-daemon.sh stop datanode
+        echo "[OK]"
+        ;;
+
+  *)
+        echo "Usage: /etc/init.d/hadoop_datanode {start|stop}"
+        exit 1
+esac
+
+exit 0
+EOF
+}}}
+{{{
+$ chmod a+x hadoop_datanode
+$ sudo /opt/drbl/sbin/drbl-cp-host hadoop_datanode /etc/init.d/
+$ sudo /opt/drbl/bin/drbl-doit update-rc.d hadoop_datanode defaults 99
+}}}
+ * shutdown DRBL clients
+ * reboot DRBL server
+ * use "Wake on LAN" for DRBL clients
+ * browse http://192.168.1.254:50070 for DFS status