Changes between Version 5 and Version 6 of YMU110509/Lab9

Timestamp:

Jun 20, 2011, 2:52:34 PM (13 years ago)

Author:

jazz

Comment:

--

YMU110509/Lab9

@@ -55 +55 @@
  * 接著，讓我們在本地端先驗證一下 testmapper.sh 的運作
 {{{
-~$ head -n 10 sample-01.txt > sample-00.txt
+~$ head -n 2 sample-01.txt > sample-00.txt
 ~$ cat > testmapper.sh << EOF
 #!/bin/bash
 
-id="h998"
+id="`whoami`"
 mkdir -p /tmp/\$id
 host=\`hostname\`
@@ -68 +68 @@
   input=\$line
   filename=\`basename \$input\`
-  echo "\$uid@$host:\$pwd> hadoop fs -get \$input /tmp/\$id/\$filename"
-  echo "\$uid@$host:\$pwd> velveth output-\$filename 17 -fasta -short /tmp/\$id/\$filename"
-  echo "\$uid@$host:\$pwd> hadoop fs -put output-\$filename ."
+  echo "\$uid@\$host:\$pwd> hadoop fs -get \$input /tmp/\$id/\$filename"
+  echo "\$uid@\$host:\$pwd> velveth output-\$filename 17 -fasta -short /tmp/\$id/\$filename"
+  echo "\$uid@\$host:\$pwd> hadoop fs -put output-\$filename ."
 done
 rm -rf /tmp/\$id
@@ -95 +95 @@
 == 實作透過 Hadoop Streaming 執行 99 組 velvet 運算 ==
 
- * 撰寫 velvet_mapper.pl
+ * 撰寫 mapper.sh
 {{{
-#!perl
+#!sh
+#!/bin/bash
 
+id="h998"
+mkdir -p /tmp/$id
+host=`hostname`
+pwd=`pwd`
+uid=`whoami`
+
+while read line; do 
+  input=$line
+  filename=`basename $input`
+  echo "$uid@$host> hadoop fs -get $input /tmp/$id/$filename"
+  hadoop fs -get $input /tmp/$id/$filename
+  echo "$uid@$host> velveth output-$filename 17 -fasta -short /tmp/$id/$filename"
+  velveth output-$filename 17 -fasta -short /tmp/$id/$filename
+  echo "$uid@$host> hadoop fs -put output-$filename /user/$id/."
+  hadoop fs -put output-$filename /user/$id/.
+done
+rm -rf /tmp/$id
 }}}
+ * 於本機測試 mapper.sh
+{{{
+~$ cat > mapper.sh << EOF
+#!/bin/bash
+
+id="`whoami`"
+mkdir -p /tmp/\$id
+host=\`hostname\`
+pwd=\`pwd\`
+uid=\`whoami\`
+
+while read line; do 
+  input=\$line
+  filename=\`basename \$input\`
+  echo "\$uid@\$host> hadoop fs -get \$input /tmp/\$id/\$filename"
+  hadoop fs -get \$input /tmp/\$id/\$filename
+  echo "\$uid@\$host> velveth output-\$filename 17 -fasta -short /tmp/\$id/\$filename"
+  velveth output-\$filename 17 -fasta -short /tmp/\$id/\$filename
+  echo "\$uid@\$host> hadoop fs -put output-\$filename /user/\$id/."
+  hadoop fs -put output-\$filename /user/\$id/.
+done
+rm -rf /tmp/\$id
+EOF
+~$ chmod a+x mapper.sh
+~$ cat sample-00.txt | ./mapper.sh 
+~$ hadoop fs -rmr output-*
+}}}
+ * 接著用 hadoop streaming 來執行
+{{{
+~$ hadoop jar hadoop-streaming.jar -input lab9_input -output lab9_out2 -mapper mapper.sh -file mapper.sh
+}}}