Hadoop Streaming with commands
jazz@hadoop:~$ hadoop fs -rmr input output jazz@hadoop:~$ hadoop fs -put /etc/hadoop/conf input jazz@hadoop:~$ hadoop jar hadoop-streaming.jar -input input -output output -mapper /bin/cat -reducer /usr/bin/wc
jazz@hadoop:~$ echo "sed -e \"s/ /\n/g\" | grep ." > streamingMapper.sh jazz@hadoop:~$ echo "uniq -c | awk '{print \$2 \"\t\" \$1}'" > streamingReducer.sh jazz@hadoop:~$ chmod a+x streamingMapper.sh jazz@hadoop:~$ chmod a+x streamingReducer.sh jazz@hadoop:~$ hadoop fs -rmr input output jazz@hadoop:~$ hadoop fs -put /etc/hadoop/conf input jazz@hadoop:~$ hadoop jar hadoop-streaming.jar -input input -output output -mapper streamingMapper.sh -reducer streamingReducer.sh -file streamingMapper.sh -file streamingReducer.sh
Last modified 14 years ago
Last modified on Aug 11, 2010, 12:56:59 AM