| 11 | |
| 12 | == Aggregation == |
| 13 | |
| 14 | {{{ |
| 15 | ~$ wget http://hadoop.nchc.org.tw/excite-small.log |
| 16 | ~$ pig -x local |
| 17 | grunt> log = LOAD 'excite-small.log' AS (user, timestamp, query); |
| 18 | grunt> grpd = GROUP log BY user; |
| 19 | grunt> cntd = FOREACH grpd GENERATE group, COUNT(log); |
| 20 | grunt> STORE cntd INTO 'lab8_out1'; |
| 21 | grunt> quit |
| 22 | ~$ head lab8_out1 |
| 23 | }}} |
| 24 | |
| 25 | == Filter == |
| 26 | |
| 27 | {{{ |
| 28 | ~$ pig -x local |
| 29 | grunt> log = LOAD 'excite-small.log' AS (user, timestamp, query); |
| 30 | grunt> grpd = GROUP log BY user; |
| 31 | grunt> cntd = FOREACH grpd GENERATE group, COUNT(log) AS cnt; |
| 32 | grunt> fltrd = FILTER cntd BY cnt > 50; |
| 33 | grunt> STORE fltrd INTO 'lab8_out2'; |
| 34 | grunt> quit |
| 35 | ~$ head lab8_out2 |
| 36 | }}} |
| 37 | |
| 38 | == Sorting == |
| 39 | |
| 40 | {{{ |
| 41 | ~$ pig -x local |
| 42 | grunt> log = LOAD 'excite-small.log' AS (user, timestamp, query); |
| 43 | grunt> grpd = GROUP log BY user; |
| 44 | grunt> cntd = FOREACH grpd GENERATE group, COUNT(log) AS cnt; |
| 45 | grunt> fltrd = FILTER cntd BY cnt > 50; |
| 46 | grunt> srtd = ORDER fltrd BY cnt; |
| 47 | grunt> STORE srtd INTO 'lab8_out3'; |
| 48 | grunt> quit |
| 49 | ~$ head lab8_out3 |
| 50 | }}} |
| 51 | |
| 52 | == Connecting Hadoop == |
| 53 | |
| 54 | {{{ |
| 55 | ~$ hadoop fs -put excite-small.log . |
| 56 | ~$ pig |
| 57 | grunt> log = LOAD 'excite-small.log' AS (user, timestamp, query); |
| 58 | grunt> grpd = GROUP log BY user; |
| 59 | grunt> cntd = FOREACH grpd GENERATE group, COUNT(log); |
| 60 | grunt> STORE cntd INTO 'lab8_out1'; |
| 61 | grunt> quit |
| 62 | ~$ head lab8_out1 |
| 63 | }}} |