= 2013-08-14 =

 * http://blog.cloudera.com/blog/2009/06/analyzing-apache-logs-with-pig/

 * 

{{{
#!html
<br /> register /home/dvryaboy/src/pig/trunk/piggybank.jar;<br /> DEFINE LogLoader<br /> org.apache.pig.piggybank.storage.apachelog.CombinedLogLoader();<br /> DEFINE DayExtractor<br /> org.apache.pig.piggybank.evaluation.util.apachelogparser.DateExtractor('yyyy-MM-dd');<br /> 
}}}

 * 

{{{
#!html
<br /> %default LOGS 'access_log.small'<br /> %default GEO 'GeoLiteCity.dat'<br /> 
}}}

 * 

{{{
# pig -x mapreduce -f scripts/blogparse.pig -param LOGS='/mirror.cloudera.com/logs/access_log.*'
}}}

 * 

{{{
#!html
<br /> #!/usr/bin/env perl<br /> use warnings;<br /> use strict;<br /> use Geo::IP::PurePerl;</p>
<p>my ($path)=shift;<br /> my $gi = Geo::IP::PurePerl->new($path);</p>
<p>while (<>) {<br /> chomp;<br /> if (/([^\t]*)\t(.*)/) {<br /> my ($ip, $rest) = ($1, $2);<br /> my ($country_code, undef, $country_name, $region, $city)<br /> = $gi->get_city_record($ip);<br /> print join("\t", $country_code||'', $country_name||'',<br /> $region||'', $city||'', $ip, $rest), "\n";<br /> }<br /> }<br /> 
}}}

 * 

{{{
#!html
<br /> #!/usr/bin/env bash<br /> tar -xzf geo-pack.tgz<br /> PERL5LIB=$PERL5LIB:$(pwd) ./geostream.pl $1<br /> 
}}}

 *

{{{
#!html
<br /> with_country = STREAM notbots THROUGH `ipwrapper.sh $GEO`<br /> AS (country_code, country, state, city, ip, time, uri, bytes, userAgent);</p>
<p>geo_uri_groups = GROUP with_country BY country_code;</p>
<p>geo_uri_group_counts = FOREACH geo_uri_groups GENERATE<br /> group,<br /> COUNT(with_country) AS cnt,<br /> SUM(with_country.bytes) AS total_bytes;</p>
<p>geo_uri_group_counts = ORDER geo_uri_group_counts BY cnt DESC;</p>
<p>STORE geo_uri_group_counts INTO 'by_country.tsv';<br /> 
}}}