I came across a line of hacking books today which proved fun. This hack for Google led to this modified script which runs the query live instead of pulling it from a saved results file:
#!/usr/bin/perl
# google_search.pl # Google Web Search Results exported to CSV suitable
# for import into Excel
# Usage: perl google_search.pl term1 term2 termN > results.csv
use LWP;
my $browser = LWP::UserAgent->new;
# Google doesn't like non-browser-based access:
$browser->agent('Mozilla/4.76 [en] (Win98; U)');
my $qstring = join("+", @ARGV);
my $url = 'http://www.google.com/search?hl=en&ie=UTF-8&oe=UTF-8&q=' . $qstring . '&btnG=Google+Search';
my $response = $browser->get($url);
print qq{"title","url","size","domain suffix"\n};
my($results) = (join '', $response->content) =~ m!<div>(.*?)</div>!mis;
while ( $results =~ m!<a href="?(.+?)"?>(.+?)</a>.+?\s+-\s+(\d+k)?!mgis ) {
my($url,$title, $size) = ($1||'',$2||'',$3||'');
my($suffix) = $url =~ m!\.(\w+)/!;
$title =~ s!"!""!g; # double escape " marks
$title =~ s!<.+?>!!g; # drop all HTML tags
print qq{"$title","$url","$size","$suffix"\n};
}
