User:SJK/yrget perl script

use strict; use LWP::UserAgent;
 * 1) !/usr/local/bin/perl -w

my $ua = LWP::UserAgent->new;

$ua->timeout(30); $ua->agent("SJK downloading Year in Review: " . $ua->agent); $ua->env_proxy ;
 * 1) Set attributes on UA

open ENTRIES, ") { chomp; my $url = "http://www.wikipedia.com/wiki.cgi?action=history&id=$_";

print "Getting $url... "; my $request = HTTP::Request->new('GET', $url); my $response = $ua->request($request); if ($response->is_error) { die $response->status_line. " ";    }     my $maxrev = 0; my @lines = split(/ /,$response->content);

for my $line (@lines) { if ($line =~ /Revision ([0-9]*):  $maxrev) { $maxrev = $1; }; }    }     $url = "http://www.wikipedia.com/wiki.cgi?action=edit&revision=$maxrev&id=$_";

print "Getting $url... "; $request = HTTP::Request->new('GET', $url); $response = $ua->request($request); if ($response->is_error) { die $response->status_line. " ";    }     open DATA, ">data/$_" or die "Can't open data/$_: $!"; if ($response->content =~ /]*>(.*) /s) { my $out = "#YEAR $_ REV=$maxrev ". $1;	 $out =~ s/&#39;/'/g; print DATA $out; }    close DATA; }


 * See also : Simon J Kissane