User:FairuseBot/10c-removal.pl


 * 1) !/usr/bin/perl


 * 1) 10c-removal
 * 2) A bot to remove NFCC #10c-incompliant images from pages
 * 1) A bot to remove NFCC #10c-incompliant images from pages

use strict; use warnings;

use Date::Calc; use Data::Dumper;

use libBot;

my @common_links = ("Copyright", "Copyright infringement", "Fair use", "Logo", "Trademark", "United States copyright law", "Wikimedia",                   "Computer game", "Counterfeit", "Currency", "Free software", "Portable Network Graphics", "Poster", "Public domain",                    "Screenshot", "Station identification", "United States Code", "U.S. state", "Video game", "Wikimedia Foundation",                    "Work of the United States Government"); my $common_links = join "|", @common_links;

my $test = 0;

my $homedir = '/home/mark/Desktop/wikibots/10cbot'; my $permit_interruptions = 1;	# Allow talkpage messages to stop the bot?

Pearle::init("FairuseBot", "", "$homedir/removebot.log","$homedir/removebot-cookies.txt"); Pearle::config(nullOK => 1, printlevel => 4); config(username => "FairuseBot");

if(!Pearle::login) {	exit; }

if(-e "$homedir/pid") {	# Possible other copy. Compare PIDs open PIDFILE, "<", "$homedir/pid"; my $pid = ; close PIDFILE;
 * 1) Check for a running copy

my $psresult = `ps -p $pid`; if($psresult =~ /10c-removal.pl/) {		botwarnlog("*Previous run is taking longer than normal\n"); exit; } }

open PIDFILE, ">", "$homedir/pid"; print PIDFILE $$; close PIDFILE;

my $total_images = 0; my @logs;

{	my @images; my $image; my $images_removed = 0; @images = ; Pearle::myLog(2, "Beginning set at " . time . "\n");

# Get the log if($test) {		@images = ('Image:Dummy316.png'); }	else {		my $CURRENT_DIR; my @files; # Scan the directory for log files opendir($CURRENT_DIR, $homedir) or (print "Failed: $!\n" and return); @files = readdir $CURRENT_DIR or (print "Failed: $!\n" and return); closedir $CURRENT_DIR; @files = grep {/^partial_failures.*txt$/} @files; foreach my $file (@files) {			my ($year, $month, $day) = $file =~ /_(\d{4})-(\d{1,2})-(\d{1,2})/; if(Date::Calc::Delta_Days( $year, $month, $day, (Date::Calc::Today(1))) > 5) {				open INFILE, "<:utf8", "$homedir/$file"; my @new_images = ; close INFILE; chomp @new_images; push @images, @new_images; push @logs, "$homedir/$file"; }		}	}	Pearle::myLog(3, join("\n", @images)); Pearle::myLog(3, "\n" . scalar(@images) . " images found\n"); if(scalar(@images) == 0) {		Pearle::myLog(1, "*No images to remove\n"); }

foreach $image (@images) {		my $image_url; my $image_regex = $image; my $page; my $full_comment = ""; my $removal_prefix = "Image with inadequate rationale removed:"; my $removal_comment = "Removing image with inadequate rationale"; # Fetch image info Pearle::myLog(2, "Processing image $image\n"); # Fetch the image data my $image_data; if($test) {			$image_data = Pearle::APIQuery(titles => [$image], prop => ['links', 'revisions', 'imageinfo', 'categories'], 							plnamespace => [0, 2], 							# Links							rvprop => ['content'],							# Article body							iiprop => ['user', 'comment', 'sha1'], iilimit => 500,			# Upload history							meta => 'userinfo', uiprop => ['hasmsg'], 					# Check for talkpage messages							list => 'imageusage', iutitle => $image, iunamespace => [0, 2], iulimit => 500);	# Image usage }		else {			$image_data = Pearle::APIQuery(titles => [$image], prop => ['links', 'revisions', 'imageinfo', 'categories'], 							plnamespace => [0],	 							# Links							rvprop => ['content'],							# Article body							iiprop => ['user', 'comment', 'sha1'], iilimit => 500,			# Upload history							meta => 'userinfo', uiprop => ['hasmsg'], 					# Check for talkpage messages							list => 'imageusage', iutitle => $image, iunamespace => [0], iulimit => 500);	# Image usage }		if(!defined($image_data)) {			Pearle::myLog(0, "Server did not return an appropriate response. Exiting.\n"); last; }		# Extract the list of pages where it's used. my @pages = GetPageList($image_data); my $num_pages = scalar(@pages); my @failed_pages; # Extract the categories my @categories = GetPageCategories($image_data); # Extract a list of pages this image links to. my @links = GetPageLinks($image_data); # Filter out common links @links = grep {$_ !~ /^($common_links)$/} @links;

if($permit_interruptions and DoIHaveMessages($image_data)) {			Pearle::myLog(0, "Talkpage message found; exiting on image $image.\n"); exit; }		# Sanity check: Does the image still exist? if($image_data =~ /missing=""/) {			Pearle::myLog(2, "*Image $image has been deleted.\n"); next; }		# Sanity check: Is this still tagged as non-free? if(!grep {$_ eq 'Category:All non-free media'} @categories) {			Pearle::myLog(2, "*Image $image is no longer marked as non-free.\n"); next; }		# Sanity check: Is the image used? if(scalar(@pages) == 0) {			# Orphaned fairuse image Pearle::myLog(2, "*Image $image is not used anywhere\n"); # Is this image already disputed? if(grep {$_ eq 'Category:All disputed non-free images'} @categories) {				Pearle::myLog(2, "*Image $image is already marked for deletion.\n"); }			else {				if(!grep {$_ eq 'Category:All orphaned fairuse images'} @categories) {					my $text = "\n\n"; wikilog($image, $text, "Non-free image is not used in any article\n"); }			}			next; }		# Sanity check: Is the image still tagged as disputed? if(!grep {$_ eq 'Category:All disputed non-free images'} @categories) {			Pearle::myLog(2, "*Image $image is not marked for deletion.\n"); next; }		# Remove the NFCC-failure tag and the list of pages # Blindly removing the tag is safe: # 1) If the program fails, 10cbot will pick the image up on its next pass		# 2) If the image is orphaned, or will be orphaned by removal (unlikely), 10cbot or another bot will pick it up		# 3) If the image is non-compliant on all pages, 10cbot will pick it up on the next pass		my $wikipage = Pearle::getPage($image);		my $text = $wikipage->getEditableText;		$text =~ s/\x03\x44i-missing article links[^\x04]*\x04//s;		Pearle::myLog(4, "Text after processing:\n$text\n");		$wikipage->setEditableText($text);		Pearle::postPage($wikipage, "Removing tag", 0);		Pearle::limit;		# Build the image-matching regex		my ($raw_image) = $image =~ /Image:(.*)/;		$raw_image = MakeWikiRegex($raw_image);		if($image !~ /(\.jpg|\.jpeg|\.png|\.gif|\.svg)$/i)		{			$image_regex = "[ _]*(:?[Ii][Mm][Aa][Gg][Ee]|[Mm][Ee][Dd][Ii][Aa])[ _]*:[ _]*${raw_image}[ _]*";			Pearle::myLog(2, "*Non-image media file $image found.\n"); next;			# Non-image media are too hard to work with }		else {			$image_regex = "[ _]*[Ii][Mm][Aa][Gg][Ee][ _]*:[ _]*${raw_image}[ _]*"; }		# Sanity check if(!defined($raw_image) or $image !~ /$raw_image/) {			botwarnlog("*Parse error on image $image ($raw_image)\n"); next; }		Pearle::myLog(3, "Image regex: $image_regex\n"); # Check for best-case compliance: each use has a matching direct link in the body of the text - tested Pearle::myLog(4, "Image is used in " . scalar(@pages) . " pages.\n"); Pearle::myLog(4, "Image is used on " . join("|", @pages) . "\n"); Pearle::myLog(4, "Image links to " . join("|", @links) . "\n"); foreach my $page (@links)	# Filter out pages that match a link {			@pages = grep {$_ ne $page} @pages; }		Pearle::myLog(4, "Image failed best-case test for " . scalar(@pages) . " pages.\n"); next if(scalar(@pages) == 0); # Check for liberal compliance: # For each use, remove it from the list if there's a case-insensitive match in the body text - tested foreach my $page (@pages) {			my $page_match_regex = MakeWikiRegex($page); push @failed_pages, $page unless($text =~ /$page_match_regex/i); }		@pages = @failed_pages; @failed_pages = ; Pearle::myLog(4, "Image failed text test for " . scalar(@pages) . " pages.\n"); next if(scalar(@pages) == 0); # Check for strict compliance: # For each link, chase redirects - tested if(scalar(@links) > 0) {			my $page_data = Pearle::APIQuery(titles => \@links, redirects => 1); my $parsed_xml = Pearle::getXMLParser->XMLin($page_data); my @redirects; Pearle::myLog(4, Dumper($parsed_xml)); if(exists($parsed_xml->{query}->{redirects}->{r}) and defined($parsed_xml->{query}->{redirects}->{r})) {				if(ref($parsed_xml->{query}->{redirects}->{r}) eq 'ARRAY') {					@redirects = @{$parsed_xml->{query}->{redirects}->{r}}; }				else {					@redirects = ($parsed_xml->{query}->{redirects}->{r}); }			}			foreach my $page (@pages) {				my $matched = 0; foreach my $redirect (@redirects) {					if($redirect->{to} eq $page) {						# We can get there by a redirect UpdateLink($image, $redirect->{from}, $page); Pearle::limit; $matched = 1; last; }				}				if(!$matched) {					push @failed_pages, $page; }			}			@pages = @failed_pages; @failed_pages = ; }		Pearle::myLog(4, "Image failed redirect test for " . scalar(@pages) . " pages.\n"); next if(scalar(@pages) == 0); # Check for near-compliance: # For each use, if we can get to it by means of a disambiguation page, update the link - tested foreach my $page (@links) {			# Fetch the page text and page links my $page_data = Pearle::APIQuery(titles => [$page], prop => ['links', 'revisions'], 						plnamespace => [2], 						# Links						rvprop => ['content']);					# Article body # If the page text indicates disambig, see if any of the links is one we're looking for my $page_text = GetPageText($page_data); if($page_text =~ //i) {				my @page_links = GetPageLinks($page_data); foreach my $disambig_link (@page_links) {					if(grep {$_ eq $disambig_link} @pages) {						# It's a match. Remove it from the list @pages = grep {$_ ne $disambig_link} @pages; # Post to the page my $success = UpdateLink($image, $page, $disambig_link); if(!$success) {							botwarnlog("*Failed to update disambiguation link for $image from $page to $disambig_link\n"); }						Pearle::limit; }				}			}		}		Pearle::myLog(4, "Image failed disambiguation test for " . scalar(@pages) . " pages.\n"); next if(scalar(@pages) == 0); # Test for compliance # Over-use (some compliant, some non-compliant): Remove from any non-compliant articles, OrphanBot-style. Leave a note on the article talk page. if(scalar(@pages) > 0 and $num_pages > scalar(@pages)) {			Pearle::myLog(2, "Image $image failed on " . scalar(@pages) . " pages.\n"); my $parsed_removal_comment = $removal_comment; $parsed_removal_comment =~ s/image/image/; foreach $page (@pages) {				my $hits = 0; notelog("Page for removal: $page\n"); if($hits = RemoveImageFromPage($image, $page, $image_regex, $removal_prefix, $parsed_removal_comment)) 	# Don't limit if we just touched the article {					Pearle::myLog(2, "Removed image $image from article $page ($hits times)\n"); Pearle::limit; }				$images_removed += $hits; }		}		elsif(scalar(@pages) > 0) {			# Fully-non-compliant. Should never occur, but if it does, let 10cbot pick it up on the next pass. Pearle::myLog(2, "Image $image failed on all pages\n"); }		else {			Pearle::myLog(2, "Image $image is now fully-compliant\n"); }	}	Pearle::myLog(2, "Finished with set. Removed $images_removed images.\n"); $total_images += $images_removed; }

unlink @logs;
 * 1) Remove the processed logs

unlink "$homedir/pid"