User:Magicpiano/NRBot/UpdateNRHPProgress2.js

/* jshint maxerr: 3000 */ /*  The following script places a button at the top of the NRHP Progress page WP:NRHPPROGRESS. When the button is clicked, the script begins to load each county list linked from the Progress page in the background, extract statistics about sites in each list, and updates the Progress page with the fetched data.

//var wikitext = 'error'; var ProgressStructure=[]; // ProgressStructure[table][row].StatName var TotalToQuery=0; var TotalQueried=0; var ErrorCount=0; var WarningCount="",0; // 0=status, 1=count var InitialTime=0; var ProgressDivTimer=0; // timer for updating ProgressDiv var DefaultQueryPause=1; // number of milliseconds to wait between each API query; increased by code if rate limit reached var RestrictedImageCount=0; var bIgnoreTableCount = false;

// Users authorized to run this script var AuthorizedUsers = [ "Magicpiano" ]; // nb this is for use only when g_disabled is true var CurrentMaintainer = "Magicpiano"; // User who is currently maintaining the script

function getRandomIntInclusive(min, max) { min = Math.ceil(min); max = Math.floor(max); return Math.floor(Math.random * (max - min + 1) + min); //The maximum is inclusive and the minimum is inclusive } // // WorkQueue helper // // run up to maxActiveSize work items at a time // function WorkQueue(maxActiveSize = 1, initialWorkQueue = []) {	this.maxActiveSize = maxActiveSize; this.pendingQueue = initialWorkQueue; this.activeQueue = []; this.completedQueue = []; this.isRunning = false; if (initialWorkQueue.length > 0) this.run; }

// determine if we have room to run something more, and if there is something more to run WorkQueue.prototype.runNext = function {	return ((this.activeQueue.length < this.maxActiveSize) && this.pendingQueue.length > 0); };

WorkQueue.prototype.createPromise = function(work, retryCount=0) {	var p = new Promise((result, reject) => work.then(result).catch(reject)); p.work = work; p.retryCount = retryCount; }

// add a work item WorkQueue.prototype.push = function(p, retryCount=0) {	this.pendingQueue.push(this.createPromise(p,retryCount)); };

// add a list of work items WorkQueue.prototype.pushList = function(plist, retryCount=0) {	for (var idx in plist) {		const p = plist[idx]; this.pendingQueue.push(this.createPromise(p,retryCount)); } };

WorkQueue.prototype.getCompletedCount = function {	return this.completedQueue.length; } // run (or attempt to) the queue WorkQueue.prototype.run = function {	if (this.isRunning) return; this.isRunning = true; this.nFailed = 0; this.nSucceeded = 0; this.nCompleted = 0; this.nEntries = this.pendingQueue.length; var ranAtLeastOnce = false; const wq = this; try { while (this.runNext) {			if (!ranAtLeastOnce) {				console.log('Start running queue',this); ranAtLeastOnce = true; }			const promise = this.pendingQueue.shift; console.log('Running promise',promise); promise .then((result) => {					wq.nSucceeded++;				}) .catch((e) => {					console.log('Promise failed',promise,e);					if (promise.retryCount > 0)					{						console.log('Retrying');						wq.push(promise.work, promise.retryCount-1);					}				}) .finally( => {					wq.completedQueue.push(removeFromArray(wq.activeQueue, promise));					console.log('Finished promise',promise);					wq.run;			}); this.activeQueue.push(promise); }	}	finally { this.isRunning = false; if (ranAtLeastOnce) console.log('Stop running queue',this); } };

function removeFromArray(arr, item) {	var idx = arr.indexOf(item); if (idx < 0) return; arr.splice(idx,1); }

async function makeJob(work, state) {	return await work(state); }

function ProgressButton { var bNotHere = true; if (mw.config.get('wgPageName')=="Wikipedia:WikiProject_National_Register_of_Historic_Places/Progress") bNotHere = false; if (mw.config.get('wgPageName')=="User:Magicpiano/NRBot/UpdateNRHPProgressTester") {   	bNotHere = false; bIgnoreTableCount = true; }	// the above notwithstanding, if in edit mode we also ignore if (location.href.indexOf('action')!=-1) bNotHere = true; if (bNotHere) return; var button = document.getElementById("NRHPProgressUpdateButton2"); if (button !== null) return; button=document.createElement("input"); button.setAttribute("type", "button"); button.setAttribute("value", "Update Statistics 2"); button.setAttribute("onclick", "ClickUpdateNRHPProgress"); button.id = "NRHPProgressUpdateButton2"; var content=document.getElementById('mw-content-text');

content.parentNode.insertBefore(button, content); }

var g_disabled = true;

function CheckPermission { var username = mw.user.getName; console.log("UpdateNRHPProgress: run by user "+username); if (username == CurrentMaintainer) return true; for (var i=0; i< AuthorizedUsers.length; i++) { if (username == AuthorizedUsers[i]) { return true; }	}	if (g_disabled) { alert("Script is a work in progress."); //alert("Script is currently disabled for maintenance."); return false; } }

async function ClickUpdateNRHPProgress {  // after button is clicked, disable it and fetch source of Progress page if (!CheckPermission) { return; }   var button2 = document.getElementById("NRHPProgressUpdateButton2"); button2.disabled = true; var ProgressDiv = document.getElementById("ProgressDiv"); if (ProgressDiv === null) { ProgressDiv = document.createElement("div"); ProgressDiv.setAttribute("id", "ProgressDiv"); ProgressDiv.setAttribute("style", "width:500px; border:1px solid black; padding:5px; background:#ffffff"); button2.parentNode.insertBefore(ProgressDiv, button2); }   ProgressDiv.innerHTML = "Initializing...";

var pagesource = await getPageSource(mw.config.get('wgPageName')); // TODO not really needed here? SetupTables; InitialTime=new Date; // record starting time UpdateProgressDiv; processTableList(ProgressStructure); //LoadList(1,0); // begin querying first page }

// create array of table structure to be populated later function SetupTables { var table=document.getElementsByClassName('wikitable sortable');

// initialize globals TotalToQuery=0; TotalQueried=0; ErrorCount=0; WarningCount="",0; // 0=status, 1=count InitialTime=0; ProgressDivTimer=0; // timer for updating ProgressDiv DefaultQueryPause=1; // number of milliseconds to wait between each API query; increased by code if rate limit reached RestrictedImageCount=0;

// Expected table count: // There should be 61 tables, as follows: // The national table, 50 states, and the following 10 non-state entities: // District of Columbia, Puerto Rico, Virgin Islands, Guam, American Samoa, // Northern Mariana Islands, Federated States of Micronesia, Palau, Marshall Islands // Minor Outlying Islands // This number should only need adjustmnent if the number of states or non-state entities having // separate tables is changed. An alert here is more likely caused by some sort of formatting change // or error in the table. //   // Ideally, the above search should be restrictable (by class in some way?) to only the specific tables of interest. //   if ((!bIgnoreTableCount) && table.length !== 61) { alert('Incorrect table count in progress page: expected 61, saw '+table.length+'.'); return; }

// set up national totals var tr=table[0].getElementsByTagName("tr"); var i,j, td; ProgressStructure[0]=[]; for (j=1; j<tr.length-3; j++) { td=tr[j].getElementsByTagName("td"); ProgressStructure[0][j-1]={ ID: td[0].innerHTML, Total: 0, Illustrated: 0, Articled: 0, Stubs: 0, NRISonly: 0, StartPlus: 0, Unassessed: 0, Untagged: 0 };   }

// special row for Tangier, Morocco td=tr[tr.length-3].getElementsByTagName("td"); /* This code copies the existing settings for the American Legation in Tangier into the table. This process is prone to error if it is modified. Because the article is an illustrated Start+ article, the settings hardcoded further down are unlikely to change. */   /* // To copy previous contents instead of hardcoding: ProgressStructure[0][tr.length-4]={}; ProgressStructure[0][tr.length-4].ID="Tangier, Morocco"; ProgressStructure[0][tr.length-4].Total=parseFloat(td[1].innerHTML.replace(",","")); ProgressStructure[0][tr.length-4].Illustrated=parseFloat(td[2].innerHTML.replace(",","")); ProgressStructure[0][tr.length-4].Articled=parseFloat(td[4].innerHTML.replace(",","")); ProgressStructure[0][tr.length-4].Stubs=parseFloat(td[6].innerHTML.replace(",","")); ProgressStructure[0][tr.length-4].NRISonly=parseFloat(td[7].innerHTML.replace(",","")); ProgressStructure[0][tr.length-4].StartPlus=parseFloat(td[8].innerHTML.replace(",","")); ProgressStructure[0][tr.length-4].Unassessed=parseFloat(td[10].innerHTML.replace(",","")); ProgressStructure[0][tr.length-4].Untagged=parseFloat(td[11].innerHTML.replace(",","")); */	/* American Legation: Illustrated, Articled, not stub, not NRIS, Start+, Assessed, Tagged */ ProgressStructure[0][tr.length-4]={ ID: "Tangier, Morocco", Total: 1, Illustrated: 1, Articled: 1, Stubs: 0, NRISonly: 0, StartPlus: 1, Unassessed: 0, Untagged: 0 };

// duplicates row td=tr[tr.length-2].getElementsByTagName("td"); ProgressStructure[0][tr.length-3]={ ID: "National Duplicates", Total: parseFloat(td[0].innerHTML.replace(",","")), Illustrated: parseFloat(td[1].innerHTML.replace(",","")), Articled: parseFloat(td[3].innerHTML.replace(",","")), Stubs: parseFloat(td[5].innerHTML.replace(",","")), NRISonly: parseFloat(td[6].innerHTML.replace(",","")), StartPlus: parseFloat(td[7].innerHTML.replace(",","")), Unassessed: parseFloat(td[9].innerHTML.replace(",","")), Untagged: parseFloat(td[10].innerHTML.replace(",","")) };

// national totals ProgressStructure[0][tr.length-2]={ ID: "National Totals", Total: 0, Illustrated: 0, Articled:0, Stubs: 0, NRISonly: 0, StartPlus: 0, Unassessed: 0, Untagged: 0 };

// now data for each state for (i=1; i<table.length; i++) { tr=table[i].getElementsByTagName("tr"); ProgressStructure[i]=[]; for (j=1; j<tr.length-2; j++) { // skip title row, statewide duplicates, and totals row td=tr[j].getElementsByTagName("td");// fill in existing data in case error ProgressStructure[i][j-1]={}; ProgressStructure[i][j-1].ID=td[0].innerHTML.substr(0,5); ProgressStructure[i][j-1].Total=parseFloat(td[3].innerHTML.replace(",","")); ProgressStructure[i][j-1].Illustrated=parseFloat(td[4].innerHTML.replace(",","")); ProgressStructure[i][j-1].Articled=parseFloat(td[6].innerHTML.replace(",","")); ProgressStructure[i][j-1].Stubs=parseFloat(td[8].innerHTML.replace(",","")); ProgressStructure[i][j-1].NRISonly=parseFloat(td[9].innerHTML.replace(",","")); ProgressStructure[i][j-1].StartPlus=parseFloat(td[10].innerHTML.replace(",","")); ProgressStructure[i][j-1].Unassessed=parseFloat(td[12].innerHTML.replace(",","")); ProgressStructure[i][j-1].Untagged=parseFloat(td[13].innerHTML.replace(",","")); var link=td[1].getElementsByTagName("a"); if (link.length!==0 && link[0].href.search("#")==-1) { link=decodeURI(link[0].href).split("/"); link=link[link.length-1].replace(/_/g," "); ProgressStructure[i][j-1].Link=link;

ProgressStructure[i][j-1].ArticleQueried=0; // for querying later ProgressStructure[i][j-1].TalkQueried=0; } else { if (ProgressStructure[i][j-1].ID!="ddddd") { // if no link and not duplicate, must be totals row, so we can zero it

ProgressStructure[i][j-1].Total=0; ProgressStructure[i][j-1].Illustrated=0; ProgressStructure[i][j-1].Articled=0; ProgressStructure[i][j-1].Stubs=0; ProgressStructure[i][j-1].NRISonly=0; ProgressStructure[i][j-1].StartPlus=0; ProgressStructure[i][j-1].Unassessed=0; ProgressStructure[i][j-1].Untagged=0; }           }        }

// duplicates row td=tr[tr.length-2].getElementsByTagName("td");

ProgressStructure[i][tr.length-3]={ ID: ProgressStructure[0][i-1].ID+" Duplicates", Total: parseFloat(td[0].innerHTML.replace(",","")), Illustrated: parseFloat(td[1].innerHTML.replace(",","")), Articled: parseFloat(td[3].innerHTML.replace(",","")), Stubs: parseFloat(td[5].innerHTML.replace(",","")), NRISonly: parseFloat(td[6].innerHTML.replace(",","")), StartPlus: parseFloat(td[7].innerHTML.replace(",","")), Unassessed: parseFloat(td[9].innerHTML.replace(",","")), Untagged: parseFloat(td[10].innerHTML.replace(",","")) };       // state totals ProgressStructure[i][tr.length-2]={ ID: ProgressStructure[0][i-1].ID+" Totals", Total: 0, Illustrated: 0, Articled:0, Stubs: 0, NRISonly: 0, StartPlus: 0, Unassessed: 0, Untagged: 0 };   }    for (i=1; i<ProgressStructure.length; i++) { // count total number of rows to check for (j=0; j<ProgressStructure[i].length-2; j++) { if (typeof ProgressStructure[i][j].Link!="undefined") TotalToQuery++; // don't count duplicates and total rows }   }    TotalQueried=0;

var ProgressDiv=document.getElementById("ProgressDiv"); ProgressDiv.innerHTML+=" Done! ";

var ProgressSpan=document.createElement("span"); ProgressSpan.setAttribute("id", "ProgressSpan"); ProgressDiv.appendChild(ProgressSpan); ProgressSpan.innerHTML = "Querying county data... 0 (0%) of "+TotalToQuery+" lists checked.";

var TimeSpan=document.createElement("span"); TimeSpan.setAttribute("id", "TimeSpan"); ProgressDiv.appendChild(TimeSpan); TimeSpan.innerHTML = "";

}

function processTableList(ps) {	console.log("processTableList"); var curtable; var tabletitle; var wq = new WorkQueue(3); var tablecount = 0; for (curtable = 0; curtable<2 /*ps.length-1*/; curtable++) {		tablecount++; var state = { progressState: ps, curtable: curtable };		wq.push(makeJob(processTable,state)); } }

async function processTable(state) {	var curtable = state.curtable; var ps = state.progressState; var tabletitle; console.log('processTable',curtable); var rowcount = 0;

for (currow = 0; currow<ps[curtable].length-2; currow++) {	   if (typeof ps[curtable][currow].Link=="undefined") {	   	// skip duplicate and total rows continue; }	   rowcount++; tabletitle=ps[curtable][currow].Link; console.log('processTable doing '+tabletitle); }	// TODO return promise that resolves when all rows are processed return rowcount; } /*

for (currow = 0; currow {				processTableRow(state, resolve, reject);				}); promise.psState = state; state.promise = promise; promise.nTries = 5; wq.push(promise); }	} }

function processTableRow1(ps,tabletitle,curtable,currow, resolve, reject) {	console.log('processTableRow1',tabletitle,curtable,currow); resolve(true); }

async function processTableRow(promise, resolve, reject) {	var ps = promise.progressState; var tabletitle = promise.tabletitle; var curtable = promise.curtable; var currow = promise.currow; console.log("processTableRow",tabletitle,curtable,currow); var listingSource, listingTalkSource; var results; try { listingSource = getPageSource(tabletitle); listingTalkSource = getPageSource('Talk:'+tabletitle); results = await Promise.allSettled([listingSource,listingTalkSource]); console.log('Promise results', results); listingSource = getPageTextFromQueryResponse(results[0].value); listingTalkSource = getPageTextFromQueryResponse(results[1].value); if (listingTalkSource == null) listingTalkSource = ""; // TODO also need query results concerning redirects (see WikiTextFetched) console.log('Results of fetching listing article source and talk page',tabletitle, listingSource.length, listingTalkSource.length); //processListingSources(ps,tabletitle,curtable,currow, listingSource, listingTalkSource); resolve(true); }	catch (e) { console.log('Error getting source for listing article or talk page',tabletitle,e); //ProgressFatalError2(0,tabletitle,curtable,currow,e); reject(e); } }

async function processListingSources(ps,tabletitle, curtable, currow, listingSource, listingTalkSource) {	console.log('processListingSources', tabletitle); // TODO WikiTextFetched }

// load next list to query function LoadList(currentTable,currentRow) { // check if we need to go to the next table if (currentRow>ProgressStructure[currentTable].length-3) { currentRow=0; currentTable++; }   // check if there are no more tables if (currentTable>ProgressStructure.length-1) return;

if (typeof ProgressStructure[currentTable][currentRow].Link=="undefined") { // skip duplicate and total rows LoadList(currentTable,currentRow+1); return; }

var title=ProgressStructure[currentTable][currentRow].Link;

setTimeout(function{ // short delay to prevent API overload       getProgressListWikitext(title,currentTable,currentRow);        LoadList(currentTable,currentRow+1);    }, DefaultQueryPause); return; }

function WikitextFetched(ajaxResponse,status,title,currentTable,currentRow) { // console.log("WikitextFetched: table="+currentTable+" row="+currentRow+" title="+title+" status="+status); if (status!="success") { NewWarning("Wikitext "+ajaxResponse.errorThrown); setTimeout(function{ // try again after delay if rate limit reached           getProgressListWikitext(title,currentTable,currentRow);        }, 250); return; }   // won't get here unless successful var responseText, pagetext; var tabletext, regex, i, j;   try { responseText=JSON.parse(ajaxResponse.responseText); pagetext=responseText.query.pages[responseText.query.pageids[0]].revisions[0]["*"]; }   catch (e) { console.log("WikiTextFetched: Exception parsing "+title+": "+e); ProgressFatalError(0,title,currentTable,currentRow); return; }   var StartIndex; // console.log("WikiTextFetched: Parsing out "+title+" to find relevant table"); try { if (responseText.query.redirects) { // if redirect, find section var SectionName="Undefined"; for (var r in responseText.query.redirects) { if (typeof responseText.query.redirects[r].tofragment!="undefined") SectionName=responseText.query.redirects[r].tofragment.replace(/.27/g,"'"); }	       regex = new RegExp("=[ ]*(\\[\\[(.*?\\|)?[ ]*)?"+SectionName+"([ ]*\\]\\])?[ ]*=", "g"); var sectionheader=pagetext.match(regex); if (sectionheader === null || sectionheader === undefined) { // if no section found, check if one of known empty counties var ID = ProgressStructure[currentTable][currentRow].ID; //console.log("WikiTextFetched: List appears to be empty: title="+title+" id="+ID); // list last check date: 2020-11-09 var EmptyCounties=["01061", // Geneva County AL	           	"02270", // Kusilvak Census Area AK	            	"08014", // Broomfield County CO	            	"12067", // Lafayette County FL	            	"20081", // Haskell County KS	            	"20175", // Seward County KS	            	"20187", // Stanton County KS	            	"20189", // Stevens County KS	            	"26051", // Gladwin County MI	            	"26079", // Kalkaska County MI	            	"26119", // Montmorency County MI	            	"26129", // Ogemaw County MI	            	"26133", // Osceola County MI	            	"31009", // Blaine County NE	            	"31113", // Logan County NE	            	"31117", // McPherson County NE	            	"38085", // Sioux County ND	            	"42023", // Cameron County PA	            	"48017", // Bailey County TX	            	"48023", // Baylor County TX	            	"48033", // Borden County TX	            	"48069", // Castro County TX	            	"48079", // Cochran County TX	            	"48103", // Crane County TX	            	"48107", // Crosby County TX	            	"48119", // Delta County TX	            	"48131", // Duval County TX	            	"48155", // Foard County TX	            	"48165", // Gaines County TX	            	"48207", // Haskell County TX	            	"48219", // Hockley County TX	            	"48247", // Jim Hogg County TX	            	"48269", // King County TX	            	"48279", // Lamb County TX	            	"48341", // Moore County TX	            	"48389", // Reeves County TX	            	"48415", // Scurry County TX	            	"48421", // Sherman County TX	            	"48433", // Stonewall County TX	            	"48437", // Swisher County TX	            	"48445", // Terry County TX	            	"48461", // Upton County TX	            	"48475", // Ward County TX	            	"48501", // Yoakum County TX	            	"51735"  // Poquoson VA	            ]; var errorcode = 0; for (var k=0; k<EmptyCounties.length; k++) { if (ID==EmptyCounties[k]) {errorcode=-1;} }	           if (errorcode!==0) { // must be an empty county ProgressStructure[currentTable][currentRow].Total=0; ProgressStructure[currentTable][currentRow].Illustrated=0; ProgressStructure[currentTable][currentRow].Articled=0; ProgressStructure[currentTable][currentRow].Stubs=0; ProgressStructure[currentTable][currentRow].NRISonly=0; ProgressStructure[currentTable][currentRow].StartPlus=0; ProgressStructure[currentTable][currentRow].Unassessed=0; ProgressStructure[currentTable][currentRow].Untagged=0; ProgressStructure[currentTable][currentRow].Link=title; TotalQueried++; if (TotalQueried==TotalToQuery) CalculateProgressTotals; return; }	           // if we're here, must have been a redirect with no section, and not a known empty county sectionheader=pagetext.match(/{{NRHP header/g); // then look for tables without a section if (sectionheader===null||sectionheader.length>1) { // if still can't find a table or find multiple tables, fatal error ProgressFatalError(0,title,currentTable,currentRow); }	       }	        StartIndex=pagetext.indexOf(sectionheader[0]); var sectiontext=pagetext.substr(StartIndex,pagetext.indexOf("\n==",StartIndex)-StartIndex); // only look at relevant section StartIndex=sectiontext.indexOf("{{NRHP header"); if (StartIndex==-1) { if (sectiontext.indexOf("{{NRHP row")!=-1) { ProgressFatalError(2,title,currentTable,currentRow); // incorrectly formatted table } else { // must be an empty county console.log("WikiTextFetched: county appears to be empty: "+title); ProgressStructure[currentTable][currentRow].Total=0; ProgressStructure[currentTable][currentRow].Illustrated=0; ProgressStructure[currentTable][currentRow].Articled=0; ProgressStructure[currentTable][currentRow].Stubs=0; ProgressStructure[currentTable][currentRow].NRISonly=0; ProgressStructure[currentTable][currentRow].StartPlus=0; ProgressStructure[currentTable][currentRow].Unassessed=0; ProgressStructure[currentTable][currentRow].Untagged=0; ProgressStructure[currentTable][currentRow].Link=title; TotalQueried++; if (TotalQueried==TotalToQuery) CalculateProgressTotals; return; }	       }

var EndIndex = sectiontext.indexOf("\n|}",StartIndex); // EndIndex should exist, but a malformed file may not have it	       if (EndIndex === -1) { console.log("WikiTextFetched: missing end of table in "+title); ProgressFatalError(2,title,currentTable,currentRow); // list is malformed return; }	       tabletext=sectiontext.substr(StartIndex,EndIndex-StartIndex); } else { // if not a redirect, default to first table on page StartIndex=pagetext.indexOf("{{NRHP header"); if (StartIndex==-1) { ProgressFatalError(1,title,currentTable,currentRow); // no list found return; }	       var EndIndex = pagetext.indexOf("\n|}",StartIndex); // EndIndex should exist, but a malformed file may not have it	       if (EndIndex === -1) { console.log("WikiTextFetched: missing end of table in "+title); ProgressFatalError(2,title,currentTable,currentRow); // list is malformed return; }	       tabletext=pagetext.substr(StartIndex,EndIndex-StartIndex); }   }    catch (e) { console.log("WikiTextFetched: Exception searching for table in "+title+": "+e); ProgressFatalError(0,title,currentTable,currentRow); return;

}

// now that tabletext has only relevant table, extract rows var Rows=[]; var str = "{{"; var start=0; var commentstart=0; //console.log("WikiTextFetched: Extracting rows from "+title); try { while (true) { commentstart=tabletext.indexOf("",commentstart); commentstart=tabletext.indexOf("<!--",start); start=tabletext.indexOf(str,start); }	       if (start==-1) break; var open=1; var index=start+str.length; while (open!==0 && index<tabletext.length) { // make sure to find correct matching close brackets for row template if (tabletext.substr(index,2)=="}}") { open--; index++; } else if (tabletext.substr(index,2)=="{{") { open++; index++; }	           index++; }	       var template=tabletext.substr(start,index-start); regex = new RegExp("{{[\\s]*NRHP row(\\s)*\\|", "g"); if (template.match(regex)!==null) Rows.push(template); // make sure it's the row template and not some other one else { console.log("WikiTextFetched: failed to find 'NRHP row' in "+template); // ProgressFatalError(0,title,currentTable,currentRow); // return; }	       start++; }	   for (i=0; i((?!<[ ]*?/[ ]*?(nowiki|pre)[ ]*?>)(.|\\n))*?"+Rows[i].replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&")+"(.|\\n)*?<[ ]*?/[ ]*?(nowiki|pre)[ ]*?>", "g"); if (tabletext.match(regex)!==null) {Rows.splice(i,1); i--;} }   }    catch (e) { console.log("WikiTextFetched: Exception extracting rows from "+title+": "+e); ProgressFatalError(0,title,currentTable,currentRow); return;

}

// now begin querying statistics var Stats={"ID":ProgressStructure[currentTable][currentRow].ID, "Total":Rows.length, "Illustrated":0, "Articled":0, "Stubs":0, "NRISonly":0, "StartPlus":0, "Unassessed":0, "Untagged":0, "Link":title};

var Titles=[]; console.log("WikiTextFetched: query statistics for "+title+": "+Rows.length+" rows"); try { for (i=0; i/g,"").trim; if (test!=="") { //console.log('NRHPstats: found image in row '+NRHPstats_currentRow+': '+test); var test2 = test.match(/Address restricted/gi); if (test2!==null && test2!=="") { RestrictedImageCount++; console.log('NRHPstats: image on row '+i+' appears to be address restricted image (ignoring #'+RestrictedImageCount+'): '+test); Stats.Illustrated++; // TODO keep for now } else { Stats.Illustrated++; // only true if image param there and non-blank }	           }	        }	        var article=ThisRow.match(/\|[ ]*?article[ ]*?=[ ]*?.*?[\n|\|]/g); var blank=ThisRow.match(/\|[ ]*?article[ ]*?=[ ]*?[\n|\|]/g);                              // default to name param if article if (article===null||blank!==null) article=ThisRow.match(/\|[ ]*?name[ ]*?=[ ]*?.*?[\n|\|]/g); // blank or missing // strip param name, final line break article=article[0].replace(/\|[ ]*?(article|name)[ ]*?=[ ]*?/g,"").replace(/[\n|\|]/g,"").replace(/<\!\-\-(.|[\r\n])*?\-\-\>/g,"").trim; article=decodeURIComponent(article.split("#")[0].trim);    // corrections for weird titles try { Titles.push(article); }	       catch (e) { console.log("WikiTextFetched: Exception parsing "+title+": "+e); ProgressFatalError(0,title,currentTable,currentRow); }	   }    }    catch (e) { console.log("WikiTextFetched: Exception extracting statistics from rows from "+title+": "+e); ProgressFatalError(0,title,currentTable,currentRow); return; }

StartIndex=0; LoadNextProgressQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow); return; }

// ready next batch of articles to query function LoadNextProgressQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow) { if (StartIndex==Stats.Total) { // all queries begun for this list return; }   var TempTitles; // must have some more rows to query if (Stats.Total-StartIndex>50) { TempTitles=Titles.slice(StartIndex,StartIndex+50); } else { TempTitles=Titles.slice(StartIndex); }   StartIndex+=TempTitles.length;

setTimeout(function{ // short delay to prevent API overload       QueryProgressStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);        LoadNextProgressQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow);    }, DefaultQueryPause); return; }

// query next batch of articles function QueryProgressStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow) { var TitleList=TempTitles.join("|"); // console.log("QueryProgressStats: titles="+TitleList); $.ajax({       dataType: "json",        url: mw.util.wikiScript('api'),        data: {            format: 'json',            action: 'query',            prop: 'categories',            clcategories: 'Category:All disambiguation pages|Category:All articles sourced only to NRIS',            cllimit: 'max',            titles: TitleList,            redirects: 'true'        },        error: function(ArticlejsonObject,status,errorThrown) {        	ArticlejsonObject.errorThrown=errorThrown;        	console.log("QueryProgressStats: error fetching "+TempTitles);        },        complete: function(ArticlejsonObject,status) {        	try {                ProgressArticleChecked(ArticlejsonObject,status,Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);        	}        	catch (e) {        		console.log("QueryProgressStats: Eception checking article "+TempTitles+": "+e);		        ProgressFatalError(0,title,currentTable,currentRow); }       }    });    return; }

// parse API response for article query function ProgressArticleChecked(ArticlejsonObject,status,Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow) { if (status!="success") { NewWarning("Articles "+ArticlejsonObject.errorThrown);

setTimeout(function{ // try again after delay if rate limit reached           QueryProgressStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);        }, 250); return; }   // won't get here unless successful ProgressStructure[currentTable][currentRow].ArticleQueried+=TempTitles.length;

var responseText=JSON.parse(ArticlejsonObject.responseText); var i;   if (responseText.query.normalized) { // normalize any weird titles for (var n in responseText.query.normalized) { for (i=0; i<TempTitles.length; i++) { if (TempTitles[i]==responseText.query.normalized[n].from) TempTitles[i]=responseText.query.normalized[n].to; }           for (i=0; i<Titles.length; i++) { // also update in Titles array to prepare to query talk pages if (Titles[i]==responseText.query.normalized[n].from) Titles[i]=responseText.query.normalized[n].to; }       }    }    if (responseText.query.redirects) { // resolve any redirects also for (var r in responseText.query.redirects) { for (i=0; i<TempTitles.length; i++) { if (TempTitles[i]==responseText.query.redirects[r].from) TempTitles[i]=responseText.query.redirects[r].to; }           for (i=0; i<Titles.length; i++) { // also update in Titles array to prepare to query talk pages if (Titles[i]==responseText.query.redirects[r].from) Titles[i]=responseText.query.redirects[r].to; }       }    }

// now determine the number of bluelinks and NRIS-only articles for (var page in responseText.query.pages) { var articled=true;  // default to articled, not NRIS-only var NRISonly=false; var pagetitle=responseText.query.pages[page].title; if (typeof responseText.query.pages[page].missing!="undefined") { // redlink=unarticled articled=false; }       if (responseText.query.pages[page].categories) { for (var category in responseText.query.pages[page].categories) { if (responseText.query.pages[page].categories[category].title=="Category:All disambiguation pages") { // dab=unarticled articled=false; }               if (responseText.query.pages[page].categories[category].title.indexOf("sourced only to NRIS")!=-1) { // mark as NRIS-only NRISonly=true; }           }        }        for (i=0; i<TempTitles.length; i++) { // if page is duplicated, count it multiple times if (TempTitles[i]==pagetitle) { if (articled) Stats.Articled++; if (NRISonly) Stats.NRISonly++; }       }    }

if (ProgressStructure[currentTable][currentRow].ArticleQueried==Stats.Total) { // after querying all articles, query talk pages for (i=0; i50) { TempTitles=Titles.slice(StartIndex,StartIndex+50); } else { TempTitles=Titles.slice(StartIndex); }   StartIndex+=TempTitles.length;

setTimeout(function{ // short delay to prevent API overload       QueryProgressTalkStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);        LoadNextProgressTalkQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow);    }, DefaultQueryPause); return; }

// query the next batch of talk pages function QueryProgressTalkStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow) { var catlist='Category:FA-Class National Register of Historic Places articles|Category:A-Class National Register of Historic Places '; catlist+='articles|Category:GA-Class National Register of Historic Places articles|Category:B-Class National Register of Historic '; catlist+='Places articles|Category:C-Class National Register of Historic Places articles|Category:Start-Class National Register of '; catlist+='Historic Places articles|Category:Stub-Class National Register of Historic Places articles|Category:Unassessed National '; catlist+='Register of Historic Places articles|Category:List-Class National Register of Historic Places articles|Category:Redirect-'; catlist+='Class National Register of Historic Places articles';

var TitleList=TempTitles.join("|"); $.ajax({       dataType: "json",        url: mw.util.wikiScript('api'),        data: {            format: 'json',            action: 'query',            prop: 'categories',            clcategories: catlist,            cllimit: 'max',            titles: TitleList,            redirects: 'true'        },        error: function(ArticlejsonObject,status,errorThrown) {        	ArticlejsonObject.errorThrown=errorThrown;        	console.log("QueryProgressTalkStats: Error fetching "+TitleList);        },        complete: function(ArticlejsonObject,status) {        	try {                ProgressTalkChecked(ArticlejsonObject,status,Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);            }            catch (e) {            	alert("QueryProgressTalkStats: Exception processing "+TitleList+": "+e);		        ProgressFatalError(0,title,currentTable,currentRow);            } }   });    return; }

// parse API response for talk page query function ProgressTalkChecked(ArticlejsonObject,status,Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow) { if (status!="success") { NewWarning("Talk "+ArticlejsonObject.errorThrown);

setTimeout(function{ // try again after delay if rate limit reached           QueryProgressTalkStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);        }, 250); return; }   // won't get here unless successful ProgressStructure[currentTable][currentRow].TalkQueried+=TempTitles.length;

// now determine quality statistics var responseText=JSON.parse(ArticlejsonObject.responseText); for (var page in responseText.query.pages) { if (typeof responseText.query.pages[page].missing!="undefined") continue; // skip talk page if not articled

var untagged=true; // default to untagged var articled=true; // assume articled to check for link to other county/MPS lists var stub=false; var unassessed=false; var startPlus=false; var pagetitle=responseText.query.pages[page].title; if (responseText.query.pages[page].categories) { untagged=false; // if cat hit, mark as tagged for (var category in responseText.query.pages[page].categories) { var CatTitle=responseText.query.pages[page].categories[category].title; if (CatTitle.indexOf("Stub")!=-1) { stub=true; // mark as stub }               if (CatTitle.indexOf("Unassessed")!=-1||CatTitle.indexOf("Redirect")!=-1) { unassessed=true; // mark as unassessed }               if  (CatTitle.indexOf("List")!=-1) { // count links to other county/MPS lists as unarticled; other list-class as stubs if (responseText.query.pages[page].title.indexOf("National Register of Historic Places")!=-1){ articled=false; } else { stub=true; }               }            }        }        if (articled&&!untagged&&!unassessed&&!stub) { // if articled, tagged, and assessed, but not stub, must be Start+ startPlus=true; }

for (var i=0; i<TempTitles.length; i++) { if (TempTitles[i]==pagetitle) { if (!articled) Stats.Articled--; // reduce the count of articled if links to lists if (untagged) Stats.Untagged++; if (stub) Stats.Stubs++; if (unassessed) Stats.Unassessed++; if (startPlus) Stats.StartPlus++; }       }    }

if (ProgressStructure[currentTable][currentRow].TalkQueried==Stats.Total) { ProgressStructure[currentTable][currentRow].Total=Stats.Total; ProgressStructure[currentTable][currentRow].Illustrated=Stats.Illustrated; ProgressStructure[currentTable][currentRow].Articled=Stats.Articled; ProgressStructure[currentTable][currentRow].Stubs=Stats.Stubs; ProgressStructure[currentTable][currentRow].NRISonly=Stats.NRISonly; ProgressStructure[currentTable][currentRow].StartPlus=Stats.StartPlus; ProgressStructure[currentTable][currentRow].Unassessed=Stats.Unassessed; ProgressStructure[currentTable][currentRow].Untagged=Stats.Untagged; ProgressStructure[currentTable][currentRow].Link=title;

TotalQueried++; if (TotalQueried==TotalToQuery) CalculateProgressTotals; } }

// keep track of warnings encountered while querying API function NewWarning(warning) { var i, NewWarning=true; for (i=0; i<WarningCount.length; i++) { // check if already encountered error if (warning==WarningCount[i][0]||WarningCount[i][0]==="") {WarningCount[i][0]=warning; WarningCount[i][1]++; NewWarning=false;} }   if (NewWarning) WarningCount[WarningCount.length]=[warning,1]; // if new warning, make new entry

var test=0; for (i=0; i<WarningCount.length; i++) { test+=WarningCount[i][1]; }   if (test%50===0) DefaultQueryPause++; // for every 50 errors encountered, increase time between each query to throttle speed }

var g_fetchErrors = {};

function ProgressFatalError2(code,title, curtable, currow, ex) {	console.log('ProgressFatalError2: ',code,title,curtable,currow, ex); var errorArray = ['No county section found for ','No list found for ','Incorrectly formatted list for ']; if (!g_fetchErrors.hasOwnProperty(title)) g_fetchErrors[title] = 0; g_fetchErrors[title]++; var retry = true; if (g_fetchErrors[title] > 5) { retry=confirm(errorArray[code]+title+"!\n\nCancel=Skip                  OK=Retry"); if (retry) g_fetchErrors[title] = 0; // reset counter for this title }   if (retry) { processTableRow(title,curtable,currow); } else { TotalQueried++; ErrorCount++; if (TotalQueried==TotalToQuery) CalculateProgressTotals; }   return; } // these errors require user input; can't just be ignored function ProgressFatalError(code,title,currentTable,currentRow) { var errorArray = ['No county section found for ','No list found for ','Incorrectly formatted list for ']; if (!g_fetchErrors.hasOwnProperty(title)) g_fetchErrors[title] = 0; g_fetchErrors[title]++; var retry = true; if (g_fetchErrors[title] > 5) { retry=confirm(errorArray[code]+title+"!\n\nCancel=Skip                  OK=Retry"); if (retry) g_fetchErrors[title] = 0; // reset counter for this title }   if (retry) { getProgressListWikitext(title,currentTable,currentRow); } else { TotalQueried++; ErrorCount++; if (TotalQueried==TotalToQuery) CalculateProgressTotals; }   return; }

// update ProgressDiv to let user know what's going on function UpdateProgressDiv { var ProgressSpan=document.getElementById("ProgressSpan"); var TimeSpan=document.getElementById("TimeSpan");

var PercentQueried=Math.round(TotalQueried/TotalToQuery*1000)/10; ProgressSpan.innerHTML = "Querying county data... "+TotalQueried+" ("+PercentQueried+"%) of "+TotalToQuery+" lists checked.";

var TimeRemainingStr=""; if (TotalQueried>100) { var CurrentTime=new Date; var SecondsElapsed = (CurrentTime-InitialTime)/1000; var Average = SecondsElapsed/TotalQueried; SecondsElapsed=Math.round(SecondsElapsed); var MinutesElapsed = 0; while (SecondsElapsed>=60) { SecondsElapsed-=60; MinutesElapsed++; }       var SecondsRemaining = Math.round(Average*(TotalToQuery-TotalQueried)); var MinutesRemaining = 0; while (SecondsRemaining>=60) { SecondsRemaining-=60; MinutesRemaining++; }

TimeRemainingStr = ""; if (MinutesRemaining!==0) TimeRemainingStr=MinutesRemaining+" min "; TimeRemainingStr+=SecondsRemaining+" sec"; var TimeElapsedStr = ""; if (MinutesElapsed!==0) TimeElapsedStr=MinutesElapsed+" min "; TimeElapsedStr+=SecondsElapsed+" sec"; TimeRemainingStr+=" ("+TimeElapsedStr+" elapsed)"; } else { TimeRemainingStr="Calculating..."; }   TimeSpan.innerHTML=" Estimated time remaining: "+TimeRemainingStr;

if (TotalQueried!=TotalToQuery) { // update ProgressDiv only at regular intervals to prevent CPU overload; stop once done ProgressDivTimer=setTimeout(function{           UpdateProgressDiv;        }, 500); }   return; }

// after all querying complete, calculate totals for states and counties with multiple sublists function CalculateProgressTotals { clearTimeout(ProgressDivTimer); var ProgressSpan=document.getElementById("ProgressSpan"); var TimeSpan=document.getElementById("TimeSpan");

ProgressSpan.innerHTML = "Querying county data... Done! "+TotalToQuery+" lists checked.";

var CurrentTime=new Date; var SecondsElapsed = (CurrentTime-InitialTime)/1000; SecondsElapsed=Math.round(SecondsElapsed); var MinutesElapsed = 0; while (SecondsElapsed>=60) { SecondsElapsed-=60; MinutesElapsed++; }   TimeSpan.innerHTML=" Time elapsed: "; if (MinutesElapsed!==0) TimeSpan.innerHTML+=MinutesElapsed+" min "; TimeSpan.innerHTML+=SecondsElapsed+" sec";

var TotalsIndex, NationalTotalsIndex; for (var i=1; i<ProgressStructure.length; i++) { // i=table number; skip national table until end for (var j=0; j<ProgressStructure[i].length-2; j++) { TotalsIndex=ProgressStructure[i].length-1; if (!isNaN(parseFloat(ProgressStructure[i][j].ID))) { // if regular county without sublists, add to totals ProgressStructure[i][TotalsIndex].Total+=ProgressStructure[i][j].Total; ProgressStructure[i][TotalsIndex].Illustrated+=ProgressStructure[i][j].Illustrated; ProgressStructure[i][TotalsIndex].Articled+=ProgressStructure[i][j].Articled; ProgressStructure[i][TotalsIndex].Stubs+=ProgressStructure[i][j].Stubs; ProgressStructure[i][TotalsIndex].NRISonly+=ProgressStructure[i][j].NRISonly; ProgressStructure[i][TotalsIndex].StartPlus+=ProgressStructure[i][j].StartPlus; ProgressStructure[i][TotalsIndex].Unassessed+=ProgressStructure[i][j].Unassessed; ProgressStructure[i][TotalsIndex].Untagged+=ProgressStructure[i][j].Untagged; } else if (ProgressStructure[i][j].ID=="-") { // if county sublist, find total county row and add there var CountyTotalsIndex=j+1; while (CountyTotalsIndex<ProgressStructure[i].length-2) { if (!isNaN(parseFloat(ProgressStructure[i][CountyTotalsIndex].ID))) break; CountyTotalsIndex++; }               ProgressStructure[i][CountyTotalsIndex].Total+=ProgressStructure[i][j].Total; ProgressStructure[i][CountyTotalsIndex].Illustrated+=ProgressStructure[i][j].Illustrated; ProgressStructure[i][CountyTotalsIndex].Articled+=ProgressStructure[i][j].Articled; ProgressStructure[i][CountyTotalsIndex].Stubs+=ProgressStructure[i][j].Stubs; ProgressStructure[i][CountyTotalsIndex].NRISonly+=ProgressStructure[i][j].NRISonly; ProgressStructure[i][CountyTotalsIndex].StartPlus+=ProgressStructure[i][j].StartPlus; ProgressStructure[i][CountyTotalsIndex].Unassessed+=ProgressStructure[i][j].Unassessed; ProgressStructure[i][CountyTotalsIndex].Untagged+=ProgressStructure[i][j].Untagged; } else if (ProgressStructure[i][j].ID=="ddddd") { // if county duplicate row, subtract from county total ProgressStructure[i][j+1].Total-=ProgressStructure[i][j].Total; ProgressStructure[i][j+1].Illustrated-=ProgressStructure[i][j].Illustrated; ProgressStructure[i][j+1].Articled-=ProgressStructure[i][j].Articled; ProgressStructure[i][j+1].Stubs-=ProgressStructure[i][j].Stubs; ProgressStructure[i][j+1].NRISonly-=ProgressStructure[i][j].NRISonly; ProgressStructure[i][j+1].StartPlus-=ProgressStructure[i][j].StartPlus; ProgressStructure[i][j+1].Unassessed-=ProgressStructure[i][j].Unassessed; ProgressStructure[i][j+1].Untagged-=ProgressStructure[i][j].Untagged; } else { // unknown ID; skip it               alert("Error! Unknown ID="+ProgressStructure[i][j].ID+" in Table "+i+", Row "+j+". Skipping this list in totals."); }       }        // subtract state duplicates ProgressStructure[i][TotalsIndex].Total-=ProgressStructure[i][TotalsIndex-1].Total; ProgressStructure[i][TotalsIndex].Illustrated-=ProgressStructure[i][TotalsIndex-1].Illustrated; ProgressStructure[i][TotalsIndex].Articled-=ProgressStructure[i][TotalsIndex-1].Articled; ProgressStructure[i][TotalsIndex].Stubs-=ProgressStructure[i][TotalsIndex-1].Stubs; ProgressStructure[i][TotalsIndex].NRISonly-=ProgressStructure[i][TotalsIndex-1].NRISonly; ProgressStructure[i][TotalsIndex].StartPlus-=ProgressStructure[i][TotalsIndex-1].StartPlus; ProgressStructure[i][TotalsIndex].Unassessed-=ProgressStructure[i][TotalsIndex-1].Unassessed; ProgressStructure[i][TotalsIndex].Untagged-=ProgressStructure[i][TotalsIndex-1].Untagged;

// record state totals in national table ProgressStructure[0][i-1].Total=ProgressStructure[i][TotalsIndex].Total; ProgressStructure[0][i-1].Illustrated=ProgressStructure[i][TotalsIndex].Illustrated; ProgressStructure[0][i-1].Articled=ProgressStructure[i][TotalsIndex].Articled; ProgressStructure[0][i-1].Stubs=ProgressStructure[i][TotalsIndex].Stubs; ProgressStructure[0][i-1].NRISonly=ProgressStructure[i][TotalsIndex].NRISonly; ProgressStructure[0][i-1].StartPlus=ProgressStructure[i][TotalsIndex].StartPlus; ProgressStructure[0][i-1].Unassessed=ProgressStructure[i][TotalsIndex].Unassessed; ProgressStructure[0][i-1].Untagged=ProgressStructure[i][TotalsIndex].Untagged;

// add state totals to national totals NationalTotalsIndex=ProgressStructure[0].length-1; ProgressStructure[0][NationalTotalsIndex].Total+=ProgressStructure[0][i-1].Total; ProgressStructure[0][NationalTotalsIndex].Illustrated+=ProgressStructure[0][i-1].Illustrated; ProgressStructure[0][NationalTotalsIndex].Articled+=ProgressStructure[0][i-1].Articled; ProgressStructure[0][NationalTotalsIndex].Stubs+=ProgressStructure[0][i-1].Stubs; ProgressStructure[0][NationalTotalsIndex].NRISonly+=ProgressStructure[0][i-1].NRISonly; ProgressStructure[0][NationalTotalsIndex].StartPlus+=ProgressStructure[0][i-1].StartPlus; ProgressStructure[0][NationalTotalsIndex].Unassessed+=ProgressStructure[0][i-1].Unassessed; ProgressStructure[0][NationalTotalsIndex].Untagged+=ProgressStructure[0][i-1].Untagged; }   // special row for Tangier, Morocco ProgressStructure[0][NationalTotalsIndex].Total+=ProgressStructure[0][NationalTotalsIndex-2].Total; ProgressStructure[0][NationalTotalsIndex].Illustrated+=ProgressStructure[0][NationalTotalsIndex-2].Illustrated; ProgressStructure[0][NationalTotalsIndex].Articled+=ProgressStructure[0][NationalTotalsIndex-2].Articled; ProgressStructure[0][NationalTotalsIndex].Stubs+=ProgressStructure[0][NationalTotalsIndex-2].Stubs; ProgressStructure[0][NationalTotalsIndex].NRISonly+=ProgressStructure[0][NationalTotalsIndex-2].NRISonly; ProgressStructure[0][NationalTotalsIndex].StartPlus+=ProgressStructure[0][NationalTotalsIndex-2].StartPlus; ProgressStructure[0][NationalTotalsIndex].Unassessed+=ProgressStructure[0][NationalTotalsIndex-2].Unassessed; ProgressStructure[0][NationalTotalsIndex].Untagged+=ProgressStructure[0][NationalTotalsIndex-2].Untagged;

// subtract national duplicates ProgressStructure[0][NationalTotalsIndex].Total-=ProgressStructure[0][NationalTotalsIndex-1].Total; ProgressStructure[0][NationalTotalsIndex].Illustrated-=ProgressStructure[0][NationalTotalsIndex-1].Illustrated; ProgressStructure[0][NationalTotalsIndex].Articled-=ProgressStructure[0][NationalTotalsIndex-1].Articled; ProgressStructure[0][NationalTotalsIndex].Stubs-=ProgressStructure[0][NationalTotalsIndex-1].Stubs; ProgressStructure[0][NationalTotalsIndex].NRISonly-=ProgressStructure[0][NationalTotalsIndex-1].NRISonly; ProgressStructure[0][NationalTotalsIndex].StartPlus-=ProgressStructure[0][NationalTotalsIndex-1].StartPlus; ProgressStructure[0][NationalTotalsIndex].Unassessed-=ProgressStructure[0][NationalTotalsIndex-1].Unassessed; ProgressStructure[0][NationalTotalsIndex].Untagged-=ProgressStructure[0][NationalTotalsIndex-1].Untagged;

setTimeout(function {ParseProgressSource},1); // small delay for non-Firefox browsers to update screen }

// update progress page source with queried totals async function ParseProgressSource { var pagesourceresponse = await getPageSource(mw.config.get('wgPageName')); console.log("Page source query",pagesourceresponse); var pagesource = getPageTextFromQueryResponse(pagesourceresponse); var newpagesource=pagesource; var TableStartIndex=pagesource.indexOf("==State totals"); var test; var RowEndIndex; var oldRow; var firstColumn; var newRow; var temp; var str; for (var i=0; i<ProgressStructure.length; i++) { TableStartIndex=pagesource.indexOf("{|",TableStartIndex+1);   // find next table in old page source var TableEndIndex=pagesource.indexOf("|}",TableStartIndex)+2; var oldTable=pagesource.substr(TableStartIndex,TableEndIndex-TableStartIndex); var newTable=oldTable;

var RowStartIndex=0; for (var j=0; j<ProgressStructure[i].length-2; j++) { RowStartIndex=oldTable.indexOf("\n|-",RowStartIndex+1); // find next row in old table if (ProgressStructure[i][j].ID=="ddddd") continue;  // skip duplicate rows RowEndIndex=oldTable.indexOf("\n|-",RowStartIndex+1); oldRow=oldTable.substr(RowStartIndex,RowEndIndex-RowStartIndex); firstColumn=oldRow.indexOf("\n|"); var stop=4;       // skip one cell for national table, three for states if (i===0) stop=2; for (var inc=0; inc999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); // add thousands separators newRow+="\n| "+str; // illustrated str=temp.Illustrated.toString; if (temp.Illustrated>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); newRow+="\n| "+str; // illustrated percent if (temp.Total===0) { str="-"; } else { str = Math.round(temp.Illustrated/temp.Total*1000)/10; test = str.toString.indexOf("."); if (test==-1 && str!=100 && str!==0) str+=".0"; // force decimal str+="%"; }           newRow+="\n| "+str; // articled str=temp.Articled.toString; if (temp.Articled>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); newRow+="\n| "+str; // articled percent if (temp.Total===0) { str="-"; } else { str = Math.round(temp.Articled/temp.Total*1000)/10; test = str.toString.indexOf("."); if (test==-1 && str!=100 && str!==0) str+=".0"; str+="%"; }           newRow+="\n| "+str; // stubs str=temp.Stubs.toString; if (temp.Stubs>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); newRow+="\n| "+str; // NRIS-only str=temp.NRISonly.toString; if (temp.NRISonly>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); newRow+="\n| "+str; // Start+ str=temp.StartPlus.toString; if (temp.StartPlus>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); newRow+="\n| "+str; // Start+ percent if (temp.Total===0) { str="-"; } else { str = Math.round(temp.StartPlus/temp.Total*1000)/10; test = str.toString.indexOf("."); if (test==-1 && str!=100 && str!==0) str+=".0"; // force decimal str+="%"; }           newRow+="\n| "+str; // unassessed str=temp.Unassessed.toString; if (temp.Unassessed>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); newRow+="\n| "+str; // untagged str=temp.Untagged.toString; if (temp.Untagged>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); newRow+="\n| "+str; // net quality if (temp.Total===0) { str="-"; } else { str=temp.StartPlus+0.5*temp.Stubs+0.5*temp.Unassessed-0.5*temp.Untagged-0.75*temp.NRISonly; str=Math.round((0.75*str/temp.Total+0.25*temp.Illustrated/temp.Total)*1000)/10; if (str<0) str=0; test=str.toString.indexOf("."); if (test==-1 && str!=100 && str!==0) str+=".0"; str+="%"; }           newRow+="\n| "+str;

// update new table with new row newTable=newTable.replace(oldRow,newRow); }       RowStartIndex=oldTable.indexOf("\n|-",RowStartIndex+1); // skip duplicate row RowStartIndex=oldTable.indexOf("\n|-",RowStartIndex+1); RowEndIndex=oldTable.indexOf("\n|}",RowStartIndex+1); oldRow=oldTable.substr(RowStartIndex,RowEndIndex-RowStartIndex); firstColumn=oldRow.indexOf("\n!"); // totals row uses ! instead of | firstColumn=oldRow.indexOf("\n!",firstColumn+1); // skip first cell

// build up new totals row newRow=oldRow.substr(0,firstColumn); temp=ProgressStructure[i][ProgressStructure[i].length-1]; // total str=temp.Total.toString; if (temp.Total>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); // add thousands separators newRow+="\n! "+str; // illustrated str=temp.Illustrated.toString; if (temp.Illustrated>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); newRow+="\n! "+str; // illustrated percent if (temp.Total===0) { str="-"; } else { str = Math.round(temp.Illustrated/temp.Total*1000)/10; test = str.toString.indexOf("."); if (test==-1 && str!=100 && str!==0) str+=".0"; // force decimal str+="%"; }       newRow+="\n! "+str; // articled str=temp.Articled.toString; if (temp.Articled>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); newRow+="\n! "+str; // articled percent if (temp.Total===0) { str="-"; } else { str = Math.round(temp.Articled/temp.Total*1000)/10; test = str.toString.indexOf("."); if (test==-1 && str!=100 && str!==0) str+=".0"; // force decimal str+="%"; }       newRow+="\n! "+str; // stubs str=temp.Stubs.toString; if (temp.Stubs>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); newRow+="\n! "+str; // NRIS-only str=temp.NRISonly.toString; if (temp.NRISonly>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); newRow+="\n! "+str; // Start+ str=temp.StartPlus.toString; if (temp.StartPlus>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); newRow+="\n! "+str; // Start+ percent if (temp.Total===0) { str="-"; } else { str = Math.round(temp.StartPlus/temp.Total*1000)/10; test = str.toString.indexOf("."); if (test==-1 && str!=100 && str!==0) str+=".0"; // force decimal str+="%"; }       newRow+="\n! "+str; // unassessed str=temp.Unassessed.toString; if (temp.Unassessed>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); newRow+="\n! "+str; // untagged str=temp.Untagged.toString; if (temp.Untagged>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); newRow+="\n! "+str; // net quality if (temp.Total===0) { str="-"; } else { str=temp.StartPlus+0.5*temp.Stubs+0.5*temp.Unassessed-0.5*temp.Untagged-0.75*temp.NRISonly; str=Math.round((0.75*str/temp.Total+0.25*temp.Illustrated/temp.Total)*1000)/10; if (str<0) str=0; test=str.toString.indexOf("."); if (test==-1 && str!=100 && str!==0) str+=".0"; str+="%"; }       newRow+="\n! "+str;

// update new page source with new table newTable=newTable.replace(oldRow,newRow); newpagesource=newpagesource.replace(oldTable,newTable); }

// now edit page to provide new source var ProgressDiv=document.getElementById("ProgressDiv"); ProgressDiv.innerHTML+=" Editing page (this might take up to one minute)... "; InitializeEdit(newpagesource); }

// initialize edit function InitializeEdit(pagesource) { var d=new Date; var months=['January','February','March','April','May','June','July','August','September','October','November','December']; var year=d.getYear; if (year < 1000) year += 1900; var DateStr=months[d.getMonth]+" "+d.getDate+", "+year;

regex=/(January|February|March|April|May|June|July|August|September|October|November|December) [0-9]{1,2}, [0-9]{4}/g; var tempstring=pagesource.split('==County totals=='); // ignore dates in lead (e.g. date of last map update) pagesource=tempstring[0]+'==County totals=='+tempstring[1].replace(regex,DateStr); // update date strings above tables

var ErrorStr = ''; if (ErrorCount>0) ErrorStr = " Errors encountered for "+ErrorCount+" counties, which were skipped. Human attention needed."; var summary='Updating county data as of '+DateStr+' using script.'+ErrorStr; editPage(pagesource,mw.config.get('wgPageName'),summary); }

function PageEdited(ajaxResponse,status,pagesource) { var ProgressDiv=document.getElementById("ProgressDiv"); if (status!="success") { var retry=confirm("Error: "+ajaxResponse.errorThrown+" while editing page!\n\nCancel=Abort                  OK=Retry"); if (retry) { ProgressDiv.innerHTML+="Retrying... "; InitializeEdit(pagesource); // try again } else { ProgressDiv.innerHTML+="Edit failure! Script aborted!"; }       return; }   var responseText=JSON.parse(ajaxResponse.responseText); var diff=responseText.edit.newrevid; var linkStr="//en.wikipedia.org/w/index.php?diff="+diff; ProgressDiv.innerHTML+="Page edited! Click here for diff."; console.log("Number of restricted images found: "+RestrictedImageCount);

// output technical information to console var WarningText="NRHP Progress Warnings: "; for (var i=0; i<WarningCount.length; i++) { WarningText+=WarningCount[i][0]+" ("+WarningCount[i][1]+"), "; }   if (WarningCount[0][0]!=="") { WarningText=WarningText.substr(0,WarningText.length-2); } else { WarningText="NRHP Progress Warnings: none"; }   console.log(WarningText); }

function editPage(text,title,summary) { // edit page when done var edit_token; edit_token = mw.user.tokens.get( 'editToken' ); if (edit_token === null) { edit_token = mw.user.tokens.get( 'csrfToken'); }   $.ajax({        dataType: 'json',        url: mw.util.wikiScript( 'api' ),        type: 'POST',        data: {            format: 'json',            action: 'edit',            title: title,            text: text,            summary: summary,            token: edit_token        },        success: function( data ) {            if (data && data.edit && data.edit.result && data.edit.result == 'Success') {                console.log('Page edit succeeded');            } else {            	var msg;            	if (data && data.error) {            		msg = "Error editing page: ";            		msg += " code=" + data.error.code;            		msg += " info=" + data.error.info;        		} else {        			msg = "Error editing page (no data or error)";        		}                alert(msg);            }        },        error: function(ajaxResponse,status,errorThrown) {        	ajaxResponse.errorThrown=errorThrown; console.log("EditPage: Error editing "+title); },       complete: function(ajaxResponse,status) {PageEdited(ajaxResponse,status,text)} }); }

function getPageTextFromQueryResponse(response) {	var txt; var query = response.query; console.log('getPageText',response); for (var pageid in query.pages) { if (pageid == "-1") return null; // this means txt=query.pages[pageid].revisions[0].slots.main['*']; }

//console.log('Body:'); //console.log(txt); return txt; }

async function getPageSource(page) { console.log("getPageSource page",page); var api = new mw.Api; var rsp;

// nb this will throw an exception if it fails rsp = await api.get({           format: 'json',            action: 'query',            prop: 'revisions',            rvprop: 'content',            rvslots: '*',            titles: page,            indexpageids: true,            redirects: 'true'        });

console.log("getPageSource response",rsp);

return rsp; }

function getProgressListWikitext(title,currentTable,currentRow) {  // asynchronous fetch of each list's wikitext console.log("GetProgressListWikiText: fetch "+title); $.ajax({       dataType: "json",        url: mw.util.wikiScript('api'),        data: {            format: 'json',            action: 'query',            prop: 'revisions',            rvprop: 'content',            titles: title,            indexpageids: true,            redirects: 'true'        },        error: function(ajaxResponse,status,errorThrown) {        	console.log("GetProgressListWikiText: Error fetching "+title);        	ajaxResponse.errorThrown=errorThrown;        },        complete: function(ajaxResponse,status) {        	console.log("GetProgressListWikitext: Status fetching "+title+": "+status);        	try {	        	WikitextFetched(ajaxResponse,status,title,currentTable,currentRow);        	}        	catch (e) {        		alert("Exception after fetching "+title+": "+e);		        ProgressFatalError(0,title,currentTable,currentRow);        	}        }    }); }

//jQuery(ProgressButton); $.when($.ready).then(ProgressButton);