User:Enterprisey/check-interface-edits.rs

"#, username, title, revid, timestamp, summary.replace("<", "&lt;").replace(">", "&gt;"), diff))       }    } }

fn make_map(params: &[(&str, &str)]) -> HashMap { params.iter.map(|(k, v)| (k.to_string, v.to_string)).collect }

async fn delay(f: T) -> ::Output { futures::future::join(f, tokio::time::delay_for(std::time::Duration::from_secs(2))).await.0 }

async fn get_site_urls<'a>(api: &'a Api, checked_already: &'a Vec) -> impl Stream + 'a { api.get_query_api_json_limit_iter(&make_map(&[ ("action", "sitematrix"), ("smlangprop", "site"), ("smstateprop", "url"), ("smlimit", "max"), ("formatversion", "2"), ]), /* limit */ None) .await .flat_map(move |response| {       let response = response.unwrap;        stream::iter( response["sitematrix"] .as_object .expect("[get_site_urls] not object") .values .flat_map(|language| {                   language["site"].as_array.into_iter.flatten.filter_map(|site| if site.is_null || site["closed"].is_null { site["url"].as_str.and_then(|url|                           if checked_already.binary_search(&url.to_string).is_err {                                Some(url.to_string)                            } else {                                None                            }) } else { None }                   )                })                .collect::> )   }) }

async fn get_interface_histories(api: &Api) -> Vec { delay(api.get_query_api_json_limit_iter(&make_map(&[       ("action", "query"),        ("list", "allrevisions"),        ("arvprop", "ids|timestamp|user|comment|contentmodel"),        ("arvlimit", "max"),        ("arvnamespace", "8"), // 8 = MediaWiki        ("arvdir", "newer"), // oldest first        ("arvstart", "2020-10-19T00:00:00Z"), // don't ask where I got this particular timestamp from        ("arvend", "now"),        ("formatversion", "2"),    ]), /* limit */ None)) .await .flat_map(|response| {       stream::iter(response.unwrap["query"]["allrevisions"].as_array.unwrap.clone)            .flat_map(|page| stream::iter(page["revisions"].as_array.unwrap.clone) .then(|revision| async move {                       let revid: u32 = revision["revid"].as_i64.unwrap.try_into.unwrap;                        let diff_data = &delay(api.get_query_api_json(&make_map(&[ ("action", "compare"), ("fromrev", &revid.to_string), ("torelative", "prev"), ("prop", "diff|title"), ("formatversion", "2"), ]))).await.unwrap["compare"];                       let rev = Rev {                            username: revision["user"].as_str.unwrap_or("no user").to_string,                            title: diff_data["totitle"].as_str.unwrap.to_string, // totitle has a namespace                            revid,                            timestamp: revision["timestamp"].as_str.unwrap.to_string,                            diff: diff_data["body"].as_str.unwrap.to_string,                            summary: revision["comment"].as_str.unwrap_or("no summary").to_string,                        };                        let filter = match revision["contentmodel"].as_str.unwrap {                            "javascript" => FilterDiff::NoFilter,                            _ => FilterDiff::UrlsOnly,                        };                        if let Some(html) = rev.to_html(filter) { println!("{}", html); }                       rev })       )    }).collect.await }

async fn main { let mut mediawiki_site_api = Api::new("https://www.mediawiki.org/w/api.php").await.expect("couldn't make API"); mediawiki_site_api.set_user_agent(&*USER_AGENT);
 * 1) [tokio::main]

// just create it, but empty let mut checked_already: Vec = std::fs::read_to_string("checked.txt").unwrap.split("\n").map(ToString::to_string).collect; checked_already.sort_unstable;

println!(r#"<!DOCTYPE html>  check-interface-edits   "#); get_site_urls(&mediawiki_site_api, &checked_already).await.for_each(|site_url| async move {       println!(" {} ", site_url);        let mut api = Api::new_from_builder(&format!("{}/w/api.php", site_url), reqwest::ClientBuilder::default.timeout(std::time::Duration::from_secs(2))).await.expect("couldn't make API");        api.set_user_agent(&*USER_AGENT);        get_interface_histories(&api).await;        eprintln!("{}", site_url);    }).await; println!(" "); }