import requests
import json
import re
domain = 'https://en.wikipedia.beta.wmflabs.org'
current_spam_blacklist = requests.get(domain + '/wiki/MediaWiki:Spam-blacklist?action=raw').text
current_blocked_domains = requests.get(domain + '/wiki/MediaWiki:BlockedExternalDomains.json?action=raw')
query_res = requests.get(domain + '/w/api.php', params= { 'action': 'query', 'format': 'json', 'prop': 'revisions', 'titles': 'MediaWiki:Spam-blacklist', 'rvslots': 'main', 'rvprops': 'ids'}).json()['query']['pages']
spam_blacklist_rev_id = query_res[list(query_res.keys())[0]]['revisions'][0]['revid']
if current_blocked_domains.status_code == 404:
current_blocked_domains = []
else:
current_blocked_domains = current_blocked_domains.json()
new_spam_blacklist = []
for line in current_spam_blacklist.split('\n'):
if line.startswith('#'):
new_spam_blacklist.append(line)
domain_regex = line.strip()
if domain_regex.startswith('\\b') and domain_regex.endswith('\\b'):
domain_regex = domain_regex[2:-2]
else:
new_spam_blacklist.append(line)
continue
if '/' in domain_regex:
new_spam_blacklist.append(line)
continue
if re.findall(r'[^\\][\.\$\^\{\[\(\|\)\*\+\?]', domain_regex):
new_spam_blacklist.append(line)
continue
if re.findall(r'\\[^\.]', domain_regex):
new_spam_blacklist.append(line)
continue
current_blocked_domains.append(
{ 'domain': domain_regex.replace('\\', ''), 'notes': 'Moved from [[Special:PermaLink/{}|MediaWiki:Spam-blacklist]]'.format(spam_blacklist_rev_id)}
)
print('\n'.join(new_spam_blacklist))
print(json.dumps(current_blocked_domains, ensure_ascii=False, indent='\t'))