RSconfig.py
mydate=0
validuser=False
archive=False
nameprefix=""
movetab=False
origpage=""
pagename=""
pagepage=""
pagetext=""
pagezise=0
destpagename=""
destpagepage=""
destpagetext=""
destpagesize=0
insert=0
changed=0
Main Program
from wikitools import *
import time
import urllib
import json
import userpassbot #Bot password
import warnings
import re
import mwparserfromhell
import datetime
import sys
import RSconfig
site = wiki.Wiki() #Tell Python to use the English Wikipedia's API
site.login(userpassbot.username, userpassbot.password) #login
#routine to autoswitch some of the output - as filenames have accented chars!
def pnt(s):
try:
print(s)
except UnicodeEncodeError:
print(s.encode('utf-8'))
def startAllowed():
textpage = page.Page(site, "User:RonBot/6/Run").getWikiText()
if textpage == "Run":
return "run"
else:
return "no"
def allow_bots(text, user):
user = user.lower().strip()
text = mwparserfromhell.parse(text)
for tl in text.filter_templates():
if tl.name.matches(['bots', 'nobots']):
break
else:
return True
print "template found" #Have we found one
for param in tl.params:
bots = [x.lower().strip() for x in param.value.split(",")]
if param.name == 'allow':
print "We have an ALLOW" # allow found
if ''.join(bots) == 'none': return False
for bot in bots:
if bot in (user, 'all'):
return True
elif param.name == 'deny':
print "We have a DENY" # deny found
if ''.join(bots) == 'none':
print "none - true"
return True
for bot in bots:
if bot in (user, 'all'):
pnt(bot)
pnt(user)
print "all - false"
return False
if (tl.name.matches('nobots') and len(tl.params) == 0):
print "match - false"
return False
return True
def StrToDate(datestr):
timestamp = datetime.datetime.strptime(datestr, '%Y-%m-%dT%H:%M:%SZ')
return timestamp
def YoungestDate(datein):
datein=StrToDate(datein)
if datein>RSconfig.mydate:
RSconfig.mydate=datein
RSconfig.validuser=True
return
def GetLastContrib(user):
params = {'action':'query',
'list':'usercontribs',
'uclimit':1,
'ucuser':user,
'ucdir':"older"
}
#print "GLC.params"
request = api.APIRequest(site, params) #Set the API request
#print "GLC.request"
result = request.query(False)
#print result, len(result)
if len(result)>2:
founddate=date=result['query']['usercontribs'][0]['timestamp']
print "Last Normal Edit", StrToDate(founddate)
YoungestDate(founddate)
else:
print "No normal contrib"
return
def GetLastDeleted(user):
params = {'action':'query',
'list':'alldeletedrevisions',
'adrprop':'timestamp',
'adrlimit':1,
'adruser':user,
'adrdir':"older"
}
#print "GLD.params"
request = api.APIRequest(site, params) #Set the API request
#print "GLD.request"
result = request.query(False)
#print result, len(result)
if len(result)>2:
founddate=result['query']['alldeletedrevisions'][0]['revisions'][0]['timestamp']
print "Last Deleted Edit", StrToDate(founddate)
YoungestDate(founddate)
else:
print "No deleted contrib"
return
def ExamineUser(user):
if len(user)>0:
GetLastContrib(user)
GetLastDeleted(user)
return
def FindUsers(text):
RSconfig.movetab=False
RSconfig.mydate=StrToDate("2000-01-01T00:00:00Z") #new lot of user, set mydate to way back.
RSconfig.validuser=False
usertext=re.sub(r'![\s]*?[Ss]cope="row"[\s\S]*?\|',"",text) #remove column 1
bracket=0 #now find end of column 2
for x in range(0,len(usertext)-1):
s=usertext[x]
found=1
if s=="|":
found=0
if bracket+found==0:
enduser=x
break
if s=="[":
bracket=bracket+1
if s=="]":
bracket=bracket-1
if s=="{":
bracket=bracket+1
if s=="}":
bracket=bracket-1
textcol2=usertext[0:enduser] # this is all column 2 - above code ignores "|" inside [|] or {|}
expirydate=datetime.datetime.utcnow()+datetime.timedelta(days=365)
for match in re.finditer(r'\[\[[Uu]ser:[\s\S]*?[\|\]]',textcol2): #[User:XXXX]
user=textcol2[match.start()+2:match.end()-1]
colon=user.index(":")
user=user[colon+1:]
pnt("UserA =" + user)
if "username" not in user.lower():
ExamineUser(user)
for match in re.finditer(r'\{\{[Uu]\|[\s\S]*?\}\}',textcol2): #{{U:XXXX}}
user="User:"+textcol2[match.start()+4:match.end()-2]
colon=user.index(":")
user=user[colon+1:]
pnt("UserB =" + user)
if "username" not in user.lower():
ExamineUser(user)
for match in re.finditer(r'\{\{[Nn]oping\|[\s\S]*?\}\}',textcol2):#{{noping|XXXX}}
user="User:"+textcol2[match.start()+9:match.end()-2]
colon=user.index(":")
user=user[colon+1:]
pnt("UserC =" + user)
if "username" not in user.lower():
ExamineUser(user)
if "dynamic" in textcol2.lower(): #Dynamic
print "DYNAMIC FLAG"
RSconfig.mydate=datetime.datetime.utcnow()
if RSconfig.validuser:
mylist=[]
textlen=len(usertext)-1
print"textlen",textlen
for x in range(textlen,0,-1):
if usertext[x]=="|":
mylist.append(x)
print mylist # mylist[1] to end is final column
textlastcol=usertext[mylist[1]:]
textlastcollow=textlastcol.lower()
if textlastcollow.find("indefinite")>=0:
print expirydate
else:
datefound=re.findall(r'20\d\d[\s-]\d\d[\s-]\d\d',textlastcol)
if len(datefound)>0:
expirydate=StrToDate(datefound[0]+"T00:00:00Z")
print"Last Group Edit", RSconfig.mydate
print"ExpiryDate", expirydate
if not RSconfig.archive:
if RSconfig.mydate < datetime.datetime.utcnow()-datetime.timedelta(days=730):
print "THIS ROW TO BE ARCHIVED - EDITS TWO YEARS OLD"
RSconfig.movetab=True
if expirydate < datetime.datetime.utcnow():
print "THIS ROW TO BE ARCHIVED - EXPIRED DATE"
RSconfig.movetab=True
else:
if expirydate > datetime.datetime.utcnow():
if RSconfig.mydate > datetime.datetime.utcnow()-datetime.timedelta(days=730):
print "THIS ROW TO BE UN ARCHIVED - NOT EXPIRED, and NEW EDITS"
RSconfig.movetab=True
return
def MoveToDest(startp,endp,text):
print "MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM"
print len(text)
print len(RSconfig.pagetext)
#tabline1=text[startp:endp]
tabline=RSconfig.pagetext[startp:endp]
print tabline
print"****************"
pagetext=RSconfig.pagetext[0:startp-1]+"\n\n"+RSconfig.pagetext[endp+1:]
RSconfig.pagetext=pagetext
print repr(tabline)
print "MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM"
pagetext=RSconfig.destpagetext[0:RSconfig.insert]+"\n\n"+tabline+"\n\n"+RSconfig.destpagetext[RSconfig.insert+1:]
RSconfig.destpagetext=pagetext
return
def CheckTab(start, end, text):
RSconfig.movetab=False
print start, end
tabline=text[start:end]
print repr(tabline)
print "++++++++++++++++++++++++++Start of Table Section+++++++++++++++++++++++++++++++++++++++"
print repr(tabline)
print "++++++++++++++++++++++++++End of Table Section+++++++++++++++++++++++++++++++++++++++"
FindUsers(tabline)
print "==========================End of Section Data======================================="
if RSconfig.movetab:
print "Eliminate", start, end
MoveToDest(start, end, text)
RSconfig.changed=+1
return
def ProcessPage(subpage):
RSconfig.changed=0
RSconfig.origpage=subpage
#set to get the two pages
if RSconfig.archive:
RSconfig.pagename = RSconfig.namprefix+"/Archive/"+subpage
RSconfig.destpagename = RSconfig.namprefix+"/"+RSconfig.origpage
editsum="(Task 6) Active Users - Remove from Archive to go to "+RSconfig.destpagename
editsumdest="(Task 6) Active Users - Returning from Archive ("+RSconfig.pagename+"}"
else:
RSconfig.pagename = RSconfig.namprefix+"/"+RSconfig.origpage
RSconfig.destpagename = RSconfig.namprefix+"/Archive/"+subpage
editsum="(Task 6) Inactive Users - Remove to archive ("+RSconfig.destpagename+")"
editsumdest="(Task 6) Inactive Users - Add to Archive from "+RSconfig.pagename
print RSconfig.pagename
print RSconfig.destpagename
RSconfig.pagepage = page.Page(site, RSconfig.pagename)
RSconfig.pagetext = RSconfig.pagepage.getWikiText()
RSconfig.pagesize=len(RSconfig.pagetext)
RSconfig.destpagepage = page.Page(site, RSconfig.destpagename)
RSconfig.destpagetext = RSconfig.destpagepage.getWikiText()
RSconfig.destpagesize=len(RSconfig.destpagetext)
print "Page Sizes from and to", RSconfig.pagesize, RSconfig.destpagesize
#Check pages for nobots - unlikly to be here
stop = allow_bots(RSconfig.pagetext, "RonBot")
if not stop:
return
stop = allow_bots(RSconfig.destpagetext, "RonBot")
if not stop:
return
#find the table start and the table end
tabtop=RSconfig.pagetext.find('{|')-1
tabbot=RSconfig.pagetext.rfind('|}')-1
desttabtop=RSconfig.destpagetext.find('{|')-1
desttabbot=RSconfig.destpagetext.rfind('|}')-1
#Fix the table end to a new row start and space out the table rows
RSconfig.pagetext=RSconfig.pagetext[0:tabbot]+'\n|-\n! scope="row"\nBOTEND'+RSconfig.pagetext[tabbot+1:]
RSconfig.destpagetext=RSconfig.destpagetext[0:desttabbot]+'\n|-\n! scope="row"\nBOTEND'+RSconfig.destpagetext[desttabbot+1:]
RSconfig.pagetext=re.sub(r'\|-\n!','|-\n\n!',RSconfig.pagetext)
RSconfig.destpagetext=re.sub(r'\|-\n!','|-\n\n!',RSconfig.destpagetext)
#Get the table row starts
destpositionlist=[]
for match in re.finditer(r'(!)[\s]*?[Ss]cope="row"',RSconfig.destpagetext):
end = match.start()-1
if end>desttabtop:
destpositionlist.append(end)
RSconfig.insert=destpositionlist[0]-1 # Insert point is everything before the first "!"
positionlist=[]
for match in re.finditer(r'(!)[\s]*?[Ss]cope="row"',RSconfig.pagetext):
end = match.start()-1
if end>tabtop:
positionlist.append(end)
#Final match is positionlist[len(positionlist)-1]
rows=len(positionlist)
end = positionlist[rows-1]#end of table marker (dummy row start put in earlier)
print end
for a in range(rows-2,0,-1): # MAIN Loop to check each tabline in reverse order
start=positionlist[a]
print start,RSconfig.pagetext[start] # all are "!"
CheckTab(start,end,RSconfig.pagetext)
end=start-1
RSconfig.lastrow=False
if RSconfig.changed>0:
#Need to restore the correct table end, and remove excess blank lines added
print "===================="
RSconfig.pagetext=re.sub(r'\|-\n*?! scope="row"\nBOTEND\|\}','|}',RSconfig.pagetext)
RSconfig.pagetext=re.sub(r'\|-\n*?\!','|-\n!',RSconfig.pagetext)
print RSconfig.pagetext
RSconfig.destpagetext=re.sub(r'\|-\n*?! scope="row"\nBOTEND\|\}','|}',RSconfig.destpagetext)
RSconfig.destpagetext=re.sub(r'\|-\n*?\!','|-\n!',RSconfig.destpagetext)
print RSconfig.destpagetext
print "TIME TO CHECK FROM PAGE"
#Get current page sizes - has someone edited while processing? - check with saved size on first load
pagepage = page.Page(site, RSconfig.pagename)
pagetext = pagepage.getWikiText()
pagesize=len(pagetext)
if pagesize<>RSconfig.pagesize:
print "Page Has been edited - aborting"
return
print "TIME TO CHECK DEST PAGE"
destpagepage = page.Page(site, RSconfig.destpagename)
destpagetext = destpagepage.getWikiText()
destpagesize=len(destpagetext)
if destpagesize<>RSconfig.destpagesize:
print "Page Has been edited abort"
return
RSconfig.pagepage.edit(text=RSconfig.pagetext, bot=True, summary=editsum) #(DO NOT UNCOMMENT UNTIL BOT IS APPROVED)
RSconfig.destpagepage.edit(text=RSconfig.destpagetext, bot=True, summary=editsumdest) #(DO NOT UNCOMMENT UNTIL BOT IS APPROVED)
print "===================="
else:
print "NO CHANGES TO PAGE NEEDED"
return
def main():
RSconfig.namprefix="Wikipedia:Editing restrictions"
go = startAllowed() #Check if task is enabled
if go == "no":
sys.exit('Disabled Task')
#parameters for API request
print "main is go"
#Send each subpage in turn
RSconfig.archive=False
print "####################################################################"
print "Process - Wikipedia:Editing restrictions/Placed by the Arbitration Committee"
ProcessPage('Placed by the Arbitration Committee')
print "####################################################################"
print "Wikipedia:Editing restrictions/Placed by the Wikipedia community"
ProcessPage('Placed by the Wikipedia community')
print "####################################################################"
print "Wikipedia:Editing restrictions/Voluntary"
ProcessPage('Voluntary')
print "####################################################################"
print "Wikipedia:Editing restrictions/Unblock conditions"
ProcessPage('Unblock conditions')
RSconfig.archive=True
print "####################################################################"
print "Process - Wikipedia:Editing restrictions/Archive/Placed by the Arbitration Committee"
ProcessPage('Placed by the Arbitration Committee')
print "####################################################################"
print "Wikipedia:Editing restrictions/Archive/Placed by the Wikipedia community"
ProcessPage('Placed by the Wikipedia community')
print "####################################################################"
print "Wikipedia:Editing restrictions/Archive/Voluntary"
ProcessPage('Voluntary')
print "####################################################################"
print "Wikipedia:Editing restrictions/Archive/Unblock conditions"
ProcessPage('Unblock conditions')
if __name__ == "__main__":
with warnings.catch_warnings():
warnings.simplefilter("ignore", FutureWarning)
main()