File:littleimagegif.py
#! /usr/bin/env python
from PIL import Image
from PIL import ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True
from PIL import ImageSequence
import pyexiv2
import cStringIO
import mwclient
import uuid
import urllib
import cgi
import sys
import urllib2
import requests
import math
import tempfile
import os
import re
import io
#routine to autoswitch some of the output - as filenames in, say, filep.unprefixedtitle have accented chars!
def pnt(s):
try:
print(s)
except UnicodeEncodeError:
print(s.encode('utf-8'))
# CC-BY-SA Theopolisme
def gimme_image(filename,compound_site,pxl,theimage):
"""This function creates the new image, runs
metadata(), and passes along the new image's
filename.
"""
end = None
SVGAction=0
site = mwclient.Site(compound_site)
extension = os.path.splitext(theimage)[1]
extension_caps = extension[1:].upper()
temppath = "c:\\python27\\bot\\tempsvg\\"
#ONLY work with SVG
if extension_caps <> "SVG":
results = "SKIP"
return results
#Get the image
image_1 = site.Images[theimage]
image_2 = str(image_1.imageinfo['url'])
response = requests.get(image_2)
contents = response.text
temp_file1 = temppath+str(uuid.uuid4()) + extension
temp_file2 = filename + extension
print temp_file1
print temp_file2
#Save the SVG locally as temp_file1
f = io.open(temp_file1,'w', encoding='utf-8')
f.write(contents)
f.close
#Find the "<svg........>" section
tagstart=contents.find('<svg',0)
tagend=contents.find('>',tagstart)
print tagstart, tagend
#Copy the section into SVGtagRaw
SVGtag=contents[tagstart:tagend+1]
pnt(SVGtag)
#Put unworkable checks here
SVGbad1=SVGtag.find("wide")
print SVGbad1
if SVGbad1>0:
print "Bad code wide found"
results="MANUAL"
return results
SVGbad2=SVGtag.find("%")
print SVGbad2
if SVGbad2>0:
print "Bad code percent found"
results="MANUAL"
return results
#Initialise deciml W and H
DecW=0
DecH=0
#Have we a height and width?
SVGheight=0
SVGhEnd=0
SVGwidth=0
SVGwEnd=0
SVGheight=SVGtag.find('height="',1)
if SVGheight>0:
ImgScale=1
SVGhEnd=SVGtag.find('"',SVGheight+8)
print SVGheight,SVGhEnd
imgHeight=SVGtag[SVGheight+8:SVGhEnd]
print imgHeight
dims=re.sub("[^a-zA-Z]","", imgHeight)
print dims #px|mm|pt|pc|cm|in
if dims=='pt':
ImgScale=1.25
if dims=='mm':
ImgScale=3.5433075
if dims=='in':
ImgScale=90
if dims=='cm':
ImgScale=35.433075
if dims=='pc':
ImgScale=15
DecH=ImgScale*float(re.sub("[^0123456789\.]","",imgHeight))
print DecH, ImgScale
#Have we a width?
SVGwidth=SVGtag.find('width="',1)
if SVGwidth>0:
SVGAction=1 # W+H found
SVGwEnd=SVGtag.find('"',SVGwidth+7)
print SVGwidth,SVGwEnd
imgWidth=SVGtag[SVGwidth+7:SVGwEnd]
print imgWidth
DecW=ImgScale*float(re.sub("[^0123456789\.]","",imgWidth))
print DecW
#Have we a viewBox?
SVGvb=SVGtag.find('viewBox="',1)
if SVGvb>0:
SVGAction=SVGAction+2
SVGvbEnd=SVGtag.find('"',SVGvb+9)
print SVGvb,SVGvbEnd
imgVB1=SVGtag[SVGvb:SVGvbEnd+1]
print imgVB1
print "~~~~~~~~~~~~~~~~~~~~~~~"
imgVB=re.sub(","," ",imgVB1) # some SVG files use comma delimiter
SVGvb1=imgVB.find(' ',1)
SVGvb2=imgVB.find(' ',SVGvb1+1)
SVGvb3=imgVB.find(' ',SVGvb2+1)
print SVGvb1, SVGvb2, SVGvb3
SVGval1=float(re.sub("[^0123456789\.]","",imgVB[0:SVGvb1]))
SVGval2=float(re.sub("[^0123456789\.]","",imgVB[SVGvb1+1:SVGvb2]))
SVGval3=float(re.sub("[^0123456789\.]","",imgVB[SVGvb2+1:SVGvb3]))
SVGval4=float(re.sub("[^0123456789\.]","",imgVB[SVGvb3+1:end]))
print SVGval1, SVGval2, SVGval3, SVGval4
if DecW==0:
DecW=SVGval3
DecH=SVGval4
print DecW, DecH
#Selection of action to perform
print "ACTION", SVGAction
if SVGAction==0:
print "Action is Zero - ERROR"
results = "ERROR"
return results
#Calcualte new size
basewidth = float(math.sqrt((pxl * float(DecW))/(DecH)))
print "basewidth",basewidth
wpercent = float(basewidth/float(DecW))
print "wpercent",wpercent
hsize = float(float(DecH)*wpercent)
print "hsize",hsize
original_pixel = DecW * DecH
print "original_pixel",original_pixel
modified_pixel = basewidth * hsize
print "modified_pixel",modified_pixel
#Get Original and New width and height
OrgW=SVGtag[SVGwidth:SVGwEnd+1]
print OrgW
NewW='width="'+str(basewidth)+'"'
print NewW
OrgH=SVGtag[SVGheight:SVGhEnd+1]
print OrgH
NewH='height="'+str(hsize)+'"'
print NewH
#Options are (1) W+H, (2) VB, (3) Both
if SVGAction==1:
#Only W+H, needs a VB to be added, and adjusted W and H
NewH=NewH+' viewBox="0 0 '+str(DecW)+' '+str(DecH)+'"'
print NewH
SVGtag1=SVGtag.replace(OrgW, NewW)
SVGtag2=SVGtag1.replace(OrgH, NewH)
pnt(SVGtag2)
contents2=contents.replace(SVGtag,SVGtag2)
#print contents2
if SVGAction==2:
#Only VB, leave VB and add W+H
imgVB2=imgVB+' '+NewW+' '+NewH
SVGtag2=SVGtag.replace(imgVB,imgVB2)
pnt(SVGtag2)
contents2=contents.replace(SVGtag,SVGtag2)
#print contents2
if SVGAction==3:
#Only Both, replace W+H
if wpercent>0.95: #Less than 5% reduction, and we skip
results = "PIXEL"
return results
SVGtag1=SVGtag.replace(OrgW, NewW)
SVGtag2=SVGtag1.replace(OrgH, NewH)
pnt(SVGtag2)
contents2=contents.replace(SVGtag,SVGtag2)
#print contents2
#Save to temp_file2 locally
f2=io.open(temp_file2,'w', encoding='utf-8')
f2.write(contents2)
f2.close
print filename + extension
print "Image saved to disk at " + filename + extension
#pass local file name to calling
results = filename + extension
filelist = [ f for f in os.listdir("c:\\python27\\bot\\") if f.startswith(temp_file1) ]
for fa in filelist: os.remove(fa)
return results
#!/usr/bin/python
from PIL import Image
from xml.dom import minidom
import cStringIO
import mwclient
import uuid
import urllib
import os.path
import cgi
import littleimagesvg
import sys
import urllib2
import re
import time
import random
import logging
import userpassbot
import requests.packages.urllib3
requests.packages.urllib3.disable_warnings()
import mwparserfromhell
#routine to autoswitch some of the output - as filenames in, say, filep.unprefixedtitle have accented chars!
def pnt(s):
try:
print(s)
except UnicodeEncodeError:
print(s.encode('utf-8'))
# CC-BY-SA Theopolisme
def startAllowed():
checkname="User:RonBot/4/Run"
page = site.Pages[checkname]
textpage = page.text()
if textpage == "Run":
return "yes"
else:
return "no"
def allow_bots(pagetext, username):
user = username.lower().strip()
text = mwparserfromhell.parse(pagetext)
for tl in text.filter_templates():
if tl.name in ('bots', 'nobots'):
break
else:
return True
for param in tl.params:
bots = [x.lower().strip() for x in param.value.split(",")]
if param.name == 'allow':
if ''.join(bots) == 'none': return False
for bot in bots:
if bot in (user, 'all'):
return True
elif param.name == 'deny':
if ''.join(bots) == 'none': return True
for bot in bots:
if bot in (user, 'all'):
return False
return True
def are_you_still_there(theimage):
""" This function makes sure that
a given image is still tagged with
{{non-free reduce}}.
"""
img_name = "File:" + theimage
page = site.Pages[img_name]
text = page.text()
r1 = re.compile(r'\{\{[Nn]on.?free-?\s*[Rr]educe.*?\}\}')
r2 = re.compile(r'\{\{[Rr]educe.*?\}\}')
r3 = re.compile(r'\{\{[Cc]omic-ovrsize-img.*?\}\}')
r4 = re.compile(r'\{\{[Ff]air.?[Uu]se.?[Rr]educe.*?\}\}')
r5 = re.compile(r'\{\{[Ii]mage-toobig.*?\}\}')
r6 = re.compile(r'\{\{[Nn]fr.*?\}\}')
r7 = re.compile(r'\{\{[Ss]maller image.*?\}\}')
if r1.search(text) is not None:
return True
elif r2.search(text) is not None:
return True
elif r3.search(text) is not None:
return True
elif r4.search(text) is not None:
return True
elif r5.search(text) is not None:
return True
elif r6.search(text) is not None:
return True
elif r7.search(text) is not None:
return True
else:
return False
def image_routine(images):
""" This function does most of the work:
* First, checks the checkpage using sokay()
* Then makes sure the image file still exists using are_you_still_there()
* Next it actually resizes the image.
* As long as the resize works, we reupload the file.
* Then we update the page with {{non-free reduced}}.
* And repeat!
"""
temppath = "c:\\python27\\bot\\tempsvg\\"
svgdone = 0
for theimage in images:
print "."
print "."
print "Working on " + theimage.encode('ascii', 'ignore')
go = startAllowed()
if go == "yes":
if are_you_still_there(theimage) == True:
#Stop if there's nobots
img_name = "File:" + theimage
page = site.Pages[img_name]
text = page.text()
allow_bots(text, "RonBot")
print "main.allowbots"
desired_megapixel = float(0.1)
print "desired_megapixel", desired_megapixel
pxl = desired_megapixel * 1000000
print "pxl", pxl
compound_site = 'en.wikipedia.org'
filename = temppath+str(uuid.uuid4())
file = littleimagesvg.gimme_image(filename,compound_site,pxl,theimage)
print "####################"
pnt(file)
print "####################"
if file == "SKIP":
print "Skipping File."
if file == "ERROR":
print "ERROR File."
if file == "PIXEL":
print "Removing tag...already reduced..."
img_name = "File:" + theimage
page = site.Pages[img_name]
text = page.text()
text = re.sub(r'\{\{[Nn]on.?free-?\s*[Rr]educe.*?\}\}', '', text)
text = re.sub(r'\{\{[Rr]educe.*?\}\}', '', text)
text = re.sub(r'\{\{[Cc]omic-ovrsize-img.*?\}\}', '', text)
text = re.sub(r'\{\{[Ff]air.?[Uu]se.?[Rr]educe.*?\}\}', '', text)
text = re.sub(r'\{\{[Ii]mage-toobig.*?\}\}', '', text)
text = re.sub(r'\{\{[Nn]fr.*?\}\}', '', text)
text = re.sub(r'\{\{[Ss]maller image.*?\}\}', '', text)
page.save(text, bot=True, summary = "(Task 4) Removing {{[[Template:Non-free reduce|Non-free reduce]]}} since file is already adequately reduced") #comment fo no edits
if file == "MANUAL":
print "Changing tag...cannot reduced..."
img_name = "File:" + theimage
page = site.Pages[img_name]
text = page.text()
if "{{Non-free manual svg reduce}}" not in text:
text = re.sub(r'\{\{[Nn]on.?free-?\s*[Rr]educe.*?\}\}', '{{Non-free manual svg reduce}}', text)
text = re.sub(r'\{\{[Rr]educe.*?\}\}', '{{Non-free manual svg reduce}}', text)
text = re.sub(r'\{\{[Cc]omic-ovrsize-img.*?\}\}', '{{Non-free manual svg reduce}}', text)
text = re.sub(r'\{\{[Ff]air.?[Uu]se.?[Rr]educe.*?\}\}', '{{Non-free manual svg reduce}}', text)
text = re.sub(r'\{\{[Ii]mage-toobig.*?\}\}', '{{Non-free manual svg reduce}}', text)
text = re.sub(r'\{\{[Nn]fr.*?\}\}', '{{Non-free manual svg reduce}}', text)
text = re.sub(r'\{\{[Ss]maller image.*?\}\}', '{{Non-free manual svg reduce}}', text)
page.save(text, bot=True, summary = "(Task 4) Tagging with [[Template:Non-free manual svg reduce]] - bot cannot reduce")
elif file not in ("ERROR", "PIXEL", "SKIP", "MANUAL"):
Fcomment="Reduce nominal size to fit NFC guideline"
pnt(theimage)
pnt(Fcomment)
try:
site.upload(open(file,"rb"), filename=theimage, ignore=True, description=Fcomment, comment=Fcomment) #comment for no edits
svgdone += 1
print "Uploaded!", svgdone
filelist = [ f for f in os.listdir(".") if f.startswith(filename) ]
for fa in filelist: os.remove(fa)
img_name = "File:" + theimage
page = site.Pages[img_name]
text = page.text()
text = re.sub(r'\{\{[Nn]on.?free-?\s*[Rr]educe.*?\}\}', '{{subst:orfurrev}}', text)
text = re.sub(r'\{\{[Rr]educe.*?\}\}', '{{subst:orfurrev}}', text)
text = re.sub(r'\{\{[Cc]omic-ovrsize-img.*?\}\}', '{{subst:orfurrev}}', text)
text = re.sub(r'\{\{[Ff]air.?[Uu]se.?[Rr]educe.*?\}\}', '{{subst:orfurrev}}', text)
text = re.sub(r'\{\{[Ii]mage-toobig.*?\}\}', '{{subst:orfurrev}}', text)
text = re.sub(r'\{\{[Nn]fr.*?\}\}', '{{subst:orfurrev}}', text)
text = re.sub(r'\{\{[Ss]maller image.*?\}\}', '{{subst:orfurrev}}', text)
page.save(text, bot=True, summary = "(Task 4) Tagging with [[Template:orfurrev]]") #comment fo no edits
print "Tagged!", svgdone
if svgdone==500:
break
except:
print "Unknown error. Image skipped."
filelist = [ f for f in os.listdir(".") if f.startswith(filename) ]
for fa in filelist: os.remove(fa)
print "Changing tag...cannot reduced..."
img_name = "File:" + theimage
page = site.Pages[img_name]
text = page.text()
if "{{Non-free manual svg reduce}}" not in text:
text = re.sub(r'\{\{[Nn]on.?free-?\s*[Rr]educe.*?\}\}', '{{Non-free manual svg reduce}}', text)
text = re.sub(r'\{\{[Rr]educe.*?\}\}', '{{Non-free manual svg reduce}}', text)
text = re.sub(r'\{\{[Cc]omic-ovrsize-img.*?\}\}', '{{Non-free manual svg reduce}}', text)
text = re.sub(r'\{\{[Ff]air.?[Uu]se.?[Rr]educe.*?\}\}', '{{Non-free manual svg reduce}}', text)
text = re.sub(r'\{\{[Ii]mage-toobig.*?\}\}', '{{Non-free manual svg reduce}}', text)
text = re.sub(r'\{\{[Nn]fr.*?\}\}', '{{Non-free manual svg reduce}}', text)
text = re.sub(r'\{\{[Ss]maller image.*?\}\}', '{{Non-free manual svg reduce}}', text)
page.save(text, bot=True, summary = "(Task 4) Tagging with [[Template:Non-free manual svg reduce]] - bot cannot reduce")
else:
print "Image skipped."
filelist = [ f for f in os.listdir(".") if f.startswith(filename) ]
for fa in filelist: os.remove(fa)
else:
print "Gah, looks like someone removed the tag."
else:
print "Ah, darn - looks like the bot was disabled."
return
def main():
"""This defines and fills a global
variable for the site, and then calls
get_images() to assemble an initial
selection of images to work with. Then
it runs image_rountine() on this selection.
"""
global site
site = mwclient.Site('en.wikipedia.org')
site.login(userpassbot.username, userpassbot.password)
zam = mwclient.listing.Category(site, "Category:Wikipedia non-free svg file size reduction requests")
glob = zam.members()
flub = []
for image in glob:
zip = image.page_title
pnt(zip)
#rough and ready sort out of svg images - will save a lot of processing time - might let some wrong ones through, not an issue.
if ".svg" in zip.lower():
flub.append(zip)
image_routine(flub)
print "We're DONE!"
if __name__ == '__main__':
main()