#!/usr/bin/python
import sys, os
print os.environ['HOME']
sys.path.append(os.environ['HOME'] + '/dschwen_bot/pywikipedia')
import wikipedia
import MySQLdb
import pyexiv2
import re
import math
import string
import unicodedata
import htmlentitydefs
import marshal
import urllib
from urllib import FancyURLopener
from PHPUnserialize import *
from datetime import timedelta
from datetime import datetime
# look at images of the last two days
dt = timedelta(3)
cut = datetime.now() - dt
class MyOpener(FancyURLopener):
version = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11'
myopener = MyOpener()
urllib.urlopen = MyOpener().open
urllib.urlretrieve = MyOpener().retrieve
def unescape_charref(ref) :
name = ref[2:-1]
base = 10
if name.startswith("x") :
name = name[1:]
base = 16
return unichr(int(name, base))
def replace_entities(match) :
ent = match.group()
if ent[1] == "#":
return unescape_charref(ent)
repl = htmlentitydefs.name2codepoint.get(ent[1:-1])
if repl is not None :
repl = unichr(repl)
else :
repl = ent
return repl
def unescape(data) :
return re.sub(r"&#?[A-Za-z0-9]+?;", replace_entities, data)
gpstrackusers = set( [ 'Ikiwaner' ] );
loc1RE = re.compile( '\{\{[Ll]ocation[ _]dec\|([^\|]+)\|([^\|]+)[\|\}]' )
loc2RE = re.compile( '\{\{[Ll]ocation[ _]dec\|([^\}\{]+)\}\}' )
loc3RE = re.compile( '\{\{[Ll]ocation\|([^\}\{]+)\}\}' )
nolocRE = re.compile( '\[\[[Cc]ategory:[Ll]ocation[ _]not[ _]applicable\]\]' )
gpsRE = re.compile( '\{\{(Template:|template:|10:|)[Gg]PS[_ ]EXIF\}\}(\n|)' )
#gpsRE = re.compile( '\{\{Template:[Gg]PS[ _]EXIF\}\}' )
latrefRE = re.compile( '^[NnSs]$' )
lonrefRE = re.compile( '^[EeOoWw]$' )
site = wikipedia.getSite()
try:
f = open( "badlist.gps", "rb" )
badlist = marshal.load( f )
f.close()
except:
badlist = {}
try:
f = open( "taglist.gps", "rb" )
taglist = marshal.load( f )
f.close()
except:
taglist = {}
try:
connection = MySQLdb.connect(host="commonswiki-p.db.ts.wikimedia.org", user="xxx", passwd="xxx", db="commonswiki_p" )
#connection = MySQLdb.connect(host="sql-s1.toolserver.org", user="xxx", passwd="xxx", db="commonswiki_p" )
cursor = connection.cursor()
cursor.execute( "create temporary table u_dschwen.dump (tl_from int, score int)" )
print "Looking for GPS EXIF data (images > %s)" % cut.strftime( "%Y%m%d%H%M%S" )
#cursor.execute( "insert into u_dschwen.dump SELECT page_id, 0 from image, page where img_name = page_title and img_name = 'Freaky_Age_4_Luc_Viatour.jpg'" )
cursor.execute( "insert into u_dschwen.dump SELECT page_id, 0 from image, page where img_timestamp > '%s' and img_name = page_title and page_namespace = 6 and ( img_metadata like '%%\"GPSAltitude\"%%' or img_metadata like '%%\"GPSLatitudeRef\"%%' )" % cut.strftime( "%Y%m%d%H%M%S" ) )
print "Looking for {{Location}}";
cursor.execute( "insert into u_dschwen.dump select tl_from, 1 from templatelinks where tl_namespace = 10 and tl_title = 'Location'" )
print "Looking for {{Location dec}}";
cursor.execute( "insert into u_dschwen.dump select tl_from, 1 from templatelinks where tl_namespace = 10 and tl_title = 'Location_dec'" )
print "Looking for {{GPS EXIF}}";
cursor.execute( "insert into u_dschwen.dump select tl_from, 0 from templatelinks where tl_namespace = 10 and tl_title = 'GPS_EXIF'" )
print "subtracting..."
cursor.execute( "select page_title, tl_from, SUM( score ) as s from u_dschwen.dump, page where page_id = tl_from group by tl_from having s = 0" )
print "fetching results..."
data = cursor.fetchall()
fields = cursor.description
cursor.close()
connection.close()
for row in range(len(data)):
name = data[row][0]
taglist[ name ] = True
file = open( "taglist.gps", "wb" )
marshal.dump( taglist, file )
file.close()
except MySQLdb.OperationalError, message:
errorMessage = "Error %d:\n%s" % (message[ 0 ], message[ 1 ] )
#
# get potential images from taglist
#
for name in taglist.keys() :
if taglist[ name ] and not ( name in badlist ):
#exif = PHPUnserialize().unserialize(data[row][1])
print name
decomposed_string = unicodedata.normalize( 'NFD', name.decode('utf-8') )
#page = wikipedia.Page(site, 'Image:' + decomposed_string.encode('utf-8') )
page = wikipedia.Page(site, 'Image:' + name.decode('utf-8') )
text = ""
if page.exists() :
text = page.get()
# remove {{GPS EXIF}}
oldtext = text
text = gpsRE.sub( '', text )
# Location not applicable
if nolocRE.search( text ) != None :
taglist[ name ] = False;
print "Location not applicable"
continue
# already contains a Location
if string.find(text, '{{Location' ) >= 0 :
# already contains a generated Location
if string.find(text, 'source:exif' ) >= 0 :
print "HMM, %s looks already processed" % name;
if oldtext != text :
wikipedia.setAction("removed gps exif request template")
page.put(text)
print "removed a superfluous gps exif request template"
taglist[ name ] = False;
continue
# extract location to compare to exif
lat_in_dec = 0
lon_in_dec = 0
#{{Location dec|47.5059|-122.0343|type:forest_region:US}}
for match in loc1RE.findall(text) :
lat_in_dec = float( match[0] )
lon_in_dec = float( match[1] )
# already contains a suggestion
if string.find(text, '<!-- EXIF_BOT' ) >= 0 :
print "HMM, looks already processed";
if oldtext != text :
wikipedia.setAction("removed gps exif request template")
page.put(text)
print "removed a superfluous gps exif request template"
taglist[ name ] = False;
continue
print "downloading http://commons.wikimedia.org/wiki/Special:Filepath/%s ..." % name
try:
urllib.urlretrieve( ( "http://commons.wikimedia.org/wiki/Special:Filepath/%s" % urllib.quote(name) ), ".temp.jpg" )
except Exception, e:
print "Exception while downloading:", e
continue
print "analyzing GPS EXIF data ...";
try :
image = pyexiv2.Image( ".temp.jpg" )
image.readMetadata()
#print image.exifKeys()
#print image['Exif.GPSInfo.GPSAltitudeRef']
temp = '{{Location'
lat_dec = 0.0;
for i in range(0, 3):
val = float(image['Exif.GPSInfo.GPSLatitude'][i].numerator) / float(image['Exif.GPSInfo.GPSLatitude'][i].denominator)
temp += ( "|%f" % ( val ) ).rstrip('0').rstrip('.')
lat_dec = lat_dec * 60.0 + val
ref = image['Exif.GPSInfo.GPSLatitudeRef']
if ref == 'S' :
lat_dec = -lat_dec
elif ref != 'N' :
print "Broken lattitude ref!"
temp = "<!-- GPS: Broken lattitude ref! --><br>" + temp
ref = 'N'
temp += '|' + ref
lon_dec = 0.0;
for i in range(0, 3):
val = float(image['Exif.GPSInfo.GPSLongitude'][i].numerator) / float(image['Exif.GPSInfo.GPSLongitude'][i].denominator)
temp += ( "|%f" % ( val ) ).rstrip('0').rstrip('.')
lon_dec = lon_dec * 60.0 + val
ref = image['Exif.GPSInfo.GPSLongitudeRef']
if ref == 'W' :
lon_dec = -lon_dec
elif ref != 'E' :
print "Broken longitude ref!"
temp = "<!-- GPS: Broken longitude ref! --><br>" + temp
ref = 'E'
temp += '|' + ref
except :
print "Broken Tag"
taglist[ name ] = False;
badlist[ name ] = True;
continue
#
# Jump through several hoops to try to determine a heading and make [User:Ikiwaner] happy
#
heading = '?'
try:
val = float(image['Exif.GPSInfo.GPSImgDirection'].numerator) / float(image['Exif.GPSInfo.GPSImgDirection'].denominator)
heading = ( "%f" % ( val ) ).rstrip('0').rstrip('.')
print "Heading found :-)"
except:
print "No heading found :-("
try:
val = float(image['Exif.GPSInfo.GPSTrack'].numerator) / float(image['Exif.GPSInfo.GPSTrack'].denominator)
try:
connection = MySQLdb.connect(host="commonswiki-p.db.ts.wikimedia.org", user="dschwen", passwd="6iemq2c!--TfJFAI", db="commonswiki_p" )
cursor = connection.cursor()
print "lets see if the user is known..."
cursor.execute( "select img_user_text from image where img_name = '%s'" % name )
data = cursor.fetchall()
fields = cursor.description
cursor.close()
connection.close()
uploader = data[0][0]
print "Uploaded by: %s" % uploader
if uploader in gpstrackusers :
heading = ( "%f" % ( val ) ).rstrip('0').rstrip('.')
except:
print "Database error"
except:
print "No dir of movement found either :-("
if heading == 0 :
heading = '?'
#
# deal with missing altitude data
#
try:
alt = float(image['Exif.GPSInfo.GPSAltitude'].numerator) / float(image['Exif.GPSInfo.GPSAltitude'].denominator)
try:
if int( image['Exif.GPSInfo.GPSAltitudeRef'] ) == 1 :
alt = -alt
except:
print "no AltitudeRef, assuming above sea level!"
temp += ( "|alt:%f" % alt ).rstrip('0').rstrip('.') + "_"
except:
print "No altitude data"
temp += "|"
temp += ( 'source:exif_heading:%s}}' % ( heading ) )
print temp
if lon_dec == 0.0 and lat_dec == 0.0 :
print "apparently INVALID GPS data!"
taglist[ name ] = False;
badlist[ name ] = True;
continue
lat_dec /= 3600.0
lon_dec /= 3600.0
#print "Old: %f,%f" % ( lat_in_dec, lon_in_dec )
#print "New: %f,%f" % ( lat_dec, lon_dec )
if string.find(text, '{{Location' ) < 0 :
print "YAY! tagging..."
wikipedia.setAction("creating {{Location}} from EXIF data, please visit [[Commons:Geocoding]] for further information")
infopos = string.find(text, '{{Information' )
if infopos < 0 :
text2 = temp + "\n" + text
else :
last = ''
infopos += 2
curl = 1
squr = 0
print infopos
while infopos < len(text) :
c = text[infopos]
if c == '[' and last == '[' :
squr += 1
#print "[ %d" % squr
last = ''
if c == ']' and last == ']' :
squr -= 1
#print "] %d" % squr
last = ''
if c == '{' and last == '{' :
curl += 1
#print "{ %d" % curl
last = ''
if c == '}' and last == '}' :
curl -= 1
#print "} %d" % curl
last = ''
last = c
infopos += 1
if curl == 0 and squr <= 0 :
break
text2 = text[:infopos] + "\n" + temp + text[infopos:]
page.put(text2)
taglist[ name ] = False;
else :
if string.find(text, '<!-- EXIF_BOT' ) < 0 and string.find(text, 'source:exif' ) < 0 :
if math.fabs( lat_in_dec - lat_dec ) < 0.0001 and math.fabs( lon_in_dec - lon_dec ) < 0.0001 :
print "OK, existing geocoding seems reasonably accurate"
taglist[ name ] = False;
continue
print "OK, just inserting hidden suggestion"
wikipedia.setAction("adding suggested {{Location}} from EXIF data")
temp = ( "%f|%f check EWNS!\n" % ( lat_dec, lon_dec ) ) + temp;
text = '<!-- EXIF_BOT suggests: ' + temp + " -->\n" + text.replace( '{{GPS EXIF}}', '' )
page.put(text)
taglist[ name ] = False;
else :
print "HMM, looks already processed";
if oldtext != text :
wikipedia.setAction("removed gps exif request template")
page.put(text)
print "removed a superfluous gps exif request template"
taglist[ name ] = False;
file = open( "badlist.gps", "wb" )
marshal.dump( badlist, file )
file.close()
file = open( "taglist.gps", "wb" )
marshal.dump( taglist, file )
file.close()
file = open( "badlist.gps", "wb" )
marshal.dump( badlist, file )
file.close()
file = open( "taglist.gps", "wb" )
marshal.dump( taglist, file )
file.close()
