pyrungps.py

   1 #!/usr/bin/env python
   2 # coding: UTF-8
   3
   4 import urllib2
   5 #import sys
   6 import os
   7 from lxml import html,etree
   8 from optparse import OptionParser
   9 from datetime import date,datetime
  10 from mx.DateTime.ISO import ParseDateTimeUTC
  11 from parsegpx import write_parsed_to_db
  12 import pygpx
  13
  14 import render_tiles
  15
  16 import generate_image
  17
  18 def get_page(uname,year,month):
  19
  20   trainings = []
  21
  22   req = urllib2.Request("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), None, {'User-agent': 'Mozilla/5.0'})
  23   page = urllib2.urlopen(req).read()
  24   dom = html.document_fromstring(page)
  25
  26   for element, attribute, link, pos in dom.iterlinks():
  27     if attribute == "href":
  28       if link.startswith("/trainings/"):
  29         dummy, dummy, link = link.split('/')
  30         name, id = link.split('_')
  31         trainings.append([ urllib2.unquote(name), id ])
  32
  33   return trainings
  34
  35 def get_gpx_track(trid,name):
  36
  37   print "trid=",trid
  38
  39   req = urllib2.urlopen("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid))
  40
  41   xml = etree.parse(req)
  42
  43   return xml
  44
  45 def get_osm_list(username,password,year,month):
  46
  47   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
  48   passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
  49   passman.add_password(None, url, username, password)
  50   authhandler = urllib2.HTTPBasicAuthHandler(passman)
  51   opener = urllib2.build_opener(authhandler)
  52   urllib2.install_opener(opener)
  53
  54   req = urllib2.urlopen(url)
  55   xml = etree.parse(req)
  56
  57   res=[]
  58
  59   for gpx_file in xml.getroot():
  60     attrib = dict(gpx_file.attrib)
  61     timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
  62     id = attrib['id']
  63     description=None
  64     for descr in gpx_file.iter('description'):
  65       description=descr.text
  66     sport=None
  67     for tag in gpx_file.iter('tag'):
  68       sport=tag.text
  69     if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
  70       print id,description,sport,timestamp
  71       record={ 'id': id, 'sport': sport }
  72       res.append(record)
  73
  74     return res
  75
  76 def get_osm_gpx(username,password,track_id):
  77
  78   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
  79   passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
  80   passman.add_password(None, url, username, password)
  81   authhandler = urllib2.HTTPBasicAuthHandler(passman)
  82   opener = urllib2.build_opener(authhandler)
  83   urllib2.install_opener(opener)
  84
  85   req = urllib2.urlopen(url)
  86   xml = etree.parse(req)
  87
  88   return xml
  89
  90 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
  91
  92     training_list = get_osm_list(username,password,year,month)
  93
  94     for training in training_list:
  95       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])
  96       dirname = "%s/%04d/%02d" % (dir,year,month)
  97
  98       if os.path.exists(filename) and not force:
  99
 100         if verbose:
 101           print "training %s exists, skipping" % (filename)
 102
 103       else:
 104
 105         try:
 106           os.makedirs(dirname)
 107         except:
 108           None
 109
 110         xml = get_osm_gpx(username,password,training['id'])
 111
 112         if verbose:
 113           print "writing training %s" % (filename)
 114
 115         gpx = pygpx.GPX()
 116         gpx.ReadTree(xml)
 117
 118         gpx.author = username
 119         gpx.FixNames(training['sport'])
 120         gpx.ProcessTrackSegs()
 121         for track in gpx.tracks:
 122           track.sport=training['sport']
 123
 124         xml = gpx.XMLTree();
 125         f = open(filename,"w")
 126         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 127         f.close
 128         write_parsed_to_db(db,gpx,filename)
 129         try:
 130           render_tiles.queue_render(db,filename)
 131         except:
 132           None
 133
 134 def sync_folder(username,year,month,dir=".",verbose=False,force=False):
 135
 136     training_list = get_page(username,year,month)
 137     for tr in training_list:
 138
 139       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1])
 140       dirname = "%s/%04d/%02d" % (dir,year,(month+1))
 141
 142       if os.path.exists(filename) and not force:
 143
 144         if verbose:
 145           print "training %s exists, skipping" % (filename)
 146
 147       else:
 148
 149         try:
 150           os.makedirs(dirname)
 151         except:
 152           None
 153
 154         xml=get_gpx_track(tr[1],tr[0])
 155
 156         if verbose:
 157           print "writing training %s" % (filename)
 158
 159         gpx = pygpx.GPX()
 160         gpx.ReadTree(xml)
 161
 162         gpx.FixNames(tr[0])
 163         gpx.ProcessTrackSegs()
 164
 165         xml = gpx.XMLTree();
 166         f = open(filename,"w")
 167         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 168         f.close
 169         write_parsed_to_db(db,gpx,filename)
 170         try:
 171           render_tiles.queue_render(db,filename)
 172         except:
 173           None
 174
 175 def main():
 176
 177     global db;
 178     parser = OptionParser()
 179     parser.add_option("-d", "--dir", dest="dirname",
 180       help="write data to directory", metavar="DIR")
 181     parser.add_option("-q", "--quiet",
 182       action="store_false", dest="verbose", default=True,
 183       help="don't print status messages to stdout")
 184     parser.add_option("-f", "--force",
 185       action="store_true", dest="force", default=False,
 186       help="rewrite all files")
 187     parser.add_option("-y", "--yearmonth", dest="yearmonth",
 188       help="year and month in YYYY-MM format", metavar="YYYY-MM")
 189     parser.add_option("-u", "--username", dest="username",
 190       help="Run.GPS username")
 191     parser.add_option("-o", "--osm-username", dest="osmname",
 192       help="OpenStreetMap username")
 193     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
 194       help="OpenStreetMap password")
 195     (options, args) = parser.parse_args()
 196
 197     username = options.username
 198     osmname = options.osmname
 199     osmpwd = options.osmpasswd
 200     if not (username or (osmname and osmpwd)):
 201       print "Run.GPS username or OSM username/password is mandatory!"
 202       return
 203
 204     if username:
 205       pusername = username+'@Run.GPS'
 206     elif osmname:
 207       pusername = osmname+'@OSM'
 208
 209     try:
 210       if options.yearmonth:
 211         year,month = options.yearmonth.split('-')
 212         month = int(month) -1
 213         year = int(year)
 214         if month<0 or month>11:
 215           raise invalid_number
 216       else:
 217         year = None
 218         month = None
 219     except:
 220       print "Year and month should be in YYYY-MM format!"
 221       return
 222
 223     if options.dirname:
 224       outdir = options.dirname
 225     else:
 226       outdir = '.'
 227
 228     db = outdir + '/gpx.db'
 229
 230     if year:
 231       if options.verbose:
 232         print "retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir)
 233       if username:
 234         sync_folder(username,year,month,outdir,options.verbose,options.force)
 235       elif osmname:
 236         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)
 237     else:
 238       if options.verbose:
 239         print "retrieving two last months for user %s to %s" % (pusername,outdir)
 240       now = date.today()
 241       current_year = now.year
 242       current_month = now.month
 243       if username:
 244         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
 245       elif osmname:
 246         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
 247       current_month = current_month -1
 248       if current_month == 0:
 249         current_month = 12
 250         current_year = current_year -1
 251       if username:
 252         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
 253       elif osmname:
 254         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
 255
 256     generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-transparent.xml',400,400)
 257
 258 if __name__ == "__main__":
 259
 260     main()
 261