pyrungps.py

   1 #!/usr/bin/env python
   2 # coding: UTF-8
   3
   4 import requests
   5
   6 #import sys
   7 import os
   8 from urllib2 import quote,unquote
   9 from lxml import html,etree
  10 from optparse import OptionParser
  11 from datetime import date,datetime
  12 from mx.DateTime.ISO import ParseDateTimeUTC
  13 from parsegpx import write_parsed_to_db,check_db_for_training
  14 import pygpx
  15
  16 from tempfile import NamedTemporaryFile
  17
  18 from pprint import pprint
  19
  20 import render_tiles
  21
  22 import generate_image
  23
  24 def get_page(uname,year,month):
  25
  26   trainings = []
  27
  28   req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'})
  29   print req.text
  30   page = req.text.encode('utf-8')
  31   dom = html.document_fromstring(page)
  32
  33   for element, attribute, link, pos in dom.iterlinks():
  34     if attribute == "href":
  35       if link.startswith("/trainings/"):
  36         dummy, dummy, link = link.split('/')
  37         name, id = link.split('_')
  38         trainings.append([ unquote(name), id ])
  39
  40   return trainings
  41
  42 def get_gpx_track(trid,name):
  43
  44   print "trid=",trid
  45
  46   req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid))
  47   xml = etree.fromstring(req.text.encode('utf-8'))
  48
  49   return xml
  50
  51 def get_osm_list(username,password,year,month):
  52
  53   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
  54
  55   req = requests.get(url,auth=(username,password))
  56
  57   xml = etree.fromstring(req.text.encode('utf-8'))
  58
  59   res=[]
  60
  61   for gpx_file in xml:
  62     attrib = dict(gpx_file.attrib)
  63     try:
  64       timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
  65       id = attrib['id']
  66       description=None
  67       for descr in gpx_file.iter('description'):
  68         description=descr.text
  69       sport=None
  70       for tag in gpx_file.iter('tag'):
  71         sport=tag.text
  72       if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
  73         record={ 'id': id, 'sport': sport }
  74         res.append(record)
  75     except:
  76       None
  77
  78   return res
  79
  80 def get_osm_gpx(username,password,track_id):
  81
  82   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
  83
  84   req = requests.get(url,auth=(username,password))
  85   xml = etree.fromstring(req.text.encode('utf-8'))
  86
  87   return xml
  88
  89 def get_db_gpx(dbx,track_id):
  90
  91   md, res = dbx.files_download(track_id)
  92
  93   fs = NamedTemporaryFile(suffix='.tcx',delete = False)
  94   tmp_tcx_name=fs.name
  95   fs.write(res.content)
  96   fs.close()
  97   fs = NamedTemporaryFile(suffix='.gpx',delete = False)
  98   tmp_gpx_name=fs.name
  99   fs.close()
 100
 101   os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name)
 102
 103   with open (tmp_gpx_name, "r") as gpxfile:
 104     data=gpxfile.read()
 105
 106   os.remove(tmp_gpx_name)
 107   os.remove(tmp_tcx_name)
 108
 109   xml = etree.fromstring(data.encode('utf-8'))
 110   return xml
 111
 112 def get_dbx_list(dbx,username,year,month):
 113
 114     gpx_list_id = None
 115
 116     for entry in dbx.files_list_folder('').entries:
 117       if entry.name == u'Приложения':
 118         for entry_app in dbx.files_list_folder(entry.id).entries:
 119           if entry_app.name == u'Run.GPS':
 120             gpx_list_id=entry_app.id
 121             break
 122         break
 123
 124     res = []
 125
 126     if gpx_list_id:
 127
 128       for file in dbx.files_list_folder(gpx_list_id).entries:
 129
 130         filename,ext = os.path.splitext(file.name)
 131         if ext == '.tcx':
 132           try:
 133             fyear = int(filename[0:4])
 134             fmonth = int(filename[5:7])
 135             if fyear == year and fmonth == month:
 136               sport = filename[18:]
 137               timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S')
 138               record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
 139               res.append(record)
 140           except:
 141             None
 142
 143     return res
 144
 145
 146 def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False):
 147
 148     training_list = get_dbx_list(dbx,username,year,month)
 149
 150     for training in training_list:
 151
 152       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:])
 153       dirname = "%s/%04d/%02d" % (dir,year,month)
 154
 155       if os.path.exists(filename) and not force:
 156
 157         if verbose:
 158           print "training %s exists, skipping" % (filename)
 159
 160       else:
 161
 162         try:
 163           os.makedirs(dirname)
 164         except:
 165           None
 166
 167         xml = get_db_gpx(dbx,training['id'])
 168
 169         if verbose:
 170           print "writing training %s" % (filename)
 171
 172         gpx = pygpx.GPX()
 173         gpx.ReadTree(xml)
 174
 175         sport = training['sport']
 176         timestamp = gpx.tracks[0].start_time()
 177
 178         print sport, timestamp
 179
 180         if check_db_for_training(db,sport,timestamp):
 181           print "The same training found"
 182           continue
 183
 184         gpx.author = username
 185         gpx.FixNames(training['sport'])
 186         gpx.ProcessTrackSegs()
 187         for track in gpx.tracks:
 188           track.sport=training['sport']
 189
 190         xml = gpx.XMLTree();
 191         f = open(filename,"w")
 192         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 193         f.close
 194         write_parsed_to_db(db,gpx,filename)
 195         try:
 196           render_tiles.queue_render(db,filename)
 197         except:
 198           None
 199
 200
 201 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
 202
 203     training_list = get_osm_list(username,password,year,month)
 204
 205     for training in training_list:
 206
 207       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])
 208       dirname = "%s/%04d/%02d" % (dir,year,month)
 209
 210       if os.path.exists(filename) and not force:
 211
 212         if verbose:
 213           print "training %s exists, skipping" % (filename)
 214
 215       else:
 216
 217         try:
 218           os.makedirs(dirname)
 219         except:
 220           None
 221
 222         xml = get_osm_gpx(username,password,training['id'])
 223
 224         if verbose:
 225           print "writing training %s" % (filename)
 226
 227         gpx = pygpx.GPX()
 228         gpx.ReadTree(xml)
 229
 230         sport = training['sport']
 231         timestamp = gpx.tracks[0].start_time()
 232
 233         if check_db_for_training(db,sport,timestamp):
 234           print "The same training found"
 235           continue
 236
 237         gpx.author = username
 238         gpx.FixNames(training['sport'])
 239         gpx.ProcessTrackSegs()
 240         for track in gpx.tracks:
 241           track.sport=training['sport']
 242
 243         xml = gpx.XMLTree();
 244         f = open(filename,"w")
 245         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 246         f.close
 247         write_parsed_to_db(db,gpx,filename)
 248         try:
 249           render_tiles.queue_render(db,filename)
 250         except:
 251           None
 252
 253 def sync_folder(username,year,month,dir=".",verbose=False,force=False):
 254
 255     training_list = get_page(username,year,month)
 256     for tr in training_list:
 257      try:
 258
 259       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1])
 260       dirname = "%s/%04d/%02d" % (dir,year,(month+1))
 261
 262       if os.path.exists(filename) and not force:
 263
 264         if verbose:
 265           print "training %s exists, skipping" % (filename)
 266
 267       else:
 268
 269         try:
 270           os.makedirs(dirname)
 271         except:
 272           None
 273
 274         xml=get_gpx_track(tr[1],tr[0])
 275
 276         if verbose:
 277           print "writing training %s" % (filename)
 278
 279         gpx = pygpx.GPX()
 280         gpx.ReadTree(xml)
 281
 282         sport = tr[0]
 283         timestamp = gpx.tracks[0].start_time()
 284
 285         if check_db_for_training(db,sport,timestamp):
 286           print "The same training found"
 287           continue
 288
 289         gpx.FixNames(tr[0])
 290         gpx.ProcessTrackSegs()
 291
 292         xml = gpx.XMLTree();
 293         f = open(filename,"w")
 294         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 295         f.close
 296         write_parsed_to_db(db,gpx,filename)
 297         try:
 298           render_tiles.queue_render(db,filename)
 299         except:
 300           None
 301
 302      except:
 303        raise
 304
 305 def main():
 306
 307     global db;
 308     parser = OptionParser()
 309     parser.add_option("-d", "--dir", dest="dirname",
 310       help="write data to directory", metavar="DIR")
 311     parser.add_option("-q", "--quiet",
 312       action="store_false", dest="verbose", default=True,
 313       help="don't print status messages to stdout")
 314     parser.add_option("-f", "--force",
 315       action="store_true", dest="force", default=False,
 316       help="rewrite all files")
 317     parser.add_option("-y", "--yearmonth", dest="yearmonth",
 318       help="year and month in YYYY-MM format", metavar="YYYY-MM")
 319     parser.add_option("-u", "--username", dest="username",
 320       help="Run.GPS username")
 321     parser.add_option("-o", "--osm-username", dest="osmname",
 322       help="OpenStreetMap username")
 323     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
 324       help="OpenStreetMap password")
 325     parser.add_option("-b", "--dropbox", dest="dropboxauth",
 326       help="DropBox Auth token")
 327     (options, args) = parser.parse_args()
 328
 329     username = options.username
 330     osmname = options.osmname
 331     osmpwd = options.osmpasswd
 332     dbauth = options.dropboxauth
 333     if not (username or (osmname and osmpwd) or dbauth):
 334       print "Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!"
 335       return
 336
 337     if username:
 338       pusername = username+'@Run.GPS'
 339     elif osmname:
 340       pusername = osmname+'@OSM'
 341     elif dbauth:
 342       import dropbox
 343       dbx = dropbox.Dropbox(dbauth)
 344       pusername = dbx.users_get_current_account().email+'@DBX'
 345
 346     try:
 347       if options.yearmonth:
 348         year,month = options.yearmonth.split('-')
 349         month = int(month) -1
 350         year = int(year)
 351         if month<0 or month>11:
 352           raise invalid_number
 353       else:
 354         year = None
 355         month = None
 356     except:
 357       print "Year and month should be in YYYY-MM format!"
 358       return
 359
 360     if options.dirname:
 361       outdir = options.dirname
 362     else:
 363       outdir = '.'
 364
 365     db = outdir + '/gpx.db'
 366
 367     if year:
 368       if options.verbose:
 369         print "retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir)
 370       if username:
 371         sync_folder(username,year,month,outdir,options.verbose,options.force)
 372       elif osmname:
 373         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)
 374       elif dbauth:
 375         sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force)
 376     else:
 377       if options.verbose:
 378         print "retrieving two last months for user %s to %s" % (pusername ,outdir)
 379       now = date.today()
 380       current_year = now.year
 381       current_month = now.month
 382       if username:
 383         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
 384       elif osmname:
 385         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
 386       elif dbauth:
 387         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)
 388       current_month = current_month -1
 389       if current_month == 0:
 390         current_month = 12
 391         current_year = current_year -1
 392       if username:
 393         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
 394       elif osmname:
 395         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
 396       elif dbauth:
 397         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)
 398
 399     generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-transparent.xml',400,400)
 400
 401 if __name__ == "__main__":
 402
 403     main()
 404