pyrungps/pyrungps_sync.py

   1 #!/usr/bin/env python
   2 # coding: UTF-8
   3
   4 import requests
   5
   6 import urllib3
   7 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
   8
   9 #import sys
  10 import os
  11 from urllib.parse import quote,unquote
  12 from lxml import html,etree
  13 from optparse import OptionParser
  14 from datetime import date,datetime
  15 from dateutil.parser import isoparse
  16 from pyrungps.parsegpx import write_parsed_to_db,check_db_for_training
  17 import pyrungps.pygpx
  18
  19 from tempfile import NamedTemporaryFile
  20
  21 from pprint import pprint
  22
  23 import pyrungps.render_tiles
  24
  25 import pyrungps.generate_image
  26
  27 def get_page(uname,year,month):
  28
  29   trainings = []
  30
  31   req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'}, verify=False)
  32   page = req.text.encode('utf-8')
  33   dom = html.document_fromstring(page)
  34
  35   for element, attribute, link, pos in dom.iterlinks():
  36     if attribute == "href":
  37       if link.startswith("/trainings/"):
  38         dummy, dummy, link = link.split('/')
  39         name, id = link.split('_')
  40         trainings.append([ unquote(name), id ])
  41
  42   return trainings
  43
  44 def get_gpx_track(trid,name):
  45
  46   print("trid=",trid)
  47
  48   req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid), verify=False)
  49   xml = etree.fromstring(req.text.encode('utf-8'))
  50
  51   return xml
  52
  53 def get_osm_list(username,password,year,month):
  54
  55   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
  56
  57   req = requests.get(url,auth=(username,password))
  58
  59   xml = etree.fromstring(req.text.encode('utf-8'))
  60
  61   res=[]
  62
  63   for gpx_file in xml:
  64     attrib = dict(gpx_file.attrib)
  65     try:
  66       timestamp=datetime.fromtimestamp(isoparse(attrib['timestamp']))
  67       id = attrib['id']
  68       description=None
  69       for descr in gpx_file.iter('description'):
  70         description=descr.text
  71       sport=None
  72       for tag in gpx_file.iter('tag'):
  73         sport=tag.text
  74       if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
  75         record={ 'id': id, 'sport': sport }
  76         res.append(record)
  77     except:
  78       None
  79
  80   return res
  81
  82 def get_osm_gpx(username,password,track_id):
  83
  84   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
  85
  86   req = requests.get(url,auth=(username,password))
  87   xml = etree.fromstring(req.text.encode('utf-8'))
  88
  89   return xml
  90
  91 def get_db_gpx(dbx,track_id):
  92
  93   md, res = dbx.files_download(track_id)
  94
  95   fs = NamedTemporaryFile(suffix='.tcx',delete = False)
  96   tmp_tcx_name=fs.name
  97   fs.write(res.content)
  98   fs.close()
  99   fs = NamedTemporaryFile(suffix='.gpx',delete = False)
 100   tmp_gpx_name=fs.name
 101   fs.close()
 102
 103   os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name)
 104
 105   with open (tmp_gpx_name, "r") as gpxfile:
 106     data=gpxfile.read()
 107
 108   os.remove(tmp_gpx_name)
 109   os.remove(tmp_tcx_name)
 110
 111   xml = etree.fromstring(data.encode('utf-8'))
 112   return xml
 113
 114 def get_dbx_list(dbx,username,year,month):
 115
 116     gpx_list_id = None
 117
 118     for entry in dbx.files_list_folder('').entries:
 119       if entry.name == u'Приложения':
 120         for entry_app in dbx.files_list_folder(entry.id).entries:
 121           if entry_app.name == u'Run.GPS':
 122             print("id="+entry_app.id)
 123             gpx_list_id=entry_app.id
 124             break
 125         break
 126
 127     res = []
 128
 129     if gpx_list_id:
 130
 131       for file in dbx.files_list_folder(gpx_list_id).entries:
 132
 133         filename,ext = os.path.splitext(file.name)
 134         if ext == '.tcx':
 135           try:
 136             fyear = int(filename[0:4])
 137             fmonth = int(filename[5:7])
 138             if fyear == year and fmonth == month:
 139               sport = filename[41:]
 140               timestamp = datetime.strptime(filename[0:19],'%Y-%m-%dT%H_%M_%S')
 141               record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
 142               res.append(record)
 143           except:
 144             try:
 145               fyear = int(filename[0:4])
 146               fmonth = int(filename[5:7])
 147               if fyear == year and fmonth == month:
 148                 sport = filename[18:]
 149                 timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S')
 150                 record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
 151                 res.append(record)
 152             except:
 153               None
 154
 155     return res
 156
 157
 158 def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False):
 159
 160     training_list = get_dbx_list(dbx,username,year,month)
 161
 162     for training in training_list:
 163
 164       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:])
 165       dirname = "%s/%04d/%02d" % (dir,year,month)
 166
 167       if os.path.exists(filename) and not force:
 168
 169         if verbose:
 170           print("training %s exists, skipping" % (filename))
 171
 172       else:
 173
 174         try:
 175           os.makedirs(dirname)
 176         except:
 177           None
 178
 179         xml = get_db_gpx(dbx,training['id'])
 180
 181         if verbose:
 182           print("writing training %s" % (filename))
 183
 184         gpx = pygpx.GPX()
 185         gpx.ReadTree(xml)
 186
 187         sport = training['sport']
 188         timestamp = gpx.tracks[0].start_time()
 189
 190         print(sport, timestamp)
 191
 192         if check_db_for_training(db,sport,timestamp):
 193           print("The same training found")
 194           continue
 195
 196         gpx.author = username
 197         gpx.FixNames(training['sport'])
 198         gpx.ProcessTrackSegs()
 199         for track in gpx.tracks:
 200           track.sport=training['sport']
 201
 202         xml = gpx.XMLTree();
 203         f = open(filename,"wb")
 204         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 205         f.close
 206         write_parsed_to_db(db,gpx,filename)
 207         try:
 208           render_tiles.queue_render(db,filename)
 209         except:
 210           None
 211
 212
 213 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
 214
 215     training_list = get_osm_list(username,password,year,month)
 216
 217     for training in training_list:
 218
 219       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])
 220       dirname = "%s/%04d/%02d" % (dir,year,month)
 221
 222       if os.path.exists(filename) and not force:
 223
 224         if verbose:
 225           print("training %s exists, skipping" % (filename))
 226
 227       else:
 228
 229         try:
 230           os.makedirs(dirname)
 231         except:
 232           None
 233
 234         xml = get_osm_gpx(username,password,training['id'])
 235
 236         if verbose:
 237           print("writing training %s" % (filename))
 238
 239         gpx = pygpx.GPX()
 240         gpx.ReadTree(xml)
 241
 242         sport = training['sport']
 243         timestamp = gpx.tracks[0].start_time()
 244
 245         if check_db_for_training(db,sport,timestamp):
 246           print("The same training found")
 247           continue
 248
 249         gpx.author = username
 250         gpx.FixNames(training['sport'])
 251         gpx.ProcessTrackSegs()
 252         for track in gpx.tracks:
 253           track.sport=training['sport']
 254
 255         xml = gpx.XMLTree();
 256         f = open(filename,"wb")
 257         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 258         f.close
 259         write_parsed_to_db(db,gpx,filename)
 260         try:
 261           render_tiles.queue_render(db,filename)
 262         except:
 263           None
 264
 265 def sync_folder(username,year,month,dir=".",verbose=False,force=False):
 266
 267     training_list = get_page(username,year,month)
 268     for tr in training_list:
 269      try:
 270
 271       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1])
 272       dirname = "%s/%04d/%02d" % (dir,year,(month+1))
 273
 274       if os.path.exists(filename) and not force:
 275
 276         if verbose:
 277           print("training %s exists, skipping" % (filename))
 278
 279       else:
 280
 281         try:
 282           os.makedirs(dirname)
 283         except:
 284           None
 285
 286         xml=get_gpx_track(tr[1],tr[0])
 287
 288         if verbose:
 289           print("writing training %s" % (filename))
 290
 291         gpx = pygpx.GPX()
 292         gpx.ReadTree(xml)
 293
 294         sport = tr[0]
 295         timestamp = gpx.tracks[0].start_time()
 296
 297         if check_db_for_training(db,sport,timestamp):
 298           print("The same training found")
 299           continue
 300
 301         gpx.FixNames(tr[0])
 302         gpx.ProcessTrackSegs()
 303
 304         xml = gpx.XMLTree();
 305         f = open(filename,"wb")
 306         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 307         f.close
 308         write_parsed_to_db(db,gpx,filename)
 309         try:
 310           render_tiles.queue_render(db,filename)
 311         except:
 312           None
 313
 314      except:
 315        raise
 316
 317 def main():
 318
 319     global db;
 320     parser = OptionParser()
 321     parser.add_option("-d", "--dir", dest="dirname",
 322       help="write data to directory", metavar="DIR")
 323     parser.add_option("-q", "--quiet",
 324       action="store_false", dest="verbose", default=True,
 325       help="don't print status messages to stdout")
 326     parser.add_option("-f", "--force",
 327       action="store_true", dest="force", default=False,
 328       help="rewrite all files")
 329     parser.add_option("-y", "--yearmonth", dest="yearmonth",
 330       help="year and month in YYYY-MM format", metavar="YYYY-MM")
 331     parser.add_option("-u", "--username", dest="username",
 332       help="Run.GPS username")
 333     parser.add_option("-o", "--osm-username", dest="osmname",
 334       help="OpenStreetMap username")
 335     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
 336       help="OpenStreetMap password")
 337     parser.add_option("-b", "--dropbox", dest="dropboxauth",
 338       help="DropBox Auth token")
 339     (options, args) = parser.parse_args()
 340
 341     username = options.username
 342     osmname = options.osmname
 343     osmpwd = options.osmpasswd
 344     dbauth = options.dropboxauth
 345     if not (username or (osmname and osmpwd) or dbauth):
 346       print("Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!")
 347       return
 348
 349     if username:
 350       pusername = username+'@Run.GPS'
 351     elif osmname:
 352       pusername = osmname+'@OSM'
 353     elif dbauth:
 354       import dropbox
 355       dbx = dropbox.Dropbox(dbauth)
 356       pusername = dbx.users_get_current_account().email+'@DBX'
 357
 358     try:
 359       if options.yearmonth:
 360         year,month = options.yearmonth.split('-')
 361         month = int(month) -1
 362         year = int(year)
 363         if month<0 or month>11:
 364           raise invalid_number
 365       else:
 366         year = None
 367         month = None
 368     except:
 369       print("Year and month should be in YYYY-MM format!")
 370       return
 371
 372     if options.dirname:
 373       outdir = options.dirname
 374     else:
 375       outdir = '.'
 376
 377     db = outdir + '/gpx.db'
 378
 379     if year:
 380       if options.verbose:
 381         print("retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir))
 382       if username:
 383         sync_folder(username,year,month,outdir,options.verbose,options.force)
 384       elif osmname:
 385         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)
 386       elif dbauth:
 387         sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force)
 388     else:
 389       if options.verbose:
 390         print("retrieving two last months for user %s to %s" % (pusername ,outdir))
 391       now = date.today()
 392       current_year = now.year
 393       current_month = now.month
 394       if username:
 395         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
 396       elif osmname:
 397         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
 398       elif dbauth:
 399         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)
 400       current_month = current_month -1
 401       if current_month == 0:
 402         current_month = 12
 403         current_year = current_year -1
 404       if username:
 405         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
 406       elif osmname:
 407         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
 408       elif dbauth:
 409         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)
 410
 411     generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-imposm.xml',640,640)
 412
 413 if __name__ == "__main__":
 414
 415     main()
 416