pyrungps/pyrungps_sync.py

   1 #!/usr/bin/env python
   2 # coding: UTF-8
   3
   4 import requests
   5
   6 import urllib3
   7 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
   8
   9 #import sys
  10 import os
  11 from urllib.parse import quote,unquote
  12 from lxml import html,etree
  13 from optparse import OptionParser
  14 from datetime import date,datetime
  15 from dateutil.parser import isoparse
  16 from pyrungps.parsegpx import write_parsed_to_db,check_db_for_training
  17 from pyrungps.pygpx import GPX
  18 from io import BytesIO
  19
  20 from tempfile import NamedTemporaryFile
  21
  22 from pprint import pprint
  23
  24 from pyrungps.render_tiles import queue_render
  25
  26 from pyrungps.generate_image import render_all
  27
  28 def get_page(uname,year,month):
  29
  30   trainings = []
  31
  32   req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'}, verify=False)
  33   page = req.text.encode('utf-8')
  34   dom = html.document_fromstring(page)
  35
  36   for element, attribute, link, pos in dom.iterlinks():
  37     if attribute == "href":
  38       if link.startswith("/trainings/"):
  39         dummy, dummy, link = link.split('/')
  40         name, *ids = link.split('_')
  41         id = ids[-1]
  42         trainings.append([ unquote(name), id ])
  43
  44   return trainings
  45
  46 def get_gpx_track(trid,name):
  47
  48   print("trid=",trid)
  49
  50   req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid), verify=False)
  51
  52   xml = etree.fromstring(req.text.encode('utf-8'))
  53
  54   return xml
  55
  56 def get_osm_list(username,password,year,month):
  57
  58   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
  59
  60   req = requests.get(url,auth=(username,password))
  61
  62   xml = etree.fromstring(req.text.encode('utf-8'))
  63
  64   res=[]
  65
  66   for gpx_file in xml:
  67     attrib = dict(gpx_file.attrib)
  68     try:
  69       timestamp=datetime.fromtimestamp(isoparse(attrib['timestamp']))
  70       id = attrib['id']
  71       description=None
  72       for descr in gpx_file.iter('description'):
  73         description=descr.text
  74       sport=None
  75       for tag in gpx_file.iter('tag'):
  76         sport=tag.text
  77       if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
  78         record={ 'id': id, 'sport': sport }
  79         res.append(record)
  80     except:
  81       None
  82
  83   return res
  84
  85 def get_osm_gpx(username,password,track_id):
  86
  87   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
  88
  89   req = requests.get(url,auth=(username,password))
  90   xml = etree.fromstring(req.text.encode('utf-8'))
  91
  92   return xml
  93
  94 def get_db_gpx(dbx,track_id):
  95
  96   md, res = dbx.files_download(track_id)
  97
  98   fs = NamedTemporaryFile(suffix='.tcx',delete = False)
  99   tmp_tcx_name=fs.name
 100   fs.write(res.content)
 101   fs.close()
 102   fs = NamedTemporaryFile(suffix='.gpx',delete = False)
 103   tmp_gpx_name=fs.name
 104   fs.close()
 105
 106   os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name)
 107
 108   with open (tmp_gpx_name, "r") as gpxfile:
 109     data=gpxfile.read()
 110
 111   os.remove(tmp_gpx_name)
 112   os.remove(tmp_tcx_name)
 113
 114   xml = etree.fromstring(data.encode('utf-8'))
 115   return xml
 116
 117 def get_dbx_list(dbx,username,year,month):
 118
 119     gpx_list_id = None
 120
 121     for entry in dbx.files_list_folder('').entries:
 122       if entry.name == u'Приложения':
 123         for entry_app in dbx.files_list_folder(entry.id).entries:
 124           if entry_app.name == u'Run.GPS':
 125             print("id="+entry_app.id)
 126             gpx_list_id=entry_app.id
 127             break
 128         break
 129
 130     res = []
 131
 132     if gpx_list_id:
 133
 134       for file in dbx.files_list_folder(gpx_list_id).entries:
 135
 136         filename,ext = os.path.splitext(file.name)
 137         if ext == '.tcx':
 138           try:
 139             fyear = int(filename[0:4])
 140             fmonth = int(filename[5:7])
 141             if fyear == year and fmonth == month:
 142               sport = filename[41:]
 143               timestamp = datetime.strptime(filename[0:19],'%Y-%m-%dT%H_%M_%S')
 144               record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
 145               res.append(record)
 146           except:
 147             try:
 148               fyear = int(filename[0:4])
 149               fmonth = int(filename[5:7])
 150               if fyear == year and fmonth == month:
 151                 sport = filename[18:]
 152                 timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S')
 153                 record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
 154                 res.append(record)
 155             except:
 156               None
 157
 158     return res
 159
 160
 161 def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False):
 162
 163     training_list = get_dbx_list(dbx,username,year,month)
 164
 165     for training in training_list:
 166
 167       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:])
 168       dirname = "%s/%04d/%02d" % (dir,year,month)
 169
 170       if os.path.exists(filename) and not force:
 171
 172         if verbose:
 173           print("training %s exists, skipping" % (filename))
 174
 175       else:
 176
 177         try:
 178           os.makedirs(dirname)
 179         except:
 180           None
 181
 182         xml = get_db_gpx(dbx,training['id'])
 183
 184         if verbose:
 185           print("writing training %s" % (filename))
 186
 187         gpx = GPX()
 188         gpx.ReadTree(xml)
 189
 190         sport = training['sport']
 191         timestamp = gpx.tracks[0].start_time()
 192
 193         print(sport, timestamp)
 194
 195         if check_db_for_training(db,sport,timestamp):
 196           print("The same training found")
 197           continue
 198
 199         gpx.author = username
 200         gpx.FixNames(training['sport'])
 201         gpx.ProcessTrackSegs()
 202         for track in gpx.tracks:
 203           track.sport=training['sport']
 204
 205         xml = gpx.XMLTree();
 206         f = open(filename,"wb")
 207         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 208         f.close
 209         write_parsed_to_db(db,gpx,filename)
 210         try:
 211           queue_render(db,filename)
 212         except:
 213           None
 214
 215
 216 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
 217
 218     training_list = get_osm_list(username,password,year,month)
 219
 220     for training in training_list:
 221
 222       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])
 223       dirname = "%s/%04d/%02d" % (dir,year,month)
 224
 225       if os.path.exists(filename) and not force:
 226
 227         if verbose:
 228           print("training %s exists, skipping" % (filename))
 229
 230       else:
 231
 232         try:
 233           os.makedirs(dirname)
 234         except:
 235           None
 236
 237         xml = get_osm_gpx(username,password,training['id'])
 238
 239         if verbose:
 240           print("writing training %s" % (filename))
 241
 242         gpx = GPX()
 243         gpx.ReadTree(xml)
 244
 245         sport = training['sport']
 246         timestamp = gpx.tracks[0].start_time()
 247
 248         if check_db_for_training(db,sport,timestamp):
 249           print("The same training found")
 250           continue
 251
 252         gpx.author = username
 253         gpx.FixNames(training['sport'])
 254         gpx.ProcessTrackSegs()
 255         for track in gpx.tracks:
 256           track.sport=training['sport']
 257
 258         xml = gpx.XMLTree();
 259         f = open(filename,"wb")
 260         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 261         f.close
 262         write_parsed_to_db(db,gpx,filename)
 263         try:
 264           queue_render(db,filename)
 265         except:
 266           None
 267
 268 def sync_folder(username,year,month,dir=".",verbose=False,force=False):
 269
 270     training_list = get_page(username,year,month)
 271     for tr in training_list:
 272      try:
 273
 274       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1])
 275       dirname = "%s/%04d/%02d" % (dir,year,(month+1))
 276
 277       if os.path.exists(filename) and not force:
 278
 279         if verbose:
 280           print("training %s exists, skipping" % (filename))
 281
 282       else:
 283
 284         try:
 285           os.makedirs(dirname)
 286         except:
 287           None
 288
 289         xml=get_gpx_track(tr[1],tr[0])
 290
 291         if verbose:
 292           print("writing training %s" % (filename))
 293
 294         gpx = GPX()
 295         gpx.ReadTree(xml)
 296
 297         sport = tr[0]
 298         timestamp = gpx.tracks[0].start_time()
 299
 300         if check_db_for_training(db,sport,timestamp):
 301           print("The same training found")
 302           continue
 303
 304         gpx.FixNames(tr[0])
 305         gpx.ProcessTrackSegs()
 306
 307         xml = gpx.XMLTree();
 308         f = open(filename,"wb")
 309         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 310         f.close
 311         write_parsed_to_db(db,gpx,filename)
 312         try:
 313           queue_render(db,filename)
 314         except:
 315           None
 316
 317      except:
 318        raise
 319
 320 def main():
 321
 322     global db;
 323     parser = OptionParser()
 324     parser.add_option("-d", "--dir", dest="dirname",
 325       help="write data to directory", metavar="DIR")
 326     parser.add_option("-q", "--quiet",
 327       action="store_false", dest="verbose", default=True,
 328       help="don't print status messages to stdout")
 329     parser.add_option("-f", "--force",
 330       action="store_true", dest="force", default=False,
 331       help="rewrite all files")
 332     parser.add_option("-y", "--yearmonth", dest="yearmonth",
 333       help="year and month in YYYY-MM format", metavar="YYYY-MM")
 334     parser.add_option("-u", "--username", dest="username",
 335       help="Run.GPS username")
 336     parser.add_option("-o", "--osm-username", dest="osmname",
 337       help="OpenStreetMap username")
 338     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
 339       help="OpenStreetMap password")
 340     parser.add_option("-b", "--dropbox", dest="dropboxauth",
 341       help="DropBox Auth token")
 342     (options, args) = parser.parse_args()
 343
 344     username = options.username
 345     osmname = options.osmname
 346     osmpwd = options.osmpasswd
 347     dbauth = options.dropboxauth
 348     if not (username or (osmname and osmpwd) or dbauth):
 349       print("Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!")
 350       return
 351
 352     if username:
 353       pusername = username+'@Run.GPS'
 354     elif osmname:
 355       pusername = osmname+'@OSM'
 356     elif dbauth:
 357       import dropbox
 358       dbx = dropbox.Dropbox(dbauth)
 359       pusername = dbx.users_get_current_account().email+'@DBX'
 360
 361     try:
 362       if options.yearmonth:
 363         year,month = options.yearmonth.split('-')
 364         month = int(month) -1
 365         year = int(year)
 366         if month<0 or month>11:
 367           raise invalid_number
 368       else:
 369         year = None
 370         month = None
 371     except:
 372       print("Year and month should be in YYYY-MM format!")
 373       return
 374
 375     if options.dirname:
 376       outdir = options.dirname
 377     else:
 378       outdir = '.'
 379
 380     db = outdir + '/gpx.db'
 381
 382     if year:
 383       if options.verbose:
 384         print("retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir))
 385       if username:
 386         sync_folder(username,year,month,outdir,options.verbose,options.force)
 387       elif osmname:
 388         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)
 389       elif dbauth:
 390         sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force)
 391     else:
 392       if options.verbose:
 393         print("retrieving two last months for user %s to %s" % (pusername ,outdir))
 394       now = date.today()
 395       current_year = now.year
 396       current_month = now.month
 397       if username:
 398         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
 399       elif osmname:
 400         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
 401       elif dbauth:
 402         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)
 403       current_month = current_month -1
 404       if current_month == 0:
 405         current_month = 12
 406         current_year = current_year -1
 407       if username:
 408         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
 409       elif osmname:
 410         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
 411       elif dbauth:
 412         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)
 413
 414     render_all(db,'/etc/mapnik-osm-carto-data/veloroad-imposm.xml',640,640)
 415
 416 if __name__ == "__main__":
 417
 418     main()
 419