pyrungps/pyrungps_sync.py

   1 #!/usr/bin/env python
   2 # coding: UTF-8
   3
   4 import requests
   5
   6 import urllib3
   7 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
   8
   9 #import sys
  10 import os
  11 from urllib.parse import quote,unquote
  12 from lxml import html,etree
  13 from optparse import OptionParser
  14 from datetime import date,datetime
  15 from dateutil.parser import isoparse
  16 from pyrungps.parsegpx import write_parsed_to_db,check_db_for_training
  17 from pyrungps.pygpx import GPX
  18 from io import BytesIO
  19
  20 from tempfile import NamedTemporaryFile
  21
  22 from pprint import pprint
  23
  24 from pyrungps.render_tiles import queue_render
  25
  26 from pyrungps.generate_image import render_all
  27
  28 def get_page(uname,year,month):
  29
  30   trainings = []
  31
  32   req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'}, verify=False)
  33   page = req.text.encode('utf-8')
  34   dom = html.document_fromstring(page)
  35
  36   for element, attribute, link, pos in dom.iterlinks():
  37     if attribute == "href":
  38       if link.startswith("/trainings/"):
  39         dummy, dummy, link = link.split('/')
  40         name, *ids = link.split('_')
  41         id = ids[-1]
  42         trainings.append([ unquote(name), id ])
  43
  44   return trainings
  45
  46 def get_gpx_track(trid,name):
  47
  48   print("trid=",trid)
  49
  50   req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid), verify=False)
  51
  52   xml = etree.fromstring(req.text.encode('utf-8'))
  53
  54   return xml
  55
  56 def get_osm_list(username,password,year,month):
  57
  58   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
  59
  60   req = requests.get(url,auth=(username,password))
  61
  62   xml = etree.fromstring(req.text.encode('utf-8'))
  63
  64   res=[]
  65
  66   for gpx_file in xml:
  67     attrib = dict(gpx_file.attrib)
  68     try:
  69       timestamp=datetime.fromtimestamp(isoparse(attrib['timestamp']))
  70       id = attrib['id']
  71       description=None
  72       for descr in gpx_file.iter('description'):
  73         description=descr.text
  74       sport=None
  75       for tag in gpx_file.iter('tag'):
  76         sport=tag.text
  77       if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
  78         record={ 'id': id, 'sport': sport }
  79         res.append(record)
  80     except:
  81       None
  82
  83   return res
  84
  85 def get_osm_gpx(username,password,track_id):
  86
  87   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
  88
  89   req = requests.get(url,auth=(username,password))
  90   xml = etree.fromstring(req.text.encode('utf-8'))
  91
  92   return xml
  93
  94 def get_db_gpx(dbx,track_id):
  95
  96   md, res = dbx.files_download(track_id)
  97
  98   fs = NamedTemporaryFile(suffix='.tcx',delete = False)
  99   tmp_tcx_name=fs.name
 100   fs.write(res.content)
 101   fs.close()
 102   fs = NamedTemporaryFile(suffix='.gpx',delete = False)
 103   tmp_gpx_name=fs.name
 104   fs.close()
 105
 106   os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name)
 107
 108   with open (tmp_gpx_name, "r") as gpxfile:
 109     data=gpxfile.read()
 110
 111   os.remove(tmp_gpx_name)
 112   os.remove(tmp_tcx_name)
 113
 114   xml = etree.fromstring(data.encode('utf-8'))
 115   return xml
 116
 117 def get_dbx_list(dbx,username,year,month):
 118
 119     gpx_list_id = None
 120
 121     for entry in dbx.files_list_folder('').entries:
 122       if entry.name == u'Приложения':
 123         for entry_app in dbx.files_list_folder(entry.id).entries:
 124           if entry_app.name == u'Run.GPS':
 125             print("id="+entry_app.id)
 126             gpx_list_id=entry_app.id
 127             break
 128         break
 129
 130     res = []
 131
 132     if gpx_list_id:
 133
 134       for file in dbx.files_list_folder(gpx_list_id).entries:
 135
 136         filename,ext = os.path.splitext(file.name)
 137         if ext == '.tcx':
 138           try:
 139             fyear = int(filename[0:4])
 140             fmonth = int(filename[5:7])
 141             if fyear == year and fmonth == month:
 142               sport = filename[41:]
 143               timestamp = datetime.strptime(filename[0:19],'%Y-%m-%dT%H_%M_%S')
 144               record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
 145               res.append(record)
 146           except:
 147             try:
 148               fyear = int(filename[0:4])
 149               fmonth = int(filename[5:7])
 150               if fyear == year and fmonth == month:
 151                 sport = filename[18:]
 152                 timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S')
 153                 record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
 154                 res.append(record)
 155             except:
 156               None
 157
 158     return res
 159
 160
 161 def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False):
 162
 163     training_list = get_dbx_list(dbx,username,year,month)
 164
 165     for training in training_list:
 166
 167       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:])
 168       dirname = "%s/%04d/%02d" % (dir,year,month)
 169
 170       if os.path.exists(filename) and not force:
 171
 172         if verbose:
 173           print("training %s exists, skipping" % (filename))
 174
 175       else:
 176
 177         try:
 178           os.makedirs(dirname)
 179         except:
 180           None
 181
 182         xml = get_db_gpx(dbx,training['id'])
 183
 184         if verbose:
 185           print("writing training %s" % (filename))
 186
 187         gpx = GPX()
 188         gpx.ReadTree(xml)
 189
 190         sport = training['sport']
 191         timestamp = gpx.tracks[0].start_time()
 192
 193         if not(timestamp):
 194           print("Empty training found, skipping...")
 195           continue
 196
 197         print(sport, timestamp)
 198
 199         if check_db_for_training(db,sport,timestamp):
 200           print("The same training found")
 201           continue
 202
 203         gpx.author = username
 204         gpx.FixNames(training['sport'])
 205         gpx.ProcessTrackSegs()
 206         for track in gpx.tracks:
 207           track.sport=training['sport']
 208
 209         xml = gpx.XMLTree();
 210         f = open(filename,"wb")
 211         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 212         f.close
 213         write_parsed_to_db(db,gpx,filename)
 214         try:
 215           queue_render(db,filename)
 216         except:
 217           None
 218
 219
 220 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
 221
 222     training_list = get_osm_list(username,password,year,month)
 223
 224     for training in training_list:
 225
 226       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])
 227       dirname = "%s/%04d/%02d" % (dir,year,month)
 228
 229       if os.path.exists(filename) and not force:
 230
 231         if verbose:
 232           print("training %s exists, skipping" % (filename))
 233
 234       else:
 235
 236         try:
 237           os.makedirs(dirname)
 238         except:
 239           None
 240
 241         xml = get_osm_gpx(username,password,training['id'])
 242
 243         if verbose:
 244           print("writing training %s" % (filename))
 245
 246         gpx = GPX()
 247         gpx.ReadTree(xml)
 248
 249         sport = training['sport']
 250         timestamp = gpx.tracks[0].start_time()
 251
 252         if check_db_for_training(db,sport,timestamp):
 253           print("The same training found")
 254           continue
 255
 256         gpx.author = username
 257         gpx.FixNames(training['sport'])
 258         gpx.ProcessTrackSegs()
 259         for track in gpx.tracks:
 260           track.sport=training['sport']
 261
 262         xml = gpx.XMLTree();
 263         f = open(filename,"wb")
 264         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 265         f.close
 266         write_parsed_to_db(db,gpx,filename)
 267         try:
 268           queue_render(db,filename)
 269         except:
 270           None
 271
 272 def sync_folder(username,year,month,dir=".",verbose=False,force=False):
 273
 274     training_list = get_page(username,year,month)
 275     for tr in training_list:
 276      try:
 277
 278       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1])
 279       dirname = "%s/%04d/%02d" % (dir,year,(month+1))
 280
 281       if os.path.exists(filename) and not force:
 282
 283         if verbose:
 284           print("training %s exists, skipping" % (filename))
 285
 286       else:
 287
 288         try:
 289           os.makedirs(dirname)
 290         except:
 291           None
 292
 293         xml=get_gpx_track(tr[1],tr[0])
 294
 295         if verbose:
 296           print("writing training %s" % (filename))
 297
 298         gpx = GPX()
 299         gpx.ReadTree(xml)
 300
 301         sport = tr[0]
 302         timestamp = gpx.tracks[0].start_time()
 303
 304         if check_db_for_training(db,sport,timestamp):
 305           print("The same training found")
 306           continue
 307
 308         gpx.FixNames(tr[0])
 309         gpx.ProcessTrackSegs()
 310
 311         xml = gpx.XMLTree();
 312         f = open(filename,"wb")
 313         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 314         f.close
 315         write_parsed_to_db(db,gpx,filename)
 316         try:
 317           queue_render(db,filename)
 318         except:
 319           None
 320
 321      except:
 322        raise
 323
 324 def main():
 325
 326     global db;
 327     parser = OptionParser()
 328     parser.add_option("-d", "--dir", dest="dirname",
 329       help="write data to directory", metavar="DIR")
 330     parser.add_option("-q", "--quiet",
 331       action="store_false", dest="verbose", default=True,
 332       help="don't print status messages to stdout")
 333     parser.add_option("-f", "--force",
 334       action="store_true", dest="force", default=False,
 335       help="rewrite all files")
 336     parser.add_option("-y", "--yearmonth", dest="yearmonth",
 337       help="year and month in YYYY-MM format", metavar="YYYY-MM")
 338     parser.add_option("-u", "--username", dest="username",
 339       help="Run.GPS username")
 340     parser.add_option("-o", "--osm-username", dest="osmname",
 341       help="OpenStreetMap username")
 342     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
 343       help="OpenStreetMap password")
 344     parser.add_option("-b", "--dropbox", dest="dropboxauth",
 345       help="DropBox Auth token")
 346     (options, args) = parser.parse_args()
 347
 348     username = options.username
 349     osmname = options.osmname
 350     osmpwd = options.osmpasswd
 351     dbauth = options.dropboxauth
 352     if not (username or (osmname and osmpwd) or dbauth):
 353       print("Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!")
 354       return
 355
 356     if username:
 357       pusername = username+'@Run.GPS'
 358     elif osmname:
 359       pusername = osmname+'@OSM'
 360     elif dbauth:
 361       import dropbox
 362       dbx = dropbox.Dropbox(dbauth)
 363       pusername = dbx.users_get_current_account().email+'@DBX'
 364
 365     try:
 366       if options.yearmonth:
 367         year,month = options.yearmonth.split('-')
 368         month = int(month) -1
 369         year = int(year)
 370         if month<0 or month>11:
 371           raise invalid_number
 372       else:
 373         year = None
 374         month = None
 375     except:
 376       print("Year and month should be in YYYY-MM format!")
 377       return
 378
 379     if options.dirname:
 380       outdir = options.dirname
 381     else:
 382       outdir = '.'
 383
 384     db = outdir + '/gpx.db'
 385
 386     if year:
 387       if options.verbose:
 388         print("retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir))
 389       if username:
 390         sync_folder(username,year,month,outdir,options.verbose,options.force)
 391       elif osmname:
 392         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)
 393       elif dbauth:
 394         sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force)
 395     else:
 396       if options.verbose:
 397         print("retrieving two last months for user %s to %s" % (pusername ,outdir))
 398       now = date.today()
 399       current_year = now.year
 400       current_month = now.month
 401       if username:
 402         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
 403       elif osmname:
 404         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
 405       elif dbauth:
 406         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)
 407       current_month = current_month -1
 408       if current_month == 0:
 409         current_month = 12
 410         current_year = current_year -1
 411       if username:
 412         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
 413       elif osmname:
 414         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
 415       elif dbauth:
 416         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)
 417
 418     render_all(db,'/etc/mapnik-osm-carto-data/veloroad-imposm.xml',640,640)
 419
 420 if __name__ == "__main__":
 421
 422     main()
 423