pyrungps.py

   1 #!/usr/bin/env python
   2 # coding: UTF-8
   3
   4 import requests
   5
   6 #import sys
   7 import os
   8 from urllib2 import quote,unquote
   9 from lxml import html,etree
  10 from optparse import OptionParser
  11 from datetime import date,datetime
  12 from mx.DateTime.ISO import ParseDateTimeUTC
  13 from parsegpx import write_parsed_to_db,check_db_for_training
  14 import pygpx
  15
  16 from tempfile import NamedTemporaryFile
  17
  18 from pprint import pprint
  19
  20 import render_tiles
  21
  22 import generate_image
  23
  24 def get_page(uname,year,month):
  25
  26   trainings = []
  27
  28   req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'})
  29   page = req.text.encode('utf-8')
  30   dom = html.document_fromstring(page)
  31
  32   for element, attribute, link, pos in dom.iterlinks():
  33     if attribute == "href":
  34       if link.startswith("/trainings/"):
  35         dummy, dummy, link = link.split('/')
  36         name, id = link.split('_')
  37         trainings.append([ unquote(name), id ])
  38
  39   return trainings
  40
  41 def get_gpx_track(trid,name):
  42
  43   print "trid=",trid
  44
  45   req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid))
  46   xml = etree.fromstring(req.text.encode('utf-8'))
  47
  48   return xml
  49
  50 def get_osm_list(username,password,year,month):
  51
  52   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
  53
  54   req = requests.get(url,auth=(username,password))
  55
  56   xml = etree.fromstring(req.text.encode('utf-8'))
  57
  58   res=[]
  59
  60   for gpx_file in xml:
  61     attrib = dict(gpx_file.attrib)
  62     try:
  63       timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
  64       id = attrib['id']
  65       description=None
  66       for descr in gpx_file.iter('description'):
  67         description=descr.text
  68       sport=None
  69       for tag in gpx_file.iter('tag'):
  70         sport=tag.text
  71       if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
  72         record={ 'id': id, 'sport': sport }
  73         res.append(record)
  74     except:
  75       None
  76
  77   return res
  78
  79 def get_osm_gpx(username,password,track_id):
  80
  81   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
  82
  83   req = requests.get(url,auth=(username,password))
  84   xml = etree.fromstring(req.text.encode('utf-8'))
  85
  86   return xml
  87
  88 def get_db_gpx(dbx,track_id):
  89
  90   md, res = dbx.files_download(track_id)
  91
  92   fs = NamedTemporaryFile(suffix='.tcx',delete = False)
  93   tmp_tcx_name=fs.name
  94   fs.write(res.content)
  95   fs.close()
  96   fs = NamedTemporaryFile(suffix='.gpx',delete = False)
  97   tmp_gpx_name=fs.name
  98   fs.close()
  99
 100   os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name)
 101
 102   with open (tmp_gpx_name, "r") as gpxfile:
 103     data=gpxfile.read()
 104
 105   os.remove(tmp_gpx_name)
 106   os.remove(tmp_tcx_name)
 107
 108   xml = etree.fromstring(data.encode('utf-8'))
 109   return xml
 110
 111 def get_dbx_list(dbx,username,year,month):
 112
 113     gpx_list_id = None
 114
 115     for entry in dbx.files_list_folder('').entries:
 116       if entry.name == u'Приложения':
 117         for entry_app in dbx.files_list_folder(entry.id).entries:
 118           if entry_app.name == u'Run.GPS':
 119             gpx_list_id=entry_app.id
 120             break
 121         break
 122
 123     res = []
 124
 125     if gpx_list_id:
 126
 127       for file in dbx.files_list_folder(gpx_list_id).entries:
 128
 129         filename,ext = os.path.splitext(file.name)
 130         if ext == '.tcx':
 131           try:
 132             fyear = int(filename[0:4])
 133             fmonth = int(filename[5:7])
 134             if fyear == year and fmonth == month:
 135               sport = filename[18:]
 136               timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S')
 137               record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
 138               res.append(record)
 139           except:
 140             None
 141
 142     return res
 143
 144
 145 def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False):
 146
 147     training_list = get_dbx_list(dbx,username,year,month)
 148
 149     for training in training_list:
 150
 151       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:])
 152       dirname = "%s/%04d/%02d" % (dir,year,month)
 153
 154       if os.path.exists(filename) and not force:
 155
 156         if verbose:
 157           print "training %s exists, skipping" % (filename)
 158
 159       else:
 160
 161         try:
 162           os.makedirs(dirname)
 163         except:
 164           None
 165
 166         xml = get_db_gpx(dbx,training['id'])
 167
 168         if verbose:
 169           print "writing training %s" % (filename)
 170
 171         gpx = pygpx.GPX()
 172         gpx.ReadTree(xml)
 173
 174         sport = training['sport']
 175         timestamp = gpx.tracks[0].start_time()
 176
 177         print sport, timestamp
 178
 179         if check_db_for_training(db,sport,timestamp):
 180           print "The same training found"
 181           continue
 182
 183         gpx.author = username
 184         gpx.FixNames(training['sport'])
 185         gpx.ProcessTrackSegs()
 186         for track in gpx.tracks:
 187           track.sport=training['sport']
 188
 189         xml = gpx.XMLTree();
 190         f = open(filename,"w")
 191         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 192         f.close
 193         write_parsed_to_db(db,gpx,filename)
 194         try:
 195           render_tiles.queue_render(db,filename)
 196         except:
 197           None
 198
 199
 200 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
 201
 202     training_list = get_osm_list(username,password,year,month)
 203
 204     for training in training_list:
 205
 206       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])
 207       dirname = "%s/%04d/%02d" % (dir,year,month)
 208
 209       if os.path.exists(filename) and not force:
 210
 211         if verbose:
 212           print "training %s exists, skipping" % (filename)
 213
 214       else:
 215
 216         try:
 217           os.makedirs(dirname)
 218         except:
 219           None
 220
 221         xml = get_osm_gpx(username,password,training['id'])
 222
 223         if verbose:
 224           print "writing training %s" % (filename)
 225
 226         gpx = pygpx.GPX()
 227         gpx.ReadTree(xml)
 228
 229         sport = training['sport']
 230         timestamp = gpx.tracks[0].start_time()
 231
 232         if check_db_for_training(db,sport,timestamp):
 233           print "The same training found"
 234           continue
 235
 236         gpx.author = username
 237         gpx.FixNames(training['sport'])
 238         gpx.ProcessTrackSegs()
 239         for track in gpx.tracks:
 240           track.sport=training['sport']
 241
 242         xml = gpx.XMLTree();
 243         f = open(filename,"w")
 244         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 245         f.close
 246         write_parsed_to_db(db,gpx,filename)
 247         try:
 248           render_tiles.queue_render(db,filename)
 249         except:
 250           None
 251
 252 def sync_folder(username,year,month,dir=".",verbose=False,force=False):
 253
 254     training_list = get_page(username,year,month)
 255     for tr in training_list:
 256      try:
 257
 258       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1])
 259       dirname = "%s/%04d/%02d" % (dir,year,(month+1))
 260
 261       if os.path.exists(filename) and not force:
 262
 263         if verbose:
 264           print "training %s exists, skipping" % (filename)
 265
 266       else:
 267
 268         try:
 269           os.makedirs(dirname)
 270         except:
 271           None
 272
 273         xml=get_gpx_track(tr[1],tr[0])
 274
 275         if verbose:
 276           print "writing training %s" % (filename)
 277
 278         gpx = pygpx.GPX()
 279         gpx.ReadTree(xml)
 280
 281         sport = tr[0]
 282         timestamp = gpx.tracks[0].start_time()
 283
 284         if check_db_for_training(db,sport,timestamp):
 285           print "The same training found"
 286           continue
 287
 288         gpx.FixNames(tr[0])
 289         gpx.ProcessTrackSegs()
 290
 291         xml = gpx.XMLTree();
 292         f = open(filename,"w")
 293         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
 294         f.close
 295         write_parsed_to_db(db,gpx,filename)
 296         try:
 297           render_tiles.queue_render(db,filename)
 298         except:
 299           None
 300
 301      except:
 302        raise
 303
 304 def main():
 305
 306     global db;
 307     parser = OptionParser()
 308     parser.add_option("-d", "--dir", dest="dirname",
 309       help="write data to directory", metavar="DIR")
 310     parser.add_option("-q", "--quiet",
 311       action="store_false", dest="verbose", default=True,
 312       help="don't print status messages to stdout")
 313     parser.add_option("-f", "--force",
 314       action="store_true", dest="force", default=False,
 315       help="rewrite all files")
 316     parser.add_option("-y", "--yearmonth", dest="yearmonth",
 317       help="year and month in YYYY-MM format", metavar="YYYY-MM")
 318     parser.add_option("-u", "--username", dest="username",
 319       help="Run.GPS username")
 320     parser.add_option("-o", "--osm-username", dest="osmname",
 321       help="OpenStreetMap username")
 322     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
 323       help="OpenStreetMap password")
 324     parser.add_option("-b", "--dropbox", dest="dropboxauth",
 325       help="DropBox Auth token")
 326     (options, args) = parser.parse_args()
 327
 328     username = options.username
 329     osmname = options.osmname
 330     osmpwd = options.osmpasswd
 331     dbauth = options.dropboxauth
 332     if not (username or (osmname and osmpwd) or dbauth):
 333       print "Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!"
 334       return
 335
 336     if username:
 337       pusername = username+'@Run.GPS'
 338     elif osmname:
 339       pusername = osmname+'@OSM'
 340     elif dbauth:
 341       import dropbox
 342       dbx = dropbox.Dropbox(dbauth)
 343       pusername = dbx.users_get_current_account().email+'@DBX'
 344
 345     try:
 346       if options.yearmonth:
 347         year,month = options.yearmonth.split('-')
 348         month = int(month) -1
 349         year = int(year)
 350         if month<0 or month>11:
 351           raise invalid_number
 352       else:
 353         year = None
 354         month = None
 355     except:
 356       print "Year and month should be in YYYY-MM format!"
 357       return
 358
 359     if options.dirname:
 360       outdir = options.dirname
 361     else:
 362       outdir = '.'
 363
 364     db = outdir + '/gpx.db'
 365
 366     if year:
 367       if options.verbose:
 368         print "retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir)
 369       if username:
 370         sync_folder(username,year,month,outdir,options.verbose,options.force)
 371       elif osmname:
 372         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)
 373       elif dbauth:
 374         sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force)
 375     else:
 376       if options.verbose:
 377         print "retrieving two last months for user %s to %s" % (pusername ,outdir)
 378       now = date.today()
 379       current_year = now.year
 380       current_month = now.month
 381       if username:
 382         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
 383       elif osmname:
 384         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
 385       elif dbauth:
 386         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)
 387       current_month = current_month -1
 388       if current_month == 0:
 389         current_month = 12
 390         current_year = current_year -1
 391       if username:
 392         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
 393       elif osmname:
 394         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
 395       elif dbauth:
 396         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)
 397
 398     generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-transparent.xml',400,400)
 399
 400 if __name__ == "__main__":
 401
 402     main()
 403