X-Git-Url: https://git.rvb.name/pyrungps.git/blobdiff_plain/50682c1f7ad6f88896b241d11ab47b57c49364d2..c49b871ebc265b1a2030482ca68f89ed920884ff:/pyrungps.py diff --git a/pyrungps.py b/pyrungps.py index 76d52f1..bfff892 100644 --- a/pyrungps.py +++ b/pyrungps.py @@ -1,21 +1,32 @@ #!/usr/bin/env python # coding: UTF-8 -import urllib2 +import requests + #import sys import os +from urllib2 import quote,unquote from lxml import html,etree from optparse import OptionParser -from datetime import date -from parsegpx import write_parsed_to_db +from datetime import date,datetime +from mx.DateTime.ISO import ParseDateTimeUTC +from parsegpx import write_parsed_to_db,check_db_for_training import pygpx +from tempfile import NamedTemporaryFile + +from pprint import pprint + +import render_tiles + +import generate_image + def get_page(uname,year,month): trainings = [] - req = urllib2.Request("http://www.gps-sport.net/embedCalendar.jsp?userName=%s&year=%s&month=%s"% (uname,year,month), None, {'User-agent': 'Mozilla/5.0'}) - page = urllib2.urlopen(req).read() + req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'}) + page = req.text.encode('utf-8') dom = html.document_fromstring(page) for element, attribute, link, pos in dom.iterlinks(): @@ -23,24 +34,229 @@ def get_page(uname,year,month): if link.startswith("/trainings/"): dummy, dummy, link = link.split('/') name, id = link.split('_') - trainings.append([ urllib2.unquote(name), id ]) + trainings.append([ unquote(name), id ]) return trainings def get_gpx_track(trid,name): - req = urllib2.urlopen("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s" % (trid)) + print "trid=",trid + + req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid)) + xml = etree.fromstring(req.text.encode('utf-8')) + + return xml + +def get_osm_list(username,password,year,month): + + url = "https://www.openstreetmap.org/api/0.6/user/gpx_files" + + req = requests.get(url,auth=(username,password)) - xml = etree.parse(req) + xml = etree.fromstring(req.text.encode('utf-8')) + + res=[] + + for gpx_file in xml: + attrib = dict(gpx_file.attrib) + try: + timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp'])) + id = attrib['id'] + description=None + for descr in gpx_file.iter('description'): + description=descr.text + sport=None + for tag in gpx_file.iter('tag'): + sport=tag.text + if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track': + record={ 'id': id, 'sport': sport } + res.append(record) + except: + None + + return res + +def get_osm_gpx(username,password,track_id): + + url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data" + + req = requests.get(url,auth=(username,password)) + xml = etree.fromstring(req.text.encode('utf-8')) return xml +def get_db_gpx(dbx,track_id): + + md, res = dbx.files_download(track_id) + + fs = NamedTemporaryFile(suffix='.tcx',delete = False) + tmp_tcx_name=fs.name + fs.write(res.content) + fs.close() + fs = NamedTemporaryFile(suffix='.gpx',delete = False) + tmp_gpx_name=fs.name + fs.close() + + os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name) + + with open (tmp_gpx_name, "r") as gpxfile: + data=gpxfile.read() + + os.remove(tmp_gpx_name) + os.remove(tmp_tcx_name) + + xml = etree.fromstring(data.encode('utf-8')) + return xml + +def get_dbx_list(dbx,username,year,month): + + gpx_list_id = None + + for entry in dbx.files_list_folder('').entries: + if entry.name == u'Приложения': + for entry_app in dbx.files_list_folder(entry.id).entries: + if entry_app.name == u'Run.GPS': + gpx_list_id=entry_app.id + break + break + + res = [] + + if gpx_list_id: + + for file in dbx.files_list_folder(gpx_list_id).entries: + + filename,ext = os.path.splitext(file.name) + if ext == '.tcx': + try: + fyear = int(filename[0:4]) + fmonth = int(filename[5:7]) + if fyear == year and fmonth == month: + sport = filename[18:] + timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S') + record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport } + res.append(record) + except: + None + + return res + + +def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False): + + training_list = get_dbx_list(dbx,username,year,month) + + for training in training_list: + + filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:]) + dirname = "%s/%04d/%02d" % (dir,year,month) + + if os.path.exists(filename) and not force: + + if verbose: + print "training %s exists, skipping" % (filename) + + else: + + try: + os.makedirs(dirname) + except: + None + + xml = get_db_gpx(dbx,training['id']) + + if verbose: + print "writing training %s" % (filename) + + gpx = pygpx.GPX() + gpx.ReadTree(xml) + + sport = training['sport'] + timestamp = gpx.tracks[0].start_time() + + print sport, timestamp + + if check_db_for_training(db,sport,timestamp): + print "The same training found" + continue + + gpx.author = username + gpx.FixNames(training['sport']) + gpx.ProcessTrackSegs() + for track in gpx.tracks: + track.sport=training['sport'] + + xml = gpx.XMLTree(); + f = open(filename,"w") + f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) + f.close + write_parsed_to_db(db,gpx,filename) + try: + render_tiles.queue_render(db,filename) + except: + None + + +def sync_osm(username,password,year,month,dir=".",verbose=False,force=False): + + training_list = get_osm_list(username,password,year,month) + + for training in training_list: + + filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id']) + dirname = "%s/%04d/%02d" % (dir,year,month) + + if os.path.exists(filename) and not force: + + if verbose: + print "training %s exists, skipping" % (filename) + + else: + + try: + os.makedirs(dirname) + except: + None + + xml = get_osm_gpx(username,password,training['id']) + + if verbose: + print "writing training %s" % (filename) + + gpx = pygpx.GPX() + gpx.ReadTree(xml) + + sport = training['sport'] + timestamp = gpx.tracks[0].start_time() + + if check_db_for_training(db,sport,timestamp): + print "The same training found" + continue + + gpx.author = username + gpx.FixNames(training['sport']) + gpx.ProcessTrackSegs() + for track in gpx.tracks: + track.sport=training['sport'] + + xml = gpx.XMLTree(); + f = open(filename,"w") + f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) + f.close + write_parsed_to_db(db,gpx,filename) + try: + render_tiles.queue_render(db,filename) + except: + None + def sync_folder(username,year,month,dir=".",verbose=False,force=False): training_list = get_page(username,year,month) for tr in training_list: + try: - filename = "%s/%s_%s.gpx" % (dir,tr[0],tr[1]) + filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1]) + dirname = "%s/%04d/%02d" % (dir,year,(month+1)) if os.path.exists(filename) and not force: @@ -48,7 +264,12 @@ def sync_folder(username,year,month,dir=".",verbose=False,force=False): print "training %s exists, skipping" % (filename) else: - + + try: + os.makedirs(dirname) + except: + None + xml=get_gpx_track(tr[1],tr[0]) if verbose: @@ -57,6 +278,13 @@ def sync_folder(username,year,month,dir=".",verbose=False,force=False): gpx = pygpx.GPX() gpx.ReadTree(xml) + sport = tr[0] + timestamp = gpx.tracks[0].start_time() + + if check_db_for_training(db,sport,timestamp): + print "The same training found" + continue + gpx.FixNames(tr[0]) gpx.ProcessTrackSegs() @@ -65,6 +293,13 @@ def sync_folder(username,year,month,dir=".",verbose=False,force=False): f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) f.close write_parsed_to_db(db,gpx,filename) + try: + render_tiles.queue_render(db,filename) + except: + None + + except: + raise def main(): @@ -82,13 +317,31 @@ def main(): help="year and month in YYYY-MM format", metavar="YYYY-MM") parser.add_option("-u", "--username", dest="username", help="Run.GPS username") + parser.add_option("-o", "--osm-username", dest="osmname", + help="OpenStreetMap username") + parser.add_option("-p", "--osm-passwd", dest="osmpasswd", + help="OpenStreetMap password") + parser.add_option("-b", "--dropbox", dest="dropboxauth", + help="DropBox Auth token") (options, args) = parser.parse_args() username = options.username - if not username: - print "Run.GPS username is mandatory!" + osmname = options.osmname + osmpwd = options.osmpasswd + dbauth = options.dropboxauth + if not (username or (osmname and osmpwd) or dbauth): + print "Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!" return + if username: + pusername = username+'@Run.GPS' + elif osmname: + pusername = osmname+'@OSM' + elif dbauth: + import dropbox + dbx = dropbox.Dropbox(dbauth) + pusername = dbx.users_get_current_account().email+'@DBX' + try: if options.yearmonth: year,month = options.yearmonth.split('-') @@ -112,20 +365,37 @@ def main(): if year: if options.verbose: - print "retrieving trainings for user %s, year %s, month %s to %s" % (username,year,month+1,outdir) - sync_folder(username,year,month,outdir,options.verbose,options.force) + print "retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir) + if username: + sync_folder(username,year,month,outdir,options.verbose,options.force) + elif osmname: + sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force) + elif dbauth: + sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force) else: if options.verbose: - print "retrieving two last months for user %s to %s" % (username,outdir) + print "retrieving two last months for user %s to %s" % (pusername ,outdir) now = date.today() current_year = now.year current_month = now.month - sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force) + if username: + sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force) + elif osmname: + sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force) + elif dbauth: + sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force) current_month = current_month -1 if current_month == 0: current_month = 12 current_year = current_year -1 - sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force) + if username: + sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force) + elif osmname: + sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force) + elif dbauth: + sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force) + + generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-transparent.xml',400,400) if __name__ == "__main__":