X-Git-Url: https://git.rvb.name/pyrungps.git/blobdiff_plain/45de8e7785e41e798de2f30fcff3be900d027055..81020ed69398f00cdb0564a132dfdf0285e9b844:/pyrungps.py diff --git a/pyrungps.py b/pyrungps.py index db24972..0f0dc56 100644 --- a/pyrungps.py +++ b/pyrungps.py @@ -1,24 +1,32 @@ #!/usr/bin/env python # coding: UTF-8 -import urllib2 +import requests + #import sys import os +from urllib2 import quote,unquote from lxml import html,etree from optparse import OptionParser from datetime import date,datetime from mx.DateTime.ISO import ParseDateTimeUTC -from parsegpx import write_parsed_to_db +from parsegpx import write_parsed_to_db,check_db_for_training import pygpx +from tempfile import NamedTemporaryFile + +from pprint import pprint + import render_tiles +import generate_image + def get_page(uname,year,month): trainings = [] - req = urllib2.Request("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), None, {'User-agent': 'Mozilla/5.0'}) - page = urllib2.urlopen(req).read() + req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'}) + page = req.text.encode() dom = html.document_fromstring(page) for element, attribute, link, pos in dom.iterlinks(): @@ -26,7 +34,7 @@ def get_page(uname,year,month): if link.startswith("/trainings/"): dummy, dummy, link = link.split('/') name, id = link.split('_') - trainings.append([ urllib2.unquote(name), id ]) + trainings.append([ unquote(name), id ]) return trainings @@ -34,62 +42,167 @@ def get_gpx_track(trid,name): print "trid=",trid - req = urllib2.urlopen("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid)) - - xml = etree.parse(req) + req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid)) + xml = etree.fromstring(req.text.encode()) return xml def get_osm_list(username,password,year,month): url = "https://www.openstreetmap.org/api/0.6/user/gpx_files" - passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - passman.add_password(None, url, username, password) - authhandler = urllib2.HTTPBasicAuthHandler(passman) - opener = urllib2.build_opener(authhandler) - urllib2.install_opener(opener) - req = urllib2.urlopen(url) - xml = etree.parse(req) + req = requests.get(url,auth=(username,password)) + + xml = etree.fromstring(req.text.encode()) res=[] - for gpx_file in xml.getroot(): + for gpx_file in xml: attrib = dict(gpx_file.attrib) - timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp'])) - id = attrib['id'] - description=None - for descr in gpx_file.iter('description'): - description=descr.text - sport=None - for tag in gpx_file.iter('tag'): - sport=tag.text - if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track': - print id,description,sport,timestamp - record={ 'id': id, 'sport': sport } - res.append(record) + try: + timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp'])) + id = attrib['id'] + description=None + for descr in gpx_file.iter('description'): + description=descr.text + sport=None + for tag in gpx_file.iter('tag'): + sport=tag.text + if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track': + record={ 'id': id, 'sport': sport } + res.append(record) + except: + None - return res + return res def get_osm_gpx(username,password,track_id): url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data" - passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - passman.add_password(None, url, username, password) - authhandler = urllib2.HTTPBasicAuthHandler(passman) - opener = urllib2.build_opener(authhandler) - urllib2.install_opener(opener) - req = urllib2.urlopen(url) - xml = etree.parse(req) + req = requests.get(url,auth=(username,password)) + xml = etree.fromstring(req.text.encode()) return xml +def get_db_gpx(dbx,track_id): + + md, res = dbx.files_download(track_id) + + fs = NamedTemporaryFile(suffix='.tcx',delete = False) + tmp_tcx_name=fs.name + fs.write(res.content) + fs.close() + fs = NamedTemporaryFile(suffix='.gpx',delete = False) + tmp_gpx_name=fs.name + fs.close() + + os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name) + + with open (tmp_gpx_name, "r") as gpxfile: + data=gpxfile.read() + + os.remove(tmp_gpx_name) + os.remove(tmp_tcx_name) + + xml = etree.fromstring(data.encode()) + return xml + +def get_dbx_list(dbx,username,year,month): + + gpx_list_id = None + + for entry in dbx.files_list_folder('').entries: + if entry.name == u'Приложения': + for entry_app in dbx.files_list_folder(entry.id).entries: + if entry_app.name == u'Run.GPS': + gpx_list_id=entry_app.id + break + break + + res = [] + + if gpx_list_id: + + for file in dbx.files_list_folder(gpx_list_id).entries: + + filename,ext = os.path.splitext(file.name) + if ext == '.tcx': + try: + fyear = int(filename[0:4]) + fmonth = int(filename[5:7]) + if fyear == year and fmonth == month: + sport = filename[18:] + timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S') + record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport } + res.append(record) + except: + None + + return res + + +def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False): + + training_list = get_dbx_list(dbx,username,year,month) + + for training in training_list: + + filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:]) + dirname = "%s/%04d/%02d" % (dir,year,month) + + if os.path.exists(filename) and not force: + + if verbose: + print "training %s exists, skipping" % (filename) + + else: + + try: + os.makedirs(dirname) + except: + None + + xml = get_db_gpx(dbx,training['id']) + + if verbose: + print "writing training %s" % (filename) + + gpx = pygpx.GPX() + gpx.ReadTree(xml) + + sport = training['sport'] + timestamp = gpx.tracks[0].start_time() + + print sport, timestamp + + if check_db_for_training(db,sport,timestamp): + print "The same training found" + continue + + gpx.author = username + gpx.FixNames(training['sport']) + gpx.ProcessTrackSegs() + for track in gpx.tracks: + track.sport=training['sport'] + + xml = gpx.XMLTree(); + f = open(filename,"w") + f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) + f.close + write_parsed_to_db(db,gpx,filename) + try: + render_tiles.queue_render(db,filename) + except: + None + + def sync_osm(username,password,year,month,dir=".",verbose=False,force=False): training_list = get_osm_list(username,password,year,month) for training in training_list: + filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id']) dirname = "%s/%04d/%02d" % (dir,year,month) @@ -99,7 +212,7 @@ def sync_osm(username,password,year,month,dir=".",verbose=False,force=False): print "training %s exists, skipping" % (filename) else: - + try: os.makedirs(dirname) except: @@ -113,12 +226,19 @@ def sync_osm(username,password,year,month,dir=".",verbose=False,force=False): gpx = pygpx.GPX() gpx.ReadTree(xml) + sport = training['sport'] + timestamp = gpx.tracks[0].start_time() + + if check_db_for_training(db,sport,timestamp): + print "The same training found" + continue + gpx.author = username gpx.FixNames(training['sport']) gpx.ProcessTrackSegs() for track in gpx.tracks: track.sport=training['sport'] - + xml = gpx.XMLTree(); f = open(filename,"w") f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) @@ -190,19 +310,26 @@ def main(): help="OpenStreetMap username") parser.add_option("-p", "--osm-passwd", dest="osmpasswd", help="OpenStreetMap password") + parser.add_option("-b", "--dropbox", dest="dropboxauth", + help="DropBox Auth token") (options, args) = parser.parse_args() username = options.username osmname = options.osmname osmpwd = options.osmpasswd - if not (username or (osmname and osmpwd)): - print "Run.GPS username or OSM username/password is mandatory!" + dbauth = options.dropboxauth + if not (username or (osmname and osmpwd) or dbauth): + print "Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!" return if username: pusername = username+'@Run.GPS' elif osmname: pusername = osmname+'@OSM' + elif dbauth: + import dropbox + dbx = dropbox.Dropbox(dbauth) + pusername = dbx.users_get_current_account().email+'@DBX' try: if options.yearmonth: @@ -232,9 +359,11 @@ def main(): sync_folder(username,year,month,outdir,options.verbose,options.force) elif osmname: sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force) + elif dbauth: + sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force) else: if options.verbose: - print "retrieving two last months for user %s to %s" % (pusername,outdir) + print "retrieving two last months for user %s to %s" % (pusername ,outdir) now = date.today() current_year = now.year current_month = now.month @@ -242,6 +371,8 @@ def main(): sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force) elif osmname: sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force) + elif dbauth: + sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force) current_month = current_month -1 if current_month == 0: current_month = 12 @@ -250,6 +381,10 @@ def main(): sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force) elif osmname: sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force) + elif dbauth: + sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force) + + generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-transparent.xml',400,400) if __name__ == "__main__":