X-Git-Url: https://git.rvb.name/pyrungps.git/blobdiff_plain/1690b8b7c8ccfdf60be2b33b314e6c26bdef86e6..94f62add438cdf546fdae207ede72c05daf66c00:/pyrungps.py diff --git a/pyrungps.py b/pyrungps.py deleted file mode 100644 index ed3fce5..0000000 --- a/pyrungps.py +++ /dev/null @@ -1,416 +0,0 @@ -#!/usr/bin/env python -# coding: UTF-8 - -import requests - -from urllib3.exceptions import InsecureRequestWarning -requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) - -#import sys -import os -from urllib2 import quote,unquote -from lxml import html,etree -from optparse import OptionParser -from datetime import date,datetime -from mx.DateTime.ISO import ParseDateTimeUTC -from parsegpx import write_parsed_to_db,check_db_for_training -import pygpx - -from tempfile import NamedTemporaryFile - -from pprint import pprint - -import render_tiles - -import generate_image - -def get_page(uname,year,month): - - trainings = [] - - req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'}, verify=False) - page = req.text.encode('utf-8') - dom = html.document_fromstring(page) - - for element, attribute, link, pos in dom.iterlinks(): - if attribute == "href": - if link.startswith("/trainings/"): - dummy, dummy, link = link.split('/') - name, id = link.split('_') - trainings.append([ unquote(name), id ]) - - return trainings - -def get_gpx_track(trid,name): - - print "trid=",trid - - req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid), verify=False) - xml = etree.fromstring(req.text.encode('utf-8')) - - return xml - -def get_osm_list(username,password,year,month): - - url = "https://www.openstreetmap.org/api/0.6/user/gpx_files" - - req = requests.get(url,auth=(username,password)) - - xml = etree.fromstring(req.text.encode('utf-8')) - - res=[] - - for gpx_file in xml: - attrib = dict(gpx_file.attrib) - try: - timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp'])) - id = attrib['id'] - description=None - for descr in gpx_file.iter('description'): - description=descr.text - sport=None - for tag in gpx_file.iter('tag'): - sport=tag.text - if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track': - record={ 'id': id, 'sport': sport } - res.append(record) - except: - None - - return res - -def get_osm_gpx(username,password,track_id): - - url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data" - - req = requests.get(url,auth=(username,password)) - xml = etree.fromstring(req.text.encode('utf-8')) - - return xml - -def get_db_gpx(dbx,track_id): - - md, res = dbx.files_download(track_id) - - fs = NamedTemporaryFile(suffix='.tcx',delete = False) - tmp_tcx_name=fs.name - fs.write(res.content) - fs.close() - fs = NamedTemporaryFile(suffix='.gpx',delete = False) - tmp_gpx_name=fs.name - fs.close() - - os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name) - - with open (tmp_gpx_name, "r") as gpxfile: - data=gpxfile.read() - - os.remove(tmp_gpx_name) - os.remove(tmp_tcx_name) - - xml = etree.fromstring(data.encode('utf-8')) - return xml - -def get_dbx_list(dbx,username,year,month): - - gpx_list_id = None - - for entry in dbx.files_list_folder('').entries: - if entry.name == u'Приложения': - for entry_app in dbx.files_list_folder(entry.id).entries: - if entry_app.name == u'Run.GPS': - print "id="+entry_app.id - gpx_list_id=entry_app.id - break - break - - res = [] - - if gpx_list_id: - - for file in dbx.files_list_folder(gpx_list_id).entries: - - filename,ext = os.path.splitext(file.name) - if ext == '.tcx': - try: - fyear = int(filename[0:4]) - fmonth = int(filename[5:7]) - if fyear == year and fmonth == month: - sport = filename[41:] - timestamp = datetime.strptime(filename[0:19],'%Y-%m-%dT%H_%M_%S') - record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport } - res.append(record) - except: - try: - fyear = int(filename[0:4]) - fmonth = int(filename[5:7]) - if fyear == year and fmonth == month: - sport = filename[18:] - timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S') - record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport } - res.append(record) - except: - None - - return res - - -def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False): - - training_list = get_dbx_list(dbx,username,year,month) - - for training in training_list: - - filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:]) - dirname = "%s/%04d/%02d" % (dir,year,month) - - if os.path.exists(filename) and not force: - - if verbose: - print "training %s exists, skipping" % (filename) - - else: - - try: - os.makedirs(dirname) - except: - None - - xml = get_db_gpx(dbx,training['id']) - - if verbose: - print "writing training %s" % (filename) - - gpx = pygpx.GPX() - gpx.ReadTree(xml) - - sport = training['sport'] - timestamp = gpx.tracks[0].start_time() - - print sport, timestamp - - if check_db_for_training(db,sport,timestamp): - print "The same training found" - continue - - gpx.author = username - gpx.FixNames(training['sport']) - gpx.ProcessTrackSegs() - for track in gpx.tracks: - track.sport=training['sport'] - - xml = gpx.XMLTree(); - f = open(filename,"w") - f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) - f.close - write_parsed_to_db(db,gpx,filename) - try: - render_tiles.queue_render(db,filename) - except: - None - - -def sync_osm(username,password,year,month,dir=".",verbose=False,force=False): - - training_list = get_osm_list(username,password,year,month) - - for training in training_list: - - filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id']) - dirname = "%s/%04d/%02d" % (dir,year,month) - - if os.path.exists(filename) and not force: - - if verbose: - print "training %s exists, skipping" % (filename) - - else: - - try: - os.makedirs(dirname) - except: - None - - xml = get_osm_gpx(username,password,training['id']) - - if verbose: - print "writing training %s" % (filename) - - gpx = pygpx.GPX() - gpx.ReadTree(xml) - - sport = training['sport'] - timestamp = gpx.tracks[0].start_time() - - if check_db_for_training(db,sport,timestamp): - print "The same training found" - continue - - gpx.author = username - gpx.FixNames(training['sport']) - gpx.ProcessTrackSegs() - for track in gpx.tracks: - track.sport=training['sport'] - - xml = gpx.XMLTree(); - f = open(filename,"w") - f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) - f.close - write_parsed_to_db(db,gpx,filename) - try: - render_tiles.queue_render(db,filename) - except: - None - -def sync_folder(username,year,month,dir=".",verbose=False,force=False): - - training_list = get_page(username,year,month) - for tr in training_list: - try: - - filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1]) - dirname = "%s/%04d/%02d" % (dir,year,(month+1)) - - if os.path.exists(filename) and not force: - - if verbose: - print "training %s exists, skipping" % (filename) - - else: - - try: - os.makedirs(dirname) - except: - None - - xml=get_gpx_track(tr[1],tr[0]) - - if verbose: - print "writing training %s" % (filename) - - gpx = pygpx.GPX() - gpx.ReadTree(xml) - - sport = tr[0] - timestamp = gpx.tracks[0].start_time() - - if check_db_for_training(db,sport,timestamp): - print "The same training found" - continue - - gpx.FixNames(tr[0]) - gpx.ProcessTrackSegs() - - xml = gpx.XMLTree(); - f = open(filename,"w") - f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) - f.close - write_parsed_to_db(db,gpx,filename) - try: - render_tiles.queue_render(db,filename) - except: - None - - except: - raise - -def main(): - - global db; - parser = OptionParser() - parser.add_option("-d", "--dir", dest="dirname", - help="write data to directory", metavar="DIR") - parser.add_option("-q", "--quiet", - action="store_false", dest="verbose", default=True, - help="don't print status messages to stdout") - parser.add_option("-f", "--force", - action="store_true", dest="force", default=False, - help="rewrite all files") - parser.add_option("-y", "--yearmonth", dest="yearmonth", - help="year and month in YYYY-MM format", metavar="YYYY-MM") - parser.add_option("-u", "--username", dest="username", - help="Run.GPS username") - parser.add_option("-o", "--osm-username", dest="osmname", - help="OpenStreetMap username") - parser.add_option("-p", "--osm-passwd", dest="osmpasswd", - help="OpenStreetMap password") - parser.add_option("-b", "--dropbox", dest="dropboxauth", - help="DropBox Auth token") - (options, args) = parser.parse_args() - - username = options.username - osmname = options.osmname - osmpwd = options.osmpasswd - dbauth = options.dropboxauth - if not (username or (osmname and osmpwd) or dbauth): - print "Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!" - return - - if username: - pusername = username+'@Run.GPS' - elif osmname: - pusername = osmname+'@OSM' - elif dbauth: - import dropbox - dbx = dropbox.Dropbox(dbauth) - pusername = dbx.users_get_current_account().email+'@DBX' - - try: - if options.yearmonth: - year,month = options.yearmonth.split('-') - month = int(month) -1 - year = int(year) - if month<0 or month>11: - raise invalid_number - else: - year = None - month = None - except: - print "Year and month should be in YYYY-MM format!" - return - - if options.dirname: - outdir = options.dirname - else: - outdir = '.' - - db = outdir + '/gpx.db' - - if year: - if options.verbose: - print "retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir) - if username: - sync_folder(username,year,month,outdir,options.verbose,options.force) - elif osmname: - sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force) - elif dbauth: - sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force) - else: - if options.verbose: - print "retrieving two last months for user %s to %s" % (pusername ,outdir) - now = date.today() - current_year = now.year - current_month = now.month - if username: - sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force) - elif osmname: - sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force) - elif dbauth: - sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force) - current_month = current_month -1 - if current_month == 0: - current_month = 12 - current_year = current_year -1 - if username: - sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force) - elif osmname: - sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force) - elif dbauth: - sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force) - - generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-imposm.xml',640,640) - -if __name__ == "__main__": - - main() - \ No newline at end of file