From 81020ed69398f00cdb0564a132dfdf0285e9b844 Mon Sep 17 00:00:00 2001 From: Roman Bazalevskiy Date: Sun, 27 May 2018 13:43:21 +0300 Subject: [PATCH] =?utf8?q?1)=20Dropbox=20API=20=D0=B8=20=D1=81=D0=BE=D0=BF?= =?utf8?q?=D1=83=D1=82=D1=81=D1=82=D0=B2=D1=83=D1=8E=D1=89=D0=B8=D0=B5=20?= =?utf8?q?=D0=B8=D0=B7=D0=BC=D0=B5=D0=BD=D0=B5=D0=BD=D0=B8=D1=8F=202)=20?= =?utf8?q?=D0=9F=D1=80=D0=BE=D0=B2=D0=B5=D1=80=D0=BA=D0=B0=20=D0=BD=D0=B0?= =?utf8?q?=20=D0=BD=D0=B0=D0=BB=D0=B8=D1=87=D0=B8=D0=B5=20=D1=82=D1=80?= =?utf8?q?=D0=B5=D0=BD=D0=B8=D1=80=D0=BE=D0=B2=D0=BA=D0=B8,=20=D0=B7=D0=B0?= =?utf8?q?=D0=B3=D1=80=D1=83=D0=B6=D0=B5=D0=BD=D0=BD=D0=BE=D0=B9=20=D0=B8?= =?utf8?q?=D0=B7=20=D0=B4=D1=80=D1=83=D0=B3=D0=BE=D0=B3=D0=BE=20=D0=B8?= =?utf8?q?=D1=81=D1=82=D0=BE=D1=87=D0=BD=D0=B8=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit --- generate_image.py | 9 +- parsegpx.py | 8 ++ pygpx.py | 3 +- pyrungps.py | 213 +++++++++++++++++++++++++++++++++++++--------- 4 files changed, 189 insertions(+), 44 deletions(-) diff --git a/generate_image.py b/generate_image.py index a4deaac..e6a5021 100644 --- a/generate_image.py +++ b/generate_image.py @@ -30,7 +30,10 @@ if not hasattr(mapnik,'mapnik_version') and not mapnik.mapnik_version() >= 600: def render_map(mapfile,map_uri,gpx_file,imgx,imgy): - xml = etree.parse(gpx_file) + with open(gpx_file,'r') as f: + data = f.read() + + xml = etree.fromstring(data) gpx = pygpx.GPX() gpx.ReadTree(xml) @@ -64,6 +67,10 @@ def render_map(mapfile,map_uri,gpx_file,imgx,imgy): style = mapnik.Style() rule = mapnik.Rule() + line_symbolizer = mapnik.LineSymbolizer() + line_symbolizer.stroke = mapnik.Color('rgb(125, 125, 0)') + rule.symbols.append(line_symbolizer) + point_symbolizer = mapnik.MarkersSymbolizer() point_symbolizer.allow_overlap = True point_symbolizer.opacity = 0.8 # semi-transparent diff --git a/parsegpx.py b/parsegpx.py index 5247c27..5299fe0 100644 --- a/parsegpx.py +++ b/parsegpx.py @@ -10,6 +10,14 @@ import pyosmname import sqlite3 import datetime +def check_db_for_training(db,sport,timestamp): + + conn = sqlite3.connect(db) + cur = conn.cursor() + + cur.execute ("select count(*) from tracks where sport=? and start_time=?" , (sport,timestamp)) + return cur.fetchall()[0][0] + def write_parsed_to_db(db,gpx,filename): conn = sqlite3.connect(db) diff --git a/pygpx.py b/pygpx.py index fea5e67..4692538 100644 --- a/pygpx.py +++ b/pygpx.py @@ -410,8 +410,7 @@ class GPX: def ReadTree(self,tree): # Загрузка из дерева etree.# - gpx_doc = tree; - root = gpx_doc.getroot() + root = tree # Test if this is a GPX file or not and if it's version 1.1 if root.tag == "{http://www.topografix.com/GPX/1/1}gpx" and root.get("version") == "1.1": diff --git a/pyrungps.py b/pyrungps.py index 6c8ea5c..0f0dc56 100644 --- a/pyrungps.py +++ b/pyrungps.py @@ -1,16 +1,22 @@ #!/usr/bin/env python # coding: UTF-8 -import urllib2 +import requests + #import sys import os +from urllib2 import quote,unquote from lxml import html,etree from optparse import OptionParser from datetime import date,datetime from mx.DateTime.ISO import ParseDateTimeUTC -from parsegpx import write_parsed_to_db +from parsegpx import write_parsed_to_db,check_db_for_training import pygpx +from tempfile import NamedTemporaryFile + +from pprint import pprint + import render_tiles import generate_image @@ -19,8 +25,8 @@ def get_page(uname,year,month): trainings = [] - req = urllib2.Request("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), None, {'User-agent': 'Mozilla/5.0'}) - page = urllib2.urlopen(req).read() + req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'}) + page = req.text.encode() dom = html.document_fromstring(page) for element, attribute, link, pos in dom.iterlinks(): @@ -28,7 +34,7 @@ def get_page(uname,year,month): if link.startswith("/trainings/"): dummy, dummy, link = link.split('/') name, id = link.split('_') - trainings.append([ urllib2.unquote(name), id ]) + trainings.append([ unquote(name), id ]) return trainings @@ -36,62 +42,167 @@ def get_gpx_track(trid,name): print "trid=",trid - req = urllib2.urlopen("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid)) - - xml = etree.parse(req) + req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid)) + xml = etree.fromstring(req.text.encode()) return xml def get_osm_list(username,password,year,month): url = "https://www.openstreetmap.org/api/0.6/user/gpx_files" - passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - passman.add_password(None, url, username, password) - authhandler = urllib2.HTTPBasicAuthHandler(passman) - opener = urllib2.build_opener(authhandler) - urllib2.install_opener(opener) - req = urllib2.urlopen(url) - xml = etree.parse(req) + req = requests.get(url,auth=(username,password)) + + xml = etree.fromstring(req.text.encode()) res=[] - for gpx_file in xml.getroot(): + for gpx_file in xml: attrib = dict(gpx_file.attrib) - timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp'])) - id = attrib['id'] - description=None - for descr in gpx_file.iter('description'): - description=descr.text - sport=None - for tag in gpx_file.iter('tag'): - sport=tag.text - if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track': - print id,description,sport,timestamp - record={ 'id': id, 'sport': sport } - res.append(record) + try: + timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp'])) + id = attrib['id'] + description=None + for descr in gpx_file.iter('description'): + description=descr.text + sport=None + for tag in gpx_file.iter('tag'): + sport=tag.text + if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track': + record={ 'id': id, 'sport': sport } + res.append(record) + except: + None - return res + return res def get_osm_gpx(username,password,track_id): url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data" - passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - passman.add_password(None, url, username, password) - authhandler = urllib2.HTTPBasicAuthHandler(passman) - opener = urllib2.build_opener(authhandler) - urllib2.install_opener(opener) - req = urllib2.urlopen(url) - xml = etree.parse(req) + req = requests.get(url,auth=(username,password)) + xml = etree.fromstring(req.text.encode()) return xml +def get_db_gpx(dbx,track_id): + + md, res = dbx.files_download(track_id) + + fs = NamedTemporaryFile(suffix='.tcx',delete = False) + tmp_tcx_name=fs.name + fs.write(res.content) + fs.close() + fs = NamedTemporaryFile(suffix='.gpx',delete = False) + tmp_gpx_name=fs.name + fs.close() + + os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name) + + with open (tmp_gpx_name, "r") as gpxfile: + data=gpxfile.read() + + os.remove(tmp_gpx_name) + os.remove(tmp_tcx_name) + + xml = etree.fromstring(data.encode()) + return xml + +def get_dbx_list(dbx,username,year,month): + + gpx_list_id = None + + for entry in dbx.files_list_folder('').entries: + if entry.name == u'Приложения': + for entry_app in dbx.files_list_folder(entry.id).entries: + if entry_app.name == u'Run.GPS': + gpx_list_id=entry_app.id + break + break + + res = [] + + if gpx_list_id: + + for file in dbx.files_list_folder(gpx_list_id).entries: + + filename,ext = os.path.splitext(file.name) + if ext == '.tcx': + try: + fyear = int(filename[0:4]) + fmonth = int(filename[5:7]) + if fyear == year and fmonth == month: + sport = filename[18:] + timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S') + record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport } + res.append(record) + except: + None + + return res + + +def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False): + + training_list = get_dbx_list(dbx,username,year,month) + + for training in training_list: + + filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:]) + dirname = "%s/%04d/%02d" % (dir,year,month) + + if os.path.exists(filename) and not force: + + if verbose: + print "training %s exists, skipping" % (filename) + + else: + + try: + os.makedirs(dirname) + except: + None + + xml = get_db_gpx(dbx,training['id']) + + if verbose: + print "writing training %s" % (filename) + + gpx = pygpx.GPX() + gpx.ReadTree(xml) + + sport = training['sport'] + timestamp = gpx.tracks[0].start_time() + + print sport, timestamp + + if check_db_for_training(db,sport,timestamp): + print "The same training found" + continue + + gpx.author = username + gpx.FixNames(training['sport']) + gpx.ProcessTrackSegs() + for track in gpx.tracks: + track.sport=training['sport'] + + xml = gpx.XMLTree(); + f = open(filename,"w") + f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) + f.close + write_parsed_to_db(db,gpx,filename) + try: + render_tiles.queue_render(db,filename) + except: + None + + def sync_osm(username,password,year,month,dir=".",verbose=False,force=False): training_list = get_osm_list(username,password,year,month) for training in training_list: + filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id']) dirname = "%s/%04d/%02d" % (dir,year,month) @@ -101,7 +212,7 @@ def sync_osm(username,password,year,month,dir=".",verbose=False,force=False): print "training %s exists, skipping" % (filename) else: - + try: os.makedirs(dirname) except: @@ -115,12 +226,19 @@ def sync_osm(username,password,year,month,dir=".",verbose=False,force=False): gpx = pygpx.GPX() gpx.ReadTree(xml) + sport = training['sport'] + timestamp = gpx.tracks[0].start_time() + + if check_db_for_training(db,sport,timestamp): + print "The same training found" + continue + gpx.author = username gpx.FixNames(training['sport']) gpx.ProcessTrackSegs() for track in gpx.tracks: track.sport=training['sport'] - + xml = gpx.XMLTree(); f = open(filename,"w") f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) @@ -192,19 +310,26 @@ def main(): help="OpenStreetMap username") parser.add_option("-p", "--osm-passwd", dest="osmpasswd", help="OpenStreetMap password") + parser.add_option("-b", "--dropbox", dest="dropboxauth", + help="DropBox Auth token") (options, args) = parser.parse_args() username = options.username osmname = options.osmname osmpwd = options.osmpasswd - if not (username or (osmname and osmpwd)): - print "Run.GPS username or OSM username/password is mandatory!" + dbauth = options.dropboxauth + if not (username or (osmname and osmpwd) or dbauth): + print "Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!" return if username: pusername = username+'@Run.GPS' elif osmname: pusername = osmname+'@OSM' + elif dbauth: + import dropbox + dbx = dropbox.Dropbox(dbauth) + pusername = dbx.users_get_current_account().email+'@DBX' try: if options.yearmonth: @@ -234,9 +359,11 @@ def main(): sync_folder(username,year,month,outdir,options.verbose,options.force) elif osmname: sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force) + elif dbauth: + sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force) else: if options.verbose: - print "retrieving two last months for user %s to %s" % (pusername,outdir) + print "retrieving two last months for user %s to %s" % (pusername ,outdir) now = date.today() current_year = now.year current_month = now.month @@ -244,6 +371,8 @@ def main(): sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force) elif osmname: sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force) + elif dbauth: + sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force) current_month = current_month -1 if current_month == 0: current_month = 12 @@ -252,6 +381,8 @@ def main(): sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force) elif osmname: sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force) + elif dbauth: + sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force) generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-transparent.xml',400,400) -- 2.34.1