From: Roman Bazalevskiy Date: Fri, 11 Dec 2020 13:17:02 +0000 (+0300) Subject: Packaged at last... X-Git-Url: https://git.rvb.name/pyrungps.git/commitdiff_plain/94f62add438cdf546fdae207ede72c05daf66c00?ds=sidebyside;hp=1690b8b7c8ccfdf60be2b33b314e6c26bdef86e6 Packaged at last... --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bee8a64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/generate_image.py b/generate_image.py deleted file mode 100644 index 12169ca..0000000 --- a/generate_image.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env python - -import mapnik - -import sys, os - -from lxml import html,etree -from optparse import OptionParser - -from pprint import pprint - -import pygpx - -# Set up projections -# spherical mercator (most common target map projection of osm data imported with osm2pgsql) -merc = mapnik.Projection('+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over') - -# long/lat in degrees, aka ESPG:4326 and "WGS 84" -longlat = mapnik.Projection('+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs') -# can also be constructed as: -#longlat = mapnik.Projection('+init=epsg:4326') - -# ensure minimum mapnik version -if not hasattr(mapnik,'mapnik_version') and not mapnik.mapnik_version() >= 600: - raise SystemExit('This script requires Mapnik >=0.6.0)') - - -def render_map(mapfile,map_uri,gpx_file,imgx,imgy): - - with open(gpx_file,'r') as f: - data = f.read() - - xml = etree.fromstring(data) - gpx = pygpx.GPX() - gpx.ReadTree(xml) - - bbox = gpx.bound_box() - pprint(bbox) - cx=(bbox[0][1]+bbox[1][1])/2 - cy=(bbox[0][0]+bbox[1][0])/2 - print cx,cy - w=(bbox[1][1]-bbox[0][1])*1.2 - h=(bbox[1][0]-bbox[0][0])*1.2 - print w,h - bounds = (cx-w/2,cy-h/2,cx+w/2,cy+h/2) - print bounds - - m = mapnik.Map(imgx,imgy) - mapnik.load_map(m,mapfile) - - m.background = mapnik.Color('rgb(255,255,255)') - - # ensure the target map projection is mercator - m.srs = merc.params() - - bbox = mapnik.Box2d(*bounds) - - pprint(bbox) - - transform = mapnik.ProjTransform(longlat,merc) - merc_bbox = transform.forward(bbox) - m.zoom_to_box(merc_bbox) - - style = mapnik.Style() - rule = mapnik.Rule() - - line_symbolizer = mapnik.LineSymbolizer() - - line_symbolizer.stroke = mapnik.Color('rgb(0,0,127)') - line_symbolizer.stroke_width = 4.0 - line_symbolizer.stroke_opacity = 0.5 - - rule.symbols.append(line_symbolizer) - - style.rules.append(rule) - m.append_style('GPS_tracking_points', style) - - layer = mapnik.Layer('GPS_tracking_points') - layer.datasource = mapnik.Ogr(file=gpx_file, layer='tracks') - layer.styles.append('GPS_tracking_points') - m.layers.append(layer) - - im = mapnik.Image(imgx,imgy) - mapnik.render(m, im) - im.save(map_uri,'png') - sys.stdout.write('output image to %s!\n' % map_uri) - - -def render_all(db,mapfile,imgx,imgy): - - import sqlite3 - from os.path import dirname - - conn = sqlite3.connect(db) - conn.text_factory = str - cur = conn.cursor() - updcur = conn.cursor() - - cur.execute ("select id,filename from tracks where preview_name is null") - - list=cur.fetchall() - - for rec in list: - - id = rec[0] - filename = rec[1] - preview_name = dirname(filename) + '/' + str(id) + '.png' - - print id,filename,preview_name - - try: - render_map(mapfile,preview_name.encode('utf8'),filename,imgx,imgy) - updcur.execute("update tracks set preview_name=? where id=?", (preview_name,id)) - conn.commit() - except: - raise - - -def main(): - - parser = OptionParser() - parser.add_option("-m", "--map", dest="mapfile", - help="use map file", metavar="MAP") - parser.add_option("-o", "--output", dest="outfile", - help="output image to file", metavar="OUT") - parser.add_option("-g", "--gpx", dest="gpxfile", - help="track to render", metavar="GPX") - parser.add_option("-x", "--x-size", dest="x", - help="image width", metavar="X") - parser.add_option("-y", "--y-size", dest="y", - help="image height", metavar="Y") - parser.add_option("-d", "--db", dest="db", - help="render all not process files in database", metavar="DB") - (options, args) = parser.parse_args() - - if options.mapfile: - mapfile = options.mapfile - else: - mapfile = "/etc/mapnik-osm-carto-data/veloroad-imposm.xml" - - if options.outfile: - map_uri = options.outfile - else: - map_uri = "image.png" - - if options.x: - imgx = int(options.x) - else: - imgx= 640 - if options.y: - imgy = int(options.y) - else: - imgy= 640 - - if options.db: - render_all(options.db,mapfile,imgx,imgy) - else: - if options.gpxfile: - gpx_file = options.gpxfile - render_map(mapfile,map_uri,gpx_file,imgx,imgy) - else: - print "No input file" - exit(1) - -if __name__ == "__main__": - - main() diff --git a/parsegpx.py b/parsegpx.py deleted file mode 100644 index c8ca599..0000000 --- a/parsegpx.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python -# coding: UTF-8 - -import sys -import os -from lxml import etree -from urllib2 import unquote -import pygpx -import pygeocode -import sqlite3 -import datetime - -def check_db_for_training(db,sport,timestamp): - - conn = sqlite3.connect(db) - conn.text_factory = str - cur = conn.cursor() - - cur.execute ("select count(*) from tracks where sport=? and start_time=?" , (sport,timestamp)) - return cur.fetchall()[0][0] - -def write_parsed_to_db(db,gpx,filename): - - conn = sqlite3.connect(db) - conn.text_factory = str - cur = conn.cursor() - - if type(filename) is str: - filename=filename.decode('UTF-8') - - cur.execute ("delete from tracks where filename=?" , (filename,)) - - tracks = gpx.tracks - - for track in tracks: - - try: - author = gpx.author - except: - author = None - - try: - name = gpx.name - if type(name) is str: - name=name.decode('UTF-8') - except: - name = None - - if author: - try: - cur.execute("insert into authors(name,description) values(?,?)", (author,'')) - print "created author %s" % (author) - except: - print "failed to create author %s" % (author) - pass - - try: - - print "processing..." - start = track.start() - if start: - printable = pygeocode.GeoName(start.lat,start.lon).printable - start_time = track.start_time() - full_duration = track.full_duration().total_seconds() - distance = track.distance() - filtered_distance = track.filtered_distance(max_speed=50) - ascent = track.elevation_gain() - descent = track.elevation_loss() - ((minlat,minlon),(maxlat,maxlon)) = track.bound_box() - params = ( - gpx.author,name,filename, - track.sport,start_time,full_duration, - distance,filtered_distance,ascent,descent, - start.lat,start.lon, - printable, - minlat,minlon,maxlat,maxlon - ) - print(params) - cur.execute(""" - insert into tracks( - author,name,filename,sport, - start_time,duration, - distance,distance_filtered, - ascent,descent, - lat,lon,printable_location,minlat,minlon,maxlat,maxlon) - values( - ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) - """ - , params ) - conn.commit() - print "created track %s" % (filename) - - except: - - raise - -def write_tree_to_db(db,tree,filename): - - gpx = pygpx.GPX() - gpx.ReadTree(tree) - - write_parsed_to_db(db,gpx,filename) - -def print_parsed_file(filename): - - gpx = pygpx.GPX() - gpx.ReadFile(filename) - - for track in gpx.tracks: - - try: - author = gpx.author - except: - author = None - - try: - name = gpx.name - except: - name = None - - start = track.start() - if start: - printable = pygeocode.GeoName(start.lat,start.lon).printable - start_time = track.start_time() - full_duration = track.full_duration().total_seconds() - distance = track.distance() - filtered_distance = track.filtered_distance(max_speed=50) - ascent = track.elevation_gain() - descent = track.elevation_loss() - ((minlat,minlon),(maxlat,maxlon)) = track.bound_box() - params = ( - gpx.author,name,filename.decode('UTF-8'), - track.sport,start_time,full_duration, - distance,filtered_distance,ascent,descent, - start.lat,start.lon, - printable, - minlat,minlon,maxlat,maxlon - ) - print(params) diff --git a/pygeocode.py b/pygeocode.py deleted file mode 100644 index f9fb0fb..0000000 --- a/pygeocode.py +++ /dev/null @@ -1,22 +0,0 @@ -# coding=UTF-8 -""" -nominatim.openstreetmap.org API -""" - -import urllib2 - -def GetDescr(lat,lon): - - reqstr = "http://maps.rvb.name/geocode.php?lat=%s&lon=%s" %(lat,lon) - headers = { 'User-Agent' : 'PyRunGPS/1.0', 'Referer' : 'https://maps.rvb.name' } - req = urllib2.Request(reqstr, None, headers) - page = urllib2.urlopen(req).read() - - return page - -class GeoName: - - def __init__(self, lat, lon): - - self.printable = GetDescr(lat,lon) - \ No newline at end of file diff --git a/pygpx.py b/pygpx.py deleted file mode 100644 index 49ccdff..0000000 --- a/pygpx.py +++ /dev/null @@ -1,662 +0,0 @@ -# coding: UTF-8 -#------------------------ -# Работа с GPX-файлами -#------------------------ -from lxml import etree -import os -import math -import datetime -from dateutil.parser import parse -from pprint import pprint - -def deg2rad(deg): -# Преобразование из градусов в радианы. - return deg / (180 / math.pi) - -tagprefix = { '1.0': '{http://www.topografix.com/GPX/1/0}', '1.1': '{http://www.topografix.com/GPX/1/1}'} - -def StripPrefix(tag,version): - nstag=tagprefix[version] - return tag.replace(nstag,'') - -def formattime(dt): - return dt.strftime("%Y-%m-%dT%H:%M:%S")+"."+str(int(dt.microsecond/1000)).zfill(3)+"Z" - -class Link: - # Ссылка. Может присутствовать в точке, треке, файле. - def __init__(self, node, version): - self.version = version - self.href = node.get("href") - self.text = None - self.type = None - if node is not None: - for child in node: - if child.tag == tagprefix[version] + "text": - self.text = child.text - elif child.tag == tagprefix[version] + "type": - self.type = child.text - else: - raise ValueError("Неизвестный тип узла: '%s' " % child.tag) - - def write(self,root): - linknode = etree.SubElement(root,"link"); - linknode.set("href",self.href) - if self.text: - etree.SubElement(linknode,"text").text = self.text - if self.type: - etree.SubElement(linknode,"type").text = self.type - -class GPXTrackPt: - # Точка с координатами. Может присутствовать в последовательности точек в треке, маршруте, а также в списке путевых точек.# - - def __init__(self, node, version): - # Извлекаем данные из GPX-тэгов.# - self.version = version - # Сначала обязательные.# - if node is not None: - self.lat = float(node.get("lat")) - self.lon = float(node.get("lon")) - self.elevation = None - self.time = None - self.speed = None - self.link = None - self.additional_info = {} - # Потом необязательные.# - if node is not None: - for child in node: - if child.tag == tagprefix[version] + "time": - self.time = parse(child.text) - elif child.tag == tagprefix[version] + "ele": - self.elevation = float(child.text) - # Стандартом скорость не предусмотрена, но де-факто в Run.GPS используется.# - elif child.tag == tagprefix[version] + "speed": - self.speed = float(child.text) - elif child.tag == tagprefix[version] + "link": - self.link = Link(child,version) - elif StripPrefix(child.tag,version) in ['magvar','geoidheight','name','cmt','desc', - 'src','sym','type','fix','sat','hdop','vdop','pdop','ageofdgpsdata','dgpsid']: - self.additional_info[StripPrefix(child.tag,version)]=child.text - elif child.tag == tagprefix[version] + "extensions": - pass - else: - raise ValueError("Неизвестный тип узла: '%s'" % child.tag) - - for i in self.additional_info.keys(): - val = self.additional_info[i]; - if val: - if not val.strip(' \n\t'): - del self.additional_info[i] - - - def write(self,root,tag ="wpt"): - wptnode = etree.SubElement(root,tag) - wptnode.set("lat",repr(self.lat)); - wptnode.set("lon",repr(self.lon)); - if self.elevation: - etree.SubElement(wptnode,"ele").text = repr(self.elevation) - if self.time: - etree.SubElement(wptnode,"time").text = formattime(self.time) - if self.speed: - etree.SubElement(wptnode,"speed").text = repr(self.speed) - if self.link: - self.link.write(wptnode) - for i in self.additional_info.keys(): - etree.SubElement(wptnode,i).text = self.additional_info[i] - - def distance(self, other): - # Расстояние между двумя точками на глобусе.# - try: - # http://www.platoscave.net/blog/2009/oct/5/calculate-distance-latitude-longitude-python/ - - radius = 6378700.0 # meters - - lat1, lon1 = self.lat, self.lon - lat2, lon2 = other.lat, other.lon - - dlat = math.radians(lat2-lat1) - dlon = math.radians(lon2-lon1) - a = math.sin(dlat/2) * math.sin(dlat/2) + math.cos(math.radians(lat1)) \ - * math.cos(math.radians(lat2)) * math.sin(dlon/2) * math.sin(dlon/2) - c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a)) - d = radius * c - - except ValueError, e: - raise ValueError(e) - return d - - def duration(self, other): - # Время между двумя точками.# - return other.time - self.time - -class GPXRoute: - # Маршрут.# - - def __init__(self, node, version): - self.name = None - self.version = version - self.rtepts = [] - self.additional_info = {} - self.link = None - if node is not None: - for child in node: - if child.tag == tagprefix[version] + "name": - self.name = child.text - elif child.tag == tagprefix[version] + "link": - self.link = Link(child,version) - elif StripPrefix(child.tag,version) in [ "cmt", "desc", "src", "number", "type" ]: - self.additional_info[StripPrefix(child.tag,version)]=child.text - elif child.tag == tagprefix[version] + "rtept": - self.rtepts.append(GPXTrackPt(child, version)) - elif child.tag == tagprefix[version] + "extensions": - pass - else: - raise ValueError("Неизвестный тип узла <%s>" % child.tag) - for i in self.additional_info.keys(): - val = self.additional_info[i]; - if val: - if not val.strip(' \n\t'): - del self.additional_info[i] - - def write(self,root): - rtenode = etree.SubElement(root,"rte") - if self.name: - etree.SubElement(rtenode,"name").text = self.name - if self.link: - self.link.write(rtenode) - for i in self.additional_info.keys(): - print "storing %s " % ( i ) - etree.SubElement(rtenode,i).text = self.additional_info[i] - for i in self.rtepts: - i.write(rtenode,"rtept") - -class GPXTrackSeg: - # Один сегмент трека.# - - def __init__(self, node, version): - self.version = version - self.trkpts = [] - self.elevation_gain = 0.0 - self.elevation_loss = 0.0 - if node is not None: - for child in node: - if child.tag == tagprefix[version] + "trkpt": - self.trkpts.append(GPXTrackPt(child, self.version)) - else: - raise ValueError("Неизвестный тип узла <%s>" % node.nodeName) - self._get_elevation() - - def write(self,root): - trksegnode = etree.SubElement(root,"trkseg") - for i in self.trkpts: - i.write(trksegnode,"trkpt") - - def _get_elevation(self): - - elev_data = [] - for pt in self.trkpts: - if pt.elevation: - elev_data.append(pt.elevation) - - gain = 0.0 - loss = 0.0 - last_elevation = None - - window_size = 5 - i = 0 - moving_averages = [] - - while i < len(elev_data) - window_size + 1: - this_window = elev_data[i : i + window_size] - window_average = sum(this_window) / window_size - moving_averages.append(window_average) - i += 1 - - if len(moving_averages)>2: - elev_data = moving_averages - - for pt in elev_data: - if last_elevation is not None: - try: - if pt > last_elevation: - gain += pt - last_elevation - else: - loss += last_elevation - pt - except: - pass - last_elevation=pt - - self.elevation_gain = gain - self.elevation_loss = loss - - def distance(self): - # Расчет длины сегмента.# - _length = 0.0 - last_pt = None - for pt in self.trkpts: - if last_pt is not None: - _length += last_pt.distance(pt) - last_pt = pt - return _length - - def filtered_distance(self,max_speed): - # Расчет длины сегмента с фильтрацией предположительно сбойных участков.# - _length = 0.0 - last_pt = None - for pt in self.trkpts: - if last_pt is not None: - _delta = last_pt.distance(pt) - _time_delta = pt.time - last_pt.time - _time_delta_s = _time_delta.days*86400 + _time_delta.seconds + _time_delta.microseconds/1000000 - if _time_delta_s > 0: - if _delta/_time_delta_s < max_speed: - _length += _delta - else: - _length += _delta - last_pt = pt - return _length - - def duration(self): - # Расчет продолжительности сегмента.# - return self.trkpts[0].duration(self.trkpts[-1]) - - def bound_box(self): - # Обрамляющий прямоугольник для сегмента.# - minlat =360 - maxlat = -360 - minlon = 360 - maxlon = -360 - for pt in self.trkpts: - if pt.latmaxlat: - maxlat=pt.lat - if pt.lonmaxlon: - maxlon=pt.lon - return ((minlat,minlon),(maxlat,maxlon)) - -class GPXTrack: - # Трек.# - - def __init__(self, node, version): - # Создаем трек из GPX-данных.# - - self.version = version - self.trksegs = [] - self.sport= None - self.additional_info = {} - self.name = None - self.link = None - - if node is not None: - for child in node: - if child.tag == tagprefix[version] + "name": - self.name = child.text - elif child.tag == tagprefix[version] + "trkseg": - if len(child) > 0: - self.trksegs.append(GPXTrackSeg(child, self.version)) - elif child.tag == tagprefix[version] + "link": - self.link = Link(child,version) - elif StripPrefix(child.tag,version) in [ "number", "desc", "cmt", "src", "type" ]: - self.additional_info[StripPrefix(child.tag,version)] = child.text - elif child.tag == tagprefix[version] + "extensions": - for ext in child: - # Из расширений извлекаем вид спорта - специфично для Run.GPS.# - if ext.tag == tagprefix[version] + "sport": - self.sport = ext.text - - for i in self.additional_info.keys(): - val = self.additional_info[i]; - if val: - if not val.strip(' \n\t'): - del self.additional_info[i] - - def write(self,root): - trknode = etree.SubElement(root,"trk") - if self.name: - etree.SubElement(trknode,"name").text = self.name - if self.sport: - etree.SubElement(etree.SubElement(trknode,"extensions"),"sport").text=self.sport - if self.link: - self.link.write(trknode) - for i in self.additional_info.keys(): - etree.SubElement(trknode,i).text = self.additional_info[i] - for i in self.trksegs: - i.write(trknode) - - def elevation_gain(self): - # Набор высоты по треку.# - return sum([trkseg.elevation_gain for trkseg in self.trksegs]) - - def elevation_loss(self): - # Сброс высоты по треку.# - return sum([trkseg.elevation_loss for trkseg in self.trksegs]) - - def distance(self): - # Длина трека.# - try: - return sum([trkseg.distance() for trkseg in self.trksegs]) - except IndexError: - print "Пустой сегмент трека, расчет длины невозможен." - - def filtered_distance(self,max_speed=100): - # Длина трека с фильтрацией сбойных участков.# - try: - return sum([trkseg.filtered_distance(max_speed=max_speed) for trkseg in self.trksegs]) - except IndexError: - print "Пустой сегмент трека, расчет длины невозможен." - - def duration(self): - # Продолжительность трека, не включая паузы между сегментами.# - dur = datetime.timedelta(0) - for trkseg in self.trksegs: - try: - dur += trkseg.duration() - except: - pass - return dur - - def full_duration(self): - # Продолжительность трека, включая паузы между сегментами.# - try: - return self.start().duration(self.end()) - except: - return None - - def start(self): - # Стартовая точка.# - try: - return self.trksegs[0].trkpts[0] - except IndexError: - return None - - def end(self): - # Финишная точка.# - try: - return self.trksegs[-1].trkpts[-1] - except IndexError: - return None - - def start_time(self): - # Время старта.# - try: - return self.start().time - except: - return None - - def end_time(self): - # Время финиша.# - try: - return self.end().time - except: - return None - - def bound_box(self): - # Обрамляющий прямоугольник для всего трека.# - minlat =360 - maxlat = -360 - minlon = 360 - maxlon = -360 - for trkseg in self.trksegs: - for pt in trkseg.trkpts: - if pt.latmaxlat: - maxlat=pt.lat - if pt.lonmaxlon: - maxlon=pt.lon - return ((minlat,minlon),(maxlat,maxlon)) - - -class GPX: - # Работа с GPX-документами.# - - def __init__(self): - # Создание пустого GPX-объекта# - PATH = os.path.dirname(__file__) - self.creator = None - self.time = None - self.tracks = [] - self.routes = [] - self.waypoints = [] - self.version = "" - self.author = None - self.name = None - self.link = None - self.copyright = None - self.meta = {} - - def ReadTree(self,tree): - # Загрузка из дерева etree.# - root = tree - - # Test if this is a GPX file or not and if it's version 1.1 - if root.tag == "{http://www.topografix.com/GPX/1/1}gpx" and root.get("version") == "1.1": - self.version = "1.1" - elif root.tag == "{http://www.topografix.com/GPX/1/0}gpx" and root.get("version") == "1.0": - self.version = "1.0" - elif root.get("version") not in [ "1.1", "1.0"]: - raise ValueError("Версия формата %s не поддерживается. Используйте GPX 1.1." % root.get("version")) - else: - raise ValueError("Неизвестный формат дерева") - # attempt to validate the xml file against the schema - - # initalize the GPX document for parsing. - self._init_version(root) - - - def ReadFile(self, fd): - # Загрузка из файла.# - gpx_doc = etree.parse(fd) - root = gpx_doc.getroot() - - # Test if this is a GPX file or not and if it's version 1.1 - if root.tag == "{http://www.topografix.com/GPX/1/1}gpx" and root.get("version") == "1.1": - self.version = "1.1" - elif root.tag == "{http://www.topografix.com/GPX/1/0}gpx" and root.get("version") == "1.0": - self.version = "1.0" - elif root.get("version") not in [ "1.1", "1.0"]: - raise ValueError("Версия формата %s не поддерживается. Используйте GPX 1.1." % root.get("version")) - else: - raise ValueError("Неизвестный формат файла") - # initalize the GPX document for parsing. - self._init_version(root) - - def _init_version(self,root): - # Инициализация объекта.# - self.creator = root.get("creator") - if root is not None: - for child in root: - if child.tag == tagprefix[self.version] + "author": - self.author = child.text - elif child.tag == tagprefix[self.version] + "trk": - self.tracks.append(GPXTrack(child, self.version)) - elif child.tag == tagprefix[self.version] + "rte": - self.routes.append(GPXRoute(child, self.version)) - elif child.tag == tagprefix[self.version] + "wpt": - self.waypoints.append(GPXTrackPt(child, self.version)) - elif child.tag == tagprefix[self.version] + "metadata": - for meta in child: - if meta.tag == tagprefix[self.version] + "author": - self.author = meta.text - elif meta.tag == tagprefix[self.version] + "name": - self.name = meta.text - elif meta.tag == tagprefix[self.version] + "bounds": - # Границы пропускаем - их будем вычислять сами заново - pass - elif meta.tag == tagprefix[self.version] + "copyright": - self.copyright = meta.text - elif meta.tag == tagprefix[self.version] + "link": - self.link = Link(meta,self.version) - else: - self.meta[StripPrefix(meta.tag,self.version)]=meta.text - - for i in self.meta.keys(): - val = self.meta[i]; - if val: - if not val.strip(' \n\t'): - del self.meta[i] - - def elevation_gain(self): - # Суммарный набор высоты.# - return sum([track.elevation_gain() for track in self.tracks]) - - def elevation_loss(self): - # Суммарная потеря высоты.# - return sum([track.elevation_loss() for track in self.tracks]) - - def distance(self): - # Суммарная дистанция.# - try: - return sum([track.distance() for track in self.tracks]) - except IndexError: - print "Пустой файл!" - - def filtered_distance(self): - # Суммарная дистанция с фильтрацией сбойных данных.# - try: - return sum([track.filtered_distance() for track in self.tracks]) - except IndexError: - print "Пустой файл!" - - def duration(self): - # Суммарная продолжительность, не включая паузы.# - dur = datetime.timedelta(0) - for track in self.tracks: - dur += track.duration() - return dur - - def full_duration(self): - # Суммарная продолжительность, включая паузы# - if hasattr(self.start(), 'duration'): - return self.start().duration(self.end()) - else: - return None - - def start(self): - # Точка старта.# - return self.tracks[0].start() - - def end(self): - # Точка финиша.# - return self.tracks[-1].end() - - def start_time(self): - # Время старта.# - if self.start(): - return self.start().time - else: - return None - - def end_time(self): - # Время финиша.# - return self.end().time - - def bound_box(self): - # Обрамляющий прямоугольник для всех треков в файле.# - minlat =360 - maxlat = -360 - minlon = 360 - maxlon = -360 - for track in self.tracks: - for trkseg in track.trksegs: - for pt in trkseg.trkpts: - if pt.latmaxlat: - maxlat=pt.lat - if pt.lonmaxlon: - maxlon=pt.lon - return ((minlat,minlon),(maxlat,maxlon)) - - def FixNames(self,linkname): - pprint(linkname) - pprint(type(linkname)) - if type(linkname) is unicode: - goodname=linkname - else: - goodname = linkname.decode('UTF-8') - badname = goodname.encode('ascii','replace') - if self.name and self.name.startswith(badname): - self.name=self.name.replace(badname,goodname) - for i in self.tracks: - if i.name and i.name.startswith(badname): - i.name = i.name.replace(badname,goodname) - - def ProcessTrackSegs(self,seconds_gap=120): - for tr in self.tracks: - # Объединяем все точки трека в единую последовательность - all_trackpoints = [] - for trseg in tr.trksegs: - all_trackpoints.extend(trseg.trkpts) - new_segs = [] - current_seg = None - prev_pt = None - for pt in all_trackpoints: - if not current_seg: - # Начало нового сегмента - current_seg = GPXTrackSeg(None,self.version) - current_seg.trkpts.append(pt) - else: - delta = pt.time - prev_pt.time - if delta.days > 0 or delta.seconds > seconds_gap: - # Нашли место разрыва - new_segs.append(current_seg) - current_seg = None - else: - current_seg.trkpts.append(pt) - prev_pt=pt - # Если есть, что закинуть в хвост - пишем - if current_seg is not None: - new_segs.append(current_seg) - # Обработали - for trs in new_segs: - trs._get_elevation() - tr.trksegs = new_segs - - - def XMLTree(self): - root = etree.Element("gpx", attrib ={"creator": "pygpx", - "version": "1.1", - "xmlns": "http://www.topografix.com/GPX/1/1"}); - - # meta subtree - - meta = etree.SubElement( root, "metadata" ); - if self.name: - metarecord = etree.SubElement( meta, "name" ) - metarecord.text = self.name - if self.author: - metarecord = etree.SubElement( meta, "author" ) - metarecord.text = self.author - if self.copyright: - metarecord = etree.SubElement( meta, "copyright" ) - metarecord.text = self.copyright - if self.link: - self.link.write(meta) - for metatag in self.meta.iterkeys(): - metarecord = etree.SubElement( meta, metatag ) - metarecord.text = self.meta[metatag] - - # wpts subtree - - for i in self.waypoints: - i.write(root) - - # routes subtree - - for i in self.routes: - i.write(root) - - # tracks subtree - - for i in self.tracks: - i.write(root) - - return root - \ No newline at end of file diff --git a/pyosmname.py b/pyosmname.py deleted file mode 100644 index e6dc29e..0000000 --- a/pyosmname.py +++ /dev/null @@ -1,107 +0,0 @@ -# coding=UTF-8 -""" -nominatim.openstreetmap.org API -""" - -import urllib2 -from lxml import etree - -def GetXMLDescr(lat,lon): - - reqstr = "http://nominatim.openstreetmap.org/reverse?lat=%s&lon=%s&accept-language=ru,en" %(lat,lon) - print reqstr - headers = { 'User-Agent' : 'PyRunGPS/1.0', 'Referer' : 'https://maps.rvb.name' } - req = urllib2.Request(reqstr, None, headers) - page = urllib2.urlopen(req).read() - - return etree.fromstring(page) - -def printable_name(addr): - - if 'country' in addr: - if 'state' in addr: - if addr['country'] == u"Российская Федерация" and addr['state'] == u"Москва": - if 'road' in addr: - str = addr['road'] + ', ' + addr['state'] + ', ' + addr['country'] - else: - str = addr['state'] + ', ' + addr['country'] - else: - if 'city' in addr: - if 'road' in addr: - str = addr['road'] + ', ' + addr['city'] + ', ' + addr['state'] + ', ' + addr['country'] - else: - str = addr['city'] + ', ' + addr['state'] + ', ' + addr['country'] - elif 'town' in addr: - if 'road' in addr: - str = addr['road'] + ', ' + addr['town'] + ', ' + addr['state'] + ', ' + addr['country'] - else: - str = addr['town'] + ', ' + addr['state'] + ', ' + addr['country'] - elif 'village' in addr: - if 'road' in addr: - str = addr['road'] + ', ' + addr['village'] + ', ' + addr['state'] + ', ' + addr['country'] - else: - str = addr['village'] + ', ' + addr['state'] + ', ' + addr['country'] - elif 'county' in addr: - if 'road' in addr: - str = addr['road'] + ', ' + addr['county'] + ', ' + addr['state'] + ', ' + addr['country'] - else: - str = addr['county'] + ', ' + addr['state'] + ', ' + addr['country'] - else: - if 'road' in addr: - str = addr['road'] + ', ' + addr['state'] + ', ' + addr['country'] - else: - str = addr['state'] + ', ' + addr['country'] - else: - if 'city' in addr: - if 'road' in addr: - str = addr['road'] + ', ' + addr['city'] + ', ' + addr['country'] - else: - str = addr['city'] + ', ' + addr['country'] - elif 'village' in addr: - if 'road' in addr: - str = addr['road'] + ', ' + addr['village'] +', ' + addr['country'] - else: - str = addr['village'] + ', ' + addr['country'] - else: - if 'road' in addr: - str = addr['road'] + ', ' + addr['country'] - else: - str = addr['country'] - else: - str = u"Планета Земля, " - if addr['lat'] > 0: - str = str + u"%s с.ш.," % (addr['lat']) - else: - str = str + u"%s ю.ш.," % (-addr['lat']) - if addr['lon'] > 0: - str = str + u"%s в.д." % (addr['lon']) - else: - str = str + u"%s з.д." % (-addr['lon']) - - return str - -class GeoName: - - def __init__(self, lat, lon): - - for elem in GetXMLDescr(lat,lon): - self.addr = {'lat':lat,'lon':lon} - self.road = '' - self.city = '' - self.country = '' - - if elem.tag == "addressparts": - for addrpart in elem: - tag = addrpart.tag - text = urllib2.unquote(addrpart.text) - self.addr[tag] = text - if tag == "road": - self.road = text - elif tag == "city": - self.city = text - elif tag == "country": - self.country = text - - self.printable = printable_name(self.addr) - - \ No newline at end of file diff --git a/pyrungps b/pyrungps deleted file mode 100755 index 314e177..0000000 --- a/pyrungps +++ /dev/null @@ -1 +0,0 @@ -#!/usr/bin/python pyrungps.py diff --git a/pyrungps.py b/pyrungps.py deleted file mode 100644 index ed3fce5..0000000 --- a/pyrungps.py +++ /dev/null @@ -1,416 +0,0 @@ -#!/usr/bin/env python -# coding: UTF-8 - -import requests - -from urllib3.exceptions import InsecureRequestWarning -requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) - -#import sys -import os -from urllib2 import quote,unquote -from lxml import html,etree -from optparse import OptionParser -from datetime import date,datetime -from mx.DateTime.ISO import ParseDateTimeUTC -from parsegpx import write_parsed_to_db,check_db_for_training -import pygpx - -from tempfile import NamedTemporaryFile - -from pprint import pprint - -import render_tiles - -import generate_image - -def get_page(uname,year,month): - - trainings = [] - - req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'}, verify=False) - page = req.text.encode('utf-8') - dom = html.document_fromstring(page) - - for element, attribute, link, pos in dom.iterlinks(): - if attribute == "href": - if link.startswith("/trainings/"): - dummy, dummy, link = link.split('/') - name, id = link.split('_') - trainings.append([ unquote(name), id ]) - - return trainings - -def get_gpx_track(trid,name): - - print "trid=",trid - - req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid), verify=False) - xml = etree.fromstring(req.text.encode('utf-8')) - - return xml - -def get_osm_list(username,password,year,month): - - url = "https://www.openstreetmap.org/api/0.6/user/gpx_files" - - req = requests.get(url,auth=(username,password)) - - xml = etree.fromstring(req.text.encode('utf-8')) - - res=[] - - for gpx_file in xml: - attrib = dict(gpx_file.attrib) - try: - timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp'])) - id = attrib['id'] - description=None - for descr in gpx_file.iter('description'): - description=descr.text - sport=None - for tag in gpx_file.iter('tag'): - sport=tag.text - if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track': - record={ 'id': id, 'sport': sport } - res.append(record) - except: - None - - return res - -def get_osm_gpx(username,password,track_id): - - url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data" - - req = requests.get(url,auth=(username,password)) - xml = etree.fromstring(req.text.encode('utf-8')) - - return xml - -def get_db_gpx(dbx,track_id): - - md, res = dbx.files_download(track_id) - - fs = NamedTemporaryFile(suffix='.tcx',delete = False) - tmp_tcx_name=fs.name - fs.write(res.content) - fs.close() - fs = NamedTemporaryFile(suffix='.gpx',delete = False) - tmp_gpx_name=fs.name - fs.close() - - os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name) - - with open (tmp_gpx_name, "r") as gpxfile: - data=gpxfile.read() - - os.remove(tmp_gpx_name) - os.remove(tmp_tcx_name) - - xml = etree.fromstring(data.encode('utf-8')) - return xml - -def get_dbx_list(dbx,username,year,month): - - gpx_list_id = None - - for entry in dbx.files_list_folder('').entries: - if entry.name == u'Приложения': - for entry_app in dbx.files_list_folder(entry.id).entries: - if entry_app.name == u'Run.GPS': - print "id="+entry_app.id - gpx_list_id=entry_app.id - break - break - - res = [] - - if gpx_list_id: - - for file in dbx.files_list_folder(gpx_list_id).entries: - - filename,ext = os.path.splitext(file.name) - if ext == '.tcx': - try: - fyear = int(filename[0:4]) - fmonth = int(filename[5:7]) - if fyear == year and fmonth == month: - sport = filename[41:] - timestamp = datetime.strptime(filename[0:19],'%Y-%m-%dT%H_%M_%S') - record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport } - res.append(record) - except: - try: - fyear = int(filename[0:4]) - fmonth = int(filename[5:7]) - if fyear == year and fmonth == month: - sport = filename[18:] - timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S') - record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport } - res.append(record) - except: - None - - return res - - -def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False): - - training_list = get_dbx_list(dbx,username,year,month) - - for training in training_list: - - filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:]) - dirname = "%s/%04d/%02d" % (dir,year,month) - - if os.path.exists(filename) and not force: - - if verbose: - print "training %s exists, skipping" % (filename) - - else: - - try: - os.makedirs(dirname) - except: - None - - xml = get_db_gpx(dbx,training['id']) - - if verbose: - print "writing training %s" % (filename) - - gpx = pygpx.GPX() - gpx.ReadTree(xml) - - sport = training['sport'] - timestamp = gpx.tracks[0].start_time() - - print sport, timestamp - - if check_db_for_training(db,sport,timestamp): - print "The same training found" - continue - - gpx.author = username - gpx.FixNames(training['sport']) - gpx.ProcessTrackSegs() - for track in gpx.tracks: - track.sport=training['sport'] - - xml = gpx.XMLTree(); - f = open(filename,"w") - f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) - f.close - write_parsed_to_db(db,gpx,filename) - try: - render_tiles.queue_render(db,filename) - except: - None - - -def sync_osm(username,password,year,month,dir=".",verbose=False,force=False): - - training_list = get_osm_list(username,password,year,month) - - for training in training_list: - - filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id']) - dirname = "%s/%04d/%02d" % (dir,year,month) - - if os.path.exists(filename) and not force: - - if verbose: - print "training %s exists, skipping" % (filename) - - else: - - try: - os.makedirs(dirname) - except: - None - - xml = get_osm_gpx(username,password,training['id']) - - if verbose: - print "writing training %s" % (filename) - - gpx = pygpx.GPX() - gpx.ReadTree(xml) - - sport = training['sport'] - timestamp = gpx.tracks[0].start_time() - - if check_db_for_training(db,sport,timestamp): - print "The same training found" - continue - - gpx.author = username - gpx.FixNames(training['sport']) - gpx.ProcessTrackSegs() - for track in gpx.tracks: - track.sport=training['sport'] - - xml = gpx.XMLTree(); - f = open(filename,"w") - f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) - f.close - write_parsed_to_db(db,gpx,filename) - try: - render_tiles.queue_render(db,filename) - except: - None - -def sync_folder(username,year,month,dir=".",verbose=False,force=False): - - training_list = get_page(username,year,month) - for tr in training_list: - try: - - filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1]) - dirname = "%s/%04d/%02d" % (dir,year,(month+1)) - - if os.path.exists(filename) and not force: - - if verbose: - print "training %s exists, skipping" % (filename) - - else: - - try: - os.makedirs(dirname) - except: - None - - xml=get_gpx_track(tr[1],tr[0]) - - if verbose: - print "writing training %s" % (filename) - - gpx = pygpx.GPX() - gpx.ReadTree(xml) - - sport = tr[0] - timestamp = gpx.tracks[0].start_time() - - if check_db_for_training(db,sport,timestamp): - print "The same training found" - continue - - gpx.FixNames(tr[0]) - gpx.ProcessTrackSegs() - - xml = gpx.XMLTree(); - f = open(filename,"w") - f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) - f.close - write_parsed_to_db(db,gpx,filename) - try: - render_tiles.queue_render(db,filename) - except: - None - - except: - raise - -def main(): - - global db; - parser = OptionParser() - parser.add_option("-d", "--dir", dest="dirname", - help="write data to directory", metavar="DIR") - parser.add_option("-q", "--quiet", - action="store_false", dest="verbose", default=True, - help="don't print status messages to stdout") - parser.add_option("-f", "--force", - action="store_true", dest="force", default=False, - help="rewrite all files") - parser.add_option("-y", "--yearmonth", dest="yearmonth", - help="year and month in YYYY-MM format", metavar="YYYY-MM") - parser.add_option("-u", "--username", dest="username", - help="Run.GPS username") - parser.add_option("-o", "--osm-username", dest="osmname", - help="OpenStreetMap username") - parser.add_option("-p", "--osm-passwd", dest="osmpasswd", - help="OpenStreetMap password") - parser.add_option("-b", "--dropbox", dest="dropboxauth", - help="DropBox Auth token") - (options, args) = parser.parse_args() - - username = options.username - osmname = options.osmname - osmpwd = options.osmpasswd - dbauth = options.dropboxauth - if not (username or (osmname and osmpwd) or dbauth): - print "Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!" - return - - if username: - pusername = username+'@Run.GPS' - elif osmname: - pusername = osmname+'@OSM' - elif dbauth: - import dropbox - dbx = dropbox.Dropbox(dbauth) - pusername = dbx.users_get_current_account().email+'@DBX' - - try: - if options.yearmonth: - year,month = options.yearmonth.split('-') - month = int(month) -1 - year = int(year) - if month<0 or month>11: - raise invalid_number - else: - year = None - month = None - except: - print "Year and month should be in YYYY-MM format!" - return - - if options.dirname: - outdir = options.dirname - else: - outdir = '.' - - db = outdir + '/gpx.db' - - if year: - if options.verbose: - print "retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir) - if username: - sync_folder(username,year,month,outdir,options.verbose,options.force) - elif osmname: - sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force) - elif dbauth: - sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force) - else: - if options.verbose: - print "retrieving two last months for user %s to %s" % (pusername ,outdir) - now = date.today() - current_year = now.year - current_month = now.month - if username: - sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force) - elif osmname: - sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force) - elif dbauth: - sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force) - current_month = current_month -1 - if current_month == 0: - current_month = 12 - current_year = current_year -1 - if username: - sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force) - elif osmname: - sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force) - elif dbauth: - sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force) - - generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-imposm.xml',640,640) - -if __name__ == "__main__": - - main() - \ No newline at end of file diff --git a/pyrungps/__init__.py b/pyrungps/__init__.py new file mode 100644 index 0000000..94b233f --- /dev/null +++ b/pyrungps/__init__.py @@ -0,0 +1,16 @@ +from .render_tiles import deg2num, queue_render, queue_tiles, process_queue +from .parsegpx import check_db_for_training, write_parsed_to_db, write_tree_to_db, print_parsed_file +from .pygeocode import GetDescr, GeoName +from .pygpx import deg2rad, StripPrefix, formattime, Link, GPXTrackPt, GPXRoute, GPXTrackSeg, GPXTrack, GPX +from .pyrungps_sync import get_page, get_gpx_track, get_osm_list, get_osm_gpx, get_db_gpx, get_dbx_list, sync_db, sync_osm, sync_folder +from .generate_image import render_map, render_all + +__all__ = [ + "deg2num", "queue_render", "queue_tiles", "process_queue", + "check_db_for_training", "write_parsed_to_db", "write_tree_to_db", "print_parsed_file", + "GetDescr", "GeoName", + "deg2rad", "StripPrefix", "formattime", "Link", "GPXTrackPt", "GPXRoute", "GPXTrackSeg", "GPXTrack", "GPX", + "get_page", "get_gpx_track", "get_osm_list", "get_osm_gpx", "get_db_gpx", "get_dbx_list", "sync_db", "sync_osm", "sync_folder", + "render_map", "render_all" + ] + \ No newline at end of file diff --git a/pyrungps/generate_image.py b/pyrungps/generate_image.py new file mode 100644 index 0000000..6e1fa3a --- /dev/null +++ b/pyrungps/generate_image.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python + +import mapnik + +import sys, os + +from lxml import html,etree +from optparse import OptionParser + +from pprint import pprint + +import pyrungps.pygpx + +# Set up projections +# spherical mercator (most common target map projection of osm data imported with osm2pgsql) +merc = mapnik.Projection('+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over') + +# long/lat in degrees, aka ESPG:4326 and "WGS 84" +longlat = mapnik.Projection('+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs') +# can also be constructed as: +#longlat = mapnik.Projection('+init=epsg:4326') + +# ensure minimum mapnik version +if not hasattr(mapnik,'mapnik_version') and not mapnik.mapnik_version() >= 600: + raise SystemExit('This script requires Mapnik >=0.6.0)') + + +def render_map(mapfile,map_uri,gpx_file,imgx,imgy): + + with open(gpx_file,'r') as f: + data = f.read() + + xml = etree.fromstring(data) + gpx = pygpx.GPX() + gpx.ReadTree(xml) + + bbox = gpx.bound_box() + pprint(bbox) + cx=(bbox[0][1]+bbox[1][1])/2 + cy=(bbox[0][0]+bbox[1][0])/2 + print(cx,cy) + w=(bbox[1][1]-bbox[0][1])*1.2 + h=(bbox[1][0]-bbox[0][0])*1.2 + print(w,h) + bounds = (cx-w/2,cy-h/2,cx+w/2,cy+h/2) + print(bounds) + + m = mapnik.Map(imgx,imgy) + mapnik.load_map(m,mapfile) + + m.background = mapnik.Color('rgb(255,255,255)') + + # ensure the target map projection is mercator + m.srs = merc.params() + + bbox = mapnik.Box2d(*bounds) + + pprint(bbox) + + transform = mapnik.ProjTransform(longlat,merc) + merc_bbox = transform.forward(bbox) + m.zoom_to_box(merc_bbox) + + style = mapnik.Style() + rule = mapnik.Rule() + + line_symbolizer = mapnik.LineSymbolizer() + + line_symbolizer.stroke = mapnik.Color('rgb(0,0,127)') + line_symbolizer.stroke_width = 4.0 + line_symbolizer.stroke_opacity = 0.5 + + rule.symbols.append(line_symbolizer) + + style.rules.append(rule) + m.append_style('GPS_tracking_points', style) + + layer = mapnik.Layer('GPS_tracking_points') + layer.datasource = mapnik.Ogr(file=gpx_file, layer='tracks') + layer.styles.append('GPS_tracking_points') + m.layers.append(layer) + + im = mapnik.Image(imgx,imgy) + mapnik.render(m, im) + im.save(map_uri,'png') + sys.stdout.write('output image to %s!\n' % map_uri) + + +def render_all(db,mapfile,imgx,imgy): + + import sqlite3 + from os.path import dirname + + conn = sqlite3.connect(db) + conn.text_factory = str + cur = conn.cursor() + updcur = conn.cursor() + + cur.execute ("select id,filename from tracks where preview_name is null") + + list=cur.fetchall() + + for rec in list: + + id = rec[0] + filename = rec[1] + preview_name = dirname(filename) + '/' + str(id) + '.png' + + print(id,filename,preview_name) + + try: + render_map(mapfile,preview_name.encode('utf8'),filename,imgx,imgy) + updcur.execute("update tracks set preview_name=? where id=?", (preview_name,id)) + conn.commit() + except: + raise + + +def main(): + + parser = OptionParser() + parser.add_option("-m", "--map", dest="mapfile", + help="use map file", metavar="MAP") + parser.add_option("-o", "--output", dest="outfile", + help="output image to file", metavar="OUT") + parser.add_option("-g", "--gpx", dest="gpxfile", + help="track to render", metavar="GPX") + parser.add_option("-x", "--x-size", dest="x", + help="image width", metavar="X") + parser.add_option("-y", "--y-size", dest="y", + help="image height", metavar="Y") + parser.add_option("-d", "--db", dest="db", + help="render all not process files in database", metavar="DB") + (options, args) = parser.parse_args() + + if options.mapfile: + mapfile = options.mapfile + else: + mapfile = "/etc/mapnik-osm-carto-data/veloroad-imposm.xml" + + if options.outfile: + map_uri = options.outfile + else: + map_uri = "image.png" + + if options.x: + imgx = int(options.x) + else: + imgx= 640 + if options.y: + imgy = int(options.y) + else: + imgy= 640 + + if options.db: + render_all(options.db,mapfile,imgx,imgy) + else: + if options.gpxfile: + gpx_file = options.gpxfile + render_map(mapfile,map_uri,gpx_file,imgx,imgy) + else: + print("No input file") + raise IOError("No input file") + +if __name__ == "__main__": + + try: + main() + except: + exit(1) diff --git a/pyrungps/parsegpx.py b/pyrungps/parsegpx.py new file mode 100644 index 0000000..c774aaa --- /dev/null +++ b/pyrungps/parsegpx.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +# coding: UTF-8 + +import sys +import os +from lxml import etree +from urllib.parse import unquote +import pyrungps.pygpx +import pyrungps.pygeocode +import sqlite3 +import datetime + +def check_db_for_training(db,sport,timestamp): + + conn = sqlite3.connect(db) + conn.text_factory = str + cur = conn.cursor() + + cur.execute ("select count(*) from tracks where sport=? and start_time=?" , (sport,timestamp)) + return cur.fetchall()[0][0] + +def write_parsed_to_db(db,gpx,filename): + + conn = sqlite3.connect(db) + conn.text_factory = str + cur = conn.cursor() + + cur.execute ("delete from tracks where filename=?" , (filename,)) + + tracks = gpx.tracks + + for track in tracks: + + try: + author = gpx.author + except: + author = None + + try: + name = gpx.name + except: + name = None + + if author: + try: + cur.execute("insert into authors(name,description) values(?,?)", (author,'')) + print("created author %s" % (author)) + except: + print("failed to create author %s" % (author)) + pass + + try: + + print("processing...") + start = track.start() + if start: + printable = pygeocode.GeoName(start.lat,start.lon).printable + start_time = track.start_time() + full_duration = track.full_duration().total_seconds() + distance = track.distance() + filtered_distance = track.filtered_distance(max_speed=50) + ascent = track.elevation_gain() + descent = track.elevation_loss() + ((minlat,minlon),(maxlat,maxlon)) = track.bound_box() + params = ( + gpx.author,name,filename, + track.sport,start_time,full_duration, + distance,filtered_distance,ascent,descent, + start.lat,start.lon, + printable, + minlat,minlon,maxlat,maxlon + ) + print(params) + cur.execute(""" + insert into tracks( + author,name,filename,sport, + start_time,duration, + distance,distance_filtered, + ascent,descent, + lat,lon,printable_location,minlat,minlon,maxlat,maxlon) + values( + ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + """ + , params ) + conn.commit() + print("created track %s" % (filename)) + + except: + + raise + +def write_tree_to_db(db,tree,filename): + + gpx = pygpx.GPX() + gpx.ReadTree(tree) + + write_parsed_to_db(db,gpx,filename) + +def print_parsed_file(filename): + + gpx = pygpx.GPX() + gpx.ReadFile(filename) + + for track in gpx.tracks: + + try: + author = gpx.author + except: + author = None + + try: + name = gpx.name + except: + name = None + + start = track.start() + if start: + printable = pygeocode.GeoName(start.lat,start.lon).printable + start_time = track.start_time() + full_duration = track.full_duration().total_seconds() + distance = track.distance() + filtered_distance = track.filtered_distance(max_speed=50) + ascent = track.elevation_gain() + descent = track.elevation_loss() + ((minlat,minlon),(maxlat,maxlon)) = track.bound_box() + params = ( + gpx.author,name,filename, + track.sport,start_time,full_duration, + distance,filtered_distance,ascent,descent, + start.lat,start.lon, + printable, + minlat,minlon,maxlat,maxlon + ) + print(params) diff --git a/pyrungps/pygeocode.py b/pyrungps/pygeocode.py new file mode 100644 index 0000000..d048622 --- /dev/null +++ b/pyrungps/pygeocode.py @@ -0,0 +1,21 @@ +# coding=UTF-8 +""" +nominatim.openstreetmap.org API +""" + +import urllib3 + +def GetDescr(lat,lon): + + reqstr = "http://maps.rvb.name/geocode.php?lat=%s&lon=%s" %(lat,lon) + headers = { 'User-Agent' : 'PyRunGPS/1.0', 'Referer' : 'https://maps.rvb.name' } + http_pool = urllib3.connection_from_url(reqstr, headers=headers) + req = http_pool.urlopen('GET',reqstr) + return req.data.decode('utf-8','ignore') + +class GeoName: + + def __init__(self, lat, lon): + + self.printable = GetDescr(lat,lon) + \ No newline at end of file diff --git a/pyrungps/pygpx.py b/pyrungps/pygpx.py new file mode 100644 index 0000000..e80077d --- /dev/null +++ b/pyrungps/pygpx.py @@ -0,0 +1,656 @@ +# coding: UTF-8 +#------------------------ +# Работа с GPX-файлами +#------------------------ +from lxml import etree +import os +import math +import datetime +from dateutil.parser import parse +from pprint import pprint + +def deg2rad(deg): +# Преобразование из градусов в радианы. + return deg / (180 / math.pi) + +tagprefix = { '1.0': '{http://www.topografix.com/GPX/1/0}', '1.1': '{http://www.topografix.com/GPX/1/1}'} + +def StripPrefix(tag,version): + nstag=tagprefix[version] + return tag.replace(nstag,'') + +def formattime(dt): + return dt.strftime("%Y-%m-%dT%H:%M:%S")+"."+str(int(dt.microsecond/1000)).zfill(3)+"Z" + +class Link: + # Ссылка. Может присутствовать в точке, треке, файле. + def __init__(self, node, version): + self.version = version + self.href = node.get("href") + self.text = None + self.type = None + if node is not None: + for child in node: + if child.tag == tagprefix[version] + "text": + self.text = child.text + elif child.tag == tagprefix[version] + "type": + self.type = child.text + else: + raise ValueError("Неизвестный тип узла: '%s' " % child.tag) + + def write(self,root): + linknode = etree.SubElement(root,"link"); + linknode.set("href",self.href) + if self.text: + etree.SubElement(linknode,"text").text = self.text + if self.type: + etree.SubElement(linknode,"type").text = self.type + +class GPXTrackPt: + # Точка с координатами. Может присутствовать в последовательности точек в треке, маршруте, а также в списке путевых точек.# + + def __init__(self, node, version): + # Извлекаем данные из GPX-тэгов.# + self.version = version + # Сначала обязательные.# + if node is not None: + self.lat = float(node.get("lat")) + self.lon = float(node.get("lon")) + self.elevation = None + self.time = None + self.speed = None + self.link = None + self.additional_info = {} + # Потом необязательные.# + if node is not None: + for child in node: + if child.tag == tagprefix[version] + "time": + self.time = parse(child.text) + elif child.tag == tagprefix[version] + "ele": + self.elevation = float(child.text) + # Стандартом скорость не предусмотрена, но де-факто в Run.GPS используется.# + elif child.tag == tagprefix[version] + "speed": + self.speed = float(child.text) + elif child.tag == tagprefix[version] + "link": + self.link = Link(child,version) + elif StripPrefix(child.tag,version) in ['magvar','geoidheight','name','cmt','desc', + 'src','sym','type','fix','sat','hdop','vdop','pdop','ageofdgpsdata','dgpsid']: + self.additional_info[StripPrefix(child.tag,version)]=child.text + elif child.tag == tagprefix[version] + "extensions": + pass + else: + raise ValueError("Неизвестный тип узла: '%s'" % child.tag) + + for i in self.additional_info.keys(): + val = self.additional_info[i]; + if val: + if not val.strip(' \n\t'): + del self.additional_info[i] + + + def write(self,root,tag ="wpt"): + wptnode = etree.SubElement(root,tag) + wptnode.set("lat",repr(self.lat)); + wptnode.set("lon",repr(self.lon)); + if self.elevation: + etree.SubElement(wptnode,"ele").text = repr(self.elevation) + if self.time: + etree.SubElement(wptnode,"time").text = formattime(self.time) + if self.speed: + etree.SubElement(wptnode,"speed").text = repr(self.speed) + if self.link: + self.link.write(wptnode) + for i in self.additional_info.keys(): + etree.SubElement(wptnode,i).text = self.additional_info[i] + + def distance(self, other): + # Расстояние между двумя точками на глобусе.# + # http://www.platoscave.net/blog/2009/oct/5/calculate-distance-latitude-longitude-python/ + + radius = 6378700.0 # meters + + lat1, lon1 = self.lat, self.lon + lat2, lon2 = other.lat, other.lon + + dlat = math.radians(lat2-lat1) + dlon = math.radians(lon2-lon1) + a = math.sin(dlat/2) * math.sin(dlat/2) + math.cos(math.radians(lat1)) \ + * math.cos(math.radians(lat2)) * math.sin(dlon/2) * math.sin(dlon/2) + c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a)) + d = radius * c + + return d + + def duration(self, other): + # Время между двумя точками.# + return other.time - self.time + +class GPXRoute: + # Маршрут.# + + def __init__(self, node, version): + self.name = None + self.version = version + self.rtepts = [] + self.additional_info = {} + self.link = None + if node is not None: + for child in node: + if child.tag == tagprefix[version] + "name": + self.name = child.text + elif child.tag == tagprefix[version] + "link": + self.link = Link(child,version) + elif StripPrefix(child.tag,version) in [ "cmt", "desc", "src", "number", "type" ]: + self.additional_info[StripPrefix(child.tag,version)]=child.text + elif child.tag == tagprefix[version] + "rtept": + self.rtepts.append(GPXTrackPt(child, version)) + elif child.tag == tagprefix[version] + "extensions": + pass + else: + raise ValueError("Неизвестный тип узла <%s>" % child.tag) + for i in self.additional_info.keys(): + val = self.additional_info[i]; + if val: + if not val.strip(' \n\t'): + del self.additional_info[i] + + def write(self,root): + rtenode = etree.SubElement(root,"rte") + if self.name: + etree.SubElement(rtenode,"name").text = self.name + if self.link: + self.link.write(rtenode) + for i in self.additional_info.keys(): + print("storing %s " % ( i )) + etree.SubElement(rtenode,i).text = self.additional_info[i] + for i in self.rtepts: + i.write(rtenode,"rtept") + +class GPXTrackSeg: + # Один сегмент трека.# + + def __init__(self, node, version): + self.version = version + self.trkpts = [] + self.elevation_gain = 0.0 + self.elevation_loss = 0.0 + if node is not None: + for child in node: + if child.tag == tagprefix[version] + "trkpt": + self.trkpts.append(GPXTrackPt(child, self.version)) + else: + raise ValueError("Неизвестный тип узла <%s>" % node.nodeName) + self._get_elevation() + + def write(self,root): + trksegnode = etree.SubElement(root,"trkseg") + for i in self.trkpts: + i.write(trksegnode,"trkpt") + + def _get_elevation(self): + + elev_data = [] + for pt in self.trkpts: + if pt.elevation: + elev_data.append(pt.elevation) + + gain = 0.0 + loss = 0.0 + last_elevation = None + + window_size = 5 + i = 0 + moving_averages = [] + + while i < len(elev_data) - window_size + 1: + this_window = elev_data[i : i + window_size] + window_average = sum(this_window) / window_size + moving_averages.append(window_average) + i += 1 + + if len(moving_averages)>2: + elev_data = moving_averages + + for pt in elev_data: + if last_elevation is not None: + try: + if pt > last_elevation: + gain += pt - last_elevation + else: + loss += last_elevation - pt + except: + pass + last_elevation=pt + + self.elevation_gain = gain + self.elevation_loss = loss + + def distance(self): + # Расчет длины сегмента.# + _length = 0.0 + last_pt = None + for pt in self.trkpts: + if last_pt is not None: + _length += last_pt.distance(pt) + last_pt = pt + return _length + + def filtered_distance(self,max_speed): + # Расчет длины сегмента с фильтрацией предположительно сбойных участков.# + _length = 0.0 + last_pt = None + for pt in self.trkpts: + if last_pt is not None: + _delta = last_pt.distance(pt) + _time_delta = pt.time - last_pt.time + _time_delta_s = _time_delta.days*86400 + _time_delta.seconds + _time_delta.microseconds/1000000 + if _time_delta_s > 0: + if _delta/_time_delta_s < max_speed: + _length += _delta + else: + _length += _delta + last_pt = pt + return _length + + def duration(self): + # Расчет продолжительности сегмента.# + return self.trkpts[0].duration(self.trkpts[-1]) + + def bound_box(self): + # Обрамляющий прямоугольник для сегмента.# + minlat =360 + maxlat = -360 + minlon = 360 + maxlon = -360 + for pt in self.trkpts: + if pt.latmaxlat: + maxlat=pt.lat + if pt.lonmaxlon: + maxlon=pt.lon + return ((minlat,minlon),(maxlat,maxlon)) + +class GPXTrack: + # Трек.# + + def __init__(self, node, version): + # Создаем трек из GPX-данных.# + + self.version = version + self.trksegs = [] + self.sport= None + self.additional_info = {} + self.name = None + self.link = None + + if node is not None: + for child in node: + if child.tag == tagprefix[version] + "name": + self.name = child.text + elif child.tag == tagprefix[version] + "trkseg": + if len(child) > 0: + self.trksegs.append(GPXTrackSeg(child, self.version)) + elif child.tag == tagprefix[version] + "link": + self.link = Link(child,version) + elif StripPrefix(child.tag,version) in [ "number", "desc", "cmt", "src", "type" ]: + self.additional_info[StripPrefix(child.tag,version)] = child.text + elif child.tag == tagprefix[version] + "extensions": + for ext in child: + # Из расширений извлекаем вид спорта - специфично для Run.GPS.# + if ext.tag == tagprefix[version] + "sport": + self.sport = ext.text + + for i in self.additional_info.keys(): + val = self.additional_info[i]; + if val: + if not val.strip(' \n\t'): + del self.additional_info[i] + + def write(self,root): + trknode = etree.SubElement(root,"trk") + if self.name: + etree.SubElement(trknode,"name").text = self.name + if self.sport: + etree.SubElement(etree.SubElement(trknode,"extensions"),"sport").text=self.sport + if self.link: + self.link.write(trknode) + for i in self.additional_info.keys(): + etree.SubElement(trknode,i).text = self.additional_info[i] + for i in self.trksegs: + i.write(trknode) + + def elevation_gain(self): + # Набор высоты по треку.# + return sum([trkseg.elevation_gain for trkseg in self.trksegs]) + + def elevation_loss(self): + # Сброс высоты по треку.# + return sum([trkseg.elevation_loss for trkseg in self.trksegs]) + + def distance(self): + # Длина трека.# + try: + return sum([trkseg.distance() for trkseg in self.trksegs]) + except IndexError: + print("Пустой сегмент трека, расчет длины невозможен.") + + def filtered_distance(self,max_speed=100): + # Длина трека с фильтрацией сбойных участков.# + try: + return sum([trkseg.filtered_distance(max_speed=max_speed) for trkseg in self.trksegs]) + except IndexError: + print("Пустой сегмент трека, расчет длины невозможен.") + + def duration(self): + # Продолжительность трека, не включая паузы между сегментами.# + dur = datetime.timedelta(0) + for trkseg in self.trksegs: + try: + dur += trkseg.duration() + except: + pass + return dur + + def full_duration(self): + # Продолжительность трека, включая паузы между сегментами.# + try: + return self.start().duration(self.end()) + except: + return None + + def start(self): + # Стартовая точка.# + try: + return self.trksegs[0].trkpts[0] + except IndexError: + return None + + def end(self): + # Финишная точка.# + try: + return self.trksegs[-1].trkpts[-1] + except IndexError: + return None + + def start_time(self): + # Время старта.# + try: + return self.start().time + except: + return None + + def end_time(self): + # Время финиша.# + try: + return self.end().time + except: + return None + + def bound_box(self): + # Обрамляющий прямоугольник для всего трека.# + minlat =360 + maxlat = -360 + minlon = 360 + maxlon = -360 + for trkseg in self.trksegs: + for pt in trkseg.trkpts: + if pt.latmaxlat: + maxlat=pt.lat + if pt.lonmaxlon: + maxlon=pt.lon + return ((minlat,minlon),(maxlat,maxlon)) + + +class GPX: + # Работа с GPX-документами.# + + def __init__(self): + # Создание пустого GPX-объекта# + PATH = os.path.dirname(__file__) + self.creator = None + self.time = None + self.tracks = [] + self.routes = [] + self.waypoints = [] + self.version = "" + self.author = None + self.name = None + self.link = None + self.copyright = None + self.meta = {} + + def ReadTree(self,tree): + # Загрузка из дерева etree.# + root = tree + + # Test if this is a GPX file or not and if it's version 1.1 + if root.tag == "{http://www.topografix.com/GPX/1/1}gpx" and root.get("version") == "1.1": + self.version = "1.1" + elif root.tag == "{http://www.topografix.com/GPX/1/0}gpx" and root.get("version") == "1.0": + self.version = "1.0" + elif root.get("version") not in [ "1.1", "1.0"]: + raise ValueError("Версия формата %s не поддерживается. Используйте GPX 1.1." % root.get("version")) + else: + raise ValueError("Неизвестный формат дерева") + # attempt to validate the xml file against the schema + + # initalize the GPX document for parsing. + self._init_version(root) + + + def ReadFile(self, fd): + # Загрузка из файла.# + gpx_doc = etree.parse(fd) + root = gpx_doc.getroot() + + # Test if this is a GPX file or not and if it's version 1.1 + if root.tag == "{http://www.topografix.com/GPX/1/1}gpx" and root.get("version") == "1.1": + self.version = "1.1" + elif root.tag == "{http://www.topografix.com/GPX/1/0}gpx" and root.get("version") == "1.0": + self.version = "1.0" + elif root.get("version") not in [ "1.1", "1.0"]: + raise ValueError("Версия формата %s не поддерживается. Используйте GPX 1.1." % root.get("version")) + else: + raise ValueError("Неизвестный формат файла") + # initalize the GPX document for parsing. + self._init_version(root) + + def _init_version(self,root): + # Инициализация объекта.# + self.creator = root.get("creator") + if root is not None: + for child in root: + if child.tag == tagprefix[self.version] + "author": + self.author = child.text + elif child.tag == tagprefix[self.version] + "trk": + self.tracks.append(GPXTrack(child, self.version)) + elif child.tag == tagprefix[self.version] + "rte": + self.routes.append(GPXRoute(child, self.version)) + elif child.tag == tagprefix[self.version] + "wpt": + self.waypoints.append(GPXTrackPt(child, self.version)) + elif child.tag == tagprefix[self.version] + "metadata": + for meta in child: + if meta.tag == tagprefix[self.version] + "author": + self.author = meta.text + elif meta.tag == tagprefix[self.version] + "name": + self.name = meta.text + elif meta.tag == tagprefix[self.version] + "bounds": + # Границы пропускаем - их будем вычислять сами заново + pass + elif meta.tag == tagprefix[self.version] + "copyright": + self.copyright = meta.text + elif meta.tag == tagprefix[self.version] + "link": + self.link = Link(meta,self.version) + else: + self.meta[StripPrefix(meta.tag,self.version)]=meta.text + + for i in self.meta.keys(): + val = self.meta[i]; + if val: + if not val.strip(' \n\t'): + del self.meta[i] + + def elevation_gain(self): + # Суммарный набор высоты.# + return sum([track.elevation_gain() for track in self.tracks]) + + def elevation_loss(self): + # Суммарная потеря высоты.# + return sum([track.elevation_loss() for track in self.tracks]) + + def distance(self): + # Суммарная дистанция.# + try: + return sum([track.distance() for track in self.tracks]) + except IndexError: + print("Пустой файл!") + + def filtered_distance(self): + # Суммарная дистанция с фильтрацией сбойных данных.# + try: + return sum([track.filtered_distance() for track in self.tracks]) + except IndexError: + print("Пустой файл!") + + def duration(self): + # Суммарная продолжительность, не включая паузы.# + dur = datetime.timedelta(0) + for track in self.tracks: + dur += track.duration() + return dur + + def full_duration(self): + # Суммарная продолжительность, включая паузы# + if hasattr(self.start(), 'duration'): + return self.start().duration(self.end()) + else: + return None + + def start(self): + # Точка старта.# + return self.tracks[0].start() + + def end(self): + # Точка финиша.# + return self.tracks[-1].end() + + def start_time(self): + # Время старта.# + if self.start(): + return self.start().time + else: + return None + + def end_time(self): + # Время финиша.# + return self.end().time + + def bound_box(self): + # Обрамляющий прямоугольник для всех треков в файле.# + minlat =360 + maxlat = -360 + minlon = 360 + maxlon = -360 + for track in self.tracks: + for trkseg in track.trksegs: + for pt in trkseg.trkpts: + if pt.latmaxlat: + maxlat=pt.lat + if pt.lonmaxlon: + maxlon=pt.lon + return ((minlat,minlon),(maxlat,maxlon)) + + def FixNames(self,linkname): + pprint(linkname) + pprint(type(linkname)) + goodname = linkname + badname = goodname.encode('ascii','replace') + if self.name and self.name.startswith(badname): + self.name=self.name.replace(badname,goodname) + for i in self.tracks: + if i.name and i.name.startswith(badname): + i.name = i.name.replace(badname,goodname) + + def ProcessTrackSegs(self,seconds_gap=120): + for tr in self.tracks: + # Объединяем все точки трека в единую последовательность + all_trackpoints = [] + for trseg in tr.trksegs: + all_trackpoints.extend(trseg.trkpts) + new_segs = [] + current_seg = None + prev_pt = None + for pt in all_trackpoints: + if not current_seg: + # Начало нового сегмента + current_seg = GPXTrackSeg(None,self.version) + current_seg.trkpts.append(pt) + else: + delta = pt.time - prev_pt.time + if delta.days > 0 or delta.seconds > seconds_gap: + # Нашли место разрыва + new_segs.append(current_seg) + current_seg = None + else: + current_seg.trkpts.append(pt) + prev_pt=pt + # Если есть, что закинуть в хвост - пишем + if current_seg is not None: + new_segs.append(current_seg) + # Обработали + for trs in new_segs: + trs._get_elevation() + tr.trksegs = new_segs + + + def XMLTree(self): + root = etree.Element("gpx", attrib ={"creator": "pygpx", + "version": "1.1", + "xmlns": "http://www.topografix.com/GPX/1/1"}); + + # meta subtree + + meta = etree.SubElement( root, "metadata" ); + if self.name: + metarecord = etree.SubElement( meta, "name" ) + metarecord.text = self.name + if self.author: + metarecord = etree.SubElement( meta, "author" ) + metarecord.text = self.author + if self.copyright: + metarecord = etree.SubElement( meta, "copyright" ) + metarecord.text = self.copyright + if self.link: + self.link.write(meta) + for metatag in self.meta.keys(): + metarecord = etree.SubElement( meta, metatag ) + metarecord.text = self.meta[metatag] + + # wpts subtree + + for i in self.waypoints: + i.write(root) + + # routes subtree + + for i in self.routes: + i.write(root) + + # tracks subtree + + for i in self.tracks: + i.write(root) + + return root + \ No newline at end of file diff --git a/pyrungps/pyrungps_sync.py b/pyrungps/pyrungps_sync.py new file mode 100644 index 0000000..b87a17e --- /dev/null +++ b/pyrungps/pyrungps_sync.py @@ -0,0 +1,416 @@ +#!/usr/bin/env python +# coding: UTF-8 + +import requests + +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +#import sys +import os +from urllib.parse import quote,unquote +from lxml import html,etree +from optparse import OptionParser +from datetime import date,datetime +from dateutil.parser import isoparse +from pyrungps.parsegpx import write_parsed_to_db,check_db_for_training +import pyrungps.pygpx + +from tempfile import NamedTemporaryFile + +from pprint import pprint + +import pyrungps.render_tiles + +import pyrungps.generate_image + +def get_page(uname,year,month): + + trainings = [] + + req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'}, verify=False) + page = req.text.encode('utf-8') + dom = html.document_fromstring(page) + + for element, attribute, link, pos in dom.iterlinks(): + if attribute == "href": + if link.startswith("/trainings/"): + dummy, dummy, link = link.split('/') + name, id = link.split('_') + trainings.append([ unquote(name), id ]) + + return trainings + +def get_gpx_track(trid,name): + + print("trid=",trid) + + req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid), verify=False) + xml = etree.fromstring(req.text.encode('utf-8')) + + return xml + +def get_osm_list(username,password,year,month): + + url = "https://www.openstreetmap.org/api/0.6/user/gpx_files" + + req = requests.get(url,auth=(username,password)) + + xml = etree.fromstring(req.text.encode('utf-8')) + + res=[] + + for gpx_file in xml: + attrib = dict(gpx_file.attrib) + try: + timestamp=datetime.fromtimestamp(isoparse(attrib['timestamp'])) + id = attrib['id'] + description=None + for descr in gpx_file.iter('description'): + description=descr.text + sport=None + for tag in gpx_file.iter('tag'): + sport=tag.text + if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track': + record={ 'id': id, 'sport': sport } + res.append(record) + except: + None + + return res + +def get_osm_gpx(username,password,track_id): + + url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data" + + req = requests.get(url,auth=(username,password)) + xml = etree.fromstring(req.text.encode('utf-8')) + + return xml + +def get_db_gpx(dbx,track_id): + + md, res = dbx.files_download(track_id) + + fs = NamedTemporaryFile(suffix='.tcx',delete = False) + tmp_tcx_name=fs.name + fs.write(res.content) + fs.close() + fs = NamedTemporaryFile(suffix='.gpx',delete = False) + tmp_gpx_name=fs.name + fs.close() + + os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name) + + with open (tmp_gpx_name, "r") as gpxfile: + data=gpxfile.read() + + os.remove(tmp_gpx_name) + os.remove(tmp_tcx_name) + + xml = etree.fromstring(data.encode('utf-8')) + return xml + +def get_dbx_list(dbx,username,year,month): + + gpx_list_id = None + + for entry in dbx.files_list_folder('').entries: + if entry.name == u'Приложения': + for entry_app in dbx.files_list_folder(entry.id).entries: + if entry_app.name == u'Run.GPS': + print("id="+entry_app.id) + gpx_list_id=entry_app.id + break + break + + res = [] + + if gpx_list_id: + + for file in dbx.files_list_folder(gpx_list_id).entries: + + filename,ext = os.path.splitext(file.name) + if ext == '.tcx': + try: + fyear = int(filename[0:4]) + fmonth = int(filename[5:7]) + if fyear == year and fmonth == month: + sport = filename[41:] + timestamp = datetime.strptime(filename[0:19],'%Y-%m-%dT%H_%M_%S') + record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport } + res.append(record) + except: + try: + fyear = int(filename[0:4]) + fmonth = int(filename[5:7]) + if fyear == year and fmonth == month: + sport = filename[18:] + timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S') + record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport } + res.append(record) + except: + None + + return res + + +def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False): + + training_list = get_dbx_list(dbx,username,year,month) + + for training in training_list: + + filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:]) + dirname = "%s/%04d/%02d" % (dir,year,month) + + if os.path.exists(filename) and not force: + + if verbose: + print("training %s exists, skipping" % (filename)) + + else: + + try: + os.makedirs(dirname) + except: + None + + xml = get_db_gpx(dbx,training['id']) + + if verbose: + print("writing training %s" % (filename)) + + gpx = pygpx.GPX() + gpx.ReadTree(xml) + + sport = training['sport'] + timestamp = gpx.tracks[0].start_time() + + print(sport, timestamp) + + if check_db_for_training(db,sport,timestamp): + print("The same training found") + continue + + gpx.author = username + gpx.FixNames(training['sport']) + gpx.ProcessTrackSegs() + for track in gpx.tracks: + track.sport=training['sport'] + + xml = gpx.XMLTree(); + f = open(filename,"wb") + f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) + f.close + write_parsed_to_db(db,gpx,filename) + try: + render_tiles.queue_render(db,filename) + except: + None + + +def sync_osm(username,password,year,month,dir=".",verbose=False,force=False): + + training_list = get_osm_list(username,password,year,month) + + for training in training_list: + + filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id']) + dirname = "%s/%04d/%02d" % (dir,year,month) + + if os.path.exists(filename) and not force: + + if verbose: + print("training %s exists, skipping" % (filename)) + + else: + + try: + os.makedirs(dirname) + except: + None + + xml = get_osm_gpx(username,password,training['id']) + + if verbose: + print("writing training %s" % (filename)) + + gpx = pygpx.GPX() + gpx.ReadTree(xml) + + sport = training['sport'] + timestamp = gpx.tracks[0].start_time() + + if check_db_for_training(db,sport,timestamp): + print("The same training found") + continue + + gpx.author = username + gpx.FixNames(training['sport']) + gpx.ProcessTrackSegs() + for track in gpx.tracks: + track.sport=training['sport'] + + xml = gpx.XMLTree(); + f = open(filename,"wb") + f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) + f.close + write_parsed_to_db(db,gpx,filename) + try: + render_tiles.queue_render(db,filename) + except: + None + +def sync_folder(username,year,month,dir=".",verbose=False,force=False): + + training_list = get_page(username,year,month) + for tr in training_list: + try: + + filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1]) + dirname = "%s/%04d/%02d" % (dir,year,(month+1)) + + if os.path.exists(filename) and not force: + + if verbose: + print("training %s exists, skipping" % (filename)) + + else: + + try: + os.makedirs(dirname) + except: + None + + xml=get_gpx_track(tr[1],tr[0]) + + if verbose: + print("writing training %s" % (filename)) + + gpx = pygpx.GPX() + gpx.ReadTree(xml) + + sport = tr[0] + timestamp = gpx.tracks[0].start_time() + + if check_db_for_training(db,sport,timestamp): + print("The same training found") + continue + + gpx.FixNames(tr[0]) + gpx.ProcessTrackSegs() + + xml = gpx.XMLTree(); + f = open(filename,"wb") + f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True)) + f.close + write_parsed_to_db(db,gpx,filename) + try: + render_tiles.queue_render(db,filename) + except: + None + + except: + raise + +def main(): + + global db; + parser = OptionParser() + parser.add_option("-d", "--dir", dest="dirname", + help="write data to directory", metavar="DIR") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + parser.add_option("-f", "--force", + action="store_true", dest="force", default=False, + help="rewrite all files") + parser.add_option("-y", "--yearmonth", dest="yearmonth", + help="year and month in YYYY-MM format", metavar="YYYY-MM") + parser.add_option("-u", "--username", dest="username", + help="Run.GPS username") + parser.add_option("-o", "--osm-username", dest="osmname", + help="OpenStreetMap username") + parser.add_option("-p", "--osm-passwd", dest="osmpasswd", + help="OpenStreetMap password") + parser.add_option("-b", "--dropbox", dest="dropboxauth", + help="DropBox Auth token") + (options, args) = parser.parse_args() + + username = options.username + osmname = options.osmname + osmpwd = options.osmpasswd + dbauth = options.dropboxauth + if not (username or (osmname and osmpwd) or dbauth): + print("Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!") + return + + if username: + pusername = username+'@Run.GPS' + elif osmname: + pusername = osmname+'@OSM' + elif dbauth: + import dropbox + dbx = dropbox.Dropbox(dbauth) + pusername = dbx.users_get_current_account().email+'@DBX' + + try: + if options.yearmonth: + year,month = options.yearmonth.split('-') + month = int(month) -1 + year = int(year) + if month<0 or month>11: + raise invalid_number + else: + year = None + month = None + except: + print("Year and month should be in YYYY-MM format!") + return + + if options.dirname: + outdir = options.dirname + else: + outdir = '.' + + db = outdir + '/gpx.db' + + if year: + if options.verbose: + print("retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir)) + if username: + sync_folder(username,year,month,outdir,options.verbose,options.force) + elif osmname: + sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force) + elif dbauth: + sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force) + else: + if options.verbose: + print("retrieving two last months for user %s to %s" % (pusername ,outdir)) + now = date.today() + current_year = now.year + current_month = now.month + if username: + sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force) + elif osmname: + sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force) + elif dbauth: + sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force) + current_month = current_month -1 + if current_month == 0: + current_month = 12 + current_year = current_year -1 + if username: + sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force) + elif osmname: + sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force) + elif dbauth: + sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force) + + generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-imposm.xml',640,640) + +if __name__ == "__main__": + + main() + \ No newline at end of file diff --git a/pyrungps/render_tiles.py b/pyrungps/render_tiles.py new file mode 100755 index 0000000..ea342e8 --- /dev/null +++ b/pyrungps/render_tiles.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python +# coding: UTF-8 + +import sqlite3 +import math +from pprint import pprint + +def queue_render(db,filename,forced_max_zoom=None): + + conn = sqlite3.connect(db) + conn.text_factory = str + cur = conn.cursor() + + cur.execute("select minlat,minlon,maxlat,maxlon from tracks where filename=?" , (filename,)) + minlat,minlon,maxlat,maxlon=cur.fetchone() + queue_tiles(db,minlat,minlon,maxlat,maxlon,forced_max_zoom) + +def deg2num(lat_deg, lon_deg, zoom): + lat_rad = math.radians(lat_deg) + n = 2.0 ** zoom + xtile = int((lon_deg + 180.0) / 360.0 * n) + ytile = int((1.0 - math.log(math.tan(lat_rad) + (1 / math.cos(lat_rad))) / math.pi) / 2.0 * n) + return (xtile, ytile) + +def queue_tiles(db,minlat,minlon,maxlat,maxlon,forced_max_zoom=None): + + conn = sqlite3.connect(db) + + # определяем примерный стартовый зум + + minzoom=8 + + if forced_max_zoom: + maxzoom=forced_max_zoom + else: + maxzoom=minzoom + while True: + minx,miny=deg2num(minlat,minlon,maxzoom) + maxx,maxy=deg2num(maxlat,maxlon,maxzoom) + + print(maxzoom,':',minx,'-',maxx,'/',miny,'-',maxy) + if (maxx-minx>16) or (maxy-miny>12) or (maxzoom==16): + break + else: + maxzoom=maxzoom+1 + + if maxzoommaxx: + tx=minx + maxx=minx + minx=tx + + if miny>maxy: + ty=miny + maxy=miny + miny=ty + + print(zoom,minx,miny,maxx,maxy) + + maps = map.split(',') + + for map_name in maps: + + command = 'render_list -a -m '+map+ \ + ' -z '+str(zoom)+' -Z '+str(zoom)+ \ + ' -x '+str(minx)+' -X '+str(maxx)+ \ + ' -y '+str(miny)+' -Y '+str(maxy) + if force: + command = command+ ' --force' + + print(command ) + + if not system(command) == 0: + return + + dcur=conn.cursor() + dcur.execute('delete from render_queue where id=?',(id,)) + conn.commit() + +def main(): + + from optparse import OptionParser + + parser = OptionParser() + parser.add_option("-d", "--data", dest="directory", + help="Data directory", metavar="DIR") + parser.add_option("-m", "--map", dest="map", + help="Map name", metavar="MAP") + parser.add_option("-z", "--zoom", dest="zoom", + help="Maximal zoom (forced), used with coordinates pairs (minlat minlon maxlat maxlon) or filename in arguments", metavar="MAP") + parser.add_option("-f", "--force", dest="force", + help="Force tile regeneration (on/off), default off") + parser.add_option("-r", "--renderer", dest="renderer", + help="Rendering backend: tirex or renderd") + (options, args) = parser.parse_args() + + db=options.directory+'/gpx.db' + map=options.map + zoom=options.zoom + if not zoom: + zoom=12 + force=(options.force=='on') + + if options.renderer: + renderer=options.renderer + else: + print("Using default rendering backend...") + renderer="default" + + if len(args)==1: + filename,=args + print("Rendering file: "+filename+"\n") + queue_render(db,filename) + elif len(args)==4: + minlat,minlon,maxlat,maxlon=args + print("Rendering region "+minlat+'..'+maxlat+' / '+minlon+'..'+maxlon+"\n") + queue_tiles(db,float(minlat),float(minlon),float(maxlat),float(maxlon),int(zoom)) + + if map: + print("Processing map(s) "+map+"\n") + process_queue(db,map,force,renderer) + +if __name__ == "__main__": + + main() + \ No newline at end of file diff --git a/pyrungps_sync b/pyrungps_sync new file mode 100755 index 0000000..314e177 --- /dev/null +++ b/pyrungps_sync @@ -0,0 +1 @@ +#!/usr/bin/python pyrungps.py diff --git a/render_tiles.py b/render_tiles.py deleted file mode 100755 index 443ae80..0000000 --- a/render_tiles.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python -# coding: UTF-8 - -import sqlite3 -import math -from pprint import pprint - -def queue_render(db,filename,forced_max_zoom=None): - - conn = sqlite3.connect(db) - conn.text_factory = str - cur = conn.cursor() - - cur.execute("select minlat,minlon,maxlat,maxlon from tracks where filename=?" , (filename.decode('UTF-8'),)) - minlat,minlon,maxlat,maxlon=cur.fetchone() - queue_tiles(db,minlat,minlon,maxlat,maxlon,forced_max_zoom) - -def deg2num(lat_deg, lon_deg, zoom): - lat_rad = math.radians(lat_deg) - n = 2.0 ** zoom - xtile = int((lon_deg + 180.0) / 360.0 * n) - ytile = int((1.0 - math.log(math.tan(lat_rad) + (1 / math.cos(lat_rad))) / math.pi) / 2.0 * n) - return (xtile, ytile) - -def queue_tiles(db,minlat,minlon,maxlat,maxlon,forced_max_zoom=None): - - conn = sqlite3.connect(db) - - # определяем примерный стартовый зум - - minzoom=8 - - if forced_max_zoom: - maxzoom=forced_max_zoom - else: - maxzoom=minzoom - while True: - minx,miny=deg2num(minlat,minlon,maxzoom) - maxx,maxy=deg2num(maxlat,maxlon,maxzoom) - - print maxzoom,':',minx,'-',maxx,'/',miny,'-',maxy - if (maxx-minx>16) or (maxy-miny>12) or (maxzoom==16): - break - else: - maxzoom=maxzoom+1 - - if maxzoommaxx: - tx=minx - maxx=minx - minx=tx - - if miny>maxy: - ty=miny - maxy=miny - miny=ty - - print zoom,minx,miny,maxx,maxy - - maps = map.split(',') - - for map_name in maps: - - command = 'render_list -a -m '+map+ \ - ' -z '+str(zoom)+' -Z '+str(zoom)+ \ - ' -x '+str(minx)+' -X '+str(maxx)+ \ - ' -y '+str(miny)+' -Y '+str(maxy) - if force: - command = command+ ' --force' - - print command - - if system(command)<>0: - return - - dcur=conn.cursor() - dcur.execute('delete from render_queue where id=?',(id,)) - conn.commit() - -def main(): - - from optparse import OptionParser - - parser = OptionParser() - parser.add_option("-d", "--data", dest="directory", - help="Data directory", metavar="DIR") - parser.add_option("-m", "--map", dest="map", - help="Map name", metavar="MAP") - parser.add_option("-z", "--zoom", dest="zoom", - help="Maximal zoom (forced), used with coordinates pairs (minlat minlon maxlat maxlon) or filename in arguments", metavar="MAP") - parser.add_option("-f", "--force", dest="force", - help="Force tile regeneration (on/off), default off") - parser.add_option("-r", "--renderer", dest="renderer", - help="Rendering backend: tirex or renderd") - (options, args) = parser.parse_args() - - db=options.directory+'/gpx.db' - map=options.map - zoom=options.zoom - if not zoom: - zoom=12 - force=(options.force=='on') - - if options.renderer: - renderer=options.renderer - else: - print "Using default rendering backend..." - renderer="default" - - if len(args)==1: - filename,=args - print "Rendering file: "+filename+"\n" - queue_render(db,filename) - elif len(args)==4: - minlat,minlon,maxlat,maxlon=args - print "Rendering region "+minlat+'..'+maxlat+' / '+minlon+'..'+maxlon+"\n" - queue_tiles(db,float(minlat),float(minlon),float(maxlat),float(maxlon),int(zoom)) - - if map: - print "Processing map(s) "+map+"\n" - process_queue(db,map,force,renderer) - -if __name__ == "__main__": - - main() - \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..1ccb8be --- /dev/null +++ b/setup.py @@ -0,0 +1,13 @@ +from setuptools import setup, find_packages + +setup( + name='pyrungps', + version='0.1', + description='Simple python tool to sync gpx tracks from GPS-SPORT site', + url='http://rvb.name/gpx', + author='Roman Bazalevskiy', + author_email='rvb@rvb.name', + license='WTFPL?', + packages=['pyrungps'], + zip_safe=False +)