#!/usr/bin/env python
# coding: UTF-8
-import urllib2
+import requests
+
#import sys
import os
+from urllib2 import quote,unquote
from lxml import html,etree
from optparse import OptionParser
from datetime import date,datetime
from mx.DateTime.ISO import ParseDateTimeUTC
-from parsegpx import write_parsed_to_db
+from parsegpx import write_parsed_to_db,check_db_for_training
import pygpx
+from tempfile import NamedTemporaryFile
+
+from pprint import pprint
+
import render_tiles
import generate_image
trainings = []
- req = urllib2.Request("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), None, {'User-agent': 'Mozilla/5.0'})
- page = urllib2.urlopen(req).read()
+ req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'})
+ page = req.text.encode()
dom = html.document_fromstring(page)
for element, attribute, link, pos in dom.iterlinks():
if link.startswith("/trainings/"):
dummy, dummy, link = link.split('/')
name, id = link.split('_')
- trainings.append([ urllib2.unquote(name), id ])
+ trainings.append([ unquote(name), id ])
return trainings
print "trid=",trid
- req = urllib2.urlopen("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid))
-
- xml = etree.parse(req)
+ req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid))
+ xml = etree.fromstring(req.text.encode())
return xml
def get_osm_list(username,password,year,month):
url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
- passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
- passman.add_password(None, url, username, password)
- authhandler = urllib2.HTTPBasicAuthHandler(passman)
- opener = urllib2.build_opener(authhandler)
- urllib2.install_opener(opener)
- req = urllib2.urlopen(url)
- xml = etree.parse(req)
+ req = requests.get(url,auth=(username,password))
+
+ xml = etree.fromstring(req.text.encode())
res=[]
- for gpx_file in xml.getroot():
+ for gpx_file in xml:
attrib = dict(gpx_file.attrib)
- timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
- id = attrib['id']
- description=None
- for descr in gpx_file.iter('description'):
- description=descr.text
- sport=None
- for tag in gpx_file.iter('tag'):
- sport=tag.text
- if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
- print id,description,sport,timestamp
- record={ 'id': id, 'sport': sport }
- res.append(record)
+ try:
+ timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
+ id = attrib['id']
+ description=None
+ for descr in gpx_file.iter('description'):
+ description=descr.text
+ sport=None
+ for tag in gpx_file.iter('tag'):
+ sport=tag.text
+ if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
+ record={ 'id': id, 'sport': sport }
+ res.append(record)
+ except:
+ None
- return res
+ return res
def get_osm_gpx(username,password,track_id):
url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
- passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
- passman.add_password(None, url, username, password)
- authhandler = urllib2.HTTPBasicAuthHandler(passman)
- opener = urllib2.build_opener(authhandler)
- urllib2.install_opener(opener)
- req = urllib2.urlopen(url)
- xml = etree.parse(req)
+ req = requests.get(url,auth=(username,password))
+ xml = etree.fromstring(req.text.encode())
return xml
+def get_db_gpx(dbx,track_id):
+
+ md, res = dbx.files_download(track_id)
+
+ fs = NamedTemporaryFile(suffix='.tcx',delete = False)
+ tmp_tcx_name=fs.name
+ fs.write(res.content)
+ fs.close()
+ fs = NamedTemporaryFile(suffix='.gpx',delete = False)
+ tmp_gpx_name=fs.name
+ fs.close()
+
+ os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name)
+
+ with open (tmp_gpx_name, "r") as gpxfile:
+ data=gpxfile.read()
+
+ os.remove(tmp_gpx_name)
+ os.remove(tmp_tcx_name)
+
+ xml = etree.fromstring(data.encode())
+ return xml
+
+def get_dbx_list(dbx,username,year,month):
+
+ gpx_list_id = None
+
+ for entry in dbx.files_list_folder('').entries:
+ if entry.name == u'Приложения':
+ for entry_app in dbx.files_list_folder(entry.id).entries:
+ if entry_app.name == u'Run.GPS':
+ gpx_list_id=entry_app.id
+ break
+ break
+
+ res = []
+
+ if gpx_list_id:
+
+ for file in dbx.files_list_folder(gpx_list_id).entries:
+
+ filename,ext = os.path.splitext(file.name)
+ if ext == '.tcx':
+ try:
+ fyear = int(filename[0:4])
+ fmonth = int(filename[5:7])
+ if fyear == year and fmonth == month:
+ sport = filename[18:]
+ timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S')
+ record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
+ res.append(record)
+ except:
+ None
+
+ return res
+
+
+def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False):
+
+ training_list = get_dbx_list(dbx,username,year,month)
+
+ for training in training_list:
+
+ filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:])
+ dirname = "%s/%04d/%02d" % (dir,year,month)
+
+ if os.path.exists(filename) and not force:
+
+ if verbose:
+ print "training %s exists, skipping" % (filename)
+
+ else:
+
+ try:
+ os.makedirs(dirname)
+ except:
+ None
+
+ xml = get_db_gpx(dbx,training['id'])
+
+ if verbose:
+ print "writing training %s" % (filename)
+
+ gpx = pygpx.GPX()
+ gpx.ReadTree(xml)
+
+ sport = training['sport']
+ timestamp = gpx.tracks[0].start_time()
+
+ print sport, timestamp
+
+ if check_db_for_training(db,sport,timestamp):
+ print "The same training found"
+ continue
+
+ gpx.author = username
+ gpx.FixNames(training['sport'])
+ gpx.ProcessTrackSegs()
+ for track in gpx.tracks:
+ track.sport=training['sport']
+
+ xml = gpx.XMLTree();
+ f = open(filename,"w")
+ f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
+ f.close
+ write_parsed_to_db(db,gpx,filename)
+ try:
+ render_tiles.queue_render(db,filename)
+ except:
+ None
+
+
def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
training_list = get_osm_list(username,password,year,month)
for training in training_list:
+
filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])
dirname = "%s/%04d/%02d" % (dir,year,month)
print "training %s exists, skipping" % (filename)
else:
-
+
try:
os.makedirs(dirname)
except:
gpx = pygpx.GPX()
gpx.ReadTree(xml)
+ sport = training['sport']
+ timestamp = gpx.tracks[0].start_time()
+
+ if check_db_for_training(db,sport,timestamp):
+ print "The same training found"
+ continue
+
gpx.author = username
gpx.FixNames(training['sport'])
gpx.ProcessTrackSegs()
for track in gpx.tracks:
track.sport=training['sport']
-
+
xml = gpx.XMLTree();
f = open(filename,"w")
f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
help="OpenStreetMap username")
parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
help="OpenStreetMap password")
+ parser.add_option("-b", "--dropbox", dest="dropboxauth",
+ help="DropBox Auth token")
(options, args) = parser.parse_args()
username = options.username
osmname = options.osmname
osmpwd = options.osmpasswd
- if not (username or (osmname and osmpwd)):
- print "Run.GPS username or OSM username/password is mandatory!"
+ dbauth = options.dropboxauth
+ if not (username or (osmname and osmpwd) or dbauth):
+ print "Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!"
return
if username:
pusername = username+'@Run.GPS'
elif osmname:
pusername = osmname+'@OSM'
+ elif dbauth:
+ import dropbox
+ dbx = dropbox.Dropbox(dbauth)
+ pusername = dbx.users_get_current_account().email+'@DBX'
try:
if options.yearmonth:
sync_folder(username,year,month,outdir,options.verbose,options.force)
elif osmname:
sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)
+ elif dbauth:
+ sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force)
else:
if options.verbose:
- print "retrieving two last months for user %s to %s" % (pusername,outdir)
+ print "retrieving two last months for user %s to %s" % (pusername ,outdir)
now = date.today()
current_year = now.year
current_month = now.month
sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
elif osmname:
sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
+ elif dbauth:
+ sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)
current_month = current_month -1
if current_month == 0:
current_month = 12
sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
elif osmname:
sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
+ elif dbauth:
+ sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)
generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-transparent.xml',400,400)
projects / pyrungps.git / blobdiff