+++ /dev/null
-#!/usr/bin/env python
-# coding: UTF-8
-
-import requests
-
-from urllib3.exceptions import InsecureRequestWarning
-requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
-
-#import sys
-import os
-from urllib2 import quote,unquote
-from lxml import html,etree
-from optparse import OptionParser
-from datetime import date,datetime
-from mx.DateTime.ISO import ParseDateTimeUTC
-from parsegpx import write_parsed_to_db,check_db_for_training
-import pygpx
-
-from tempfile import NamedTemporaryFile
-
-from pprint import pprint
-
-import render_tiles
-
-import generate_image
-
-def get_page(uname,year,month):
-
- trainings = []
-
- req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'}, verify=False)
- page = req.text.encode('utf-8')
- dom = html.document_fromstring(page)
-
- for element, attribute, link, pos in dom.iterlinks():
- if attribute == "href":
- if link.startswith("/trainings/"):
- dummy, dummy, link = link.split('/')
- name, id = link.split('_')
- trainings.append([ unquote(name), id ])
-
- return trainings
-
-def get_gpx_track(trid,name):
-
- print "trid=",trid
-
- req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid), verify=False)
- xml = etree.fromstring(req.text.encode('utf-8'))
-
- return xml
-
-def get_osm_list(username,password,year,month):
-
- url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
-
- req = requests.get(url,auth=(username,password))
-
- xml = etree.fromstring(req.text.encode('utf-8'))
-
- res=[]
-
- for gpx_file in xml:
- attrib = dict(gpx_file.attrib)
- try:
- timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
- id = attrib['id']
- description=None
- for descr in gpx_file.iter('description'):
- description=descr.text
- sport=None
- for tag in gpx_file.iter('tag'):
- sport=tag.text
- if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
- record={ 'id': id, 'sport': sport }
- res.append(record)
- except:
- None
-
- return res
-
-def get_osm_gpx(username,password,track_id):
-
- url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
-
- req = requests.get(url,auth=(username,password))
- xml = etree.fromstring(req.text.encode('utf-8'))
-
- return xml
-
-def get_db_gpx(dbx,track_id):
-
- md, res = dbx.files_download(track_id)
-
- fs = NamedTemporaryFile(suffix='.tcx',delete = False)
- tmp_tcx_name=fs.name
- fs.write(res.content)
- fs.close()
- fs = NamedTemporaryFile(suffix='.gpx',delete = False)
- tmp_gpx_name=fs.name
- fs.close()
-
- os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name)
-
- with open (tmp_gpx_name, "r") as gpxfile:
- data=gpxfile.read()
-
- os.remove(tmp_gpx_name)
- os.remove(tmp_tcx_name)
-
- xml = etree.fromstring(data.encode('utf-8'))
- return xml
-
-def get_dbx_list(dbx,username,year,month):
-
- gpx_list_id = None
-
- for entry in dbx.files_list_folder('').entries:
- if entry.name == u'Приложения':
- for entry_app in dbx.files_list_folder(entry.id).entries:
- if entry_app.name == u'Run.GPS':
- print "id="+entry_app.id
- gpx_list_id=entry_app.id
- break
- break
-
- res = []
-
- if gpx_list_id:
-
- for file in dbx.files_list_folder(gpx_list_id).entries:
-
- filename,ext = os.path.splitext(file.name)
- if ext == '.tcx':
- try:
- fyear = int(filename[0:4])
- fmonth = int(filename[5:7])
- if fyear == year and fmonth == month:
- sport = filename[41:]
- timestamp = datetime.strptime(filename[0:19],'%Y-%m-%dT%H_%M_%S')
- record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
- res.append(record)
- except:
- try:
- fyear = int(filename[0:4])
- fmonth = int(filename[5:7])
- if fyear == year and fmonth == month:
- sport = filename[18:]
- timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S')
- record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
- res.append(record)
- except:
- None
-
- return res
-
-
-def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False):
-
- training_list = get_dbx_list(dbx,username,year,month)
-
- for training in training_list:
-
- filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:])
- dirname = "%s/%04d/%02d" % (dir,year,month)
-
- if os.path.exists(filename) and not force:
-
- if verbose:
- print "training %s exists, skipping" % (filename)
-
- else:
-
- try:
- os.makedirs(dirname)
- except:
- None
-
- xml = get_db_gpx(dbx,training['id'])
-
- if verbose:
- print "writing training %s" % (filename)
-
- gpx = pygpx.GPX()
- gpx.ReadTree(xml)
-
- sport = training['sport']
- timestamp = gpx.tracks[0].start_time()
-
- print sport, timestamp
-
- if check_db_for_training(db,sport,timestamp):
- print "The same training found"
- continue
-
- gpx.author = username
- gpx.FixNames(training['sport'])
- gpx.ProcessTrackSegs()
- for track in gpx.tracks:
- track.sport=training['sport']
-
- xml = gpx.XMLTree();
- f = open(filename,"w")
- f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
- f.close
- write_parsed_to_db(db,gpx,filename)
- try:
- render_tiles.queue_render(db,filename)
- except:
- None
-
-
-def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
-
- training_list = get_osm_list(username,password,year,month)
-
- for training in training_list:
-
- filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])
- dirname = "%s/%04d/%02d" % (dir,year,month)
-
- if os.path.exists(filename) and not force:
-
- if verbose:
- print "training %s exists, skipping" % (filename)
-
- else:
-
- try:
- os.makedirs(dirname)
- except:
- None
-
- xml = get_osm_gpx(username,password,training['id'])
-
- if verbose:
- print "writing training %s" % (filename)
-
- gpx = pygpx.GPX()
- gpx.ReadTree(xml)
-
- sport = training['sport']
- timestamp = gpx.tracks[0].start_time()
-
- if check_db_for_training(db,sport,timestamp):
- print "The same training found"
- continue
-
- gpx.author = username
- gpx.FixNames(training['sport'])
- gpx.ProcessTrackSegs()
- for track in gpx.tracks:
- track.sport=training['sport']
-
- xml = gpx.XMLTree();
- f = open(filename,"w")
- f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
- f.close
- write_parsed_to_db(db,gpx,filename)
- try:
- render_tiles.queue_render(db,filename)
- except:
- None
-
-def sync_folder(username,year,month,dir=".",verbose=False,force=False):
-
- training_list = get_page(username,year,month)
- for tr in training_list:
- try:
-
- filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1])
- dirname = "%s/%04d/%02d" % (dir,year,(month+1))
-
- if os.path.exists(filename) and not force:
-
- if verbose:
- print "training %s exists, skipping" % (filename)
-
- else:
-
- try:
- os.makedirs(dirname)
- except:
- None
-
- xml=get_gpx_track(tr[1],tr[0])
-
- if verbose:
- print "writing training %s" % (filename)
-
- gpx = pygpx.GPX()
- gpx.ReadTree(xml)
-
- sport = tr[0]
- timestamp = gpx.tracks[0].start_time()
-
- if check_db_for_training(db,sport,timestamp):
- print "The same training found"
- continue
-
- gpx.FixNames(tr[0])
- gpx.ProcessTrackSegs()
-
- xml = gpx.XMLTree();
- f = open(filename,"w")
- f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
- f.close
- write_parsed_to_db(db,gpx,filename)
- try:
- render_tiles.queue_render(db,filename)
- except:
- None
-
- except:
- raise
-
-def main():
-
- global db;
- parser = OptionParser()
- parser.add_option("-d", "--dir", dest="dirname",
- help="write data to directory", metavar="DIR")
- parser.add_option("-q", "--quiet",
- action="store_false", dest="verbose", default=True,
- help="don't print status messages to stdout")
- parser.add_option("-f", "--force",
- action="store_true", dest="force", default=False,
- help="rewrite all files")
- parser.add_option("-y", "--yearmonth", dest="yearmonth",
- help="year and month in YYYY-MM format", metavar="YYYY-MM")
- parser.add_option("-u", "--username", dest="username",
- help="Run.GPS username")
- parser.add_option("-o", "--osm-username", dest="osmname",
- help="OpenStreetMap username")
- parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
- help="OpenStreetMap password")
- parser.add_option("-b", "--dropbox", dest="dropboxauth",
- help="DropBox Auth token")
- (options, args) = parser.parse_args()
-
- username = options.username
- osmname = options.osmname
- osmpwd = options.osmpasswd
- dbauth = options.dropboxauth
- if not (username or (osmname and osmpwd) or dbauth):
- print "Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!"
- return
-
- if username:
- pusername = username+'@Run.GPS'
- elif osmname:
- pusername = osmname+'@OSM'
- elif dbauth:
- import dropbox
- dbx = dropbox.Dropbox(dbauth)
- pusername = dbx.users_get_current_account().email+'@DBX'
-
- try:
- if options.yearmonth:
- year,month = options.yearmonth.split('-')
- month = int(month) -1
- year = int(year)
- if month<0 or month>11:
- raise invalid_number
- else:
- year = None
- month = None
- except:
- print "Year and month should be in YYYY-MM format!"
- return
-
- if options.dirname:
- outdir = options.dirname
- else:
- outdir = '.'
-
- db = outdir + '/gpx.db'
-
- if year:
- if options.verbose:
- print "retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir)
- if username:
- sync_folder(username,year,month,outdir,options.verbose,options.force)
- elif osmname:
- sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)
- elif dbauth:
- sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force)
- else:
- if options.verbose:
- print "retrieving two last months for user %s to %s" % (pusername ,outdir)
- now = date.today()
- current_year = now.year
- current_month = now.month
- if username:
- sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
- elif osmname:
- sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
- elif dbauth:
- sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)
- current_month = current_month -1
- if current_month == 0:
- current_month = 12
- current_year = current_year -1
- if username:
- sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
- elif osmname:
- sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)
- elif dbauth:
- sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)
-
- generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-imposm.xml',640,640)
-
-if __name__ == "__main__":
-
- main()
-
\ No newline at end of file
projects / pyrungps.git / blobdiff