X-Git-Url: https://git.rvb.name/pyrungps.git/blobdiff_plain/1690b8b7c8ccfdf60be2b33b314e6c26bdef86e6..94f62add438cdf546fdae207ede72c05daf66c00:/pyrungps/pyrungps_sync.py

diff --git a/pyrungps/pyrungps_sync.py b/pyrungps/pyrungps_sync.py
new file mode 100644
index 0000000..b87a17e
--- /dev/null
+++ b/pyrungps/pyrungps_sync.py
@@ -0,0 +1,416 @@
+#!/usr/bin/env python
+# coding: UTF-8
+
+import requests
+
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+#import sys
+import os
+from urllib.parse import quote,unquote
+from lxml import html,etree
+from optparse import OptionParser
+from datetime import date,datetime
+from dateutil.parser import isoparse
+from pyrungps.parsegpx import write_parsed_to_db,check_db_for_training
+import pyrungps.pygpx
+
+from tempfile import NamedTemporaryFile
+
+from pprint import pprint
+
+import pyrungps.render_tiles
+
+import pyrungps.generate_image
+
+def get_page(uname,year,month):
+  
+  trainings = []
+
+  req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'}, verify=False)
+  page = req.text.encode('utf-8')
+  dom = html.document_fromstring(page)
+
+  for element, attribute, link, pos in dom.iterlinks():
+    if attribute == "href":
+      if link.startswith("/trainings/"):
+        dummy, dummy, link = link.split('/')
+        name, id = link.split('_')
+        trainings.append([ unquote(name), id ])
+      
+  return trainings      
+
+def get_gpx_track(trid,name):
+
+  print("trid=",trid)
+
+  req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid), verify=False)
+  xml = etree.fromstring(req.text.encode('utf-8'))
+
+  return xml
+
+def get_osm_list(username,password,year,month):
+
+  url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
+
+  req = requests.get(url,auth=(username,password))
+  
+  xml = etree.fromstring(req.text.encode('utf-8'))
+
+  res=[]
+
+  for gpx_file in xml:
+    attrib = dict(gpx_file.attrib)
+    try:
+      timestamp=datetime.fromtimestamp(isoparse(attrib['timestamp']))
+      id = attrib['id']
+      description=None
+      for descr in gpx_file.iter('description'):
+        description=descr.text
+      sport=None
+      for tag in gpx_file.iter('tag'):
+        sport=tag.text
+      if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
+        record={ 'id': id, 'sport': sport }
+        res.append(record)
+    except:
+      None    
+      
+  return res      
+
+def get_osm_gpx(username,password,track_id):
+
+  url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
+
+  req = requests.get(url,auth=(username,password))
+  xml = etree.fromstring(req.text.encode('utf-8'))
+
+  return xml
+
+def get_db_gpx(dbx,track_id):
+
+  md, res = dbx.files_download(track_id)
+
+  fs = NamedTemporaryFile(suffix='.tcx',delete = False)
+  tmp_tcx_name=fs.name
+  fs.write(res.content)
+  fs.close()
+  fs = NamedTemporaryFile(suffix='.gpx',delete = False)
+  tmp_gpx_name=fs.name
+  fs.close()
+  
+  os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name)
+
+  with open (tmp_gpx_name, "r") as gpxfile:
+    data=gpxfile.read()
+    
+  os.remove(tmp_gpx_name)  
+  os.remove(tmp_tcx_name)
+
+  xml = etree.fromstring(data.encode('utf-8'))
+  return xml
+
+def get_dbx_list(dbx,username,year,month):
+
+    gpx_list_id = None
+
+    for entry in dbx.files_list_folder('').entries:
+      if entry.name == u'Приложения':
+        for entry_app in dbx.files_list_folder(entry.id).entries:
+          if entry_app.name == u'Run.GPS':
+            print("id="+entry_app.id)
+            gpx_list_id=entry_app.id
+            break
+        break
+
+    res = []
+        
+    if gpx_list_id:
+    
+      for file in dbx.files_list_folder(gpx_list_id).entries:
+      
+        filename,ext = os.path.splitext(file.name)
+        if ext == '.tcx':
+          try:
+            fyear = int(filename[0:4])
+            fmonth = int(filename[5:7])
+            if fyear == year and fmonth == month:
+              sport = filename[41:]
+              timestamp = datetime.strptime(filename[0:19],'%Y-%m-%dT%H_%M_%S')
+              record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
+              res.append(record)
+          except:
+            try:
+              fyear = int(filename[0:4])
+              fmonth = int(filename[5:7])
+              if fyear == year and fmonth == month:
+                sport = filename[18:]
+                timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S')
+                record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
+                res.append(record)
+            except:
+              None
+
+    return res        
+      
+
+def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False):
+
+    training_list = get_dbx_list(dbx,username,year,month)
+
+    for training in training_list:
+    
+      filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:])   
+      dirname = "%s/%04d/%02d" % (dir,year,month)
+      
+      if os.path.exists(filename) and not force:
+
+        if verbose:
+          print("training %s exists, skipping" % (filename))
+
+      else:  
+
+        try:
+          os.makedirs(dirname)
+        except:
+          None
+
+        xml = get_db_gpx(dbx,training['id'])
+
+        if verbose:
+          print("writing training %s" % (filename))
+
+        gpx = pygpx.GPX()
+        gpx.ReadTree(xml)
+
+        sport = training['sport']
+        timestamp = gpx.tracks[0].start_time()
+        
+        print(sport, timestamp)
+        
+        if check_db_for_training(db,sport,timestamp):
+          print("The same training found")
+          continue
+
+        gpx.author = username
+        gpx.FixNames(training['sport'])
+        gpx.ProcessTrackSegs()
+        for track in gpx.tracks:
+          track.sport=training['sport']
+
+        xml = gpx.XMLTree();
+        f = open(filename,"wb")
+        f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
+        f.close
+        write_parsed_to_db(db,gpx,filename)
+        try:
+          render_tiles.queue_render(db,filename)
+        except:
+          None  
+
+ 
+def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
+
+    training_list = get_osm_list(username,password,year,month)
+
+    for training in training_list:
+    
+      filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])   
+      dirname = "%s/%04d/%02d" % (dir,year,month)
+
+      if os.path.exists(filename) and not force:
+
+        if verbose:
+          print("training %s exists, skipping" % (filename))
+
+      else:  
+      
+        try:
+          os.makedirs(dirname)    
+        except:
+          None
+
+        xml = get_osm_gpx(username,password,training['id'])
+
+        if verbose:
+          print("writing training %s" % (filename))
+
+        gpx = pygpx.GPX()
+        gpx.ReadTree(xml)
+
+        sport = training['sport']
+        timestamp = gpx.tracks[0].start_time()
+        
+        if check_db_for_training(db,sport,timestamp):
+          print("The same training found")
+          continue
+
+        gpx.author = username
+        gpx.FixNames(training['sport'])
+        gpx.ProcessTrackSegs()
+        for track in gpx.tracks:
+          track.sport=training['sport']
+
+        xml = gpx.XMLTree();
+        f = open(filename,"wb")
+        f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
+        f.close
+        write_parsed_to_db(db,gpx,filename)
+        try:
+          render_tiles.queue_render(db,filename)
+        except:
+          None  
+
+def sync_folder(username,year,month,dir=".",verbose=False,force=False):
+
+    training_list = get_page(username,year,month)
+    for tr in training_list:
+     try:
+
+      filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1])   
+      dirname = "%s/%04d/%02d" % (dir,year,(month+1))
+
+      if os.path.exists(filename) and not force:
+
+        if verbose:
+          print("training %s exists, skipping" % (filename))
+
+      else:  
+
+        try:
+          os.makedirs(dirname)    
+        except:
+          None
+          
+        xml=get_gpx_track(tr[1],tr[0])
+
+        if verbose:
+          print("writing training %s" % (filename))
+
+        gpx = pygpx.GPX()
+        gpx.ReadTree(xml)
+
+        sport = tr[0]
+        timestamp = gpx.tracks[0].start_time()
+        
+        if check_db_for_training(db,sport,timestamp):
+          print("The same training found")
+          continue
+
+        gpx.FixNames(tr[0])
+        gpx.ProcessTrackSegs()
+        
+        xml = gpx.XMLTree();
+        f = open(filename,"wb")
+        f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
+        f.close
+        write_parsed_to_db(db,gpx,filename)
+        try:
+          render_tiles.queue_render(db,filename)
+        except:
+          None  
+
+     except:
+       raise
+
+def main():
+
+    global db;
+    parser = OptionParser()
+    parser.add_option("-d", "--dir", dest="dirname",
+      help="write data to directory", metavar="DIR")
+    parser.add_option("-q", "--quiet",
+      action="store_false", dest="verbose", default=True,
+      help="don't print status messages to stdout")
+    parser.add_option("-f", "--force",
+      action="store_true", dest="force", default=False,
+      help="rewrite all files")
+    parser.add_option("-y", "--yearmonth", dest="yearmonth",
+      help="year and month in YYYY-MM format", metavar="YYYY-MM")                                                          
+    parser.add_option("-u", "--username", dest="username",
+      help="Run.GPS username")                                                          
+    parser.add_option("-o", "--osm-username", dest="osmname",
+      help="OpenStreetMap username")                                                          
+    parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
+      help="OpenStreetMap password")                                                          
+    parser.add_option("-b", "--dropbox", dest="dropboxauth",
+      help="DropBox Auth token")                                                          
+    (options, args) = parser.parse_args()
+
+    username = options.username
+    osmname = options.osmname
+    osmpwd = options.osmpasswd
+    dbauth = options.dropboxauth
+    if not (username or (osmname and osmpwd) or dbauth):
+      print("Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!")
+      return
+
+    if username:
+      pusername = username+'@Run.GPS'
+    elif osmname:
+      pusername = osmname+'@OSM'  
+    elif dbauth:
+      import dropbox
+      dbx = dropbox.Dropbox(dbauth)  
+      pusername = dbx.users_get_current_account().email+'@DBX'
+
+    try:
+      if options.yearmonth:
+        year,month = options.yearmonth.split('-')
+        month = int(month) -1
+        year = int(year)
+        if month<0 or month>11:
+          raise invalid_number
+      else:
+        year = None
+        month = None
+    except:
+      print("Year and month should be in YYYY-MM format!")
+      return 
+      
+    if options.dirname:  
+      outdir = options.dirname
+    else:
+      outdir = '.'
+    
+    db = outdir + '/gpx.db'
+    
+    if year:
+      if options.verbose:
+        print("retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir))
+      if username:
+        sync_folder(username,year,month,outdir,options.verbose,options.force)
+      elif osmname:
+        sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)  
+      elif dbauth:
+        sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force)  
+    else:
+      if options.verbose:
+        print("retrieving two last months for user %s to %s" % (pusername ,outdir))
+      now = date.today()
+      current_year = now.year
+      current_month = now.month
+      if username:
+        sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
+      elif osmname:
+        sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
+      elif dbauth:
+        sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  
+      current_month = current_month -1
+      if current_month == 0:
+        current_month = 12
+        current_year = current_year -1
+      if username:
+        sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
+      elif osmname:  
+        sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
+      elif dbauth:
+        sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  
+
+    generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-imposm.xml',640,640)
+
+if __name__ == "__main__":
+
+    main()                    
+    
\ No newline at end of file