From: Roman Bazalevskiy <rvb@rvb.name>
Date: Sun, 27 May 2018 10:43:21 +0000 (+0300)
Subject: 1) Dropbox API и сопутствующие изменения
X-Git-Url: https://git.rvb.name/pyrungps.git/commitdiff_plain/81020ed69398f00cdb0564a132dfdf0285e9b844?ds=inline;hp=425be6e68f3aa66f99ec3085e4d2dc3d3c7fd491

1) Dropbox API и сопутствующие изменения
2) Проверка на наличие тренировки, загруженной из другого источника
---

diff --git a/generate_image.py b/generate_image.py
index a4deaac..e6a5021 100644
--- a/generate_image.py
+++ b/generate_image.py
@@ -30,7 +30,10 @@ if not hasattr(mapnik,'mapnik_version') and not mapnik.mapnik_version() >= 600:
 
 def render_map(mapfile,map_uri,gpx_file,imgx,imgy):
 
-    xml = etree.parse(gpx_file)
+    with open(gpx_file,'r') as f:
+      data = f.read()
+
+    xml = etree.fromstring(data)
     gpx = pygpx.GPX()
     gpx.ReadTree(xml)
 
@@ -64,6 +67,10 @@ def render_map(mapfile,map_uri,gpx_file,imgx,imgy):
     style = mapnik.Style()
     rule = mapnik.Rule()
 
+    line_symbolizer = mapnik.LineSymbolizer()
+    line_symbolizer.stroke = mapnik.Color('rgb(125, 125, 0)')
+    rule.symbols.append(line_symbolizer)
+
     point_symbolizer = mapnik.MarkersSymbolizer()
     point_symbolizer.allow_overlap = True
     point_symbolizer.opacity = 0.8 # semi-transparent
diff --git a/parsegpx.py b/parsegpx.py
index 5247c27..5299fe0 100644
--- a/parsegpx.py
+++ b/parsegpx.py
@@ -10,6 +10,14 @@ import pyosmname
 import sqlite3
 import datetime
 
+def check_db_for_training(db,sport,timestamp):
+
+  conn = sqlite3.connect(db)
+  cur = conn.cursor()
+
+  cur.execute ("select count(*) from tracks where sport=? and start_time=?" , (sport,timestamp))
+  return cur.fetchall()[0][0]
+
 def write_parsed_to_db(db,gpx,filename):
 
   conn = sqlite3.connect(db)
diff --git a/pygpx.py b/pygpx.py
index fea5e67..4692538 100644
--- a/pygpx.py
+++ b/pygpx.py
@@ -410,8 +410,7 @@ class GPX:
 
     def ReadTree(self,tree):
 		# Загрузка из дерева etree.# 
-        gpx_doc = tree;    
-        root = gpx_doc.getroot()
+        root = tree
 
         # Test if this is a GPX file or not and if it's version 1.1
         if root.tag == "{http://www.topografix.com/GPX/1/1}gpx" and root.get("version") == "1.1":
diff --git a/pyrungps.py b/pyrungps.py
index 6c8ea5c..0f0dc56 100644
--- a/pyrungps.py
+++ b/pyrungps.py
@@ -1,16 +1,22 @@
 #!/usr/bin/env python
 # coding: UTF-8
 
-import urllib2
+import requests
+    
 #import sys
 import os
+from urllib2 import quote,unquote
 from lxml import html,etree
 from optparse import OptionParser
 from datetime import date,datetime
 from mx.DateTime.ISO import ParseDateTimeUTC
-from parsegpx import write_parsed_to_db
+from parsegpx import write_parsed_to_db,check_db_for_training
 import pygpx
 
+from tempfile import NamedTemporaryFile
+
+from pprint import pprint
+
 import render_tiles
 
 import generate_image
@@ -19,8 +25,8 @@ def get_page(uname,year,month):
   
   trainings = []
 
-  req = urllib2.Request("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), None, {'User-agent': 'Mozilla/5.0'})
-  page = urllib2.urlopen(req).read()
+  req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'})
+  page = req.text.encode()
   dom = html.document_fromstring(page)
 
   for element, attribute, link, pos in dom.iterlinks():
@@ -28,7 +34,7 @@ def get_page(uname,year,month):
       if link.startswith("/trainings/"):
         dummy, dummy, link = link.split('/')
         name, id = link.split('_')
-        trainings.append([ urllib2.unquote(name), id ])
+        trainings.append([ unquote(name), id ])
       
   return trainings      
 
@@ -36,62 +42,167 @@ def get_gpx_track(trid,name):
 
   print "trid=",trid
 
-  req = urllib2.urlopen("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid))
-  
-  xml = etree.parse(req)
+  req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid))
+  xml = etree.fromstring(req.text.encode())
 
   return xml
 
 def get_osm_list(username,password,year,month):
 
   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
-  passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
-  passman.add_password(None, url, username, password)
-  authhandler = urllib2.HTTPBasicAuthHandler(passman)
-  opener = urllib2.build_opener(authhandler)
-  urllib2.install_opener(opener)
 
-  req = urllib2.urlopen(url)
-  xml = etree.parse(req)
+  req = requests.get(url,auth=(username,password))
+  
+  xml = etree.fromstring(req.text.encode())
 
   res=[]
 
-  for gpx_file in xml.getroot():
+  for gpx_file in xml:
     attrib = dict(gpx_file.attrib)
-    timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
-    id = attrib['id']
-    description=None
-    for descr in gpx_file.iter('description'):
-      description=descr.text
-    sport=None
-    for tag in gpx_file.iter('tag'):
-      sport=tag.text
-    if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
-      print id,description,sport,timestamp
-      record={ 'id': id, 'sport': sport }
-      res.append(record)
+    try:
+      timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
+      id = attrib['id']
+      description=None
+      for descr in gpx_file.iter('description'):
+        description=descr.text
+      sport=None
+      for tag in gpx_file.iter('tag'):
+        sport=tag.text
+      if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
+        record={ 'id': id, 'sport': sport }
+        res.append(record)
+    except:
+      None    
       
-    return res      
+  return res      
 
 def get_osm_gpx(username,password,track_id):
 
   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
-  passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
-  passman.add_password(None, url, username, password)
-  authhandler = urllib2.HTTPBasicAuthHandler(passman)
-  opener = urllib2.build_opener(authhandler)
-  urllib2.install_opener(opener)
 
-  req = urllib2.urlopen(url)
-  xml = etree.parse(req)
+  req = requests.get(url,auth=(username,password))
+  xml = etree.fromstring(req.text.encode())
 
   return xml
 
+def get_db_gpx(dbx,track_id):
+
+  md, res = dbx.files_download(track_id)
+
+  fs = NamedTemporaryFile(suffix='.tcx',delete = False)
+  tmp_tcx_name=fs.name
+  fs.write(res.content)
+  fs.close()
+  fs = NamedTemporaryFile(suffix='.gpx',delete = False)
+  tmp_gpx_name=fs.name
+  fs.close()
+  
+  os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name)
+
+  with open (tmp_gpx_name, "r") as gpxfile:
+    data=gpxfile.read()
+    
+  os.remove(tmp_gpx_name)  
+  os.remove(tmp_tcx_name)
+
+  xml = etree.fromstring(data.encode())
+  return xml
+
+def get_dbx_list(dbx,username,year,month):
+
+    gpx_list_id = None
+
+    for entry in dbx.files_list_folder('').entries:
+      if entry.name == u'Приложения':
+        for entry_app in dbx.files_list_folder(entry.id).entries:
+          if entry_app.name == u'Run.GPS':
+            gpx_list_id=entry_app.id
+            break
+        break
+
+    res = []
+        
+    if gpx_list_id:
+    
+      for file in dbx.files_list_folder(gpx_list_id).entries:
+        
+        filename,ext = os.path.splitext(file.name)
+        if ext == '.tcx':
+          try:
+            fyear = int(filename[0:4])
+            fmonth = int(filename[5:7])
+            if fyear == year and fmonth == month:
+              sport = filename[18:]
+              timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S')
+              record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
+              res.append(record)
+          except:
+            None
+    
+    return res        
+      
+
+def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False):
+
+    training_list = get_dbx_list(dbx,username,year,month)
+
+    for training in training_list:
+    
+      filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:])   
+      dirname = "%s/%04d/%02d" % (dir,year,month)
+      
+      if os.path.exists(filename) and not force:
+
+        if verbose:
+          print "training %s exists, skipping" % (filename)
+
+      else:  
+
+        try:
+          os.makedirs(dirname)
+        except:
+          None
+
+        xml = get_db_gpx(dbx,training['id'])
+
+        if verbose:
+          print "writing training %s" % (filename)
+
+        gpx = pygpx.GPX()
+        gpx.ReadTree(xml)
+
+        sport = training['sport']
+        timestamp = gpx.tracks[0].start_time()
+        
+        print sport, timestamp
+        
+        if check_db_for_training(db,sport,timestamp):
+          print "The same training found"
+          continue
+
+        gpx.author = username
+        gpx.FixNames(training['sport'])
+        gpx.ProcessTrackSegs()
+        for track in gpx.tracks:
+          track.sport=training['sport']
+
+        xml = gpx.XMLTree();
+        f = open(filename,"w")
+        f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
+        f.close
+        write_parsed_to_db(db,gpx,filename)
+        try:
+          render_tiles.queue_render(db,filename)
+        except:
+          None  
+
+ 
 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
 
     training_list = get_osm_list(username,password,year,month)
 
     for training in training_list:
+    
       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])   
       dirname = "%s/%04d/%02d" % (dir,year,month)
 
@@ -101,7 +212,7 @@ def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
           print "training %s exists, skipping" % (filename)
 
       else:  
-
+      
         try:
           os.makedirs(dirname)    
         except:
@@ -115,12 +226,19 @@ def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
         gpx = pygpx.GPX()
         gpx.ReadTree(xml)
 
+        sport = training['sport']
+        timestamp = gpx.tracks[0].start_time()
+        
+        if check_db_for_training(db,sport,timestamp):
+          print "The same training found"
+          continue
+
         gpx.author = username
         gpx.FixNames(training['sport'])
         gpx.ProcessTrackSegs()
         for track in gpx.tracks:
           track.sport=training['sport']
-        
+
         xml = gpx.XMLTree();
         f = open(filename,"w")
         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
@@ -192,19 +310,26 @@ def main():
       help="OpenStreetMap username")                                                          
     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
       help="OpenStreetMap password")                                                          
+    parser.add_option("-b", "--dropbox", dest="dropboxauth",
+      help="DropBox Auth token")                                                          
     (options, args) = parser.parse_args()
 
     username = options.username
     osmname = options.osmname
     osmpwd = options.osmpasswd
-    if not (username or (osmname and osmpwd)):
-      print "Run.GPS username or OSM username/password is mandatory!"
+    dbauth = options.dropboxauth
+    if not (username or (osmname and osmpwd) or dbauth):
+      print "Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!"
       return
 
     if username:
       pusername = username+'@Run.GPS'
     elif osmname:
       pusername = osmname+'@OSM'  
+    elif dbauth:
+      import dropbox
+      dbx = dropbox.Dropbox(dbauth)  
+      pusername = dbx.users_get_current_account().email+'@DBX'
 
     try:
       if options.yearmonth:
@@ -234,9 +359,11 @@ def main():
         sync_folder(username,year,month,outdir,options.verbose,options.force)
       elif osmname:
         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)  
+      elif dbauth:
+        sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force)  
     else:
       if options.verbose:
-        print "retrieving two last months for user %s to %s" % (pusername,outdir)
+        print "retrieving two last months for user %s to %s" % (pusername ,outdir)
       now = date.today()
       current_year = now.year
       current_month = now.month
@@ -244,6 +371,8 @@ def main():
         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
       elif osmname:
         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
+      elif dbauth:
+        sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  
       current_month = current_month -1
       if current_month == 0:
         current_month = 12
@@ -252,6 +381,8 @@ def main():
         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
       elif osmname:  
         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
+      elif dbauth:
+        sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  
 
     generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-transparent.xml',400,400)