1) Dropbox API и сопутствующие изменения

2) Проверка на наличие тренировки, загруженной из другого источника

diff --git a/generate_image.py b/generate_image.py
index a4deaac7c7d996ddc8e15f8aea84c850c04f0fbb..e6a50210e36cf5e1d96c2540abcabbf79f2ec267 100644 (file)
--- a/generate_image.py
+++ b/generate_image.py
@@ -30,7 +30,10 @@ if not hasattr(mapnik,'mapnik_version') and not mapnik.mapnik_version() >= 600:
 
 def render_map(mapfile,map_uri,gpx_file,imgx,imgy):
 
 
 def render_map(mapfile,map_uri,gpx_file,imgx,imgy):
 

-    xml = etree.parse(gpx_file)
+    with open(gpx_file,'r') as f:
+      data = f.read()
+
+    xml = etree.fromstring(data)

     gpx = pygpx.GPX()
     gpx.ReadTree(xml)
 
     gpx = pygpx.GPX()
     gpx.ReadTree(xml)
 


@@ -64,6 +67,10 @@ def render_map(mapfile,map_uri,gpx_file,imgx,imgy):
     style = mapnik.Style()
     rule = mapnik.Rule()
 
     style = mapnik.Style()
     rule = mapnik.Rule()
 

+    line_symbolizer = mapnik.LineSymbolizer()
+    line_symbolizer.stroke = mapnik.Color('rgb(125, 125, 0)')
+    rule.symbols.append(line_symbolizer)
+

     point_symbolizer = mapnik.MarkersSymbolizer()
     point_symbolizer.allow_overlap = True
     point_symbolizer.opacity = 0.8 # semi-transparent
     point_symbolizer = mapnik.MarkersSymbolizer()
     point_symbolizer.allow_overlap = True
     point_symbolizer.opacity = 0.8 # semi-transparent

diff --git a/parsegpx.py b/parsegpx.py
index 5247c27d7ff7382966f67bbf2df36b8e3c41356c..5299fe0f1da253ea79826b75c0c8ac1247b67ce5 100644 (file)
--- a/parsegpx.py
+++ b/parsegpx.py
@@ -10,6 +10,14 @@ import pyosmname
 import sqlite3
 import datetime
 
 import sqlite3
 import datetime
 

+def check_db_for_training(db,sport,timestamp):
+
+  conn = sqlite3.connect(db)
+  cur = conn.cursor()
+
+  cur.execute ("select count(*) from tracks where sport=? and start_time=?" , (sport,timestamp))
+  return cur.fetchall()[0][0]
+

 def write_parsed_to_db(db,gpx,filename):
 
   conn = sqlite3.connect(db)
 def write_parsed_to_db(db,gpx,filename):
 
   conn = sqlite3.connect(db)

diff --git a/pygpx.py b/pygpx.py
index fea5e67db3bbca8ec93706925ab9d8949a4e2c8d..469253854e0b70c3dd169f68819c377ecccb2afe 100644 (file)
--- a/pygpx.py
+++ b/pygpx.py
@@ -410,8 +410,7 @@ class GPX:
 
     def ReadTree(self,tree):
                # Загрузка из дерева etree.# 
 
     def ReadTree(self,tree):
                # Загрузка из дерева etree.# 

-        gpx_doc = tree;    
-        root = gpx_doc.getroot()
+        root = tree

 
         # Test if this is a GPX file or not and if it's version 1.1
         if root.tag == "{http://www.topografix.com/GPX/1/1}gpx" and root.get("version") == "1.1":
 
         # Test if this is a GPX file or not and if it's version 1.1
         if root.tag == "{http://www.topografix.com/GPX/1/1}gpx" and root.get("version") == "1.1":

diff --git a/pyrungps.py b/pyrungps.py
index 6c8ea5c14e1fab54384190694bc51bb87c7bc62c..0f0dc563206b28a58116df86a2b1c2ab3342a2ae 100644 (file)
--- a/pyrungps.py
+++ b/pyrungps.py
@@ -1,16 +1,22 @@
 #!/usr/bin/env python
 # coding: UTF-8
 
 #!/usr/bin/env python
 # coding: UTF-8
 

-import urllib2
+import requests
+    

 #import sys
 import os
 #import sys
 import os

+from urllib2 import quote,unquote

 from lxml import html,etree
 from optparse import OptionParser
 from datetime import date,datetime
 from mx.DateTime.ISO import ParseDateTimeUTC
 from lxml import html,etree
 from optparse import OptionParser
 from datetime import date,datetime
 from mx.DateTime.ISO import ParseDateTimeUTC

-from parsegpx import write_parsed_to_db
+from parsegpx import write_parsed_to_db,check_db_for_training

 import pygpx
 
 import pygpx
 

+from tempfile import NamedTemporaryFile
+
+from pprint import pprint
+

 import render_tiles
 
 import generate_image
 import render_tiles
 
 import generate_image


@@ -19,8 +25,8 @@ def get_page(uname,year,month):
   
   trainings = []
 
   
   trainings = []
 

-  req = urllib2.Request("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), None, {'User-agent': 'Mozilla/5.0'})
-  page = urllib2.urlopen(req).read()
+  req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'})
+  page = req.text.encode()

   dom = html.document_fromstring(page)
 
   for element, attribute, link, pos in dom.iterlinks():
   dom = html.document_fromstring(page)
 
   for element, attribute, link, pos in dom.iterlinks():


@@ -28,7 +34,7 @@ def get_page(uname,year,month):
       if link.startswith("/trainings/"):
         dummy, dummy, link = link.split('/')
         name, id = link.split('_')
       if link.startswith("/trainings/"):
         dummy, dummy, link = link.split('/')
         name, id = link.split('_')

-        trainings.append([ urllib2.unquote(name), id ])
+        trainings.append([ unquote(name), id ])

       
   return trainings      
 
       
   return trainings      
 


@@ -36,62 +42,167 @@ def get_gpx_track(trid,name):
 
   print "trid=",trid
 
 
   print "trid=",trid
 

-  req = urllib2.urlopen("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid))
-  
-  xml = etree.parse(req)
+  req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid))
+  xml = etree.fromstring(req.text.encode())

 
   return xml
 
 def get_osm_list(username,password,year,month):
 
   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
 
   return xml
 
 def get_osm_list(username,password,year,month):
 
   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"

-  passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
-  passman.add_password(None, url, username, password)
-  authhandler = urllib2.HTTPBasicAuthHandler(passman)
-  opener = urllib2.build_opener(authhandler)
-  urllib2.install_opener(opener)

 
 

-  req = urllib2.urlopen(url)
-  xml = etree.parse(req)
+  req = requests.get(url,auth=(username,password))
+  
+  xml = etree.fromstring(req.text.encode())

 
   res=[]
 
 
   res=[]
 

-  for gpx_file in xml.getroot():
+  for gpx_file in xml:

     attrib = dict(gpx_file.attrib)
     attrib = dict(gpx_file.attrib)

-    timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
-    id = attrib['id']
-    description=None
-    for descr in gpx_file.iter('description'):
-      description=descr.text
-    sport=None
-    for tag in gpx_file.iter('tag'):
-      sport=tag.text
-    if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
-      print id,description,sport,timestamp
-      record={ 'id': id, 'sport': sport }
-      res.append(record)
+    try:
+      timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
+      id = attrib['id']
+      description=None
+      for descr in gpx_file.iter('description'):
+        description=descr.text
+      sport=None
+      for tag in gpx_file.iter('tag'):
+        sport=tag.text
+      if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
+        record={ 'id': id, 'sport': sport }
+        res.append(record)
+    except:
+      None    

       
       

-    return res      
+  return res      

 
 def get_osm_gpx(username,password,track_id):
 
   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
 
 def get_osm_gpx(username,password,track_id):
 
   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"

-  passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
-  passman.add_password(None, url, username, password)
-  authhandler = urllib2.HTTPBasicAuthHandler(passman)
-  opener = urllib2.build_opener(authhandler)
-  urllib2.install_opener(opener)

 
 

-  req = urllib2.urlopen(url)
-  xml = etree.parse(req)
+  req = requests.get(url,auth=(username,password))
+  xml = etree.fromstring(req.text.encode())

 
   return xml
 
 
   return xml
 

+def get_db_gpx(dbx,track_id):
+
+  md, res = dbx.files_download(track_id)
+
+  fs = NamedTemporaryFile(suffix='.tcx',delete = False)
+  tmp_tcx_name=fs.name
+  fs.write(res.content)
+  fs.close()
+  fs = NamedTemporaryFile(suffix='.gpx',delete = False)
+  tmp_gpx_name=fs.name
+  fs.close()
+  
+  os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name)
+
+  with open (tmp_gpx_name, "r") as gpxfile:
+    data=gpxfile.read()
+    
+  os.remove(tmp_gpx_name)  
+  os.remove(tmp_tcx_name)
+
+  xml = etree.fromstring(data.encode())
+  return xml
+
+def get_dbx_list(dbx,username,year,month):
+
+    gpx_list_id = None
+
+    for entry in dbx.files_list_folder('').entries:
+      if entry.name == u'Приложения':
+        for entry_app in dbx.files_list_folder(entry.id).entries:
+          if entry_app.name == u'Run.GPS':
+            gpx_list_id=entry_app.id
+            break
+        break
+
+    res = []
+        
+    if gpx_list_id:
+    
+      for file in dbx.files_list_folder(gpx_list_id).entries:
+        
+        filename,ext = os.path.splitext(file.name)
+        if ext == '.tcx':
+          try:
+            fyear = int(filename[0:4])
+            fmonth = int(filename[5:7])
+            if fyear == year and fmonth == month:
+              sport = filename[18:]
+              timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S')
+              record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
+              res.append(record)
+          except:
+            None
+    
+    return res        
+      
+
+def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False):
+
+    training_list = get_dbx_list(dbx,username,year,month)
+
+    for training in training_list:
+    
+      filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:])   
+      dirname = "%s/%04d/%02d" % (dir,year,month)
+      
+      if os.path.exists(filename) and not force:
+
+        if verbose:
+          print "training %s exists, skipping" % (filename)
+
+      else:  
+
+        try:
+          os.makedirs(dirname)
+        except:
+          None
+
+        xml = get_db_gpx(dbx,training['id'])
+
+        if verbose:
+          print "writing training %s" % (filename)
+
+        gpx = pygpx.GPX()
+        gpx.ReadTree(xml)
+
+        sport = training['sport']
+        timestamp = gpx.tracks[0].start_time()
+        
+        print sport, timestamp
+        
+        if check_db_for_training(db,sport,timestamp):
+          print "The same training found"
+          continue
+
+        gpx.author = username
+        gpx.FixNames(training['sport'])
+        gpx.ProcessTrackSegs()
+        for track in gpx.tracks:
+          track.sport=training['sport']
+
+        xml = gpx.XMLTree();
+        f = open(filename,"w")
+        f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
+        f.close
+        write_parsed_to_db(db,gpx,filename)
+        try:
+          render_tiles.queue_render(db,filename)
+        except:
+          None  
+
+ 

 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
 
     training_list = get_osm_list(username,password,year,month)
 
     for training in training_list:
 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
 
     training_list = get_osm_list(username,password,year,month)
 
     for training in training_list:

+    

       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])   
       dirname = "%s/%04d/%02d" % (dir,year,month)
 
       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])   
       dirname = "%s/%04d/%02d" % (dir,year,month)
 


@@ -101,7 +212,7 @@ def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
           print "training %s exists, skipping" % (filename)
 
       else:  
           print "training %s exists, skipping" % (filename)
 
       else:  

-
+      

         try:
           os.makedirs(dirname)    
         except:
         try:
           os.makedirs(dirname)    
         except:


@@ -115,12 +226,19 @@ def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
         gpx = pygpx.GPX()
         gpx.ReadTree(xml)
 
         gpx = pygpx.GPX()
         gpx.ReadTree(xml)
 

+        sport = training['sport']
+        timestamp = gpx.tracks[0].start_time()
+        
+        if check_db_for_training(db,sport,timestamp):
+          print "The same training found"
+          continue
+

         gpx.author = username
         gpx.FixNames(training['sport'])
         gpx.ProcessTrackSegs()
         for track in gpx.tracks:
           track.sport=training['sport']
         gpx.author = username
         gpx.FixNames(training['sport'])
         gpx.ProcessTrackSegs()
         for track in gpx.tracks:
           track.sport=training['sport']

-        
+

         xml = gpx.XMLTree();
         f = open(filename,"w")
         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
         xml = gpx.XMLTree();
         f = open(filename,"w")
         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))


@@ -192,19 +310,26 @@ def main():
       help="OpenStreetMap username")                                                          
     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
       help="OpenStreetMap password")                                                          
       help="OpenStreetMap username")                                                          
     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
       help="OpenStreetMap password")                                                          

+    parser.add_option("-b", "--dropbox", dest="dropboxauth",
+      help="DropBox Auth token")                                                          

     (options, args) = parser.parse_args()
 
     username = options.username
     osmname = options.osmname
     osmpwd = options.osmpasswd
     (options, args) = parser.parse_args()
 
     username = options.username
     osmname = options.osmname
     osmpwd = options.osmpasswd

-    if not (username or (osmname and osmpwd)):
-      print "Run.GPS username or OSM username/password is mandatory!"
+    dbauth = options.dropboxauth
+    if not (username or (osmname and osmpwd) or dbauth):
+      print "Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!"

       return
 
     if username:
       pusername = username+'@Run.GPS'
     elif osmname:
       pusername = osmname+'@OSM'  
       return
 
     if username:
       pusername = username+'@Run.GPS'
     elif osmname:
       pusername = osmname+'@OSM'  

+    elif dbauth:
+      import dropbox
+      dbx = dropbox.Dropbox(dbauth)  
+      pusername = dbx.users_get_current_account().email+'@DBX'

 
     try:
       if options.yearmonth:
 
     try:
       if options.yearmonth:


@@ -234,9 +359,11 @@ def main():
         sync_folder(username,year,month,outdir,options.verbose,options.force)
       elif osmname:
         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)  
         sync_folder(username,year,month,outdir,options.verbose,options.force)
       elif osmname:
         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)  

+      elif dbauth:
+        sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force)  

     else:
       if options.verbose:
     else:
       if options.verbose:

-        print "retrieving two last months for user %s to %s" % (pusername,outdir)
+        print "retrieving two last months for user %s to %s" % (pusername ,outdir)

       now = date.today()
       current_year = now.year
       current_month = now.month
       now = date.today()
       current_year = now.year
       current_month = now.month


@@ -244,6 +371,8 @@ def main():
         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
       elif osmname:
         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
       elif osmname:
         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  

+      elif dbauth:
+        sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  

       current_month = current_month -1
       if current_month == 0:
         current_month = 12
       current_month = current_month -1
       if current_month == 0:
         current_month = 12


@@ -252,6 +381,8 @@ def main():
         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
       elif osmname:  
         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
       elif osmname:  
         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  

+      elif dbauth:
+        sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  

 
     generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-transparent.xml',400,400)
 
 
     generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-transparent.xml',400,400)