1) Dropbox API и сопутствующие изменения

[pyrungps.git] / pyrungps.py

diff --git a/pyrungps.py b/pyrungps.py
index 6c8ea5c14e1fab54384190694bc51bb87c7bc62c..0f0dc563206b28a58116df86a2b1c2ab3342a2ae 100644 (file)
--- a/pyrungps.py
+++ b/pyrungps.py
@@ -1,16 +1,22 @@
 #!/usr/bin/env python
 # coding: UTF-8
 
 #!/usr/bin/env python
 # coding: UTF-8
 

-import urllib2
+import requests
+    

 #import sys
 import os
 #import sys
 import os

+from urllib2 import quote,unquote

 from lxml import html,etree
 from optparse import OptionParser
 from datetime import date,datetime
 from mx.DateTime.ISO import ParseDateTimeUTC
 from lxml import html,etree
 from optparse import OptionParser
 from datetime import date,datetime
 from mx.DateTime.ISO import ParseDateTimeUTC

-from parsegpx import write_parsed_to_db
+from parsegpx import write_parsed_to_db,check_db_for_training

 import pygpx
 
 import pygpx
 

+from tempfile import NamedTemporaryFile
+
+from pprint import pprint
+

 import render_tiles
 
 import generate_image
 import render_tiles
 
 import generate_image


@@ -19,8 +25,8 @@ def get_page(uname,year,month):
   
   trainings = []
 
   
   trainings = []
 

-  req = urllib2.Request("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), None, {'User-agent': 'Mozilla/5.0'})
-  page = urllib2.urlopen(req).read()
+  req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'})
+  page = req.text.encode()

   dom = html.document_fromstring(page)
 
   for element, attribute, link, pos in dom.iterlinks():
   dom = html.document_fromstring(page)
 
   for element, attribute, link, pos in dom.iterlinks():


@@ -28,7 +34,7 @@ def get_page(uname,year,month):
       if link.startswith("/trainings/"):
         dummy, dummy, link = link.split('/')
         name, id = link.split('_')
       if link.startswith("/trainings/"):
         dummy, dummy, link = link.split('/')
         name, id = link.split('_')

-        trainings.append([ urllib2.unquote(name), id ])
+        trainings.append([ unquote(name), id ])

       
   return trainings      
 
       
   return trainings      
 


@@ -36,62 +42,167 @@ def get_gpx_track(trid,name):
 
   print "trid=",trid
 
 
   print "trid=",trid
 

-  req = urllib2.urlopen("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid))
-  
-  xml = etree.parse(req)
+  req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid))
+  xml = etree.fromstring(req.text.encode())

 
   return xml
 
 def get_osm_list(username,password,year,month):
 
   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
 
   return xml
 
 def get_osm_list(username,password,year,month):
 
   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"

-  passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
-  passman.add_password(None, url, username, password)
-  authhandler = urllib2.HTTPBasicAuthHandler(passman)
-  opener = urllib2.build_opener(authhandler)
-  urllib2.install_opener(opener)

 
 

-  req = urllib2.urlopen(url)
-  xml = etree.parse(req)
+  req = requests.get(url,auth=(username,password))
+  
+  xml = etree.fromstring(req.text.encode())

 
   res=[]
 
 
   res=[]
 

-  for gpx_file in xml.getroot():
+  for gpx_file in xml:

     attrib = dict(gpx_file.attrib)
     attrib = dict(gpx_file.attrib)

-    timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
-    id = attrib['id']
-    description=None
-    for descr in gpx_file.iter('description'):
-      description=descr.text
-    sport=None
-    for tag in gpx_file.iter('tag'):
-      sport=tag.text
-    if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
-      print id,description,sport,timestamp
-      record={ 'id': id, 'sport': sport }
-      res.append(record)
+    try:
+      timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
+      id = attrib['id']
+      description=None
+      for descr in gpx_file.iter('description'):
+        description=descr.text
+      sport=None
+      for tag in gpx_file.iter('tag'):
+        sport=tag.text
+      if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
+        record={ 'id': id, 'sport': sport }
+        res.append(record)
+    except:
+      None    

       
       

-    return res      
+  return res      

 
 def get_osm_gpx(username,password,track_id):
 
   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
 
 def get_osm_gpx(username,password,track_id):
 
   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"

-  passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
-  passman.add_password(None, url, username, password)
-  authhandler = urllib2.HTTPBasicAuthHandler(passman)
-  opener = urllib2.build_opener(authhandler)
-  urllib2.install_opener(opener)

 
 

-  req = urllib2.urlopen(url)
-  xml = etree.parse(req)
+  req = requests.get(url,auth=(username,password))
+  xml = etree.fromstring(req.text.encode())

 
   return xml
 
 
   return xml
 

+def get_db_gpx(dbx,track_id):
+
+  md, res = dbx.files_download(track_id)
+
+  fs = NamedTemporaryFile(suffix='.tcx',delete = False)
+  tmp_tcx_name=fs.name
+  fs.write(res.content)
+  fs.close()
+  fs = NamedTemporaryFile(suffix='.gpx',delete = False)
+  tmp_gpx_name=fs.name
+  fs.close()
+  
+  os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name)
+
+  with open (tmp_gpx_name, "r") as gpxfile:
+    data=gpxfile.read()
+    
+  os.remove(tmp_gpx_name)  
+  os.remove(tmp_tcx_name)
+
+  xml = etree.fromstring(data.encode())
+  return xml
+
+def get_dbx_list(dbx,username,year,month):
+
+    gpx_list_id = None
+
+    for entry in dbx.files_list_folder('').entries:
+      if entry.name == u'Приложения':
+        for entry_app in dbx.files_list_folder(entry.id).entries:
+          if entry_app.name == u'Run.GPS':
+            gpx_list_id=entry_app.id
+            break
+        break
+
+    res = []
+        
+    if gpx_list_id:
+    
+      for file in dbx.files_list_folder(gpx_list_id).entries:
+        
+        filename,ext = os.path.splitext(file.name)
+        if ext == '.tcx':
+          try:
+            fyear = int(filename[0:4])
+            fmonth = int(filename[5:7])
+            if fyear == year and fmonth == month:
+              sport = filename[18:]
+              timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S')
+              record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
+              res.append(record)
+          except:
+            None
+    
+    return res        
+      
+
+def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False):
+
+    training_list = get_dbx_list(dbx,username,year,month)
+
+    for training in training_list:
+    
+      filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:])   
+      dirname = "%s/%04d/%02d" % (dir,year,month)
+      
+      if os.path.exists(filename) and not force:
+
+        if verbose:
+          print "training %s exists, skipping" % (filename)
+
+      else:  
+
+        try:
+          os.makedirs(dirname)
+        except:
+          None
+
+        xml = get_db_gpx(dbx,training['id'])
+
+        if verbose:
+          print "writing training %s" % (filename)
+
+        gpx = pygpx.GPX()
+        gpx.ReadTree(xml)
+
+        sport = training['sport']
+        timestamp = gpx.tracks[0].start_time()
+        
+        print sport, timestamp
+        
+        if check_db_for_training(db,sport,timestamp):
+          print "The same training found"
+          continue
+
+        gpx.author = username
+        gpx.FixNames(training['sport'])
+        gpx.ProcessTrackSegs()
+        for track in gpx.tracks:
+          track.sport=training['sport']
+
+        xml = gpx.XMLTree();
+        f = open(filename,"w")
+        f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
+        f.close
+        write_parsed_to_db(db,gpx,filename)
+        try:
+          render_tiles.queue_render(db,filename)
+        except:
+          None  
+
+ 

 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
 
     training_list = get_osm_list(username,password,year,month)
 
     for training in training_list:
 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
 
     training_list = get_osm_list(username,password,year,month)
 
     for training in training_list:

+    

       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])   
       dirname = "%s/%04d/%02d" % (dir,year,month)
 
       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])   
       dirname = "%s/%04d/%02d" % (dir,year,month)
 


@@ -101,7 +212,7 @@ def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
           print "training %s exists, skipping" % (filename)
 
       else:  
           print "training %s exists, skipping" % (filename)
 
       else:  

-
+      

         try:
           os.makedirs(dirname)    
         except:
         try:
           os.makedirs(dirname)    
         except:


@@ -115,12 +226,19 @@ def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
         gpx = pygpx.GPX()
         gpx.ReadTree(xml)
 
         gpx = pygpx.GPX()
         gpx.ReadTree(xml)
 

+        sport = training['sport']
+        timestamp = gpx.tracks[0].start_time()
+        
+        if check_db_for_training(db,sport,timestamp):
+          print "The same training found"
+          continue
+

         gpx.author = username
         gpx.FixNames(training['sport'])
         gpx.ProcessTrackSegs()
         for track in gpx.tracks:
           track.sport=training['sport']
         gpx.author = username
         gpx.FixNames(training['sport'])
         gpx.ProcessTrackSegs()
         for track in gpx.tracks:
           track.sport=training['sport']

-        
+

         xml = gpx.XMLTree();
         f = open(filename,"w")
         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
         xml = gpx.XMLTree();
         f = open(filename,"w")
         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))


@@ -192,19 +310,26 @@ def main():
       help="OpenStreetMap username")                                                          
     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
       help="OpenStreetMap password")                                                          
       help="OpenStreetMap username")                                                          
     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
       help="OpenStreetMap password")                                                          

+    parser.add_option("-b", "--dropbox", dest="dropboxauth",
+      help="DropBox Auth token")                                                          

     (options, args) = parser.parse_args()
 
     username = options.username
     osmname = options.osmname
     osmpwd = options.osmpasswd
     (options, args) = parser.parse_args()
 
     username = options.username
     osmname = options.osmname
     osmpwd = options.osmpasswd

-    if not (username or (osmname and osmpwd)):
-      print "Run.GPS username or OSM username/password is mandatory!"
+    dbauth = options.dropboxauth
+    if not (username or (osmname and osmpwd) or dbauth):
+      print "Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!"

       return
 
     if username:
       pusername = username+'@Run.GPS'
     elif osmname:
       pusername = osmname+'@OSM'  
       return
 
     if username:
       pusername = username+'@Run.GPS'
     elif osmname:
       pusername = osmname+'@OSM'  

+    elif dbauth:
+      import dropbox
+      dbx = dropbox.Dropbox(dbauth)  
+      pusername = dbx.users_get_current_account().email+'@DBX'

 
     try:
       if options.yearmonth:
 
     try:
       if options.yearmonth:


@@ -234,9 +359,11 @@ def main():
         sync_folder(username,year,month,outdir,options.verbose,options.force)
       elif osmname:
         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)  
         sync_folder(username,year,month,outdir,options.verbose,options.force)
       elif osmname:
         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)  

+      elif dbauth:
+        sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force)  

     else:
       if options.verbose:
     else:
       if options.verbose:

-        print "retrieving two last months for user %s to %s" % (pusername,outdir)
+        print "retrieving two last months for user %s to %s" % (pusername ,outdir)

       now = date.today()
       current_year = now.year
       current_month = now.month
       now = date.today()
       current_year = now.year
       current_month = now.month


@@ -244,6 +371,8 @@ def main():
         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
       elif osmname:
         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
       elif osmname:
         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  

+      elif dbauth:
+        sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  

       current_month = current_month -1
       if current_month == 0:
         current_month = 12
       current_month = current_month -1
       if current_month == 0:
         current_month = 12


@@ -252,6 +381,8 @@ def main():
         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
       elif osmname:  
         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
       elif osmname:  
         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  

+      elif dbauth:
+        sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  

 
     generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-transparent.xml',400,400)
 
 
     generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-transparent.xml',400,400)