18c2235c34ac667f1dec830e22980745a9ee2ea6
[pyrungps.git] / pyrungps / pyrungps_sync.py
1 #!/usr/bin/env python
2 # coding: UTF-8
3
4 import requests
5
6 import urllib3
7 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
8
9 #import sys
10 import os
11 from urllib.parse import quote,unquote
12 from lxml import html,etree
13 from optparse import OptionParser
14 from datetime import date,datetime
15 from dateutil.parser import isoparse
16 from pyrungps.parsegpx import write_parsed_to_db,check_db_for_training
17 from pyrungps.pygpx import GPX
18 from io import BytesIO
19
20 from tempfile import NamedTemporaryFile
21
22 from pprint import pprint
23
24 from pyrungps.render_tiles import queue_render
25
26 from pyrungps.generate_image import render_all
27
28 def get_page(uname,year,month):
29   
30   trainings = []
31
32   req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'}, verify=False)
33   page = req.text.encode('utf-8')
34   dom = html.document_fromstring(page)
35
36   for element, attribute, link, pos in dom.iterlinks():
37     if attribute == "href":
38       if link.startswith("/trainings/"):
39         dummy, dummy, link = link.split('/')
40         name, *ids = link.split('_')
41         id = ids[-1]
42         trainings.append([ unquote(name), id ])
43       
44   return trainings      
45
46 def get_gpx_track(trid,name):
47
48   print("trid=",trid)
49
50   req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid), verify=False)
51   
52   xml = etree.fromstring(req.text.encode('utf-8'))
53
54   return xml
55
56 def get_osm_list(username,password,year,month):
57
58   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
59
60   req = requests.get(url,auth=(username,password))
61   
62   xml = etree.fromstring(req.text.encode('utf-8'))
63
64   res=[]
65
66   for gpx_file in xml:
67     attrib = dict(gpx_file.attrib)
68     try:
69       timestamp=datetime.fromtimestamp(isoparse(attrib['timestamp']))
70       id = attrib['id']
71       description=None
72       for descr in gpx_file.iter('description'):
73         description=descr.text
74       sport=None
75       for tag in gpx_file.iter('tag'):
76         sport=tag.text
77       if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
78         record={ 'id': id, 'sport': sport }
79         res.append(record)
80     except:
81       None    
82       
83   return res      
84
85 def get_osm_gpx(username,password,track_id):
86
87   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
88
89   req = requests.get(url,auth=(username,password))
90   xml = etree.fromstring(req.text.encode('utf-8'))
91
92   return xml
93
94 def get_db_gpx(dbx,track_id):
95
96   md, res = dbx.files_download(track_id)
97
98   fs = NamedTemporaryFile(suffix='.tcx',delete = False)
99   tmp_tcx_name=fs.name
100   fs.write(res.content)
101   fs.close()
102   fs = NamedTemporaryFile(suffix='.gpx',delete = False)
103   tmp_gpx_name=fs.name
104   fs.close()
105   
106   os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name)
107
108   with open (tmp_gpx_name, "r") as gpxfile:
109     data=gpxfile.read()
110     
111   os.remove(tmp_gpx_name)  
112   os.remove(tmp_tcx_name)
113
114   xml = etree.fromstring(data.encode('utf-8'))
115   return xml
116
117 def get_dbx_list(dbx,username,year,month):
118
119     gpx_list_id = None
120
121     for entry in dbx.files_list_folder('').entries:
122       if entry.name == u'Приложения':
123         for entry_app in dbx.files_list_folder(entry.id).entries:
124           if entry_app.name == u'Run.GPS':
125             print("id="+entry_app.id)
126             gpx_list_id=entry_app.id
127             break
128         break
129
130     res = []
131         
132     if gpx_list_id:
133     
134       for file in dbx.files_list_folder(gpx_list_id).entries:
135       
136         filename,ext = os.path.splitext(file.name)
137         if ext == '.tcx':
138           try:
139             fyear = int(filename[0:4])
140             fmonth = int(filename[5:7])
141             if fyear == year and fmonth == month:
142               sport = filename[41:]
143               timestamp = datetime.strptime(filename[0:19],'%Y-%m-%dT%H_%M_%S')
144               record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
145               res.append(record)
146           except:
147             try:
148               fyear = int(filename[0:4])
149               fmonth = int(filename[5:7])
150               if fyear == year and fmonth == month:
151                 sport = filename[18:]
152                 timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S')
153                 record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
154                 res.append(record)
155             except:
156               None
157
158     return res        
159       
160
161 def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False):
162
163     training_list = get_dbx_list(dbx,username,year,month)
164
165     for training in training_list:
166     
167       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:])   
168       dirname = "%s/%04d/%02d" % (dir,year,month)
169       
170       if os.path.exists(filename) and not force:
171
172         if verbose:
173           print("training %s exists, skipping" % (filename))
174
175       else:  
176
177         try:
178           os.makedirs(dirname)
179         except:
180           None
181
182         xml = get_db_gpx(dbx,training['id'])
183
184         if verbose:
185           print("writing training %s" % (filename))
186
187         gpx = GPX()
188         gpx.ReadTree(xml)
189
190         sport = training['sport']
191         timestamp = gpx.tracks[0].start_time()
192         
193         print(sport, timestamp)
194         
195         if check_db_for_training(db,sport,timestamp):
196           print("The same training found")
197           continue
198
199         gpx.author = username
200         gpx.FixNames(training['sport'])
201         gpx.ProcessTrackSegs()
202         for track in gpx.tracks:
203           track.sport=training['sport']
204
205         xml = gpx.XMLTree();
206         f = open(filename,"wb")
207         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
208         f.close
209         write_parsed_to_db(db,gpx,filename)
210         try:
211           queue_render(db,filename)
212         except:
213           None  
214
215  
216 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
217
218     training_list = get_osm_list(username,password,year,month)
219
220     for training in training_list:
221     
222       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])   
223       dirname = "%s/%04d/%02d" % (dir,year,month)
224
225       if os.path.exists(filename) and not force:
226
227         if verbose:
228           print("training %s exists, skipping" % (filename))
229
230       else:  
231       
232         try:
233           os.makedirs(dirname)    
234         except:
235           None
236
237         xml = get_osm_gpx(username,password,training['id'])
238
239         if verbose:
240           print("writing training %s" % (filename))
241
242         gpx = GPX()
243         gpx.ReadTree(xml)
244
245         sport = training['sport']
246         timestamp = gpx.tracks[0].start_time()
247         
248         if check_db_for_training(db,sport,timestamp):
249           print("The same training found")
250           continue
251
252         gpx.author = username
253         gpx.FixNames(training['sport'])
254         gpx.ProcessTrackSegs()
255         for track in gpx.tracks:
256           track.sport=training['sport']
257
258         xml = gpx.XMLTree();
259         f = open(filename,"wb")
260         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
261         f.close
262         write_parsed_to_db(db,gpx,filename)
263         try:
264           queue_render(db,filename)
265         except:
266           None  
267
268 def sync_folder(username,year,month,dir=".",verbose=False,force=False):
269
270     training_list = get_page(username,year,month)
271     for tr in training_list:
272      try:
273
274       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1])   
275       dirname = "%s/%04d/%02d" % (dir,year,(month+1))
276
277       if os.path.exists(filename) and not force:
278
279         if verbose:
280           print("training %s exists, skipping" % (filename))
281
282       else:  
283
284         try:
285           os.makedirs(dirname)    
286         except:
287           None
288           
289         xml=get_gpx_track(tr[1],tr[0])
290
291         if verbose:
292           print("writing training %s" % (filename))
293
294         gpx = GPX()
295         gpx.ReadTree(xml)
296
297         sport = tr[0]
298         timestamp = gpx.tracks[0].start_time()
299         
300         if check_db_for_training(db,sport,timestamp):
301           print("The same training found")
302           continue
303
304         gpx.FixNames(tr[0])
305         gpx.ProcessTrackSegs()
306         
307         xml = gpx.XMLTree();
308         f = open(filename,"wb")
309         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
310         f.close
311         write_parsed_to_db(db,gpx,filename)
312         try:
313           queue_render(db,filename)
314         except:
315           None  
316
317      except:
318        raise
319
320 def main():
321
322     global db;
323     parser = OptionParser()
324     parser.add_option("-d", "--dir", dest="dirname",
325       help="write data to directory", metavar="DIR")
326     parser.add_option("-q", "--quiet",
327       action="store_false", dest="verbose", default=True,
328       help="don't print status messages to stdout")
329     parser.add_option("-f", "--force",
330       action="store_true", dest="force", default=False,
331       help="rewrite all files")
332     parser.add_option("-y", "--yearmonth", dest="yearmonth",
333       help="year and month in YYYY-MM format", metavar="YYYY-MM")                                                          
334     parser.add_option("-u", "--username", dest="username",
335       help="Run.GPS username")                                                          
336     parser.add_option("-o", "--osm-username", dest="osmname",
337       help="OpenStreetMap username")                                                          
338     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
339       help="OpenStreetMap password")                                                          
340     parser.add_option("-b", "--dropbox", dest="dropboxauth",
341       help="DropBox Auth token")                                                          
342     (options, args) = parser.parse_args()
343
344     username = options.username
345     osmname = options.osmname
346     osmpwd = options.osmpasswd
347     dbauth = options.dropboxauth
348     if not (username or (osmname and osmpwd) or dbauth):
349       print("Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!")
350       return
351
352     if username:
353       pusername = username+'@Run.GPS'
354     elif osmname:
355       pusername = osmname+'@OSM'  
356     elif dbauth:
357       import dropbox
358       dbx = dropbox.Dropbox(dbauth)  
359       pusername = dbx.users_get_current_account().email+'@DBX'
360
361     try:
362       if options.yearmonth:
363         year,month = options.yearmonth.split('-')
364         month = int(month) -1
365         year = int(year)
366         if month<0 or month>11:
367           raise invalid_number
368       else:
369         year = None
370         month = None
371     except:
372       print("Year and month should be in YYYY-MM format!")
373       return 
374       
375     if options.dirname:  
376       outdir = options.dirname
377     else:
378       outdir = '.'
379     
380     db = outdir + '/gpx.db'
381     
382     if year:
383       if options.verbose:
384         print("retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir))
385       if username:
386         sync_folder(username,year,month,outdir,options.verbose,options.force)
387       elif osmname:
388         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)  
389       elif dbauth:
390         sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force)  
391     else:
392       if options.verbose:
393         print("retrieving two last months for user %s to %s" % (pusername ,outdir))
394       now = date.today()
395       current_year = now.year
396       current_month = now.month
397       if username:
398         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
399       elif osmname:
400         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
401       elif dbauth:
402         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  
403       current_month = current_month -1
404       if current_month == 0:
405         current_month = 12
406         current_year = current_year -1
407       if username:
408         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
409       elif osmname:  
410         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
411       elif dbauth:
412         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  
413
414     render_all(db,'/etc/mapnik-osm-carto-data/veloroad-imposm.xml',640,640)
415
416 if __name__ == "__main__":
417
418     main()                    
419