ed3fce52b94fac40d21a84fcb49270dc5ffaaec1
[pyrungps.git] / pyrungps.py
1 #!/usr/bin/env python
2 # coding: UTF-8
3
4 import requests
5
6 from urllib3.exceptions import InsecureRequestWarning
7 requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
8     
9 #import sys
10 import os
11 from urllib2 import quote,unquote
12 from lxml import html,etree
13 from optparse import OptionParser
14 from datetime import date,datetime
15 from mx.DateTime.ISO import ParseDateTimeUTC
16 from parsegpx import write_parsed_to_db,check_db_for_training
17 import pygpx
18
19 from tempfile import NamedTemporaryFile
20
21 from pprint import pprint
22
23 import render_tiles
24
25 import generate_image
26
27 def get_page(uname,year,month):
28   
29   trainings = []
30
31   req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'}, verify=False)
32   page = req.text.encode('utf-8')
33   dom = html.document_fromstring(page)
34
35   for element, attribute, link, pos in dom.iterlinks():
36     if attribute == "href":
37       if link.startswith("/trainings/"):
38         dummy, dummy, link = link.split('/')
39         name, id = link.split('_')
40         trainings.append([ unquote(name), id ])
41       
42   return trainings      
43
44 def get_gpx_track(trid,name):
45
46   print "trid=",trid
47
48   req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid), verify=False)
49   xml = etree.fromstring(req.text.encode('utf-8'))
50
51   return xml
52
53 def get_osm_list(username,password,year,month):
54
55   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
56
57   req = requests.get(url,auth=(username,password))
58   
59   xml = etree.fromstring(req.text.encode('utf-8'))
60
61   res=[]
62
63   for gpx_file in xml:
64     attrib = dict(gpx_file.attrib)
65     try:
66       timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
67       id = attrib['id']
68       description=None
69       for descr in gpx_file.iter('description'):
70         description=descr.text
71       sport=None
72       for tag in gpx_file.iter('tag'):
73         sport=tag.text
74       if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
75         record={ 'id': id, 'sport': sport }
76         res.append(record)
77     except:
78       None    
79       
80   return res      
81
82 def get_osm_gpx(username,password,track_id):
83
84   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
85
86   req = requests.get(url,auth=(username,password))
87   xml = etree.fromstring(req.text.encode('utf-8'))
88
89   return xml
90
91 def get_db_gpx(dbx,track_id):
92
93   md, res = dbx.files_download(track_id)
94
95   fs = NamedTemporaryFile(suffix='.tcx',delete = False)
96   tmp_tcx_name=fs.name
97   fs.write(res.content)
98   fs.close()
99   fs = NamedTemporaryFile(suffix='.gpx',delete = False)
100   tmp_gpx_name=fs.name
101   fs.close()
102   
103   os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name)
104
105   with open (tmp_gpx_name, "r") as gpxfile:
106     data=gpxfile.read()
107     
108   os.remove(tmp_gpx_name)  
109   os.remove(tmp_tcx_name)
110
111   xml = etree.fromstring(data.encode('utf-8'))
112   return xml
113
114 def get_dbx_list(dbx,username,year,month):
115
116     gpx_list_id = None
117
118     for entry in dbx.files_list_folder('').entries:
119       if entry.name == u'Приложения':
120         for entry_app in dbx.files_list_folder(entry.id).entries:
121           if entry_app.name == u'Run.GPS':
122             print "id="+entry_app.id
123             gpx_list_id=entry_app.id
124             break
125         break
126
127     res = []
128         
129     if gpx_list_id:
130     
131       for file in dbx.files_list_folder(gpx_list_id).entries:
132       
133         filename,ext = os.path.splitext(file.name)
134         if ext == '.tcx':
135           try:
136             fyear = int(filename[0:4])
137             fmonth = int(filename[5:7])
138             if fyear == year and fmonth == month:
139               sport = filename[41:]
140               timestamp = datetime.strptime(filename[0:19],'%Y-%m-%dT%H_%M_%S')
141               record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
142               res.append(record)
143           except:
144             try:
145               fyear = int(filename[0:4])
146               fmonth = int(filename[5:7])
147               if fyear == year and fmonth == month:
148                 sport = filename[18:]
149                 timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S')
150                 record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
151                 res.append(record)
152             except:
153               None
154
155     return res        
156       
157
158 def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False):
159
160     training_list = get_dbx_list(dbx,username,year,month)
161
162     for training in training_list:
163     
164       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:])   
165       dirname = "%s/%04d/%02d" % (dir,year,month)
166       
167       if os.path.exists(filename) and not force:
168
169         if verbose:
170           print "training %s exists, skipping" % (filename)
171
172       else:  
173
174         try:
175           os.makedirs(dirname)
176         except:
177           None
178
179         xml = get_db_gpx(dbx,training['id'])
180
181         if verbose:
182           print "writing training %s" % (filename)
183
184         gpx = pygpx.GPX()
185         gpx.ReadTree(xml)
186
187         sport = training['sport']
188         timestamp = gpx.tracks[0].start_time()
189         
190         print sport, timestamp
191         
192         if check_db_for_training(db,sport,timestamp):
193           print "The same training found"
194           continue
195
196         gpx.author = username
197         gpx.FixNames(training['sport'])
198         gpx.ProcessTrackSegs()
199         for track in gpx.tracks:
200           track.sport=training['sport']
201
202         xml = gpx.XMLTree();
203         f = open(filename,"w")
204         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
205         f.close
206         write_parsed_to_db(db,gpx,filename)
207         try:
208           render_tiles.queue_render(db,filename)
209         except:
210           None  
211
212  
213 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
214
215     training_list = get_osm_list(username,password,year,month)
216
217     for training in training_list:
218     
219       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])   
220       dirname = "%s/%04d/%02d" % (dir,year,month)
221
222       if os.path.exists(filename) and not force:
223
224         if verbose:
225           print "training %s exists, skipping" % (filename)
226
227       else:  
228       
229         try:
230           os.makedirs(dirname)    
231         except:
232           None
233
234         xml = get_osm_gpx(username,password,training['id'])
235
236         if verbose:
237           print "writing training %s" % (filename)
238
239         gpx = pygpx.GPX()
240         gpx.ReadTree(xml)
241
242         sport = training['sport']
243         timestamp = gpx.tracks[0].start_time()
244         
245         if check_db_for_training(db,sport,timestamp):
246           print "The same training found"
247           continue
248
249         gpx.author = username
250         gpx.FixNames(training['sport'])
251         gpx.ProcessTrackSegs()
252         for track in gpx.tracks:
253           track.sport=training['sport']
254
255         xml = gpx.XMLTree();
256         f = open(filename,"w")
257         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
258         f.close
259         write_parsed_to_db(db,gpx,filename)
260         try:
261           render_tiles.queue_render(db,filename)
262         except:
263           None  
264
265 def sync_folder(username,year,month,dir=".",verbose=False,force=False):
266
267     training_list = get_page(username,year,month)
268     for tr in training_list:
269      try:
270
271       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1])   
272       dirname = "%s/%04d/%02d" % (dir,year,(month+1))
273
274       if os.path.exists(filename) and not force:
275
276         if verbose:
277           print "training %s exists, skipping" % (filename)
278
279       else:  
280
281         try:
282           os.makedirs(dirname)    
283         except:
284           None
285           
286         xml=get_gpx_track(tr[1],tr[0])
287
288         if verbose:
289           print "writing training %s" % (filename)
290
291         gpx = pygpx.GPX()
292         gpx.ReadTree(xml)
293
294         sport = tr[0]
295         timestamp = gpx.tracks[0].start_time()
296         
297         if check_db_for_training(db,sport,timestamp):
298           print "The same training found"
299           continue
300
301         gpx.FixNames(tr[0])
302         gpx.ProcessTrackSegs()
303         
304         xml = gpx.XMLTree();
305         f = open(filename,"w")
306         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
307         f.close
308         write_parsed_to_db(db,gpx,filename)
309         try:
310           render_tiles.queue_render(db,filename)
311         except:
312           None  
313
314      except:
315        raise
316
317 def main():
318
319     global db;
320     parser = OptionParser()
321     parser.add_option("-d", "--dir", dest="dirname",
322       help="write data to directory", metavar="DIR")
323     parser.add_option("-q", "--quiet",
324       action="store_false", dest="verbose", default=True,
325       help="don't print status messages to stdout")
326     parser.add_option("-f", "--force",
327       action="store_true", dest="force", default=False,
328       help="rewrite all files")
329     parser.add_option("-y", "--yearmonth", dest="yearmonth",
330       help="year and month in YYYY-MM format", metavar="YYYY-MM")                                                          
331     parser.add_option("-u", "--username", dest="username",
332       help="Run.GPS username")                                                          
333     parser.add_option("-o", "--osm-username", dest="osmname",
334       help="OpenStreetMap username")                                                          
335     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
336       help="OpenStreetMap password")                                                          
337     parser.add_option("-b", "--dropbox", dest="dropboxauth",
338       help="DropBox Auth token")                                                          
339     (options, args) = parser.parse_args()
340
341     username = options.username
342     osmname = options.osmname
343     osmpwd = options.osmpasswd
344     dbauth = options.dropboxauth
345     if not (username or (osmname and osmpwd) or dbauth):
346       print "Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!"
347       return
348
349     if username:
350       pusername = username+'@Run.GPS'
351     elif osmname:
352       pusername = osmname+'@OSM'  
353     elif dbauth:
354       import dropbox
355       dbx = dropbox.Dropbox(dbauth)  
356       pusername = dbx.users_get_current_account().email+'@DBX'
357
358     try:
359       if options.yearmonth:
360         year,month = options.yearmonth.split('-')
361         month = int(month) -1
362         year = int(year)
363         if month<0 or month>11:
364           raise invalid_number
365       else:
366         year = None
367         month = None
368     except:
369       print "Year and month should be in YYYY-MM format!"
370       return 
371       
372     if options.dirname:  
373       outdir = options.dirname
374     else:
375       outdir = '.'
376     
377     db = outdir + '/gpx.db'
378     
379     if year:
380       if options.verbose:
381         print "retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir)
382       if username:
383         sync_folder(username,year,month,outdir,options.verbose,options.force)
384       elif osmname:
385         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)  
386       elif dbauth:
387         sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force)  
388     else:
389       if options.verbose:
390         print "retrieving two last months for user %s to %s" % (pusername ,outdir)
391       now = date.today()
392       current_year = now.year
393       current_month = now.month
394       if username:
395         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
396       elif osmname:
397         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
398       elif dbauth:
399         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  
400       current_month = current_month -1
401       if current_month == 0:
402         current_month = 12
403         current_year = current_year -1
404       if username:
405         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
406       elif osmname:  
407         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
408       elif dbauth:
409         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  
410
411     generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-imposm.xml',640,640)
412
413 if __name__ == "__main__":
414
415     main()                    
416