d8aaa58736f9cf9146dd9dd46824afb04ed1b0ca
[pyrungps.git] / pyrungps.py
1 #!/usr/bin/env python
2 # coding: UTF-8
3
4 import requests
5     
6 #import sys
7 import os
8 from urllib2 import quote,unquote
9 from lxml import html,etree
10 from optparse import OptionParser
11 from datetime import date,datetime
12 from mx.DateTime.ISO import ParseDateTimeUTC
13 from parsegpx import write_parsed_to_db,check_db_for_training
14 import pygpx
15
16 from tempfile import NamedTemporaryFile
17
18 from pprint import pprint
19
20 import render_tiles
21
22 import generate_image
23
24 def get_page(uname,year,month):
25   
26   trainings = []
27
28   req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'})
29   print req.text
30   page = req.text.encode('utf-8')
31   dom = html.document_fromstring(page)
32
33   for element, attribute, link, pos in dom.iterlinks():
34     if attribute == "href":
35       if link.startswith("/trainings/"):
36         dummy, dummy, link = link.split('/')
37         name, id = link.split('_')
38         trainings.append([ unquote(name), id ])
39       
40   return trainings      
41
42 def get_gpx_track(trid,name):
43
44   print "trid=",trid
45
46   req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid))
47   xml = etree.fromstring(req.text.encode('utf-8'))
48
49   return xml
50
51 def get_osm_list(username,password,year,month):
52
53   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
54
55   req = requests.get(url,auth=(username,password))
56   
57   xml = etree.fromstring(req.text.encode('utf-8'))
58
59   res=[]
60
61   for gpx_file in xml:
62     attrib = dict(gpx_file.attrib)
63     try:
64       timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
65       id = attrib['id']
66       description=None
67       for descr in gpx_file.iter('description'):
68         description=descr.text
69       sport=None
70       for tag in gpx_file.iter('tag'):
71         sport=tag.text
72       if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
73         record={ 'id': id, 'sport': sport }
74         res.append(record)
75     except:
76       None    
77       
78   return res      
79
80 def get_osm_gpx(username,password,track_id):
81
82   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
83
84   req = requests.get(url,auth=(username,password))
85   xml = etree.fromstring(req.text.encode('utf-8'))
86
87   return xml
88
89 def get_db_gpx(dbx,track_id):
90
91   md, res = dbx.files_download(track_id)
92
93   fs = NamedTemporaryFile(suffix='.tcx',delete = False)
94   tmp_tcx_name=fs.name
95   fs.write(res.content)
96   fs.close()
97   fs = NamedTemporaryFile(suffix='.gpx',delete = False)
98   tmp_gpx_name=fs.name
99   fs.close()
100   
101   os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name)
102
103   with open (tmp_gpx_name, "r") as gpxfile:
104     data=gpxfile.read()
105     
106   os.remove(tmp_gpx_name)  
107   os.remove(tmp_tcx_name)
108
109   xml = etree.fromstring(data.encode('utf-8'))
110   return xml
111
112 def get_dbx_list(dbx,username,year,month):
113
114     gpx_list_id = None
115
116     for entry in dbx.files_list_folder('').entries:
117       if entry.name == u'Приложения':
118         for entry_app in dbx.files_list_folder(entry.id).entries:
119           if entry_app.name == u'Run.GPS':
120             gpx_list_id=entry_app.id
121             break
122         break
123
124     res = []
125         
126     if gpx_list_id:
127     
128       for file in dbx.files_list_folder(gpx_list_id).entries:
129         
130         filename,ext = os.path.splitext(file.name)
131         if ext == '.tcx':
132           try:
133             fyear = int(filename[0:4])
134             fmonth = int(filename[5:7])
135             if fyear == year and fmonth == month:
136               sport = filename[18:]
137               timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S')
138               record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
139               res.append(record)
140           except:
141             None
142     
143     return res        
144       
145
146 def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False):
147
148     training_list = get_dbx_list(dbx,username,year,month)
149
150     for training in training_list:
151     
152       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:])   
153       dirname = "%s/%04d/%02d" % (dir,year,month)
154       
155       if os.path.exists(filename) and not force:
156
157         if verbose:
158           print "training %s exists, skipping" % (filename)
159
160       else:  
161
162         try:
163           os.makedirs(dirname)
164         except:
165           None
166
167         xml = get_db_gpx(dbx,training['id'])
168
169         if verbose:
170           print "writing training %s" % (filename)
171
172         gpx = pygpx.GPX()
173         gpx.ReadTree(xml)
174
175         sport = training['sport']
176         timestamp = gpx.tracks[0].start_time()
177         
178         print sport, timestamp
179         
180         if check_db_for_training(db,sport,timestamp):
181           print "The same training found"
182           continue
183
184         gpx.author = username
185         gpx.FixNames(training['sport'])
186         gpx.ProcessTrackSegs()
187         for track in gpx.tracks:
188           track.sport=training['sport']
189
190         xml = gpx.XMLTree();
191         f = open(filename,"w")
192         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
193         f.close
194         write_parsed_to_db(db,gpx,filename)
195         try:
196           render_tiles.queue_render(db,filename)
197         except:
198           None  
199
200  
201 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
202
203     training_list = get_osm_list(username,password,year,month)
204
205     for training in training_list:
206     
207       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])   
208       dirname = "%s/%04d/%02d" % (dir,year,month)
209
210       if os.path.exists(filename) and not force:
211
212         if verbose:
213           print "training %s exists, skipping" % (filename)
214
215       else:  
216       
217         try:
218           os.makedirs(dirname)    
219         except:
220           None
221
222         xml = get_osm_gpx(username,password,training['id'])
223
224         if verbose:
225           print "writing training %s" % (filename)
226
227         gpx = pygpx.GPX()
228         gpx.ReadTree(xml)
229
230         sport = training['sport']
231         timestamp = gpx.tracks[0].start_time()
232         
233         if check_db_for_training(db,sport,timestamp):
234           print "The same training found"
235           continue
236
237         gpx.author = username
238         gpx.FixNames(training['sport'])
239         gpx.ProcessTrackSegs()
240         for track in gpx.tracks:
241           track.sport=training['sport']
242
243         xml = gpx.XMLTree();
244         f = open(filename,"w")
245         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
246         f.close
247         write_parsed_to_db(db,gpx,filename)
248         try:
249           render_tiles.queue_render(db,filename)
250         except:
251           None  
252
253 def sync_folder(username,year,month,dir=".",verbose=False,force=False):
254
255     training_list = get_page(username,year,month)
256     for tr in training_list:
257      try:
258
259       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1])   
260       dirname = "%s/%04d/%02d" % (dir,year,(month+1))
261
262       if os.path.exists(filename) and not force:
263
264         if verbose:
265           print "training %s exists, skipping" % (filename)
266
267       else:  
268
269         try:
270           os.makedirs(dirname)    
271         except:
272           None
273           
274         xml=get_gpx_track(tr[1],tr[0])
275
276         if verbose:
277           print "writing training %s" % (filename)
278
279         gpx = pygpx.GPX()
280         gpx.ReadTree(xml)
281
282         sport = tr[0]
283         timestamp = gpx.tracks[0].start_time()
284         
285         if check_db_for_training(db,sport,timestamp):
286           print "The same training found"
287           continue
288
289         gpx.FixNames(tr[0])
290         gpx.ProcessTrackSegs()
291         
292         xml = gpx.XMLTree();
293         f = open(filename,"w")
294         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
295         f.close
296         write_parsed_to_db(db,gpx,filename)
297         try:
298           render_tiles.queue_render(db,filename)
299         except:
300           None  
301
302      except:
303        raise
304
305 def main():
306
307     global db;
308     parser = OptionParser()
309     parser.add_option("-d", "--dir", dest="dirname",
310       help="write data to directory", metavar="DIR")
311     parser.add_option("-q", "--quiet",
312       action="store_false", dest="verbose", default=True,
313       help="don't print status messages to stdout")
314     parser.add_option("-f", "--force",
315       action="store_true", dest="force", default=False,
316       help="rewrite all files")
317     parser.add_option("-y", "--yearmonth", dest="yearmonth",
318       help="year and month in YYYY-MM format", metavar="YYYY-MM")                                                          
319     parser.add_option("-u", "--username", dest="username",
320       help="Run.GPS username")                                                          
321     parser.add_option("-o", "--osm-username", dest="osmname",
322       help="OpenStreetMap username")                                                          
323     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
324       help="OpenStreetMap password")                                                          
325     parser.add_option("-b", "--dropbox", dest="dropboxauth",
326       help="DropBox Auth token")                                                          
327     (options, args) = parser.parse_args()
328
329     username = options.username
330     osmname = options.osmname
331     osmpwd = options.osmpasswd
332     dbauth = options.dropboxauth
333     if not (username or (osmname and osmpwd) or dbauth):
334       print "Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!"
335       return
336
337     if username:
338       pusername = username+'@Run.GPS'
339     elif osmname:
340       pusername = osmname+'@OSM'  
341     elif dbauth:
342       import dropbox
343       dbx = dropbox.Dropbox(dbauth)  
344       pusername = dbx.users_get_current_account().email+'@DBX'
345
346     try:
347       if options.yearmonth:
348         year,month = options.yearmonth.split('-')
349         month = int(month) -1
350         year = int(year)
351         if month<0 or month>11:
352           raise invalid_number
353       else:
354         year = None
355         month = None
356     except:
357       print "Year and month should be in YYYY-MM format!"
358       return 
359       
360     if options.dirname:  
361       outdir = options.dirname
362     else:
363       outdir = '.'
364     
365     db = outdir + '/gpx.db'
366     
367     if year:
368       if options.verbose:
369         print "retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir)
370       if username:
371         sync_folder(username,year,month,outdir,options.verbose,options.force)
372       elif osmname:
373         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)  
374       elif dbauth:
375         sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force)  
376     else:
377       if options.verbose:
378         print "retrieving two last months for user %s to %s" % (pusername ,outdir)
379       now = date.today()
380       current_year = now.year
381       current_month = now.month
382       if username:
383         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
384       elif osmname:
385         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
386       elif dbauth:
387         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  
388       current_month = current_month -1
389       if current_month == 0:
390         current_month = 12
391         current_year = current_year -1
392       if username:
393         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
394       elif osmname:  
395         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
396       elif dbauth:
397         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  
398
399     generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-transparent.xml',400,400)
400
401 if __name__ == "__main__":
402
403     main()                    
404