bfff892fca2164b19b10c762e1b6175e18f642f4
[pyrungps.git] / pyrungps.py
1 #!/usr/bin/env python
2 # coding: UTF-8
3
4 import requests
5     
6 #import sys
7 import os
8 from urllib2 import quote,unquote
9 from lxml import html,etree
10 from optparse import OptionParser
11 from datetime import date,datetime
12 from mx.DateTime.ISO import ParseDateTimeUTC
13 from parsegpx import write_parsed_to_db,check_db_for_training
14 import pygpx
15
16 from tempfile import NamedTemporaryFile
17
18 from pprint import pprint
19
20 import render_tiles
21
22 import generate_image
23
24 def get_page(uname,year,month):
25   
26   trainings = []
27
28   req = requests.get("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), headers = {'User-agent': 'Mozilla/5.0'})
29   page = req.text.encode('utf-8')
30   dom = html.document_fromstring(page)
31
32   for element, attribute, link, pos in dom.iterlinks():
33     if attribute == "href":
34       if link.startswith("/trainings/"):
35         dummy, dummy, link = link.split('/')
36         name, id = link.split('_')
37         trainings.append([ unquote(name), id ])
38       
39   return trainings      
40
41 def get_gpx_track(trid,name):
42
43   print "trid=",trid
44
45   req = requests.get("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid))
46   xml = etree.fromstring(req.text.encode('utf-8'))
47
48   return xml
49
50 def get_osm_list(username,password,year,month):
51
52   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
53
54   req = requests.get(url,auth=(username,password))
55   
56   xml = etree.fromstring(req.text.encode('utf-8'))
57
58   res=[]
59
60   for gpx_file in xml:
61     attrib = dict(gpx_file.attrib)
62     try:
63       timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
64       id = attrib['id']
65       description=None
66       for descr in gpx_file.iter('description'):
67         description=descr.text
68       sport=None
69       for tag in gpx_file.iter('tag'):
70         sport=tag.text
71       if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
72         record={ 'id': id, 'sport': sport }
73         res.append(record)
74     except:
75       None    
76       
77   return res      
78
79 def get_osm_gpx(username,password,track_id):
80
81   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
82
83   req = requests.get(url,auth=(username,password))
84   xml = etree.fromstring(req.text.encode('utf-8'))
85
86   return xml
87
88 def get_db_gpx(dbx,track_id):
89
90   md, res = dbx.files_download(track_id)
91
92   fs = NamedTemporaryFile(suffix='.tcx',delete = False)
93   tmp_tcx_name=fs.name
94   fs.write(res.content)
95   fs.close()
96   fs = NamedTemporaryFile(suffix='.gpx',delete = False)
97   tmp_gpx_name=fs.name
98   fs.close()
99   
100   os.system("gpsbabel -i gtrnctr -f "+tmp_tcx_name+" -o gpx -F "+tmp_gpx_name)
101
102   with open (tmp_gpx_name, "r") as gpxfile:
103     data=gpxfile.read()
104     
105   os.remove(tmp_gpx_name)  
106   os.remove(tmp_tcx_name)
107
108   xml = etree.fromstring(data.encode('utf-8'))
109   return xml
110
111 def get_dbx_list(dbx,username,year,month):
112
113     gpx_list_id = None
114
115     for entry in dbx.files_list_folder('').entries:
116       if entry.name == u'Приложения':
117         for entry_app in dbx.files_list_folder(entry.id).entries:
118           if entry_app.name == u'Run.GPS':
119             gpx_list_id=entry_app.id
120             break
121         break
122
123     res = []
124         
125     if gpx_list_id:
126     
127       for file in dbx.files_list_folder(gpx_list_id).entries:
128         
129         filename,ext = os.path.splitext(file.name)
130         if ext == '.tcx':
131           try:
132             fyear = int(filename[0:4])
133             fmonth = int(filename[5:7])
134             if fyear == year and fmonth == month:
135               sport = filename[18:]
136               timestamp = datetime.strptime(filename[0:16],'%Y-%m-%d_%H%M%S')
137               record={ 'id': file.id, 'timestamp': timestamp, 'sport': sport }
138               res.append(record)
139           except:
140             None
141     
142     return res        
143       
144
145 def sync_db(dbx,username,year,month,dir=".",verbose=False,force=False):
146
147     training_list = get_dbx_list(dbx,username,year,month)
148
149     for training in training_list:
150     
151       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'][3:])   
152       dirname = "%s/%04d/%02d" % (dir,year,month)
153       
154       if os.path.exists(filename) and not force:
155
156         if verbose:
157           print "training %s exists, skipping" % (filename)
158
159       else:  
160
161         try:
162           os.makedirs(dirname)
163         except:
164           None
165
166         xml = get_db_gpx(dbx,training['id'])
167
168         if verbose:
169           print "writing training %s" % (filename)
170
171         gpx = pygpx.GPX()
172         gpx.ReadTree(xml)
173
174         sport = training['sport']
175         timestamp = gpx.tracks[0].start_time()
176         
177         print sport, timestamp
178         
179         if check_db_for_training(db,sport,timestamp):
180           print "The same training found"
181           continue
182
183         gpx.author = username
184         gpx.FixNames(training['sport'])
185         gpx.ProcessTrackSegs()
186         for track in gpx.tracks:
187           track.sport=training['sport']
188
189         xml = gpx.XMLTree();
190         f = open(filename,"w")
191         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
192         f.close
193         write_parsed_to_db(db,gpx,filename)
194         try:
195           render_tiles.queue_render(db,filename)
196         except:
197           None  
198
199  
200 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
201
202     training_list = get_osm_list(username,password,year,month)
203
204     for training in training_list:
205     
206       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])   
207       dirname = "%s/%04d/%02d" % (dir,year,month)
208
209       if os.path.exists(filename) and not force:
210
211         if verbose:
212           print "training %s exists, skipping" % (filename)
213
214       else:  
215       
216         try:
217           os.makedirs(dirname)    
218         except:
219           None
220
221         xml = get_osm_gpx(username,password,training['id'])
222
223         if verbose:
224           print "writing training %s" % (filename)
225
226         gpx = pygpx.GPX()
227         gpx.ReadTree(xml)
228
229         sport = training['sport']
230         timestamp = gpx.tracks[0].start_time()
231         
232         if check_db_for_training(db,sport,timestamp):
233           print "The same training found"
234           continue
235
236         gpx.author = username
237         gpx.FixNames(training['sport'])
238         gpx.ProcessTrackSegs()
239         for track in gpx.tracks:
240           track.sport=training['sport']
241
242         xml = gpx.XMLTree();
243         f = open(filename,"w")
244         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
245         f.close
246         write_parsed_to_db(db,gpx,filename)
247         try:
248           render_tiles.queue_render(db,filename)
249         except:
250           None  
251
252 def sync_folder(username,year,month,dir=".",verbose=False,force=False):
253
254     training_list = get_page(username,year,month)
255     for tr in training_list:
256      try:
257
258       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1])   
259       dirname = "%s/%04d/%02d" % (dir,year,(month+1))
260
261       if os.path.exists(filename) and not force:
262
263         if verbose:
264           print "training %s exists, skipping" % (filename)
265
266       else:  
267
268         try:
269           os.makedirs(dirname)    
270         except:
271           None
272           
273         xml=get_gpx_track(tr[1],tr[0])
274
275         if verbose:
276           print "writing training %s" % (filename)
277
278         gpx = pygpx.GPX()
279         gpx.ReadTree(xml)
280
281         sport = tr[0]
282         timestamp = gpx.tracks[0].start_time()
283         
284         if check_db_for_training(db,sport,timestamp):
285           print "The same training found"
286           continue
287
288         gpx.FixNames(tr[0])
289         gpx.ProcessTrackSegs()
290         
291         xml = gpx.XMLTree();
292         f = open(filename,"w")
293         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
294         f.close
295         write_parsed_to_db(db,gpx,filename)
296         try:
297           render_tiles.queue_render(db,filename)
298         except:
299           None  
300
301      except:
302        raise
303
304 def main():
305
306     global db;
307     parser = OptionParser()
308     parser.add_option("-d", "--dir", dest="dirname",
309       help="write data to directory", metavar="DIR")
310     parser.add_option("-q", "--quiet",
311       action="store_false", dest="verbose", default=True,
312       help="don't print status messages to stdout")
313     parser.add_option("-f", "--force",
314       action="store_true", dest="force", default=False,
315       help="rewrite all files")
316     parser.add_option("-y", "--yearmonth", dest="yearmonth",
317       help="year and month in YYYY-MM format", metavar="YYYY-MM")                                                          
318     parser.add_option("-u", "--username", dest="username",
319       help="Run.GPS username")                                                          
320     parser.add_option("-o", "--osm-username", dest="osmname",
321       help="OpenStreetMap username")                                                          
322     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
323       help="OpenStreetMap password")                                                          
324     parser.add_option("-b", "--dropbox", dest="dropboxauth",
325       help="DropBox Auth token")                                                          
326     (options, args) = parser.parse_args()
327
328     username = options.username
329     osmname = options.osmname
330     osmpwd = options.osmpasswd
331     dbauth = options.dropboxauth
332     if not (username or (osmname and osmpwd) or dbauth):
333       print "Run.GPS username or OSM username/password or Dropbox Auth token is mandatory!"
334       return
335
336     if username:
337       pusername = username+'@Run.GPS'
338     elif osmname:
339       pusername = osmname+'@OSM'  
340     elif dbauth:
341       import dropbox
342       dbx = dropbox.Dropbox(dbauth)  
343       pusername = dbx.users_get_current_account().email+'@DBX'
344
345     try:
346       if options.yearmonth:
347         year,month = options.yearmonth.split('-')
348         month = int(month) -1
349         year = int(year)
350         if month<0 or month>11:
351           raise invalid_number
352       else:
353         year = None
354         month = None
355     except:
356       print "Year and month should be in YYYY-MM format!"
357       return 
358       
359     if options.dirname:  
360       outdir = options.dirname
361     else:
362       outdir = '.'
363     
364     db = outdir + '/gpx.db'
365     
366     if year:
367       if options.verbose:
368         print "retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir)
369       if username:
370         sync_folder(username,year,month,outdir,options.verbose,options.force)
371       elif osmname:
372         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)  
373       elif dbauth:
374         sync_db(dbx,pusername,year,month+1,outdir,options.verbose,options.force)  
375     else:
376       if options.verbose:
377         print "retrieving two last months for user %s to %s" % (pusername ,outdir)
378       now = date.today()
379       current_year = now.year
380       current_month = now.month
381       if username:
382         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
383       elif osmname:
384         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
385       elif dbauth:
386         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  
387       current_month = current_month -1
388       if current_month == 0:
389         current_month = 12
390         current_year = current_year -1
391       if username:
392         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
393       elif osmname:  
394         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
395       elif dbauth:
396         sync_db(dbx,pusername,current_year,current_month,outdir,options.verbose,options.force)  
397
398     generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-transparent.xml',400,400)
399
400 if __name__ == "__main__":
401
402     main()                    
403