8d13b8ddfd8925c4da9545363772165b4540e52a
[pyrungps.git] / pyrungps.py
1 #!/usr/bin/env python
2 # coding: UTF-8
3
4 import urllib2
5 #import sys
6 import os
7 from lxml import html,etree
8 from optparse import OptionParser
9 from datetime import date,datetime
10 from mx.DateTime.ISO import ParseDateTimeUTC
11 from parsegpx import write_parsed_to_db
12 import pygpx
13
14 import render_tiles
15
16 import generate_image
17
18 def get_page(uname,year,month):
19   
20   trainings = []
21
22   req = urllib2.Request("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), None, {'User-agent': 'Mozilla/5.0'})
23   page = urllib2.urlopen(req).read()
24   dom = html.document_fromstring(page)
25
26   for element, attribute, link, pos in dom.iterlinks():
27     if attribute == "href":
28       if link.startswith("/trainings/"):
29         dummy, dummy, link = link.split('/')
30         name, id = link.split('_')
31         trainings.append([ urllib2.unquote(name), id ])
32       
33   return trainings      
34
35 def get_gpx_track(trid,name):
36
37   print "trid=",trid
38
39   req = urllib2.urlopen("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid))
40   
41   xml = etree.parse(req)
42
43   return xml
44
45 def get_osm_list(username,password,year,month):
46
47   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
48   passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
49   passman.add_password(None, url, username, password)
50   authhandler = urllib2.HTTPBasicAuthHandler(passman)
51   opener = urllib2.build_opener(authhandler)
52   urllib2.install_opener(opener)
53
54   req = urllib2.urlopen(url)
55   xml = etree.parse(req)
56
57   res=[]
58
59   for gpx_file in xml.getroot():
60     attrib = dict(gpx_file.attrib)
61     timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
62     id = attrib['id']
63     description=None
64     for descr in gpx_file.iter('description'):
65       description=descr.text
66     sport=None
67     for tag in gpx_file.iter('tag'):
68       sport=tag.text
69     if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
70       print id,description,sport,timestamp
71       record={ 'id': id, 'sport': sport }
72       res.append(record)
73       
74     return res      
75
76 def get_osm_gpx(username,password,track_id):
77
78   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
79   passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
80   passman.add_password(None, url, username, password)
81   authhandler = urllib2.HTTPBasicAuthHandler(passman)
82   opener = urllib2.build_opener(authhandler)
83   urllib2.install_opener(opener)
84
85   req = urllib2.urlopen(url)
86   xml = etree.parse(req)
87
88   return xml
89
90 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
91
92     training_list = get_osm_list(username,password,year,month)
93
94     for training in training_list:
95       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])   
96       dirname = "%s/%04d/%02d" % (dir,year,month)
97
98       if os.path.exists(filename) and not force:
99
100         if verbose:
101           print "training %s exists, skipping" % (filename)
102
103       else:  
104
105         try:
106           os.makedirs(dirname)    
107         except:
108           None
109
110         xml = get_osm_gpx(username,password,training['id'])
111
112         if verbose:
113           print "writing training %s" % (filename)
114
115         gpx = pygpx.GPX()
116         gpx.ReadTree(xml)
117
118         gpx.author = username
119         gpx.FixNames(training['sport'])
120         gpx.ProcessTrackSegs()
121         for track in gpx.tracks:
122           track.sport=training['sport']
123         
124         xml = gpx.XMLTree();
125         f = open(filename,"w")
126         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
127         f.close
128         write_parsed_to_db(db,gpx,filename)
129         try:
130           render_tiles.queue_render(db,filename)
131         except:
132           None  
133
134 def sync_folder(username,year,month,dir=".",verbose=False,force=False):
135
136     training_list = get_page(username,year,month)
137     for tr in training_list:
138
139       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1])   
140       dirname = "%s/%04d/%02d" % (dir,year,(month+1))
141
142       if os.path.exists(filename) and not force:
143
144         if verbose:
145           print "training %s exists, skipping" % (filename)
146
147       else:  
148
149         try:
150           os.makedirs(dirname)    
151         except:
152           None
153           
154         xml=get_gpx_track(tr[1],tr[0])
155
156         if verbose:
157           print "writing training %s" % (filename)
158
159         gpx = pygpx.GPX()
160         gpx.ReadTree(xml)
161
162         gpx.FixNames(tr[0])
163         gpx.ProcessTrackSegs()
164         
165         xml = gpx.XMLTree();
166         f = open(filename,"w")
167         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
168         f.close
169         write_parsed_to_db(db,gpx,filename)
170         try:
171           render_tiles.queue_render(db,filename)
172         except:
173           None  
174
175 def main():
176
177     global db;
178     parser = OptionParser()
179     parser.add_option("-d", "--dir", dest="dirname",
180       help="write data to directory", metavar="DIR")
181     parser.add_option("-q", "--quiet",
182       action="store_false", dest="verbose", default=True,
183       help="don't print status messages to stdout")
184     parser.add_option("-f", "--force",
185       action="store_true", dest="force", default=False,
186       help="rewrite all files")
187     parser.add_option("-y", "--yearmonth", dest="yearmonth",
188       help="year and month in YYYY-MM format", metavar="YYYY-MM")                                                          
189     parser.add_option("-u", "--username", dest="username",
190       help="Run.GPS username")                                                          
191     parser.add_option("-o", "--osm-username", dest="osmname",
192       help="OpenStreetMap username")                                                          
193     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
194       help="OpenStreetMap password")                                                          
195     (options, args) = parser.parse_args()
196
197     username = options.username
198     osmname = options.osmname
199     osmpwd = options.osmpasswd
200     if not (username or (osmname and osmpwd)):
201       print "Run.GPS username or OSM username/password is mandatory!"
202       return
203
204     if username:
205       pusername = username+'@Run.GPS'
206     elif osmname:
207       pusername = osmname+'@OSM'  
208
209     try:
210       if options.yearmonth:
211         year,month = options.yearmonth.split('-')
212         month = int(month) -1
213         year = int(year)
214         if month<0 or month>11:
215           raise invalid_number
216       else:
217         year = None
218         month = None
219     except:
220       print "Year and month should be in YYYY-MM format!"
221       return 
222       
223     if options.dirname:  
224       outdir = options.dirname
225     else:
226       outdir = '.'
227     
228     db = outdir + '/gpx.db'
229     
230     if year:
231       if options.verbose:
232         print "retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir)
233       if username:
234         sync_folder(username,year,month,outdir,options.verbose,options.force)
235       elif osmname:
236         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)  
237     else:
238       if options.verbose:
239         print "retrieving two last months for user %s to %s" % (pusername,outdir)
240       now = date.today()
241       current_year = now.year
242       current_month = now.month
243       if username:
244         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
245       elif osmname:
246         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
247       current_month = current_month -1
248       if current_month == 0:
249         current_month = 12
250         current_year = current_year -1
251       if username:
252         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
253       elif osmname:  
254         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
255
256    generate_image.render_all(db,'/etc/mapnik-osm-carto-data/veloroad-transparent.xml',400,400)
257
258 if __name__ == "__main__":
259
260     main()                    
261