db2497238f3abdf030d54cce4e225cc7e9a21753
[pyrungps.git] / pyrungps.py
1 #!/usr/bin/env python
2 # coding: UTF-8
3
4 import urllib2
5 #import sys
6 import os
7 from lxml import html,etree
8 from optparse import OptionParser
9 from datetime import date,datetime
10 from mx.DateTime.ISO import ParseDateTimeUTC
11 from parsegpx import write_parsed_to_db
12 import pygpx
13
14 import render_tiles
15
16 def get_page(uname,year,month):
17   
18   trainings = []
19
20   req = urllib2.Request("http://www.gps-sport.net/services/getMonthlyTrainingDataHTML_V2.jsp?userName=%s&year=%s&month=%s&rnd=0.645673"% (uname,year,month), None, {'User-agent': 'Mozilla/5.0'})
21   page = urllib2.urlopen(req).read()
22   dom = html.document_fromstring(page)
23
24   for element, attribute, link, pos in dom.iterlinks():
25     if attribute == "href":
26       if link.startswith("/trainings/"):
27         dummy, dummy, link = link.split('/')
28         name, id = link.split('_')
29         trainings.append([ urllib2.unquote(name), id ])
30       
31   return trainings      
32
33 def get_gpx_track(trid,name):
34
35   print "trid=",trid
36
37   req = urllib2.urlopen("http://www.gps-sport.net/services/trainingGPX.jsp?trainingID=%s&tz=-180" % (trid))
38   
39   xml = etree.parse(req)
40
41   return xml
42
43 def get_osm_list(username,password,year,month):
44
45   url = "https://www.openstreetmap.org/api/0.6/user/gpx_files"
46   passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
47   passman.add_password(None, url, username, password)
48   authhandler = urllib2.HTTPBasicAuthHandler(passman)
49   opener = urllib2.build_opener(authhandler)
50   urllib2.install_opener(opener)
51
52   req = urllib2.urlopen(url)
53   xml = etree.parse(req)
54
55   res=[]
56
57   for gpx_file in xml.getroot():
58     attrib = dict(gpx_file.attrib)
59     timestamp=datetime.fromtimestamp(ParseDateTimeUTC(attrib['timestamp']))
60     id = attrib['id']
61     description=None
62     for descr in gpx_file.iter('description'):
63       description=descr.text
64     sport=None
65     for tag in gpx_file.iter('tag'):
66       sport=tag.text
67     if timestamp.year==year and timestamp.month==month and description=='Run.GPS Track':
68       print id,description,sport,timestamp
69       record={ 'id': id, 'sport': sport }
70       res.append(record)
71       
72     return res      
73
74 def get_osm_gpx(username,password,track_id):
75
76   url = "https://www.openstreetmap.org/api/0.6/gpx/"+track_id+"/data"
77   passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
78   passman.add_password(None, url, username, password)
79   authhandler = urllib2.HTTPBasicAuthHandler(passman)
80   opener = urllib2.build_opener(authhandler)
81   urllib2.install_opener(opener)
82
83   req = urllib2.urlopen(url)
84   xml = etree.parse(req)
85
86   return xml
87
88 def sync_osm(username,password,year,month,dir=".",verbose=False,force=False):
89
90     training_list = get_osm_list(username,password,year,month)
91
92     for training in training_list:
93       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,month,training['sport'],training['id'])   
94       dirname = "%s/%04d/%02d" % (dir,year,month)
95
96       if os.path.exists(filename) and not force:
97
98         if verbose:
99           print "training %s exists, skipping" % (filename)
100
101       else:  
102
103         try:
104           os.makedirs(dirname)    
105         except:
106           None
107
108         xml = get_osm_gpx(username,password,training['id'])
109
110         if verbose:
111           print "writing training %s" % (filename)
112
113         gpx = pygpx.GPX()
114         gpx.ReadTree(xml)
115
116         gpx.author = username
117         gpx.FixNames(training['sport'])
118         gpx.ProcessTrackSegs()
119         for track in gpx.tracks:
120           track.sport=training['sport']
121         
122         xml = gpx.XMLTree();
123         f = open(filename,"w")
124         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
125         f.close
126         write_parsed_to_db(db,gpx,filename)
127         try:
128           render_tiles.queue_render(db,filename)
129         except:
130           None  
131
132 def sync_folder(username,year,month,dir=".",verbose=False,force=False):
133
134     training_list = get_page(username,year,month)
135     for tr in training_list:
136
137       filename = "%s/%04d/%02d/%s_%s.gpx" % (dir,year,(month+1),tr[0],tr[1])   
138       dirname = "%s/%04d/%02d" % (dir,year,(month+1))
139
140       if os.path.exists(filename) and not force:
141
142         if verbose:
143           print "training %s exists, skipping" % (filename)
144
145       else:  
146
147         try:
148           os.makedirs(dirname)    
149         except:
150           None
151           
152         xml=get_gpx_track(tr[1],tr[0])
153
154         if verbose:
155           print "writing training %s" % (filename)
156
157         gpx = pygpx.GPX()
158         gpx.ReadTree(xml)
159
160         gpx.FixNames(tr[0])
161         gpx.ProcessTrackSegs()
162         
163         xml = gpx.XMLTree();
164         f = open(filename,"w")
165         f.write(etree.tostring(xml,encoding='UTF-8',pretty_print=True))
166         f.close
167         write_parsed_to_db(db,gpx,filename)
168         try:
169           render_tiles.queue_render(db,filename)
170         except:
171           None  
172
173 def main():
174
175     global db;
176     parser = OptionParser()
177     parser.add_option("-d", "--dir", dest="dirname",
178       help="write data to directory", metavar="DIR")
179     parser.add_option("-q", "--quiet",
180       action="store_false", dest="verbose", default=True,
181       help="don't print status messages to stdout")
182     parser.add_option("-f", "--force",
183       action="store_true", dest="force", default=False,
184       help="rewrite all files")
185     parser.add_option("-y", "--yearmonth", dest="yearmonth",
186       help="year and month in YYYY-MM format", metavar="YYYY-MM")                                                          
187     parser.add_option("-u", "--username", dest="username",
188       help="Run.GPS username")                                                          
189     parser.add_option("-o", "--osm-username", dest="osmname",
190       help="OpenStreetMap username")                                                          
191     parser.add_option("-p", "--osm-passwd", dest="osmpasswd",
192       help="OpenStreetMap password")                                                          
193     (options, args) = parser.parse_args()
194
195     username = options.username
196     osmname = options.osmname
197     osmpwd = options.osmpasswd
198     if not (username or (osmname and osmpwd)):
199       print "Run.GPS username or OSM username/password is mandatory!"
200       return
201
202     if username:
203       pusername = username+'@Run.GPS'
204     elif osmname:
205       pusername = osmname+'@OSM'  
206
207     try:
208       if options.yearmonth:
209         year,month = options.yearmonth.split('-')
210         month = int(month) -1
211         year = int(year)
212         if month<0 or month>11:
213           raise invalid_number
214       else:
215         year = None
216         month = None
217     except:
218       print "Year and month should be in YYYY-MM format!"
219       return 
220       
221     if options.dirname:  
222       outdir = options.dirname
223     else:
224       outdir = '.'
225     
226     db = outdir + '/gpx.db'
227     
228     if year:
229       if options.verbose:
230         print "retrieving trainings for user %s, year %s, month %s to %s" % (pusername,year,month+1,outdir)
231       if username:
232         sync_folder(username,year,month,outdir,options.verbose,options.force)
233       elif osmname:
234         sync_osm(osmname,osmpwd,year,month+1,outdir,options.verbose,options.force)  
235     else:
236       if options.verbose:
237         print "retrieving two last months for user %s to %s" % (pusername,outdir)
238       now = date.today()
239       current_year = now.year
240       current_month = now.month
241       if username:
242         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
243       elif osmname:
244         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
245       current_month = current_month -1
246       if current_month == 0:
247         current_month = 12
248         current_year = current_year -1
249       if username:
250         sync_folder(username,current_year,current_month-1,outdir,options.verbose,options.force)
251       elif osmname:  
252         sync_osm(osmname,osmpwd,current_year,current_month,outdir,options.verbose,options.force)  
253
254 if __name__ == "__main__":
255
256     main()                    
257