From: Roman Bazalevsky <rvb@rvb.name>
Date: Tue, 27 Sep 2016 12:31:40 +0000 (+0300)
Subject: Сглаживание выбросов показаний датчиков медианным фильтром.
X-Git-Url: https://git.rvb.name/weathermon.git/commitdiff_plain/b7dcd298e5f0565be8e5ca3d9477cb6de97ae3ac?ds=inline;hp=f9168d78a2f045082d82631029236b218f0e5310

Сглаживание выбросов показаний датчиков медианным фильтром.
---

diff --git a/filter_meteo.py b/filter_meteo.py
new file mode 100644
index 0000000..b20dc41
--- /dev/null
+++ b/filter_meteo.py
@@ -0,0 +1,133 @@
+#!/usr/bin/python
+
+import MySQLdb
+import ConfigParser
+import sys
+
+from pprint import pprint
+import datetime
+
+import numpy as np
+
+import scipy.signal
+
+global database
+
+def GetTables():
+  if database:
+    c = database.cursor()
+    c.execute("SELECT s.id sid,p.id pid FROM sensors s,st_parameters p where s.st_id=p.st_id and p.id>=0")
+    return c.fetchall()
+  else:
+    print "No connection to DB"
+    exit()
+
+def Today():
+  dt = datetime.datetime.now()
+  d_truncated = datetime.date(dt.year, dt.month, dt.day)
+  return d_truncated
+  
+def Tomorrow():
+  dt = Today()
+  return dt + datetime.timedelta(days=1)
+
+def Yesterday():
+  dt = Today()
+  return dt - datetime.timedelta(days=1)
+
+def GetData(sid,pid,fromDate=Yesterday(),toDate=Today()):
+  if database:
+    c = database.cursor()
+    c.execute("SELECT id,timestamp,value FROM sensor_values WHERE sensor_id=%s and parameter_id=%s and timestamp>=%s AND timestamp<%s",[sid,pid,fromDate.strftime('%Y-%m-%d %H:%M:%S'),toDate.strftime('%Y-%m-%d %H:%M:%S')])
+    return c.fetchall()
+  else:
+    print "No connection to DB"
+    exit()
+
+def FixRecord(id,value):
+  if database:
+    c = database.cursor()
+    command="UPDATE sensor_values  SET value={} WHERE id='{}'".format(value,id)
+    print command
+    c.execute(command)
+  else:
+    print "No connection to DB"
+    exit()
+
+def ProcessTable(sid,pid):
+
+  if not current:
+    data=GetData(sid,pid)
+  else:
+    data=GetData(sid,pid,Today(),Tomorrow())  
+  sID=[]
+  sTime=[]
+  sValue=[]
+  for rec in data:
+    sID.append(rec[0])
+    sTime.append(rec[1])
+    sValue.append(rec[2])
+  sValue=np.array(sValue)
+
+  sValueFilt=scipy.signal.medfilt(sValue,5)
+
+  sValueDiff=abs(sValue-sValueFilt)
+  
+  avg=np.mean(sValueDiff)
+
+  print "Average="+str(avg)
+
+  for i in range(0,len(sTime)-1):
+    if sValueDiff[i]>avg*filterThreshold:
+      print "fixing %s : %5.2f %5.2f %5.2f" % (sTime[i],sValue[i],sValueFilt[i],sValueDiff[i])
+      FixRecord(sID[i],sValueFilt[i])      
+
+  database.commit()
+
+if len(sys.argv)==2 and sys.argv[1]=='current':
+  current=True
+else:
+  current=False
+
+
+try:
+
+  cfg = ConfigParser.RawConfigParser(allow_no_value=True)
+  cfg.readfp(open('/etc/weathermon.conf'))
+  dbhost = cfg.get("mysql","host")
+  dbuser = cfg.get("mysql","user")
+  dbpasswd = cfg.get("mysql","passwd")
+  dbdb = cfg.get("mysql","db")
+
+  filterWindow = int(cfg.get("filter","window"))
+  filterThreshold = float(cfg.get("filter","threshold"))
+   
+except:
+
+  print "Error reading configuration file"
+  exit()
+
+try:
+
+  database = MySQLdb.connect(host=dbhost,user=dbuser,passwd=dbpasswd,db=dbdb,use_unicode=True)
+  database.set_character_set('utf8')
+  c = database.cursor()
+  c.execute('SET NAMES utf8;')
+
+  print "Connected..."
+
+except:
+
+  print "Error connecting database"
+  exit()
+
+tables = GetTables()
+
+for sid,pid in tables:
+
+  print "Processing sensor %d, parameter %d " % (sid,pid)
+
+  ProcessTable(sid,pid)
+
+print "Processed "
+  
\ No newline at end of file
diff --git a/weathermon.conf b/weathermon.conf
index 3b02ae2..9018a24 100644
--- a/weathermon.conf
+++ b/weathermon.conf
@@ -2,6 +2,10 @@
 host = dbhost
 user = meteo
 passwd = password
+db = meteo
+[filter]
+window=5
+threshold=10
 [serial]
 port = /dev/ttyATH0
 [openweathermap]
diff --git a/weathermon.conf~ b/weathermon.conf~
deleted file mode 100644
index 164a1ee..0000000
--- a/weathermon.conf~
+++ /dev/null
@@ -1,17 +0,0 @@
-[mysql]
-host = estia.rvb-home.lan
-user = meteo
-passwd = snovadozhdi
-[serial]
-port = /dev/ttyATH0
-[openweathermap]
-user = rvbglas
-passwd = gdetotamdaleko
-temp = OUTDOOR.44F.TEMPERATURE
-pres = BARO.DEFAULT.PRESSURE
-humi = OUTDOOR.44F.HUMIDITY
-lat = 55.403266
-lon = 37.561651
-station = rvb.name
-[logging]
-enabled=on