diff --git a/micropython.py b/micropython.py new file mode 100644 index 0000000000000000000000000000000000000000..c7069bc951ab6e55c6ff853ea3e3ee7e809b6ca0 --- /dev/null +++ b/micropython.py @@ -0,0 +1,49 @@ +""" +This code is designed to run on an ESP32 for grabbing temperature data from +ds18x20 sensors via the onewire protocol and sending them via serial to the +connected host. It should be trivial to port it to different platforms. + +The Protocol is dead simple: + +ONEWIRE-ID1 TEMPERATURE1 +ONEWIRE-ID2 TEMPERATURE2 +... +<Empty Line> + +When a sensor has problems reading, it sends as temperature 9001. + +New sensors are only detected upon powerup, so you have to reboot in order to +extend the sensor network. + +The sensors have a parasitic-power-mode which is NOT TO BE USED here. +Please connect all three pins, and multiplex as you please. +""" + +import machine +import time +import onewire, ds18x20 +import ubinascii + +class reader: + + def __init__(self): + + self.di = machine.Pin(13) + + self.ds = ds18x20.DS18X20(onewire.OneWire(self.di)) + + #scan for sensors + self.roms = self.ds.scan() + + def run(self): + while 1: + time.sleep_ms(240) + self.ds.convert_temp() + time.sleep_ms(750) + for rom in self.roms: + print(ubinascii.hexlify(rom).decode('utf-8'), end=' ') + try: + print(self.ds.read_temp(rom)) + except: + print(9001) + print() diff --git a/plugins/collectd.py b/plugins/collectd.py new file mode 100644 index 0000000000000000000000000000000000000000..ba6dc85e807b45fe925e4e1337f835230d26d074 --- /dev/null +++ b/plugins/collectd.py @@ -0,0 +1,94 @@ +import asyncio +import time + +def init(monitor): + return PluginCollectd(monitor) + +class PluginCollectd: + """ + Implements a super simple collectd interface for only sending temperature data + """ + def __init__(self, monitor): + self.loop = asyncio.get_event_loop() + self.config = monitor.config + self.path = self.config['collectd']['socketpath'] + self._reader, self._writer = (None, None) + self.loop.run_until_complete(self.reconnect()) + + self.monitor = monitor + + self.last_store = 0 + + async def reconnect(self): + """ + optionally close and then reconnect to the unix socket + """ + if self._writer: + self._writer.close() + + self._reader, self._writer = await asyncio.open_unix_connection( + path=self.path, + loop=self.loop) + + async def _send(self, identifier, interval, timestamp, value): + """ + The collectd naming convention is: + host "/" plugin ["-" plugin instance] "/" type ["-" type instance] + Whereby: + - host: local host name + - plugin: the tab in CGP + - plugin-instance: the graph + - type the line in the graph + - type instance : if there are more than one "temperature"s + """ + + data = "PUTVAL \"{}/{}\" interval={} {}:{}\n".format( + self.config['collectd']['hostname'], + identifier, + interval, + timestamp, + value) + #print("Sending data:", data.strip()) + self._writer.write(data.encode('utf-8')) + await self._writer.drain() + + try: + line = await asyncio.wait_for(self._reader.readline(), 1) + except asyncio.TimeoutError: + print("Collectd did not respond.") + return + line = line.decode('utf-8').strip() + if not line: + print("Connection reset. reconnecting") + await self.reconnect() + else: + pass + #print("recv:", line) + + async def send_sensor_values(self, sensor): + """ + Store the temperature to collectd for fancy graphs + """ + await self._send("tail-temperature/temperature-{}".format(sensor.name), + int(self.config['collectd']['interval']), + int(sensor.last_update), + sensor.temperature) + + ## Plugin Callbacks ## + async def send_stats_graph(self, graph, stattype, stattime, statval): + """ + to be called as a plugin callback to store stuff into collectd + """ + await self._send("tail-{}/{}".format(graph, stattype), + int(self.config['collectd']['interval']), + stattime, + statval) + + async def sensor_update(self): + """ + Receive sensor data to store them regularely into collectd + """ + for sensor in self.monitor.sensors.values(): + if sensor.valid: + await self.send_sensor_values(sensor) + self.last_store = time.time() diff --git a/plugins/mail.py b/plugins/mail.py new file mode 100644 index 0000000000000000000000000000000000000000..869b625be8d4652be94827dec177886ac1dae96d --- /dev/null +++ b/plugins/mail.py @@ -0,0 +1,154 @@ +import time +from email.mime.text import MIMEText +from email.utils import formatdate +import smtplib + +UNKNOWN_SENSOR_SUBJECT = "WARNING: Unconfigured Sensor ID" +UNKNOWN_SENSOR_BODY = """Hello Guys, + +An unknown sensor has been connected to the temperature monitoring service. +Please add the following section to the list of known sensors in {config}. + +[{owid}] +name=changeme +calibration=0 + +The current temperature of the sensor is {temp} + +Regards, Temperature +""" + +SENSOR_MEASUREMENT_MISSED_SUBJECT = "WARNING: Sensor Measurement was missed" +SENSOR_MEASUREMENT_MISSED_BODY = """Hello Guys, + +A sensor measurement was missed from the temperature monitoring. +This indicates either a problem with the hardware (check the wireing!) or the config. + +ID: {owid} +NAME: {name}. + +Please go check it! + +Regards, Temperature +""" + +SENSOR_PROBLEM_SUBJECT = "WARNING: Sensor error" +SENSOR_PROBLEM_BODY = """Hello Guys, + +A sensor measurement was invalid. This might mean, that the sensor was disconnected. +Please go and check the sensor with the id + +ID: {owid} +NAME: {name}. +LAST: {temp} + +Regards, Temperature +""" + +NO_DATA_SUBJECT = "WARNING: Did not receive any data" +NO_DATA_BODY = """Helly guys, + +It has been {time} seconds since i have last received a temperature value. +This is unlikely - please come and check + +Regards, Temperature +""" + + + +def init(monitor): + """ + Plugin initialization method to be called from the outside + """ + return PluginMail(monitor) + +class PluginMail: + """ + Handle all the mail sending stuff + """ + def __init__(self, monitor): + self.monitor = monitor + self.config = self.monitor.config + + self._mail_rate_limit = {} + + async def send_mail(self, subject, body, urgent=False): + """ + Send a mail to the configured recipients + """ + msg = MIMEText(body, _charset="UTF-8") + msg['Subject'] = subject + msg['From'] = self.config['mail']['from'] + if urgent: + msg['To'] = self.config['mail']['to_urgent'] + else: + msg['To'] = self.config['mail']['to'] + msg['Date'] = formatdate(localtime=True) + + print("Notification: {}".format(subject)) + + # Ratelimit the emails + time_since_last_mail = time.time() - self._mail_rate_limit.get(subject, 0) + if time_since_last_mail < int(self.config['mail']['min_delay_between_messages']): + print("Not sending due to ratelimiting") + return + + print("Body: {}".format(body)) + + self._mail_rate_limit[subject] = time.time() + smtp = smtplib.SMTP("mail.stusta.mhn.de") + #smtp.sendmail(msg['From'], msg['To'], msg.as_string()) + smtp.quit() + + async def err_nodata(self, **kwargs): + await self.send_mail(NO_DATA_SUBJECT, NO_DATA_BODY) + + async def err_unknown_sensor(self, **kwargs): + await self.send_mail( + UNKNOWN_SENSOR_SUBJECT, + UNKNOWN_SENSOR_BODY.format(**kwargs)) + + async def err_problem_sensor(self, **kwargs): + await self.send_mail( + SENSOR_PROBLEM_SUBJECT, + SENSOR_PROBLEM_BODY.format(**kwargs)) + + async def err_missed_sensor(self, **kwargs): + await self.send_mail( + SENSOR_MEASUREMENT_MISSED_SUBJECT, + SENSOR_MEASUREMENT_MISSED_BODY.format(**kwargs)) + + + async def temperature_warning(self, source, **kwargs): + subject = "Temperaturwarnung Serverraum" + + body = """Hi Guys, + +Die Temperaturen im Serverraum werden langsam Bedenklich: + +{temperatures} + +Auslöser: {reason} + +Aktuelle Temperaturen: +{alltemperatures} + +Bitte haltet die Temperaturen im Auge und fahrt eventuell heiß laufende Server herunter + +with love, +Temperator""" + + if source == "tempdiff": + temperatures = "{name1}:{temp1}\n{name2}:{temp2}".format(**kwargs) + reason = "Differenztemperatur: {tempdiff}".format(**kwargs) + elif source == "singlehot": + temperatures="{name}:{temp}".format(**kwargs) + reason = "Einzeltemperatur zu hoch" + + alltemperatures = '\n'.join([ + "{}: {}".format(sensor.name, sensor.temperature) if sensor.valid + else "{}: INVALID".format(sensor.name) + for sensor in self.monitor.sensors.values() ]) + + await self.send_mail(subject, body.format( + temperatures=temperatures, reason=reason, alltemperatures=alltemperatures)) diff --git a/plugins/warnings.py b/plugins/warnings.py new file mode 100644 index 0000000000000000000000000000000000000000..d6fe632ebfa796f3155c39925bb8af9f453945f2 --- /dev/null +++ b/plugins/warnings.py @@ -0,0 +1,125 @@ +import time + +def init(monitor): + """ Plugin interface method """ + return PluginWarning(monitor) + +class PluginWarning: + """ + Generate all kind of warnings whenever needed and observe the sensor + if they see a problematic situation in the container + """ + def __init__(self, monitor): + self.monitor = monitor + + self.revmapping = { + sensor.name : sensor + for sensor in self.monitor.sensors.values() + } + + self.warning_conf = self.monitor.config['warning'] + + conftest = [ + self.warning_conf['floor_sensors'], + self.warning_conf['ceiling_sensors'], + self.warning_conf['floor_ceiling_diff'], + self.warning_conf['ceiling_warning_level'], + ] + del conftest + + def get_sensor(self, sensorname): + """ + Resovle the reverse mapping and get back the sensor + """ + return self.revmapping[sensorname] + + def get_stats(self, sensorlist): + """ + Calculate mininmum, maximum average and variance over the sensor names given + """ + sensors = [self.get_sensor(sensor) for sensor in sensorlist] + sensors = [sensor for sensor in sensors if sensor.valid] + + if not sensors: + return [], 0, 0, 0, 0 + + avg = sum(sensor.temperature for sensor in sensors) / len(sensors) + var = sum((sensor.temperature - avg)**2 for sensor in sensors) / len(sensors) + + sensormin = -9999 + sensormax = +9999 + for sensor in sensors: + if sensor.temperature < sensormin: + sensormin = sensormin + if sensor.temperature > sensormax: + sensormax = sensormax + + return sensors, sensormin, sensormax, avg, var + + + async def sensor_update(self): + """ + First generate stats and relay them to the collectd module, then use these stats + to decide wether it is currently critical in the container, and if so, send + warnings + """ + # Do nothing yet + floor_sensors, floor_min, floor_max, floor_avg, floor_var = self.get_stats( + self.warning_conf['floor_sensors'].split(',')) + + ceil_sensors, ceil_min, ceil_max, ceil_avg, ceil_var = self.get_stats( + self.warning_conf['ceiling_sensors'].split(',')) + + now = time.time() + if floor_sensors: + await self.monitor.call_plugin( + "send_stats_graph", graph="stats", + stattype="temperature-floormin", stattime=now, statval=floor_min) + await self.monitor.call_plugin( + "send_stats_graph", graph="stats", + stattype="temperature-floormax", stattime=now, statval=floor_max) + await self.monitor.call_plugin( + "send_stats_graph", graph="stats", + stattype="temperature-flooravg", stattime=now, statval=floor_avg) + await self.monitor.call_plugin( + "send_stats_graph", graph="stats", + stattype="temperature-floorvar", stattime=now, statval=floor_var) + + if ceil_sensors: + await self.monitor.call_plugin( + "send_stats_graph", graph="stats", + stattype="temperature-ceilmin", stattime=now, statval=ceil_min) + await self.monitor.call_plugin( + "send_stats_graph", graph="stats", + stattype="temperature-ceilmax", stattime=now, statval=ceil_max) + await self.monitor.call_plugin( + "send_stats_graph", graph="stats", + stattype="temperature-ceilavg", stattime=now, statval=ceil_avg) + await self.monitor.call_plugin( + "send_stats_graph", graph="stats", + stattype="temperature-ceilvar", stattime=now, statval=ceil_var) + + if floor_sensors and ceil_sensors: + # Else we already have sent warning messages for broken sensors + + tempdiff = ceil_avg - floor_avg + await self.monitor.call_plugin( + "send_stats_graph", graph="stats", + stattype="temperature-floor_ceil_diff", stattime=now, statval=tempdiff) + + # Here comes the warning magic + if ceil_max > int(self.warning_conf['min_ceiling_warning']): + if tempdiff > int(self.warning_conf['floor_ceiling_diff']): + await self.monitor.call_plugin("temperature_warning", + source="tempdiff", + name1="floor", + name2="ceiling", + temp1=floor_avg, + temp2=ceil_avg, + tempdiff=tempdiff) + + if ceil_avg > int(self.warning_conf['ceiling_warning_level']): + await self.monitor.call_plugin("temperature_warning", + source="singlehot", + name="ceiling", + temp=ceil_avg) diff --git a/tempermonitor.py b/tempermonitor.py index 4a1e80519b72323beaa31dcab026222f9d65894f..04c6f0109b7d57598f2602dfed93417a8e25bb81 100755 --- a/tempermonitor.py +++ b/tempermonitor.py @@ -16,67 +16,15 @@ Open issues: - Integrate USB Sensors """ - import asyncio import configparser import sys import time +import importlib from datetime import datetime -from email.mime.text import MIMEText -from email.utils import formatdate -import smtplib +from pathlib import Path import serial_asyncio - -UNKNOWN_SENSOR_SUBJECT = "WARNING: Unconfigured Sensor ID" -UNKNOWN_SENSOR_BODY = """Hello Guys, - -An unknown sensor has been connected to the temperature monitoring service. -Please add the following section to the list of known sensors in {config}. - -[{owid}] -name=changeme -calibration=0 - -The current temperature of the sensor is {temp} - -Regards, Temperature -""" - -SENSOR_MEASUREMENT_MISSED_SUBJECT = "WARNING: Sensor Measurement was missed" -SENSOR_MEASUREMENT_MISSED_BODY = """Hello Guys, - -A sensor measurement was missed from the temperature monitoring. -This indicates either a problem with the hardware (check the wireing!) or the config. - -ID: {owid} -NAME: {name}. - -Please go check it! - -Regards, Temperature -""" - -SENSOR_PROBLEM_SUBJECT = "WARNING: Sensor error" -SENSOR_PROBLEM_BODY = """Hello Guys, - -A sensor measurement was invalid. This might mean, that the sensor was disconnected. -Please go and check the sensor with the id - -ID: {owid} -NAME: {name}. -LAST: {tem} - -Regards, Temperature -""" - -NO_DATA_SUBJECT = "WARNING: Did not receive any data" -NO_DATA_BODY = """Helly guys, - -It has been {time} seconds since i have last received a temperature value. -This is unlikely - please come and check - -Regards, Temperature -""" +import serial class Sensor: """ @@ -86,6 +34,7 @@ class Sensor: self.temperature = None self.last_update = 0 self.calibration = 0 + self.valid = False try: if owid in config: @@ -104,52 +53,6 @@ class Sensor: self.temperature = float(temperature) self.last_update = time.time() -class Collectd: - """ - Implements a super simple collectd interface for only sending temperature data - """ - def __init__(self, loop, config): - self.loop = loop or asyncio.get_event_loop() - self.config = config - self.path = self.config['collectd']['socketpath'] - self._reader, self._writer = (None, None) - self.loop.run_until_complete(self.reconnect()) - - async def reconnect(self): - """ - optionally close and then reconnect to the unix socket - """ - if self._reader: - self._reader.close() - if self._writer: - self._writer.close() - - self._reader, self._writer = await asyncio.open_unix_connection( - path=self.path, - loop=self.loop) - - async def send(self, sensor): - """ - Store the temperature to collectd for fancy graphs - """ - data = "PUTVAL \"{}/{}\" interval={} {}:{}\n".format( - self.config['collectd']['hostname'], - "tail-temperature/temperature-{}".format(sensor.name), - int(self.config['collectd']['interval']), - int(sensor.last_update), - sensor.temperature) - print("Sending data:", data.strip()) - self._writer.write(data.encode('utf-8')) - await self._writer.drain() - line = (await self._reader.readline()).decode('utf-8').strip() - if not line: - print("Connection reset. reconnecting") - await self.reconnect() - else: - print("recv:", line) - - - class TempMonitor: """ Interact with the esp-one-wire interface that sends: @@ -162,25 +65,17 @@ class TempMonitor: """ def __init__(self, loop, configfile): - loop = loop or asyncio.get_event_loop() + self.loop = loop or asyncio.get_event_loop() self._configname = configfile self.config = configparser.ConfigParser() self.config.read(configfile) - self._collectd = Collectd(loop, self.config) - print("connecting to", self.config['serial']['port']) - self._reader, self._writer = loop.run_until_complete( - serial_asyncio.open_serial_connection( - url=self.config['serial']['port'], - baudrate=self.config['serial']['baudrate'], - loop=loop - )) - - self._known_sensors = {} + self.plugins = [] + self.sensors = {} self._last_store = 0 - self._mail_rate_limit = {} + self._reader, self._writer = (None, None) # Test if all necessary config fields are set, that are not part of the normal # startup @@ -192,21 +87,27 @@ class TempMonitor: self.config['mail']['to_urgent'], self.config['mail']['min_delay_between_messages'], self.config['serial']['timeout'], + self.config['serial']['port'], + self.config['serial']['baudrate'], ] del configtest + print("connecting to", self.config['serial']['port']) for owid in self.config: # Skip all known and predefined sections - if owid in ['DEFAULT', 'serial', 'collectd', 'mail']: + if owid in ['DEFAULT', 'serial', 'collectd', 'mail', 'warning']: continue - self._known_sensors[owid] = Sensor(self.config, owid) - + self.sensors[owid] = Sensor(self.config, owid) self._run_task = loop.create_task(self.run()) - async def run(self): + async def reconnect(self): """ - Read the protocol, update the sensors or trigger a collectd update + Connect to the ESP chip """ + self._reader, self._writer = await serial_asyncio.open_serial_connection( + url=self.config['serial']['port'], + baudrate=self.config['serial']['baudrate'], + loop=self.loop) # upon startup we only see garbage. (micropython starting up), # also it will produce warnings if the recording is started in the middle # of a message, so wait until the end of a message block to start the game @@ -219,6 +120,12 @@ class TempMonitor: except UnicodeError: continue + async def run(self): + """ + Read the protocol, update the sensors or trigger a collectd update + """ + await self.reconnect() + while True: # Wait for the next line try: @@ -226,14 +133,18 @@ class TempMonitor: self._reader.readline(), timeout=int(self.config['serial']['timeout'])) except asyncio.TimeoutError: - await self.send_mail(NO_DATA_SUBJECT, NO_DATA_BODY) + await self.call_plugin("err_nodata") + continue + except serial.SerialException as exc: + print("Problem with the serial connection - reconnecting: ", exc) + await self.reconnect() continue try: line = line.decode('ascii').strip() except UnicodeError: continue - print("recv:", line) + #print("recv:", line) if line == '': # Block has ended @@ -247,25 +158,22 @@ class TempMonitor: print("Invaid line received: {}\n{}".format(line, exc)) continue - sensor = self._known_sensors.get(owid, None) + sensor = self.sensors.get(owid, None) if not sensor: # If the sensor is new - notify the operators - await self.send_mail( - UNKNOWN_SENSOR_SUBJECT, - UNKNOWN_SENSOR_BODY.format( - configname=self._configname, - owid=owid, - temp=temp)) - + await self.call_plugin("err_unknown_sensor", + config=self._configname, + owid=owid, + temp=temp) elif temp > 1000 or temp < -1000: + sensor.valid = False # if the sensor is giving bullshit data - notify the operators - await self.send_mail( - SENSOR_PROBLEM_SUBJECT, - SENSOR_PROBLEM_BODY.format( - owid=owid, - name=sensor.name, - temp=temp)) + await self.call_plugin("err_problem_sensor", + owid=owid, + name=sensor.name, + temp=temp) else: + sensor.valid = True # in the unlikely event that everyting is fine: log the data sensor.update(temp) @@ -277,48 +185,48 @@ class TempMonitor: except asyncio.CancelledError: pass + async def call_plugin(self, call, *args, **kwargs): + """ + Call the given method on all plugins, proxying arguments + """ + result = {} + for plugin in self.plugins: + func = getattr(plugin, call, None) + if func: + if asyncio.iscoroutinefunction(func): + result[plugin.name] = await func(*args, **kwargs) + else: + result[plugin.name] = func(*args, **kwargs) + async def store_sensors(self): """ Prepare the sensors to be stored and maybe send an email """ - for owid, sensor in self._known_sensors.items(): - if sensor.last_update <= self._last_store: - isotime = datetime.utcfromtimestamp(sensor.last_update).isoformat() - await self.send_mail( - SENSOR_MEASUREMENT_MISSED_SUBJECT, - SENSOR_MEASUREMENT_MISSED_BODY.format( - owid=owid, - name=sensor.name, - last_update=isotime)) + sensorstr = "measurements: " + for owid, sensor in self.sensors.items(): + if sensor.valid: + sensorstr += "{}: {}; ".format(sensor.name, sensor.temperature) + if sensor.last_update <= self._last_store: + sensor.valid = False + isotime = datetime.utcfromtimestamp(sensor.last_update).isoformat() + await self.call_plugin("err_missed_sensor", + owid=owid, + name=sensor.name, + last_update=isotime) else: - await self._collectd.send(sensor) + sensorstr += "{}: INVALID; ".format(sensor.name) + print(sensorstr) + await self.call_plugin("sensor_update") self._last_store = time.time() - async def send_mail(self, subject, body, urgent=False): - """ - Send a mail to the configured recipients - """ - msg = MIMEText(body, _charset="UTF-8") - msg['Subject'] = subject - msg['From'] = self.config['mail']['from'] - if urgent: - msg['To'] = self.config['mail']['to_urgent'] - else: - msg['To'] = self.config['mail']['to'] - msg['Date'] = formatdate(localtime=True) - - print("Problem {}\n\n{}\n".format(subject, body)) - - # Ratelimit the emails - time_since_last_mail = time.time() - self._mail_rate_limit.get(subject, 0) - if time_since_last_mail < self.config['mail']['min_delay_between_messages']: - return - - self._mail_rate_limit[subject] = time.time() - smtp = smtplib.SMTP("mail.stusta.mhn.de") - smtp.sendmail(msg['From'], msg['To'], msg.as_string()) - smtp.quit() + +def setup_plugin(filename, plugin): + """ + Setup and fix plugins + """ + if not getattr(plugin, "name", None): + plugin.name = filename def main(): """ @@ -332,6 +240,19 @@ def main(): print("Configuring temperature monitoring system from {}.".format(configfile)) monitor = TempMonitor(loop, configfile) + + plugin_path = Path(__file__).resolve().parent / "plugins" + print("Loading plugins from {}".format(plugin_path)) + for filename in plugin_path.glob("*.py"): + if (plugin_path / filename).exists(): + print("loading {}".format(filename.name)) + modname = "plugins." + filename.name.split('.')[0] + module = importlib.import_module(modname) + plugin = module.init(monitor) + setup_plugin(filename, plugin) + monitor.plugins.append(plugin) + print("Loaded: {}".format(plugin.name)) + try: loop.run_forever() except KeyboardInterrupt: