plugins/collectd.py → tempermonitor/plugins/collectd.py

 import asyncio
 import time
 
-def init(monitor):
-    return PluginCollectd(monitor)
+from . import Plugin
 
-class PluginCollectd:
+
+class Collectd(Plugin):
     """
     Implements a super simple collectd interface for only sending temperature data
     """
+
     def __init__(self, monitor):
-        self.loop = asyncio.get_event_loop()
         self.config = monitor.config
         self.path = self.config['collectd']['socketpath']
         self._reader, self._writer = (None, None)
-        self.loop.run_until_complete(self.reconnect())
+        asyncio.run(self.reconnect())
 
         self.monitor = monitor
 
@@ -27,8 +27,7 @@ class PluginCollectd:
             self._writer.close()
 
         self._reader, self._writer = await asyncio.open_unix_connection(
-            path=self.path,
-            loop=self.loop)
+            path=self.path)
 
     async def _send(self, identifier, interval, timestamp, value):
         """
@@ -48,9 +47,13 @@ class PluginCollectd:
             interval,
             timestamp,
             value)
-        #print("Sending data:", data.strip())
-        self._writer.write(data.encode('utf-8'))
-        await self._writer.drain()
+        try:
+            #print("Sending data:", data.strip())
+            self._writer.write(data.encode('utf-8'))
+            await self._writer.drain()
+        except:
+            await self.reconnect()
+            return
 
         try:
             line = await asyncio.wait_for(self._reader.readline(), 1)
@@ -63,7 +66,7 @@ class PluginCollectd:
             await self.reconnect()
         else:
             pass
-            #print("recv:", line)
+            # print("recv:", line)
 
     async def send_sensor_values(self, sensor):
         """
@@ -81,7 +84,7 @@ class PluginCollectd:
         """
         await self._send("tail-{}/{}".format(graph, stattype),
                          int(self.config['collectd']['interval']),
-                         stattime,
+                         int(stattime),
                          statval)
 
     async def sensor_update(self):

plugins/mail.py → tempermonitor/plugins/mail.py

@@ -3,7 +3,9 @@ from email.mime.text import MIMEText
 from email.utils import formatdate
 import smtplib
 
-UNKNOWN_SENSOR_SUBJECT = "WARNING: Unconfigured Sensor ID"
+from . import Plugin
+
+UNKNOWN_SENSOR_SUBJECT = "WARNING: Unconfigured Sensor ID: {owid}"
 UNKNOWN_SENSOR_BODY = """Hello Guys,
 
 An unknown sensor has been connected to the temperature monitoring service.
@@ -54,18 +56,46 @@ This is unlikely - please come and check
 Regards, Temperature
 """
 
+NO_VALID_DATA_SUBJECT = "WARNING: Garbage data"
+NO_VALID_DATA_BODY = """Helly guys,
 
+We have data on the line - but it fails even the most simple verification.
+
+The last received line was:
+
+{last_line}
+
+please check if the controller is going haywire.
+
+I will try to fix this issue by reconnecting...
+
+
+Regards, Temperature
+"""
+
+SENSOR_TEMPERATURE_WARNING_SUBJECT = "Temperaturwarnung Serverraum"
+SENSOR_TEMPERATURE_WARNING_BODY = """Hi Guys,
+
+Die Temperaturen im Serverraum werden langsam Bedenklich:
+
+{temperatures}
+
+Auslöser: {reason}
+
+Aktuelle Temperaturen:
+{alltemperatures}
+
+Bitte haltet die Temperaturen im Auge und fahrt eventuell heiß laufende Server herunter
+
+with love,
+Temperator"""
 
-def init(monitor):
-    """
-    Plugin initialization method to be called from the outside
-    """
-    return PluginMail(monitor)
 
-class PluginMail:
+class Mail(Plugin):
     """
     Handle all the mail sending stuff
     """
+
     def __init__(self, monitor):
         self.monitor = monitor
         self.config = self.monitor.config
@@ -80,9 +110,10 @@ class PluginMail:
         msg['Subject'] = subject
         msg['From'] = self.config['mail']['from']
         if urgent:
-            msg['To'] = self.config['mail']['to_urgent']
+            recipients = self.config['mail']['to_urgent'].split(',')
         else:
-            msg['To'] = self.config['mail']['to']
+            recipients = self.config['mail']['to'].split(',')
+        msg['To'] = ",".join([s.strip() for s in recipients])
         msg['Date'] = formatdate(localtime=True)
 
         print("Notification: {}".format(subject))
@@ -90,22 +121,29 @@ class PluginMail:
         # Ratelimit the emails
         time_since_last_mail = time.time() - self._mail_rate_limit.get(subject, 0)
         if time_since_last_mail < int(self.config['mail']['min_delay_between_messages']):
-            print("Not sending due to ratelimiting")
+            print("Not sending due to ratelimiting: %i", time_since_last_mail)
             return
 
         print("Body: {}".format(body))
 
         self._mail_rate_limit[subject] = time.time()
         smtp = smtplib.SMTP("mail.stusta.mhn.de")
-        #smtp.sendmail(msg['From'], msg['To'], msg.as_string())
+        smtp.sendmail(msg['From'], recipients, msg.as_string())
         smtp.quit()
 
     async def err_nodata(self, **kwargs):
-        await self.send_mail(NO_DATA_SUBJECT, NO_DATA_BODY)
+        await self.send_mail(
+            NO_DATA_SUBJECT.format(**kwargs),
+            NO_DATA_BODY.format(**kwargs))
+
+    async def err_no_valid_data(self, **kwargs):
+        await self.send_mail(
+            NO_VALID_DATA_SUBJECT.format(**kwargs),
+            NO_VALID_DATA_BODY.format(**kwargs))
 
     async def err_unknown_sensor(self, **kwargs):
         await self.send_mail(
-            UNKNOWN_SENSOR_SUBJECT,
+            UNKNOWN_SENSOR_SUBJECT.format(**kwargs),
             UNKNOWN_SENSOR_BODY.format(**kwargs))
 
     async def err_problem_sensor(self, **kwargs):
@@ -118,37 +156,24 @@ class PluginMail:
             SENSOR_MEASUREMENT_MISSED_SUBJECT,
             SENSOR_MEASUREMENT_MISSED_BODY.format(**kwargs))
 
-
-    async def temperature_warning(self, source, **kwargs):
-        subject = "Temperaturwarnung Serverraum"
-
-        body = """Hi Guys,
-
-Die Temperaturen im Serverraum werden langsam Bedenklich:
-
-{temperatures}
-
-Auslöser: {reason}
-
-Aktuelle Temperaturen:
-{alltemperatures}
-
-Bitte haltet die Temperaturen im Auge und fahrt eventuell heiß laufende Server herunter
-
-with love,
-Temperator"""
-
+    async def temperature_warning(self, source, urgent=False, **kwargs):
         if source == "tempdiff":
             temperatures = "{name1}:{temp1}\n{name2}:{temp2}".format(**kwargs)
             reason = "Differenztemperatur: {tempdiff}".format(**kwargs)
         elif source == "singlehot":
-            temperatures="{name}:{temp}".format(**kwargs)
+            temperatures = "{name}:{temp}".format(**kwargs)
             reason = "Einzeltemperatur zu hoch"
 
         alltemperatures = '\n'.join([
             "{}: {}".format(sensor.name, sensor.temperature) if sensor.valid
             else "{}: INVALID".format(sensor.name)
-            for sensor in self.monitor.sensors.values() ])
+            for sensor in self.monitor.sensors.values()])
 
-        await self.send_mail(subject, body.format(
-            temperatures=temperatures, reason=reason, alltemperatures=alltemperatures))
+        await self.send_mail(
+            SENSOR_TEMPERATURE_WARNING_SUBJECT,
+            SENSOR_TEMPERATURE_WARNING_BODY.format(
+                temperatures=temperatures,
+                reason=reason,
+                alltemperatures=alltemperatures),
+            urgent=urgent
+        )

tempermonitor/plugins/prometheus.py

+import re
+import asyncio
+from prometheus_client import start_http_server, Gauge
+
+from . import Plugin
+
+stats_name_re = re.compile(r'^temperature-(?P<group>\w+)-(?P<type>\w+)$')
+
+
+class Prometheus(Plugin):
+    def __init__(self, monitor):
+        self.loop = asyncio.get_event_loop()
+        self.config = monitor.config
+        self.last_store = 0
+        self.monitor = monitor
+
+        self.sensor_metrics = Gauge(
+            name=self.config["prometheus"]["sensor_metric_name"],
+            documentation="Container Temperature Measurements",
+            labelnames=["sensor"]
+        )
+
+        self.aggregated_metrics = Gauge(
+            name=self.config["prometheus"]["aggregated_metric_name"],
+            documentation="Container Temperature Aggregations",
+            labelnames=["group", "type"]
+        )
+
+        start_http_server(
+            addr=self.config["prometheus"].get('address', 'localhost'),
+            port=int(self.config["prometheus"]["port"])
+        )
+        print("started prometheus http server")
+
+    async def send_stats_graph(self, graph, stattype, stattime, statval):
+        """
+        to be called as a plugin callback to export aggregated measurements
+        """
+        m = stats_name_re.match(stattype)
+        if not m:
+            return
+
+        self.aggregated_metrics.labels(group=m.group('group'), type=m.group('type')).set(statval)
+
+    async def sensor_update(self):
+        """
+        Receive sensor data to store them regularely into collectd
+        """
+        for sensor in self.monitor.sensors.values():
+            if sensor.valid:
+                self.sensor_metrics.labels(sensor=sensor.name).set(sensor.temperature)

plugins/warnings.py → tempermonitor/plugins/warnings.py

 import time
 
-def init(monitor):
-    """ Plugin interface method """
-    return PluginWarning(monitor)
+from . import Plugin
 
-class PluginWarning:
+
+class Warnings(Plugin):
     """
     Generate all kind of warnings whenever needed and observe the sensor
     if they see a problematic situation in the container
     """
+
     def __init__(self, monitor):
         self.monitor = monitor
 
         self.revmapping = {
-            sensor.name : sensor
+            sensor.name: sensor
             for sensor in self.monitor.sensors.values()
         }
 
@@ -44,19 +44,18 @@ class PluginWarning:
             return [], 0, 0, 0, 0
 
         avg = sum(sensor.temperature for sensor in sensors) / len(sensors)
-        var = sum((sensor.temperature - avg)**2 for sensor in sensors) / len(sensors)
+        var = sum((sensor.temperature - avg) ** 2 for sensor in sensors) / len(sensors)
 
-        sensormin = -9999
-        sensormax = +9999
+        sensormin = +9999
+        sensormax = -9999
         for sensor in sensors:
             if sensor.temperature < sensormin:
-                sensormin = sensormin
+                sensormin = sensor.temperature
             if sensor.temperature > sensormax:
-                sensormax = sensormax
+                sensormax = sensor.temperature
 
         return sensors, sensormin, sensormax, avg, var
 
-
     async def sensor_update(self):
         """
         First generate stats and relay them to the collectd module, then use these stats
@@ -74,30 +73,30 @@ class PluginWarning:
         if floor_sensors:
             await self.monitor.call_plugin(
                 "send_stats_graph", graph="stats",
-                stattype="temperature-floormin", stattime=now, statval=floor_min)
+                stattype="temperature-floor-min", stattime=now, statval=floor_min)
             await self.monitor.call_plugin(
                 "send_stats_graph", graph="stats",
-                stattype="temperature-floormax", stattime=now, statval=floor_max)
+                stattype="temperature-floor-max", stattime=now, statval=floor_max)
             await self.monitor.call_plugin(
                 "send_stats_graph", graph="stats",
-                stattype="temperature-flooravg", stattime=now, statval=floor_avg)
+                stattype="temperature-floor-avg", stattime=now, statval=floor_avg)
             await self.monitor.call_plugin(
                 "send_stats_graph", graph="stats",
-                stattype="temperature-floorvar", stattime=now, statval=floor_var)
+                stattype="temperature-floor-var", stattime=now, statval=floor_var)
 
         if ceil_sensors:
             await self.monitor.call_plugin(
                 "send_stats_graph", graph="stats",
-                stattype="temperature-ceilmin", stattime=now, statval=ceil_min)
+                stattype="temperature-ceil-min", stattime=now, statval=ceil_min)
             await self.monitor.call_plugin(
                 "send_stats_graph", graph="stats",
-                stattype="temperature-ceilmax", stattime=now, statval=ceil_max)
+                stattype="temperature-ceil-max", stattime=now, statval=ceil_max)
             await self.monitor.call_plugin(
                 "send_stats_graph", graph="stats",
-                stattype="temperature-ceilavg", stattime=now, statval=ceil_avg)
+                stattype="temperature-ceil-avg", stattime=now, statval=ceil_avg)
             await self.monitor.call_plugin(
                 "send_stats_graph", graph="stats",
-                stattype="temperature-ceilvar", stattime=now, statval=ceil_var)
+                stattype="temperature-ceil-var", stattime=now, statval=ceil_var)
 
         if floor_sensors and ceil_sensors:
             # Else we already have sent warning messages for broken sensors
@@ -105,21 +104,37 @@ class PluginWarning:
             tempdiff = ceil_avg - floor_avg
             await self.monitor.call_plugin(
                 "send_stats_graph", graph="stats",
-                stattype="temperature-floor_ceil_diff", stattime=now, statval=tempdiff)
+                stattype="temperature-floor_ceil-diff", stattime=now, statval=tempdiff)
+
+            print("floor: min {:05.2f} max {:05.2f} avg {:05.2f} var {:05.2f}".format(
+                floor_min, floor_max, floor_avg, floor_var))
+            print("ceil:  min {:05.2f} max {:05.2f} avg {:05.2f} var {:05.2f}".format(
+                ceil_min, ceil_max, ceil_avg, ceil_var))
 
             # Here comes the warning magic
-            if ceil_max > int(self.warning_conf['min_ceiling_warning']):
-                if  tempdiff > int(self.warning_conf['floor_ceiling_diff']):
-                    await self.monitor.call_plugin("temperature_warning",
-                                                source="tempdiff",
-                                                name1="floor",
-                                                name2="ceiling",
-                                                temp1=floor_avg,
-                                                temp2=ceil_avg,
-                                                tempdiff=tempdiff)
 
+            # Critical: ceiling temperature > threshold (sane default: 45)
+            if ceil_max > int(self.warning_conf['ceiling_critical_level']):
+                await self.monitor.call_plugin("temperature_warning",
+                                               source="singlehot",
+                                               name="ceiling",
+                                               temp=ceil_max,
+                                               urgent=True)
+
+            # Warning: ceiling tempareture > threshold (sane default: 40)
             if ceil_avg > int(self.warning_conf['ceiling_warning_level']):
                 await self.monitor.call_plugin("temperature_warning",
-                                            source="singlehot",
-                                            name="ceiling",
-                                            temp=ceil_avg)
+                                               source="singlehot",
+                                               name="ceiling",
+                                               temp=ceil_avg)
+
+            # Warning: temperature difference > threshold (sane default: 17)
+            if ceil_max > int(self.warning_conf['min_ceiling_warning']):
+                if tempdiff > int(self.warning_conf['floor_ceiling_diff']):
+                    await self.monitor.call_plugin("temperature_warning",
+                                                   source="tempdiff",
+                                                   name1="floor",
+                                                   name2="ceiling",
+                                                   temp1=floor_avg,
+                                                   temp2=ceil_avg,
+                                                   tempdiff=tempdiff)

tempermonitor.py → tempermonitor/tempermonitor.py

@@ -9,6 +9,7 @@ under the terms as stated in the LICENSE.md file.
 Changelog:
 
 2018-08 jotweh: reimplemented using a micropython-esp32
+2020-04 milo: added prometheus plugin
 
 Open issues:
 
@@ -20,31 +21,34 @@ import asyncio
 import configparser
 import sys
 import time
-import importlib
 from datetime import datetime
-from pathlib import Path
 import serial_asyncio
 import serial
 
+from .plugins import PLUGINS
+
+
 class Sensor:
     """
-    One instance as sensor posing as measurementproxy
+    One instance as sensor posing as measurement proxy
     """
+
     def __init__(self, config, owid):
         self.temperature = None
         self.last_update = 0
         self.calibration = 0
-        self.valid = False
+        self.valid = True
 
-        try:
-            if owid in config:
-                self.name = config[owid]['name']
-                self.calibration = config[owid]['calibration']
-            else:
-                print("Invalid Config: missing section {}".format(owid))
-        except KeyError as exc:
-            print("Invalid Config: for {}: {}".format(owid, exc))
-            raise
+        if owid not in config:
+            print(f"Invalid Config: missing section {owid}")
+            return
+
+        if 'name' not in config[owid] or 'calibration' not in config[owid]:
+            print(f"Invalid Config for: {owid}")
+            raise RuntimeError(f"Invalid Config for: {owid}")
+
+        self.name = config[owid]['name']
+        self.calibration = config[owid]['calibration']
 
     def update(self, temperature):
         """
@@ -53,6 +57,7 @@ class Sensor:
         self.temperature = float(temperature)
         self.last_update = time.time()
 
+
 class TempMonitor:
     """
     Interact with the esp-one-wire interface that sends:
@@ -80,6 +85,7 @@ class TempMonitor:
         # Test if all necessary config fields are set, that are not part of the normal
         # startup
         configtest = [
+            self.config['general']['plugins'],
             self.config['collectd']['hostname'],
             self.config['collectd']['interval'],
             self.config['mail']['from'],
@@ -95,7 +101,7 @@ class TempMonitor:
         print("connecting to", self.config['serial']['port'])
         for owid in self.config:
             # Skip all known and predefined sections
-            if owid in ['DEFAULT', 'serial', 'collectd', 'mail', 'warning']:
+            if owid in ['DEFAULT', 'serial', 'collectd', 'mail', 'warning', 'prometheus', 'general']:
                 continue
             self.sensors[owid] = Sensor(self.config, owid)
         self._run_task = loop.create_task(self.run())
@@ -104,10 +110,16 @@ class TempMonitor:
         """
         Connect to the ESP chip
         """
-        self._reader, self._writer = await serial_asyncio.open_serial_connection(
-            url=self.config['serial']['port'],
-            baudrate=self.config['serial']['baudrate'],
-            loop=self.loop)
+        try:
+            self._reader, self._writer = await serial_asyncio.open_serial_connection(
+                url=self.config['serial']['port'],
+                baudrate=self.config['serial']['baudrate'],
+                loop=self.loop)
+        except serial.SerialException:
+            print("Connection failed!")
+            self.loop.stop()
+            raise
+
         # upon startup we only see garbage. (micropython starting up),
         # also it will produce warnings if the recording is started in the middle
         # of a message, so wait until the end of a message block to start the game
@@ -125,14 +137,25 @@ class TempMonitor:
         Read the protocol, update the sensors or trigger a collectd update
         """
         await self.reconnect()
-
+        last_valid_data_received = time.time()
+        line = ""
+        reconnected_on_error = False
         while True:
             # Wait for the next line
+
+            if time.time() - last_valid_data_received > 10:
+                await self.call_plugin("err_no_valid_data", last_line=line)
+                if not reconnected_on_error:
+                    reconnected_on_error = True
+                    await self.reconnect()
+
             try:
                 line = await asyncio.wait_for(
                     self._reader.readline(),
                     timeout=int(self.config['serial']['timeout']))
+                print("Received: ", line)
             except asyncio.TimeoutError:
+                print("No Data")
                 await self.call_plugin("err_nodata")
                 continue
             except serial.SerialException as exc:
@@ -143,13 +166,17 @@ class TempMonitor:
             try:
                 line = line.decode('ascii').strip()
             except UnicodeError:
+                print("Unicode error")
                 continue
-            #print("recv:", line)
+            # print("recv:", line)
 
             if line == '':
                 # Block has ended
+                print("Done block, storing sensors")
                 await self.store_sensors()
+                print("Done")
                 continue
+
             # Try to parse the line
             try:
                 owid, temp = line.split(' ')
@@ -158,14 +185,20 @@ class TempMonitor:
                 print("Invaid line received: {}\n{}".format(line, exc))
                 continue
 
+            ## we have at least a valid line
+            last_valid_data_received = time.time()
+            reconnected_on_error = False
+
             sensor = self.sensors.get(owid, None)
             if not sensor:
                 # If the sensor is new - notify the operators
+                print("Unknown sensor")
                 await self.call_plugin("err_unknown_sensor",
                                        config=self._configname,
                                        owid=owid,
                                        temp=temp)
             elif temp > 1000 or temp < -1000:
+                print("Sensor invalid")
                 sensor.valid = False
                 # if the sensor is giving bullshit data - notify the operators
                 await self.call_plugin("err_problem_sensor",
@@ -221,37 +254,27 @@ class TempMonitor:
         self._last_store = time.time()
 
 
-def setup_plugin(filename, plugin):
-    """
-    Setup and fix plugins
-    """
-    if not getattr(plugin, "name", None):
-        plugin.name = filename
-
 def main():
     """
     Start the tempmonitor
     """
     loop = asyncio.get_event_loop()
 
-    configfile = "/etc/temperature/tempermon.ini"
+    configfile = "/etc/tempermonitor.ini"
     if len(sys.argv) == 2:
         configfile = sys.argv[1]
 
-    print("Configuring temperature monitoring system from {}.".format(configfile))
+    print(f"Configuring temperature monitoring system from {configfile}.")
     monitor = TempMonitor(loop, configfile)
 
-    plugin_path = Path(__file__).resolve().parent / "plugins"
-    print("Loading plugins from {}".format(plugin_path))
-    for filename in plugin_path.glob("*.py"):
-        if (plugin_path / filename).exists():
-            print("loading {}".format(filename.name))
-            modname = "plugins." + filename.name.split('.')[0]
-            module = importlib.import_module(modname)
-            plugin = module.init(monitor)
-            setup_plugin(filename, plugin)
-            monitor.plugins.append(plugin)
-            print("Loaded: {}".format(plugin.name))
+    active_plugins = monitor.config["general"]["plugins"].split(",")
+    print(f"Active plugins: {active_plugins}")
+
+    for plugin in active_plugins:
+        if plugin in PLUGINS:
+            p = PLUGINS[plugin](monitor)
+            monitor.plugins.append(p)
+            print(f"Loaded plugin: {plugin}")
 
     try:
         loop.run_forever()
@@ -260,5 +283,3 @@ def main():
     finally:
         loop.run_until_complete(monitor.teardown())
 
-if __name__ == "__main__":
-    main()

test/collectdmock.py

+#!/usr/bin/env python3
+
+import sys
+
+for line in sys.stdin:
+    print("Ack")
+    sys.stdout.flush()

test/run_tests.sh

+# IMPORTANT: Set the config to use the right socket path
+
+socat -d PTY,link=/tmp/temperature_pts,echo=0 "EXEC:python3 test.py ...,pty,raw",echo=0 &
+socat UNIX-LISTEN:/tmp/collectd_sock,fork EXEC:"python3 collectdmock.py"
+

test/test.py

+#!/usr/bin/env python3
+import random
+import time
+import sys
+
+delay = 0.1
+
+while True:
+    for _ in range(3):
+        print("testsensor {}".format(random.random() * 40))
+        print("sensortest {}".format(random.random() * 40))
+        print()
+        time.sleep(delay)
+
+    print("Sensor Error test", file=sys.stderr)
+    print("testsensor {}".format(9001))
+    print("sensortest {}".format(random.random() * 40))
+    print()
+    time.sleep(delay)
+
+
+    for _ in range(3):
+        print("testsensor {}".format(random.random() * 40))
+        print("sensortest {}".format(random.random() * 40))
+        print()
+        time.sleep(delay)
+
+    print("Missing sensor test", file=sys.stderr)
+    print("sensortest {}".format(random.random() * 40))
+    print()
+    time.sleep(delay)
+
+    for _ in range(3):
+        print("testsensor {}".format(random.random() * 40))
+        print("sensortest {}".format(random.random() * 40))
+        print()
+        time.sleep(delay)
+
+    print("Extra sensor test", file=sys.stderr)
+    print("idonotexist {}".format(random.random() * 40))
+    print("testsensor {}".format(random.random() * 40))
+    print("sensortest {}".format(random.random() * 40))
+    print()
+    time.sleep(delay)
+
+
+    for _ in range(3):
+        print("testsensor {}".format(random.random() * 40))
+        print("sensortest {}".format(random.random() * 40))
+        print()
+        time.sleep(delay)
+
+    print("Too Hot Test", file=sys.stderr)
+    print("testsensor {}".format(45))
+    print("sensortest {}".format(40))
+    print()
+    time.sleep(delay)
+
+    for _ in range(3):
+        print("testsensor {}".format(random.random() * 40))
+        print("sensortest {}".format(random.random() * 40))
+        print()
+        time.sleep(delay)
+
+    print("high diff test", file=sys.stderr)
+    print("testsensor {}".format(45))
+    print("sensortest {}".format(25))
+    print()
+    time.sleep(delay)