[feature] Add a check which inspects device configuration status periodically #54

[influxdb] Add support for retention_policy in read

Author: nepython

README.rst

@@ -213,6 +213,20 @@ command is used to collect these metrics.
 You may choose to disable auto creation of this check by setting
 `OPENWISP_MONITORING_AUTO_PING <#OPENWISP_MONITORING_AUTO_PING>`_ to ``False``.
 
+``Configuration applied``
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This check ensures that the `openwisp-config agent <https://github.com/openwisp/openwisp-config/>`_
+is running and applying configuration changes in a timely manner.
+
+By default, if a configuration object stays in the ``modified`` state
+for more than 5 minutes (configurable via
+`OPENWISP_MONITORING_DEVICE_CONFIG_CHECK_MAX_TIME <#OPENWISP_MONITORING_DEVICE_CONFIG_CHECK_MAX_TIME>`_),
+the check will fail and set the health status of the device to ``PROBLEM``.
+
+You may choose to disable auto creation of this check by using the
+setting `OPENWISP_MONITORING_AUTO_DEVICE_CONFIG_CHECK <#OPENWISP_MONITORING_AUTO_DEVICE_CONFIG_CHECK>`_.
+
 Settings
 --------
 
@@ -242,6 +256,33 @@ in terms of disk space.
 
 Whether ping checks are created automatically for devices.
 
+``OPENWISP_MONITORING_AUTO_DEVICE_CONFIG_CHECK``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
++--------------+-------------+
+| **type**:    | ``bool``    |
++--------------+-------------+
+| **default**: | ``True``    |
++--------------+-------------+
+
+This setting allows you to choose whether `config_applied <#configuration-applied>`_ checks should be
+created automatically for newly registered devices. It's enabled by default.
+
+``OPENWISP_MONITORING_DEVICE_CONFIG_CHECK_MAX_TIME``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
++--------------+-----------+
+| **type**:    |  ``int``  |
++--------------+-----------+
+| **default**: |   ``5``   |
++--------------+-----------+
+
+Defines the maximum amount of minutes the device configuration can stay in the *modified*
+state before its health status is changed to ``PROBLEM`` and an alert is sent.
+
+**Note**: The setting will be ignored if ``OPENWISP_MONITORING_AUTO_DEVICE_CONFIG_CHECK``
+is ``False``.
+
 ``OPENWISP_MONITORING_AUTO_CHARTS``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

openwisp_monitoring/check/apps.py

@@ -22,3 +22,12 @@ def _connect_signals(self):
                 sender=load_model('config', 'Device'),
                 dispatch_uid='auto_ping',
             )
+
+        if app_settings.AUTO_CONFIG_CHECK:
+            from .base.models import auto_config_check_receiver
+
+            post_save.connect(
+                auto_config_check_receiver,
+                sender=load_model('config', 'Device'),
+                dispatch_uid='auto_config_check',
+            )

openwisp_monitoring/check/base/models.py

@@ -8,7 +8,7 @@
 from django.utils.translation import gettext_lazy as _
 from jsonfield import JSONField
 from openwisp_monitoring.check import settings as app_settings
-from openwisp_monitoring.check.tasks import auto_create_ping
+from openwisp_monitoring.check.tasks import auto_create_config_check, auto_create_ping
 
 from openwisp_utils.base import TimeStampedEditableModel
 
@@ -97,3 +97,22 @@ def auto_ping_receiver(sender, instance, created, **kwargs):
                 object_id=str(instance.pk),
             )
         )
+
+
+def auto_config_check_receiver(sender, instance, created, **kwargs):
+    """
+    Implements OPENWISP_MONITORING_AUTO_DEVICE_CONFIG_CHECK
+    The creation step is executed in the background
+    """
+    # we need to skip this otherwise this task will be executed
+    # every time the configuration is requested via checksum
+    if not created:
+        return
+    with transaction.atomic():
+        transaction.on_commit(
+            lambda: auto_create_config_check.delay(
+                model=sender.__name__.lower(),
+                app_label=sender._meta.app_label,
+                object_id=str(instance.pk),
+            )
+        )

openwisp_monitoring/check/classes/__init__.py

@@ -1 +1,2 @@
+from .config_applied import ConfigApplied  # noqa
 from .ping import Ping  # noqa

openwisp_monitoring/check/classes/base.py

@@ -30,7 +30,7 @@ def validate_params(self):
     def check(self, store=True):
         raise NotImplementedError
 
-    def _get_or_create_metric(self):
+    def _get_or_create_metric(self, configuration=None):
         """
         Gets or creates metric
         """
@@ -44,7 +44,7 @@ def _get_or_create_metric(self):
         options = dict(
             object_id=obj_id,
             content_type=ct,
-            configuration=self.__class__.__name__.lower(),
+            configuration=configuration or self.__class__.__name__.lower(),
         )
         metric, created = Metric._get_or_create(**options)
         return metric, created

openwisp_monitoring/check/classes/config_applied.py

@@ -0,0 +1,35 @@
+from swapper import load_model
+
+from ...device.utils import SHORT_RP
+from ..settings import CONFIG_CHECK_MAX_TIME
+from .base import BaseCheck
+
+AlertSettings = load_model('monitoring', 'AlertSettings')
+
+
+class ConfigApplied(BaseCheck):
+    def check(self, store=True):
+        # If the device is down do not run config applied checks
+        if self.related_object.monitoring.status == 'critical':
+            return
+        if not hasattr(self.related_object, 'config'):
+            return
+        result = int(self.related_object.config.status == 'applied')
+        if store:
+            self._get_metric().write(
+                result, retention_policy=SHORT_RP,
+            )
+        return result
+
+    def _get_metric(self):
+        metric, created = self._get_or_create_metric(configuration='config_applied')
+        if created:
+            self._create_alert_setting(metric)
+        return metric
+
+    def _create_alert_setting(self, metric):
+        alert_s = AlertSettings(
+            metric=metric, operator='<', value=1, seconds=CONFIG_CHECK_MAX_TIME * 60
+        )
+        alert_s.full_clean()
+        alert_s.save()

openwisp_monitoring/check/migrations/0005_create_config_applied.py

@@ -0,0 +1,38 @@
+from django.db import migrations
+from openwisp_monitoring.check.settings import AUTO_CONFIG_CHECK
+from openwisp_monitoring.check.tasks import auto_create_config_check
+
+
+def add_config_applied_checks(apps, schema_editor):
+    if not AUTO_CONFIG_CHECK:
+        return
+    Device = apps.get_model('config', 'Device')
+    for device in Device.objects.all():
+        auto_create_config_check.delay(
+            model=Device.__name__.lower(),
+            app_label=Device._meta.app_label,
+            object_id=str(device.pk),
+        )
+
+
+def remove_config_applied_checks(apps, schema_editor):
+    Check = apps.get_model('check', 'Check')
+    Metric = apps.get_model('monitoring', 'Metric')
+    Check.objects.filter(
+        check='openwisp_monitoring.check.classes.ConfigApplied'
+    ).delete()
+    Metric.objects.filter(configuration='config_applied').delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('check', '0004_rename_active_to_is_active'),
+        ('monitoring', '0016_metric_configuration'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            add_config_applied_checks, reverse_code=remove_config_applied_checks
+        ),
+    ]

openwisp_monitoring/check/settings.py

@@ -3,7 +3,17 @@
 CHECK_CLASSES = getattr(
     settings,
     'OPENWISP_MONITORING_CHECK_CLASSES',
-    (('openwisp_monitoring.check.classes.Ping', 'Ping'),),
+    (
+        ('openwisp_monitoring.check.classes.Ping', 'Ping'),
+        ('openwisp_monitoring.check.classes.ConfigApplied', 'Configuration Applied'),
+    ),
 )
 AUTO_PING = getattr(settings, 'OPENWISP_MONITORING_AUTO_PING', True)
+AUTO_CONFIG_CHECK = getattr(
+    settings, 'OPENWISP_MONITORING_AUTO_DEVICE_CONFIG_CHECK', True
+)
+# Input in minutes
+CONFIG_CHECK_MAX_TIME = getattr(
+    settings, 'OPENWISP_MONITORING_DEVICE_CONFIG_CHECK_MAX_TIME', 5
+)
 MANAGEMENT_IP_ONLY = getattr(settings, 'OPENWISP_MONITORING_MANAGEMENT_IP_ONLY', True)

openwisp_monitoring/check/tasks.py

@@ -52,3 +52,27 @@ def auto_create_ping(model, app_label, object_id):
     check = Check(name='Ping', check=ping_path, content_type=ct, object_id=object_id)
     check.full_clean()
     check.save()
+
+
+@shared_task
+def auto_create_config_check(model, app_label, object_id):
+    """
+    Called by openwisp_monitoring.check.models.auto_config_check_receiver
+    """
+    Check = load_model('check', 'Check')
+    config_check_path = 'openwisp_monitoring.check.classes.ConfigApplied'
+    has_check = Check.objects.filter(
+        object_id=object_id, content_type__model='device', check=config_check_path,
+    ).exists()
+    # create new check only if necessary
+    if has_check:
+        return
+    ct = ContentType.objects.get(app_label=app_label, model=model)
+    check = Check(
+        name='Configuration Applied',
+        check=config_check_path,
+        content_type=ct,
+        object_id=object_id,
+    )
+    check.full_clean()
+    check.save()