[feature] Add a check which inspects device configuration status periodically #54

Author: nepython

README.rst

@@ -242,6 +242,47 @@ in terms of disk space.
 
 Whether ping checks are created automatically for devices.
 
+``OPENWISP_MONITORING_AUTO_CONFIG_STATUS``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
++--------------+-------------+
+| **type**:    | ``bool``    |
++--------------+-------------+
+| **default**: | ``True``    |
++--------------+-------------+
+
+Whether config_status checks are created automatically for devices.
+
+``OPENWISP_MONITORING_CONFIG_MODIFIED_MAX_TIME``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
++--------------+-----------+
+| **type**:    | ``float`` |
++--------------+-----------+
+| **default**: |   ``5``   |
++--------------+-----------+
+
+After ``config`` is ``modified``, if the ``modified`` status does not change  after a
+fixed **duration** then ``device`` health status changes to ``problem``.
+This **duration** can be set with the help of this setting. The input represents the duration in minutes.
+Thus, by default the health status of the device changes to ``problem`` after 5 minutes of ``config_modified`` status.
+
+**Note**: If the setting ``AUTO_CONFIG_STATUS`` is disabled then this setting need not be declared.
+
+``OPENWISP_MONITORING_CONFIG_STATUS_RETENTION_POLICY``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
++--------------+-------------+
+| **type**:    |  ``time``   |
++--------------+-------------+
+| **default**: | ``48h0m0s`` |
++--------------+-------------+
+
+This setting allows to modify the duration for which the metric data generated
+by ``modified_status`` check is to be retained.
+
+**Note**: If the setting ``AUTO_CONFIG_STATUS`` is disabled then this setting need not be declared.
+
 ``OPENWISP_MONITORING_AUTO_GRAPHS``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

openwisp_monitoring/check/apps.py

@@ -1,8 +1,13 @@
 from django.apps import AppConfig
 from django.utils.translation import ugettext_lazy as _
 
+from .utils import manage_config_status_retention_policy
+
 
 class CheckConfig(AppConfig):
     name = 'openwisp_monitoring.check'
     label = 'check'
     verbose_name = _('Network Monitoring Checks')
+
+    def ready(self):
+        manage_config_status_retention_policy()

openwisp_monitoring/check/base/models.py

@@ -2,8 +2,9 @@
 
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
-from django.db import models
+from django.db import models, transaction
 from django.db.models.signals import post_save
+from django.dispatch import receiver
 from django.utils.functional import cached_property
 from django.utils.module_loading import import_string
 from django.utils.translation import ugettext_lazy as _
@@ -75,27 +76,45 @@ def perform_check(self, store=True):
         return self.check_instance.check(store=True)
 
 
-if app_settings.AUTO_PING:
-    from django.db import transaction
-    from django.dispatch import receiver
+@receiver(post_save, sender=Device, dispatch_uid='auto_ping')
+def auto_ping_receiver(sender, instance, created, **kwargs):
+    """
+    Implements OPENWISP_MONITORING_AUTO_PING
+    The creation step is executed in the background
+    """
+    # we need to skip this otherwise this task will be executed
+    # every time the configuration is requested via checksum
     from openwisp_monitoring.check.tasks import auto_create_ping
 
-    @receiver(post_save, sender=Device, dispatch_uid='auto_ping')
-    def auto_ping_receiver(sender, instance, created, **kwargs):
-        """
-        Implements OPENWISP_MONITORING_AUTO_PING
-        The creation step is executed in the backround
-        """
-        # we need to skip this otherwise this task will be executed
-        # every time the configuration is requested via checksum
-        if not created:
-            return
-        with transaction.atomic():
-            transaction.on_commit(
-                lambda: auto_create_ping.delay(
-                    model=sender.__name__.lower(),
-                    app_label=sender._meta.app_label,
-                    object_id=str(instance.pk),
-                    created=created,
-                )
+    if not app_settings.AUTO_PING or not created:
+        return
+    with transaction.atomic():
+        transaction.on_commit(
+            lambda: auto_create_ping.delay(
+                model=sender.__name__.lower(),
+                app_label=sender._meta.app_label,
+                object_id=str(instance.pk),
+            )
+        )
+
+
+@receiver(post_save, sender=Device, dispatch_uid='auto_config_status')
+def auto_config_status_receiver(sender, instance, created, **kwargs):
+    """
+    Implements OPENWISP_MONITORING_AUTO_CONFIG_STATUS
+    The creation step is executed in the background
+    """
+    # we need to skip this otherwise this task will be executed
+    # every time the configuration is requested via checksum
+    from openwisp_monitoring.check.tasks import auto_create_config_status
+
+    if not app_settings.AUTO_CONFIG_STATUS or not created:
+        return
+    with transaction.atomic():
+        transaction.on_commit(
+            lambda: auto_create_config_status.delay(
+                model=sender.__name__.lower(),
+                app_label=sender._meta.app_label,
+                object_id=str(instance.pk),
             )
+        )

openwisp_monitoring/check/classes/__init__.py

@@ -1 +1,2 @@
+from .config_status import ConfigStatus  # noqa
 from .ping import Ping  # noqa

openwisp_monitoring/check/classes/base.py

@@ -0,0 +1,39 @@
+from django.contrib.contenttypes.models import ContentType
+from django.core.exceptions import ValidationError
+from swapper import load_model
+
+from openwisp_controller.config.models import Device
+
+Metric = load_model('monitoring', 'Metric')
+
+
+class BaseCheck(object):
+    def validate_instance(self):
+        # check instance is of type device
+        obj = self.related_object
+        if not obj or not isinstance(obj, Device):
+            message = 'A related device is required to perform this operation'
+            raise ValidationError({'content_type': message, 'object_id': message})
+
+    def _get_or_create_metric(self, field_name):
+        """
+        Gets or creates metric
+        """
+        check = self.check_instance
+        if check.object_id and check.content_type:
+            obj_id = check.object_id
+            ct = check.content_type
+        else:
+            obj_id = str(check.id)
+            ct = ContentType.objects.get(
+                app_label=check._meta.app_label, model=check.__class__.__name__.lower()
+            )
+        options = dict(
+            name=check.name,
+            object_id=obj_id,
+            content_type=ct,
+            field_name=field_name,
+            key=self.__class__.__name__.lower(),
+        )
+        metric, created = Metric.objects.get_or_create(**options)
+        return metric, created

openwisp_monitoring/check/classes/config_status.py

@@ -0,0 +1,69 @@
+from time import time
+
+from swapper import load_model
+
+from ...monitoring.utils import write
+from ..settings import CONFIG_MODIFIED_MAX_TIME
+from ..utils import CONFIG_STATUS_RP
+from .base import BaseCheck
+
+Graph = load_model('monitoring', 'Graph')
+Threshold = load_model('monitoring', 'Threshold')
+
+
+class ConfigStatus(BaseCheck):
+    def __init__(self, check, params):
+        self.check_instance = check
+        self.related_object = check.content_object
+        self.params = params
+
+    def validate(self):
+        self.validate_instance()
+
+    def check(self, store=True):
+        if not hasattr(self.related_object, 'config'):
+            return
+        result = 0 if self.related_object.config.status == 'applied' else 1
+        if result and self._check_modified_status_crossed_max_time():
+            # TODO: When is the status supposed to be made critical?
+            dm = self.related_object.monitoring
+            if dm.status in ['ok', 'unknown']:
+                dm.update_status('problem')
+        if store:
+            metric = self.get_metric()
+            # TODO: Find why doing the same by adding rp as an arg
+            # in metric write changes status to problem
+            write(
+                name=metric.key,
+                values={metric.field_name: result},
+                tags=metric.tags,
+                retention_policy=CONFIG_STATUS_RP,
+            )
+        return result
+
+    # TODO: Check if this can be done using threshold
+    def _check_modified_status_crossed_max_time(self):
+        # A small margin is kept to take care of border cases
+        # TODO: Find why `m` --> minute is not working correctly!
+        since = f'now() - {int(CONFIG_MODIFIED_MAX_TIME*63)}s'
+        measurement_list = self.get_metric().read(
+            since=since, limit=None, order='time DESC'
+        )
+        for measurement in measurement_list:
+            if not measurement['config_status']:
+                break
+            minutes_difference = (time() - measurement['time']) / 60
+            if minutes_difference >= CONFIG_MODIFIED_MAX_TIME:
+                return True
+        return False
+
+    def get_metric(self):
+        metric, created = self._get_or_create_metric(field_name='config_status')
+        if created:
+            self._create_threshold(metric)
+        return metric
+
+    def _create_threshold(self, metric):
+        t = Threshold(metric=metric, operator='>', value=0, seconds=0)
+        t.full_clean()
+        t.save()

openwisp_monitoring/check/classes/ping.py

@@ -1,25 +1,22 @@
 import subprocess
 
-from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ValidationError
 from jsonschema import draft7_format_checker, validate
 from jsonschema.exceptions import ValidationError as SchemaError
 from swapper import load_model
 
-from openwisp_controller.config.models import Device
-
 from ... import settings as monitoring_settings
 from .. import settings as app_settings
 from ..exceptions import OperationalError
+from .base import BaseCheck
 
 Graph = load_model('monitoring', 'Graph')
-Metric = load_model('monitoring', 'Metric')
 Threshold = load_model('monitoring', 'Threshold')
 
 
-class Ping(object):
+class Ping(BaseCheck):
     schema = {
-        '$schema': 'http://json-schema.org/draft-04/schema#',
+        '$schema': 'http://json-schema.org/draft-07/schema#',
         'type': 'object',
         'additionalProperties': False,
         'properties': {
@@ -57,13 +54,6 @@ def validate(self):
         self.validate_instance()
         self.validate_params()
 
-    def validate_instance(self):
-        # check instance is of type device
-        obj = self.related_object
-        if not obj or not isinstance(obj, Device):
-            message = 'A related device is required ' 'to perform this operation'
-            raise ValidationError({'content_type': message, 'object_id': message})
-
     def validate_params(self):
         try:
             validate(self.params, self.schema, format_checker=draft7_format_checker)
@@ -165,23 +155,7 @@ def _get_metric(self):
         """
         Gets or creates metric
         """
-        check = self.check_instance
-        if check.object_id and check.content_type:
-            obj_id = check.object_id
-            ct = check.content_type
-        else:
-            obj_id = str(check.id)
-            ct = ContentType.objects.get(
-                app_label=check._meta.app_label, model=check.__class__.__name__.lower()
-            )
-        options = dict(
-            name=check.name,
-            object_id=obj_id,
-            content_type=ct,
-            field_name='reachable',
-            key=self.__class__.__name__.lower(),
-        )
-        metric, created = Metric.objects.get_or_create(**options)
+        metric, created = self._get_or_create_metric(field_name='reachable')
         if created:
             self._create_threshold(metric)
             self._create_graphs(metric)

openwisp_monitoring/check/migrations/0001_initial.py

@@ -9,6 +9,8 @@
 import collections
 import swapper
 
+from ..settings import CHECK_CLASSES
+
 
 class Migration(migrations.Migration):
 
@@ -58,7 +60,7 @@ class Migration(migrations.Migration):
                 (
                     'check',
                     models.CharField(
-                        choices=[('openwisp_monitoring.check.classes.Ping', 'Ping')],
+                        choices=CHECK_CLASSES,
                         db_index=True,
                         help_text='Select check type',
                         max_length=128,

openwisp_monitoring/check/migrations/0003_create_ping.py

@@ -11,7 +11,6 @@ def create_device_ping(apps, schema_editor):
                 model=Device.__name__.lower(),
                 app_label=Device._meta.app_label,
                 object_id=str(device.pk),
-                created=True,
             )
 
 

openwisp_monitoring/check/migrations/0004_create_config_status.py

@@ -0,0 +1,33 @@
+from django.db import migrations
+from openwisp_monitoring.check.settings import AUTO_CONFIG_STATUS
+from openwisp_monitoring.check.tasks import auto_create_config_status
+
+
+def add_config_status_checks(apps, schema_editor):
+    if not AUTO_CONFIG_STATUS:
+        return
+    Device = apps.get_model('config', 'Device')
+    for device in Device.objects.all():
+        auto_create_config_status.delay(
+            model=Device.__name__.lower(),
+            app_label=Device._meta.app_label,
+            object_id=str(device.pk),
+        )
+
+
+def remove_config_status_checks(apps, schema_editor):
+    Check = apps.get_model('config', 'Device')
+    Check.objects.filter(name='Config Status').delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('check', '0003_create_ping'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            add_config_status_checks, reverse_code=remove_config_status_checks
+        ),
+    ]