[timeseries] Add initial support for elasticsearch #99

Author: nepython

.travis.yml

@@ -24,27 +24,30 @@ addons:
 
 services:
   - docker
-  - redis-server
 
 branches:
   only:
     - master
     - dev
 
 before_install:
-  - docker run -d --name influxdb -e INFLUXDB_DB=openwisp2 -p 8086:8086 influxdb:alpine
+  - docker-compose up -d
   - pip install -U pip wheel setuptools
   - pip install $DJANGO
   - pip install -U -r requirements-test.txt
 
 install:
-  - pip install -e .
+  - pip install -e .[influxdb,elasticsearch]
   - sh install-dev.sh
 
 script:
   - ./run-qa-checks
-  - SAMPLE_APP=1 coverage run --source=openwisp_monitoring runtests.py
-  - coverage run -a --source=openwisp_monitoring runtests.py
+  - SAMPLE_APP=1 coverage run -p --source=openwisp_monitoring runtests.py
+  - coverage run -p --source=openwisp_monitoring runtests.py
+  - elasticsearch=1 SAMPLE_APP=1 coverage run -p --source=openwisp_monitoring runtests.py
+  - elasticsearch=1 coverage run -p --source=openwisp_monitoring runtests.py
+  - coverage combine
+  - coverage report -m
 
 jobs:
   include:

Dockerfile

@@ -19,6 +19,7 @@ WORKDIR /opt/openwisp/tests/
 ENV NAME=openwisp-monitoring \
     PYTHONBUFFERED=1 \
     INFLUXDB_HOST=influxdb \
-    REDIS_HOST=redis
+    REDIS_HOST=redis \
+    ELASTICSEARCH_HOST=es01
 CMD ["sh", "docker-entrypoint.sh"]
 EXPOSE 8000

README.rst

@@ -28,7 +28,7 @@ Available Features
 * Collects and displays `device status <#device-status>`_ information like uptime, RAM status, CPU load averages,
   Interface properties and addresses, WiFi interface status and associated clients,
   Neighbors information, DHCP Leases, Disk/Flash status
-* Collection of monitoring information in a timeseries database (currently only influxdb is supported)
+* Collection of monitoring information in a timeseries database (`InfluxDB <https://www.influxdata.com/>`_ and `Elasticsearch <https://www.elastic.co/elasticsearch/>`_ are currently supported)
 * Monitoring charts for uptime, packet loss, round trip time (latency), associated wifi clients, interface traffic,
   RAM usage, CPU load, flash/disk usage
 * Charts can be viewed at resolutions of 1 day, 3 days, a week, a month and a year
@@ -50,6 +50,8 @@ beforehand.
 In case you prefer not to use Docker you can `install InfluxDB <https://docs.influxdata.com/influxdb/v1.8/introduction/install/>`_
 and Redis from your repositories, but keep in mind that the version packaged by your distribution may be different.
 
+If you wish to use ``Elasticsearch`` for storing and retrieving timeseries data then `install Elasticsearch <https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html>`_.
+
 Install spatialite and sqlite:
 
 .. code-block:: shell
@@ -110,6 +112,19 @@ Follow the setup instructions of `openwisp-controller
         'PORT': '8086',
     }
 
+In case, you wish to use ``Elasticsearch`` for timeseries data storage and retrieval,
+make use of the following settings
+
+.. code-block:: python
+    TIMESERIES_DATABASE = {
+        'BACKEND': 'openwisp_monitoring.db.backends.elasticsearch',
+        'USER': 'openwisp',
+        'PASSWORD': 'openwisp',
+        'NAME': 'openwisp2',
+        'HOST': 'localhost',
+        'PORT': '9200',
+    }
+
 ``urls.py``:
 
 .. code-block:: python
@@ -422,6 +437,9 @@ This data is only used to assess the recent status of devices, keeping
 it for a long time would not add much benefit and would cost a lot more
 in terms of disk space.
 
+**Note**: In case you use ``Elasticsearch`` then time shall be taken as integral multiple of a day.
+That means the time ``36h0m0s`` shall be interpreted as ``24h0m0s`` (integral multiple of a day).
+
 ``OPENWISP_MONITORING_AUTO_PING``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -702,18 +720,30 @@ MB (megabytes) instead of GB (Gigabytes) you can use:
                     "SUM(rx_bytes) / 1000000 AS download FROM {key} "
                     "WHERE time >= '{time}' AND content_type = '{content_type}' "
                     "AND object_id = '{object_id}' GROUP BY time(1d)"
-                )
+                ),
+                'elasticsearch': _make_query({
+                    'upload': {'sum': {'field': 'points.fields.tx_bytes'}},
+                    'download': {'avg': {'field': 'points.fields.rx_bytes'}},
+                })
             },
         }
     }
 
+    # Please declare the operations separately in case you use elasticsearch as done below
+    OPENWISP_MONITORING_ADDITIONAL_CHART_OPERATIONS = {
+        'upload': {'operator': '/', 'value': 1000000},
+        'download': {'operator': '/', 'value': 1000000},
+    }
+
 Or if you want to define a new chart configuration, which you can then
 call in your custom code (eg: a custom check class), you can do so as follows:
 
 .. code-block:: python
 
     from django.utils.translation import gettext_lazy as _
 
+    from openwisp_monitoring.db.backends.elasticsearch import _make_query
+
     OPENWISP_MONITORING_CHARTS = {
         'ram': {
             'type': 'line',
@@ -727,7 +757,12 @@ call in your custom code (eg: a custom check class), you can do so as follows:
                     "MEAN(buffered) AS buffered FROM {key} WHERE time >= '{time}' AND "
                     "content_type = '{content_type}' AND object_id = '{object_id}' "
                     "GROUP BY time(1d)"
-                )
+                ),
+                'elasticsearch': _make_query({
+                    'total': {'avg': {'field': 'points.fields.total'}},
+                    'free': {'avg': {'field': 'points.fields.free'}},
+                    'buffered': {'avg': {'field': 'points.fields.buffered'}},
+                })
             },
         }
     }

docker-compose.yml

@@ -11,6 +11,8 @@ services:
     depends_on:
       - influxdb
       - redis
+      - es01
+      - es02
 
   influxdb:
     image: influxdb:1.8-alpine
@@ -22,6 +24,45 @@ services:
       INFLUXDB_DB: openwisp2
       INFLUXDB_USER: openwisp
       INFLUXDB_USER_PASSWORD: openwisp
+  # clustered version of elasticsearch is used as that might be used in production
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.8.0
+    container_name: es01
+    environment:
+      - "node.name=es01"
+      - "discovery.seed_hosts=es02"
+      - "cluster.initial_master_nodes=es01,es02"
+      - "cluster.name=openwisp2"
+      - "bootstrap.memory_lock=true"
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - esdata01:/usr/share/elasticsearch/data
+    ports:
+      - 9200:9200
+    networks:
+      - esnet
+  es02:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.8.0
+    container_name: es02
+    environment:
+      - "node.name=es02"
+      - "discovery.seed_hosts=es01"
+      - "cluster.initial_master_nodes=es01,es02"
+      - "cluster.name=openwisp2"
+      - "bootstrap.memory_lock=true"
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - esdata02:/usr/share/elasticsearch/data
+    networks:
+      - esnet
 
   redis:
     image: redis:5.0-alpine
@@ -31,3 +72,10 @@ services:
 
 volumes:
   influxdb-data: {}
+  esdata01:
+    driver: local
+  esdata02:
+    driver: local
+
+networks:
+  esnet:

openwisp_monitoring/db/__init__.py

@@ -1,7 +1,5 @@
 from .backends import timeseries_db
 
 chart_query = timeseries_db.queries.chart_query
-default_chart_query = timeseries_db.queries.default_chart_query
-device_data_query = timeseries_db.queries.device_data_query
 
-__all__ = ['timeseries_db', 'chart_query', 'default_chart_query', 'device_data_query']
+__all__ = ['timeseries_db', 'chart_query']

openwisp_monitoring/db/backends/__init__.py

@@ -48,7 +48,8 @@ def load_backend_module(backend_name=TIMESERIES_DB['BACKEND'], module=None):
     except ImportError as e:
         # The database backend wasn't found. Display a helpful error message
         # listing all built-in database backends.
-        builtin_backends = ['influxdb']
+        builtin_backends = ['influxdb', 'elasticsearch']
+        raise e
         if backend_name not in [
             f'openwisp_monitoring.db.backends.{b}' for b in builtin_backends
         ]:

openwisp_monitoring/db/backends/elasticsearch/__init__.py

@@ -0,0 +1,3 @@
+from .queries import _make_query
+
+__all__ = ['_make_query']