Merge pull request #1075 from stackhpc/monitor-swap-usage

Add alerts for low available swap space

Author: seunghun1ee

etc/kayobe/kolla/config/prometheus/system.rules

@@ -24,6 +24,24 @@ groups:
       summary: "Prometheus exporter at {{ $labels.instance }} reports low memory"
       description: "Available memory is {{ $value }} GiB."
 
+  - alert: LowSwapSpace
+    expr: (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) < {% endraw %}{{ alertmanager_node_free_swap_warning_threshold_ratio }}{% raw %}
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Swap space at {{ $labels.instance }} reports low memory"
+      description: "Available swap space is {{ $value | humanizePercentage }}. Running out of swap space causes OOM Kills."
+
+  - alert: LowSwapSpace
+    expr: (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) < {% endraw %}{{ alertmanager_node_free_swap_critical_threshold_ratio }}{% raw %}
+    for: 1m
+    labels:
+      severity: critical
+    annotations:
+      summary: "Swap space at {{ $labels.instance }} reports low memory"
+      description: "Available swap space is {{ $value | humanizePercentage }}. Running out of swap space causes OOM Kills."
+
   - alert: HostOomKillDetected
     expr: increase(node_vmstat_oom_kill[5m]) > 0
     for: 5m

etc/kayobe/stackhpc-monitoring.yml

@@ -12,6 +12,12 @@ alertmanager_low_memory_threshold_gib: 5
 # link. Change to false to disable this alert.
 alertmanager_warn_network_bond_single_link: true
 
+# Threshold to trigger an LowSwapSpace alert on swap space depletion (ratio).
+# When the ratio of free swap space is lower than each of these values, warning
+# and critical alerts will be triggered respectively.
+alertmanager_node_free_swap_warning_threshold_ratio: 0.25
+alertmanager_node_free_swap_critical_threshold_ratio: 0.1
+
 ###############################################################################
 # Exporter configuration
 

releasenotes/notes/add-alerts-for-swap-availability-75e28ed7f913d1ec.yaml

@@ -0,0 +1,13 @@
+---
+features:
+  - |
+    Added two alerts (Warning and critical) that are triggered when the ratio
+    of (free_swap_sppace / total_swap_space) is below thresholds.
+    Each threshold can be modified by alterting value of
+    ``alertmanager_node_free_swap_warning_threshold_ratio`` and
+    ``alertmanager_node_free_swap_critical_threshold_ratio``.
+
+    Currently this solution has limitation of having one-size fits all policy.
+    This can cause unwanted alerts for the hosts which utilise swap heavily
+    Therefore it is recommended to tune the thresholds or apply silence rules
+    for the needs.