feat: allow enabling memory oversubscription in cluster

allows Memory Oversubscription to be enabled in the cluster with
`enable_mem_oversubscription` set to true
ref:
- #8
- https://developer.hashicorp.com/nomad/tutorials/advanced-scheduling/memory-oversubscription

Signed-off-by: Chinmay D. Pai <[email protected]>

Author: Thunderbottom

modules/nomad-servers/launch_template.tf

@@ -9,12 +9,13 @@ resource "aws_launch_template" "nomad_server" {
   update_default_version  = true
 
   user_data = base64encode(templatefile("${path.module}/scripts/setup_server.tftpl.sh", {
-    nomad_acl_bootstrap_token = var.nomad_acl_bootstrap_token
-    nomad_acl_enable          = var.nomad_acl_enable
-    enable_tls                = var.enable_tls
-    tls_certificates          = var.tls_certificates
-    tls_http_enable           = var.tls_http_enable
-    tls_rpc_enable            = var.tls_rpc_enable
+    nomad_acl_bootstrap_token   = var.nomad_acl_bootstrap_token
+    nomad_acl_enable            = var.nomad_acl_enable
+    enable_mem_oversubscription = var.enable_mem_oversubscription
+    enable_tls                  = var.enable_tls
+    tls_certificates            = var.tls_certificates
+    tls_http_enable             = var.tls_http_enable
+    tls_rpc_enable              = var.tls_rpc_enable
     nomad_server_cfg = templatefile("${path.module}/templates/nomad.tftpl", {
       nomad_dc                 = var.cluster_name
       aws_region               = var.aws_region

modules/nomad-servers/scripts/setup_server.tftpl.sh

@@ -165,6 +165,34 @@ bootstrap_acl() {
   fi
 }
 
+enable_mem_oversubscription() {
+  command -v jq >/dev/null 2>&1 || { log "ERROR" "jq not found in PATH. Aborting."; exit 1; }
+
+  local nomad_config
+  nomad_config=$(curl -s http://0.0.0.0:4646/v1/operator/scheduler/configuration %{ if "${nomad_acl_bootstrap_token}" != "" }-H "X-Nomad-Token: ${nomad_acl_bootstrap_token}"%{ endif })
+  if [[ "$nomad_config" == *"Permission denied"* ]]; then
+    log "ERROR" "Permission denied while enabling memory oversubscription. Please check the bootstrap token."
+  else
+    log "INFO" "Checking if Memory Over subscription is already enabled."
+    mem_oversub_enabled=$(echo "$nomad_config" | jq '.SchedulerConfig | .MemoryOversubscriptionEnabled')
+    if [ "$mem_oversub_enabled" == "false" ]; then
+      log "INFO" "Memory Oversubscription is disabled. Enabling."
+      echo "$nomad_config" | \
+        status_code=(jq '.SchedulerConfig | .MemoryOversubscriptionEnabled=true' | \
+        curl -s -X PUT --write-out %%{http_code} \
+        %{ if "${nomad_acl_bootstrap_token}" != "" }-H "X-Nomad-Token: ${nomad_acl_bootstrap_token}"%{ endif } \
+        http://0.0.0.0:4646/v1/operator/scheduler/configuration -d @-)
+      if [ "$status_code" == "200" ]; then
+        log "INFO" "Successfully enabled Memory Oversubscription!"
+      else
+        log "ERROR" "Something went wrong while updating memory oversubscription. Please run it manually."
+      fi
+    else
+      log "INFO" "Memory Oversubscription is already enabled!"
+    fi
+  fi
+}
+
 log "INFO" "Fetching EC2 Tags from AWS"
 store_tags
 
@@ -195,4 +223,9 @@ log "INFO" "Skipping ACL Bootstrap for Nomad as 'nomad_acl_enable' is not set to
 log "INFO" "Restarting services"
 restart_nomad
 
+%{ if enable_memory_oversubscription }
+log "INFO" "Enabling Memory Oversubscription for the cluster"
+enable_mem_oversubscription
+%{ endif }
+
 log "INFO" "Finished server initializing process! Enjoy Nomad!"

modules/nomad-servers/variables.tf

@@ -81,6 +81,12 @@ variable "ebs_encryption" {
   default     = true
 }
 
+variable "enable_mem_oversubscription" {
+  description = "Whether to enable Memory Oversubscription on the cluster"
+  type        = bool
+  default     = false
+}
+
 variable "enable_tls" {
   description = "Whether to enable TLS on client nodes"
   type        = bool