feat: Implement worker pool drain

Signed-off-by: Devon Crouse <[email protected]>

Author: devoncrouse

docs/src/guide/workers_drain.md

@@ -5,3 +5,62 @@
 ```javascript
 {{#include ../../../examples/workers/vars-workers-drain.auto.tfvars:4:}}
 ```
+
+## Example
+
+```
+Terraform will perform the following actions:
+
+  # module.workers_only.module.utilities[0].null_resource.drain_workers[0] will be created
+  + resource "null_resource" "drain_workers" {
+      + id       = (known after apply)
+      + triggers = {
+          + "drain_commands" = jsonencode(
+                [
+                  + "kubectl drain --timeout=900s --ignore-daemonsets=true --delete-emptydir-data=true -l oke.oraclecloud.com/pool.name=oke-vm-draining",
+                ]
+            )
+          + "drain_pools"    = jsonencode(
+                [
+                  + "oke-vm-draining",
+                ]
+            )
+        }
+    }
+
+Plan: 1 to add, 0 to change, 0 to destroy.
+```
+
+```
+module.workers_only.module.utilities[0].null_resource.drain_workers[0] (remote-exec): node/10.200.220.157 cordoned
+module.workers_only.module.utilities[0].null_resource.drain_workers[0] (remote-exec): WARNING: ignoring DaemonSet-managed Pods: kube-system/csi-oci-node-99x74, kube-system/kube-flannel-ds-spvsp, kube-system/kube-proxy-6m2kk, ...
+module.workers_only.module.utilities[0].null_resource.drain_workers[0] (remote-exec): node/10.200.220.157 drained
+module.workers_only.module.utilities[0].null_resource.drain_workers[0]: Creation complete after 18s [id=7686343707387113624]
+
+Apply complete! Resources: 1 added, 0 changed, 0 destroyed.
+```
+
+Observe that the node(s) are now disabled for scheduling, and free of workloads other than DaemonSet-managed Pods when `worker_drain_ignore_daemonsets = true` (default):
+```shell
+kubectl get nodes -l oke.oraclecloud.com/pool.name=oke-vm-draining
+NAME             STATUS                     ROLES   AGE   VERSION
+10.200.220.157   Ready,SchedulingDisabled   node    24m   v1.26.2
+
+kubectl get pods --all-namespaces --field-selector spec.nodeName=10.200.220.157
+NAMESPACE     NAME                    READY   STATUS    RESTARTS   AGE
+kube-system   csi-oci-node-99x74      1/1     Running   0          50m
+kube-system   kube-flannel-ds-spvsp   1/1     Running   0          50m
+kube-system   kube-proxy-6m2kk        1/1     Running   0          50m
+kube-system   proxymux-client-2r6lk   1/1     Running   0          50m
+```
+
+Run the following command to uncordon a previously drained worker pool. The `drain = true` setting should be removed from the `worker_pools` entry to avoid re-draining the pool when running Terraform in the future.
+```shell
+kubectl uncordon -l oke.oraclecloud.com/pool.name=oke-vm-draining
+node/10.200.220.157 uncordoned
+```
+
+## References
+* [Safely Drain a Node](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/)
+* [`kubectl drain`](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#drain)
+* [Deleting a Worker Node](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengdeletingworkernodes.htm)

examples/profiles/workers-only/main.tf

@@ -15,6 +15,7 @@ module "workers_only" {
   vcn_id              = var.vcn_id
   bastion_public_ip   = var.bastion_public_ip
   cluster_id          = var.cluster_id
+  operator_private_ip = var.operator_private_ip
   ssh_public_key_path = var.ssh_public_key_path
 
   create_vcn      = false

examples/profiles/workers-only/variables.tf

@@ -14,6 +14,11 @@ variable "config_file_profile" {
   type    = string
 }
 
+variable "operator_private_ip" {
+  default = null
+  type    = string
+}
+
 variable "worker_nsg_ids" {
   default = []
   type    = list(string)

examples/workers/vars-workers-drain.auto.tfvars

@@ -4,6 +4,10 @@
 worker_pool_mode = "node-pool"
 worker_pool_size = 1
 
+worker_drain_ignore_daemonsets = true
+worker_drain_delete_local_data = true
+worker_drain_timeout_seconds   = 900
+
 worker_pools = {
   oke-vm-active = {
     description = "Node pool with active workers",

module-utilities.tf

@@ -25,7 +25,9 @@ module "utilities" {
   ocir_secret_namespace = var.ocir_secret_namespace
   ocir_username         = var.ocir_username
 
-  providers = {
-    oci.home = oci.home
-  }
+  # Worker pool draining
+  expected_drain_count           = local.worker_drain_expected
+  worker_drain_delete_local_data = var.worker_drain_delete_local_data
+  worker_drain_ignore_daemonsets = var.worker_drain_ignore_daemonsets
+  worker_drain_timeout_seconds   = var.worker_drain_timeout_seconds
 }

module-workers.tf

@@ -3,6 +3,7 @@
 
 locals {
   worker_count_expected = coalesce(one(module.workers[*].worker_count_expected), 0)
+  worker_drain_expected = coalesce(one(module.workers[*].worker_drain_expected), 0)
 
   # Distinct list of compartments for enabled worker pools
   worker_compartments = distinct(compact([

modules/utilities/drain.tf

@@ -2,16 +2,28 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
 
 locals {
-  drain_enabled = var.expected_node_count > 0 && var.worker_pools != null
-  worker_pools_draining = (local.drain_enabled
-    ? { for k, v in var.worker_pools : k => v if tobool(lookup(v, "drain", false)) } : {}
+  drain_enabled = var.expected_drain_count > 0
+  drain_pools = (local.drain_enabled
+    ? tolist([for k, v in var.worker_pools : k if tobool(lookup(v, "drain", false))]) : []
+  )
+
+  drain_commands = formatlist(
+    format(
+      "kubectl drain %v %v %v %v",
+      format("--timeout=%vs", var.worker_drain_timeout_seconds),
+      format("--ignore-daemonsets=%v", var.worker_drain_ignore_daemonsets),
+      format("--delete-emptydir-data=%v", var.worker_drain_delete_local_data),
+      "-l oke.oraclecloud.com/pool.name=%v" # interpolation deferred to formatlist
+    ),
+    local.drain_pools
   )
 }
 
 resource "null_resource" "drain_workers" {
   count = local.drain_enabled ? 1 : 0
   triggers = {
-    drain_workers = jsonencode(sort(keys(local.worker_pools_draining)))
+    drain_pools    = jsonencode(sort(local.drain_pools))
+    drain_commands = jsonencode(local.drain_commands)
   }
 
   connection {
@@ -26,9 +38,6 @@ resource "null_resource" "drain_workers" {
   }
 
   provisioner "remote-exec" {
-    inline = [
-      "echo kubectl get nodes ...",             # TODO List nodes by label for draining pools
-      "echo kubectl drain --ignore-daemonsets", # TODO Drain nodes for draining pools
-    ]
+    inline = local.drain_commands
   }
 }

modules/utilities/variables.tf

@@ -22,6 +22,10 @@ variable "ocir_secret_name" { type = string }
 variable "ocir_secret_namespace" { type = string }
 variable "ocir_username" { type = string }
 
-# Node readiness check
+# Node readiness check, drain
 variable "await_node_readiness" { type = string }
+variable "expected_drain_count" { type = number }
 variable "expected_node_count" { type = number }
+variable "worker_drain_ignore_daemonsets" { type = bool }
+variable "worker_drain_delete_local_data" { type = bool }
+variable "worker_drain_timeout_seconds" { type = number }

modules/utilities/versions.tf

@@ -0,0 +1,18 @@
+# Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
+
+terraform {
+  required_version = ">= 1.2.0"
+
+  required_providers {
+    null = {
+      source  = "hashicorp/null"
+      version = ">= 3.2.1"
+    }
+
+    oci = {
+      source  = "oracle/oci"
+      version = ">= 4.115.0"
+    }
+  }
+}

modules/utilities/versions.tf

@@ -138,7 +138,12 @@ locals {
 
   # Number of nodes expected from enabled worker pools
   expected_node_count = length(local.enabled_worker_pools) == 0 ? 0 : sum([
-    for k, v in local.enabled_worker_pools : lookup(v, "size", 0)
+    for k, v in local.enabled_worker_pools : lookup(v, "size", var.worker_pool_size)
+  ])
+
+  # Number of nodes expected to be draining in worker pools
+  expected_drain_count = length(local.enabled_worker_pools) == 0 ? 0 : sum([
+    for k, v in local.enabled_worker_pools : tobool(v.drain) ? lookup(v, "size", var.worker_pool_size) : 0
   ])
 
   # Enabled worker_pool map entries for node pools

modules/workers/locals.tf

@@ -20,3 +20,8 @@ output "worker_count_expected" {
   description = "# of nodes expected from created worker pools"
   value       = local.expected_node_count
 }
+
+output "worker_drain_expected" {
+  description = "# of nodes expected to be draining in worker pools"
+  value       = local.expected_drain_count
+}

modules/workers/outputs.tf

@@ -43,3 +43,23 @@ variable "ocir_username" {
   description = "A username with access to the OCI Vault secret for OCIR access. Required when 'ocir_secret_id' is provided."
   type        = string
 }
+
+# Worker pool draining
+
+variable "worker_drain_ignore_daemonsets" {
+  default     = true
+  description = "Whether to ignore DaemonSet-managed Pods when draining worker pools. See <a href=https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#drain>kubectl drain</a> for more information."
+  type        = bool
+}
+
+variable "worker_drain_delete_local_data" {
+  default     = true
+  description = "Whether to accept removal of data stored locally on draining worker pools. See <a href=https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#drain>kubectl drain</a> for more information."
+  type        = bool
+}
+
+variable "worker_drain_timeout_seconds" {
+  default     = 900
+  description = "The length of time to wait before giving up on draining nodes in a pool. See <a href=https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#drain>kubectl drain</a> for more information."
+  type        = number
+}