Merge pull request #1695 from xavierleune/feature/private-ips-autoscaling

Feature: Disable public ips on autoscaled nodes

Author: mysticaltech

README.md

@@ -1015,6 +1015,7 @@ If you follow this values, in your kube.tf, please set:
 - `existing_network_id = [YOURID]` (with the brackets)
 - `network_ipv4_cidr = "10.0.0.0/9"`
 - Add `disable_ipv4 = true` and  `disable_ipv6 = true` in all machines in all nodepools (control planes + agents).
+- Add `autoscaler_disable_ipv4 = true` and `autoscaler_disable_ipv6 = true` to disable public ips on autoscaled nodes.
 
 This setup is compatible with a loadbalancer for your control planes, however you should consider to set
 `control_plane_lb_enable_public_interface = false` to keep ip private.

autoscaler-agents.tf

@@ -41,7 +41,9 @@ locals {
       cluster_config                             = base64encode(jsonencode(local.cluster_config))
       firewall_id                                = hcloud_firewall.k3s.id
       cluster_name                               = local.cluster_prefix
-      node_pools                                 = var.autoscaler_nodepools
+      node_pools                                 = var.autoscaler_nodepools,
+      disable_ipv4                               = var.autoscaler_disable_ipv4,
+      disable_ipv6                               = var.autoscaler_disable_ipv6,
   })
   # A concatenated list of all autoscaled nodes
   autoscaled_nodes = length(var.autoscaler_nodepools) == 0 ? {} : {
@@ -116,7 +118,8 @@ data "cloudinit_config" "autoscaler_config" {
         })
         install_k3s_agent_script     = join("\n", concat(local.install_k3s_agent, ["systemctl start k3s-agent"]))
         cloudinit_write_files_common = local.cloudinit_write_files_common
-        cloudinit_runcmd_common      = local.cloudinit_runcmd_common
+        cloudinit_runcmd_common      = local.cloudinit_runcmd_common,
+        private_network_only         = var.autoscaler_disable_ipv4 && var.autoscaler_disable_ipv6,
       }
     )
   }
@@ -150,7 +153,8 @@ data "cloudinit_config" "autoscaler_legacy_config" {
         })
         install_k3s_agent_script     = join("\n", concat(local.install_k3s_agent, ["systemctl start k3s-agent"]))
         cloudinit_write_files_common = local.cloudinit_write_files_common
-        cloudinit_runcmd_common      = local.cloudinit_runcmd_common
+        cloudinit_runcmd_common      = local.cloudinit_runcmd_common,
+        private_network_only         = var.autoscaler_disable_ipv4 && var.autoscaler_disable_ipv6,
       }
     )
   }

kube.tf.example

@@ -360,6 +360,10 @@ module "kube-hetzner" {
   #    # kubelet_args = ["kube-reserved=cpu=250m,memory=1500Mi,ephemeral-storage=1Gi", "system-reserved=cpu=250m,memory=300Mi"]
   #  }
   # ]
+  #
+  # To disable public ips on your autoscaled nodes, uncomment the following lines:
+  # autoscaler_disable_ipv4 = true
+  # autoscaler_disable_ipv6 = true
 
   # ⚠️ Deprecated, will be removed after a new Cluster Autoscaler version has been released which support the new way of setting labels and taints. See above.
   # Add extra labels on nodes started by the Cluster Autoscaler
@@ -1173,7 +1177,7 @@ terraform {
   required_providers {
     hcloud = {
       source  = "hetznercloud/hcloud"
-      version = ">= 1.49.1"
+      version = ">= 1.51.0"
     }
   }
 }

modules/host/versions.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     hcloud = {
       source  = "hetznercloud/hcloud"
-      version = ">= 1.49.1"
+      version = ">= 1.51.0"
     }
   }
 }

templates/autoscaler-cloudinit.yaml.tpl

@@ -42,5 +42,12 @@ runcmd:
 
 ${cloudinit_runcmd_common}
 
+# Configure default route based on public ip availability
+%{if private_network_only~}
+- [ip, route, add, default, via, '10.0.0.1', dev, 'eth0']
+%{else~}
+- [ip, route, add, default, via, '172.31.1.1', dev, 'eth0']
+%{endif~}
+
 # Start the install-k3s-agent service
 - ['/bin/bash', '/var/pre_install/install-k3s-agent.sh']

templates/autoscaler.yaml.tpl

@@ -194,6 +194,14 @@ spec:
             value: '${ipv4_subnet_id}'
           - name: HCLOUD_FIREWALL
             value: '${firewall_id}'
+          %{~ if disable_ipv4 ~}
+          - name: HCLOUD_PUBLIC_IPV4
+            value: "false"
+          %{~ endif ~}
+          %{~ if disable_ipv6 ~}
+          - name: HCLOUD_PUBLIC_IPV6
+            value: "false"
+          %{~ endif ~}
           %{~ if cluster_autoscaler_server_creation_timeout != "" ~}
           - name: HCLOUD_SERVER_CREATION_TIMEOUT
             value: '${cluster_autoscaler_server_creation_timeout}'

variables.tf

@@ -359,6 +359,19 @@ variable "autoscaler_taints" {
   default     = []
 }
 
+variable "autoscaler_disable_ipv4" {
+  description = "Disable IPv4 on nodes created by the Cluster Autoscaler."
+  type        = bool
+  default     = false
+}
+
+variable "autoscaler_disable_ipv6" {
+  description = "Disable IPv6 on nodes created by the Cluster Autoscaler."
+  type        = bool
+  default     = false
+}
+
+
 variable "hetzner_ccm_version" {
   type        = string
   default     = null

versions.tf

@@ -7,7 +7,7 @@ terraform {
     }
     hcloud = {
       source  = "hetznercloud/hcloud"
-      version = ">= 1.49.1"
+      version = ">= 1.51.0"
     }
     local = {
       source  = "hashicorp/local"