diff --git a/apis/v1alpha2/nginxproxy_types.go b/apis/v1alpha2/nginxproxy_types.go index 43b509d06d..48616f8545 100644 --- a/apis/v1alpha2/nginxproxy_types.go +++ b/apis/v1alpha2/nginxproxy_types.go @@ -1,6 +1,7 @@ package v1alpha2 import ( + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -388,6 +389,11 @@ type DeploymentSpec struct { // +optional Replicas *int32 `json:"replicas,omitempty"` + // Autoscaling defines the configuration for Horizontal Pod Autoscaling. + // + // +optional + Autoscaling HPASpec `json:"autoscaling"` + // Pod defines Pod-specific fields. // // +optional @@ -412,6 +418,56 @@ type DaemonSetSpec struct { Container ContainerSpec `json:"container"` } +// +kubebuilder:validation:XValidation:message="at least one metric must be specified when autoscaling is enabled",rule="!self.enabled || (has(self.targetCPUUtilizationPercentage) || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) && size(self.autoscalingTemplate) > 0))" +// +kubebuilder:validation:XValidation:message="minReplicas must be less than or equal to maxReplicas",rule="self.minReplicas <= self.maxReplicas" +// +kubebuilder:validation:XValidation:message="CPU utilization must be between 1 and 100",rule="!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage >= 1 && self.targetCPUUtilizationPercentage <= 100)" +// +kubebuilder:validation:XValidation:message="memory utilization must be between 1 and 100",rule="!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage >= 1 && self.targetMemoryUtilizationPercentage <= 100)" +// +// HPASpec is the configuration for the Horizontal Pod Autoscaling. +// +//nolint:lll +type HPASpec struct { + // Behavior configures the scaling behavior of the target + // in both Up and Down directions (scaleUp and scaleDown fields respectively). + // If not set, the default HPAScalingRules for scale up and scale down are used. + // + // +optional + Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"` + + // AutoscalingTemplate configures the additional scaling option. + // + // +optional + AutoscalingTemplate *[]autoscalingv2.MetricSpec `json:"autoscalingTemplate,omitempty"` + + // Target cpu utilization percentage of HPA. + // + // +optional + TargetCPUUtilizationPercentage *int32 `json:"targetCPUUtilizationPercentage,omitempty"` + + // Target memory utilization percentage of HPA. + // + // +optional + TargetMemoryUtilizationPercentage *int32 `json:"targetMemoryUtilizationPercentage,omitempty"` + + // Annotation for Horizontal Pod Autoscaler + // Annotations is an unstructured key value map stored with a resource that may be + // set by external tools to store and retrieve arbitrary metadata. They are not + // queryable and should be preserved when modifying objects. + // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + // + // +optional + HPAAnnotations map[string]string `json:"hpaAnnotations,omitempty"` + + // Minimum number of replicas. + MinReplicas int32 `json:"minReplicas"` + + // Maximum number of replicas. + MaxReplicas int32 `json:"maxReplicas"` + + // Enable or disable Horizontal Pod Autoscaler + Enabled bool `json:"enabled"` +} + // PodSpec defines Pod-specific fields. type PodSpec struct { // TerminationGracePeriodSeconds is the optional duration in seconds the pod needs to terminate gracefully. diff --git a/apis/v1alpha2/zz_generated.deepcopy.go b/apis/v1alpha2/zz_generated.deepcopy.go index 4b0f8bf9f8..e4e4981ca4 100644 --- a/apis/v1alpha2/zz_generated.deepcopy.go +++ b/apis/v1alpha2/zz_generated.deepcopy.go @@ -6,6 +6,7 @@ package v1alpha2 import ( "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1" + "k8s.io/api/autoscaling/v2" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" apisv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -78,6 +79,7 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { *out = new(int32) **out = **in } + in.Autoscaling.DeepCopyInto(&out.Autoscaling) in.Pod.DeepCopyInto(&out.Pod) in.Container.DeepCopyInto(&out.Container) } @@ -92,6 +94,54 @@ func (in *DeploymentSpec) DeepCopy() *DeploymentSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HPASpec) DeepCopyInto(out *HPASpec) { + *out = *in + if in.Behavior != nil { + in, out := &in.Behavior, &out.Behavior + *out = new(v2.HorizontalPodAutoscalerBehavior) + (*in).DeepCopyInto(*out) + } + if in.AutoscalingTemplate != nil { + in, out := &in.AutoscalingTemplate, &out.AutoscalingTemplate + *out = new([]v2.MetricSpec) + if **in != nil { + in, out := *in, *out + *out = make([]v2.MetricSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + } + if in.TargetCPUUtilizationPercentage != nil { + in, out := &in.TargetCPUUtilizationPercentage, &out.TargetCPUUtilizationPercentage + *out = new(int32) + **out = **in + } + if in.TargetMemoryUtilizationPercentage != nil { + in, out := &in.TargetMemoryUtilizationPercentage, &out.TargetMemoryUtilizationPercentage + *out = new(int32) + **out = **in + } + if in.HPAAnnotations != nil { + in, out := &in.HPAAnnotations, &out.HPAAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HPASpec. +func (in *HPASpec) DeepCopy() *HPASpec { + if in == nil { + return nil + } + out := new(HPASpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Image) DeepCopyInto(out *Image) { *out = *in diff --git a/charts/nginx-gateway-fabric/README.md b/charts/nginx-gateway-fabric/README.md index 3fb3f6230b..7c4c348c6a 100644 --- a/charts/nginx-gateway-fabric/README.md +++ b/charts/nginx-gateway-fabric/README.md @@ -264,7 +264,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `certGenerator.ttlSecondsAfterFinished` | How long to wait after the cert generator job has finished before it is removed by the job controller. | int | `30` | | `clusterDomain` | The DNS cluster domain of your Kubernetes cluster. | string | `"cluster.local"` | | `gateways` | A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. | list | `[]` | -| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"config":{},"container":{},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` | +| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"autoscaling":{"enabled":false},"autoscalingTemplate":[],"config":{},"container":{},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` | | `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` | | `nginx.container` | The container configuration for the NGINX container. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` | | `nginx.debug` | Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. | bool | `false` | @@ -288,7 +288,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `nginx.usage.resolver` | The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. | string | `""` | | `nginx.usage.secretName` | The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"nplus-license"` | | `nginx.usage.skipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` | -| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` | +| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"autoscaling":{"enabled":false},"autoscalingTemplate":[],"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` | | `nginxGateway.affinity` | The affinity of the NGINX Gateway Fabric control plane pod. | object | `{}` | | `nginxGateway.config.logging.level` | Log level. | string | `"info"` | | `nginxGateway.configAnnotations` | Set of custom annotations for NginxGateway objects. | object | `{}` | diff --git a/charts/nginx-gateway-fabric/templates/clusterrole.yaml b/charts/nginx-gateway-fabric/templates/clusterrole.yaml index 1205570535..84d32a1fc8 100644 --- a/charts/nginx-gateway-fabric/templates/clusterrole.yaml +++ b/charts/nginx-gateway-fabric/templates/clusterrole.yaml @@ -8,6 +8,9 @@ rules: - apiGroups: - "" - apps + {{- if or .Values.nginx.autoscaling.enabled .Values.nginxGateway.autoscaling.enabled }} + - autoscaling + {{- end }} resources: - secrets - configmaps @@ -15,6 +18,9 @@ rules: - services - deployments - daemonsets + {{- if or .Values.nginx.autoscaling.enabled .Values.nginxGateway.autoscaling.enabled }} + - horizontalpodautoscalers + {{- end }} verbs: - create - update diff --git a/charts/nginx-gateway-fabric/templates/hpa.yaml b/charts/nginx-gateway-fabric/templates/hpa.yaml new file mode 100644 index 0000000000..86730bcfa5 --- /dev/null +++ b/charts/nginx-gateway-fabric/templates/hpa.yaml @@ -0,0 +1,46 @@ +{{- if and (eq .Values.nginxGateway.kind "deployment") .Values.nginxGateway.autoscaling.enabled (.Capabilities.APIVersions.Has "autoscaling/v2") -}} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + {{- with .Values.nginxGateway.autoscaling.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "nginx-gateway.labels" . | nindent 4 }} + {{- with .Values.nginxGateway.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "nginx-gateway.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "nginx-gateway.fullname" . }} + minReplicas: {{ .Values.nginxGateway.autoscaling.minReplicas }} + maxReplicas: {{ .Values.nginxGateway.autoscaling.maxReplicas }} + metrics: + {{- with .Values.nginxGateway.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.nginxGateway.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.autoscalingTemplate }} + {{- toYaml . | nindent 2 }} + {{- end }} + {{- with .Values.nginxGateway.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml index b5e33292c8..29ab315c76 100644 --- a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml +++ b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml @@ -12,7 +12,33 @@ spec: kubernetes: {{- if eq .Values.nginx.kind "deployment" }} deployment: + {{- if .Values.nginx.replicas }} replicas: {{ .Values.nginx.replicas }} + {{- end }} + {{- if .Values.nginx.autoscaling.enabled }} + autoscaling: + enabled: {{ .Values.nginx.autoscaling.enabled }} + {{- if .Values.nginx.autoscaling.hpaAnnotations }} + hpaAnnotations: + {{- toYaml .Values.nginx.autoscaling.hpaAnnotations | nindent 10 }} + {{- end }} + minReplicas: {{ .Values.nginx.autoscaling.minReplicas }} + maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }} + {{- if .Values.nginx.autoscaling.targetCPUUtilizationPercentage }} + targetCPUUtilizationPercentage: {{ .Values.nginx.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }} + targetMemoryUtilizationPercentage: {{ .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} + {{- if .Values.nginx.autoscaling.behavior }} + behavior: + {{- toYaml .Values.nginx.autoscaling.behavior | nindent 10 }} + {{- end }} + {{- if .Values.nginx.autoscalingTemplate }} + autoscalingTemplate: + {{- toYaml .Values.nginx.autoscalingTemplate | nindent 8 }} + {{- end }} + {{- end }} {{- if .Values.nginx.pod }} pod: {{- toYaml .Values.nginx.pod | nindent 8 }} diff --git a/charts/nginx-gateway-fabric/values-aks.yaml b/charts/nginx-gateway-fabric/values-aks.yaml new file mode 100644 index 0000000000..31cd16b6f6 --- /dev/null +++ b/charts/nginx-gateway-fabric/values-aks.yaml @@ -0,0 +1,657 @@ +# yaml-language-server: $schema=values.schema.json + +# -- The DNS cluster domain of your Kubernetes cluster. +clusterDomain: cluster.local + +# -- The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. +nginxGateway: + # FIXME(lucacome): https://github.com/nginx/nginx-gateway-fabric/issues/2490 + + # @schema + # const: deployment + # @schema + # -- The kind of the NGINX Gateway Fabric installation - currently, only deployment is supported. + kind: deployment + + # @schema + # required: true + # type: string + # @schema + # -- The name of the GatewayClass that will be created as part of this release. Every NGINX Gateway + # Fabric must have a unique corresponding GatewayClass resource. NGINX Gateway Fabric only processes resources that + # belong to its class - i.e. have the "gatewayClassName" field resource equal to the class. + gatewayClassName: nginx-test + + # -- Set of custom annotations for the NGINX Gateway Fabric pods. + podAnnotations: {} + + # -- Set of labels to be added for NGINX Gateway Fabric deployment. + labels: {} + + # -- Set of custom annotations for GatewayClass objects. + gatewayClassAnnotations: {} + + # @schema + # pattern: ^gateway.nginx.org/.* + # required: true + # @schema + # -- The name of the Gateway controller. The controller name must be of the form: DOMAIN/PATH. The controller's domain + # is gateway.nginx.org. + gatewayControllerName: gateway.nginx.org/nginx-gateway-controller + + # The dynamic configuration for the control plane that is contained in the NginxGateway resource. + config: + logging: + # @schema + # enum: + # - info + # - debug + # - error + # @schema + # -- Log level. + level: info + + # -- Set of custom annotations for NginxGateway objects. + configAnnotations: {} + + # -- The service configuration for the NGINX Gateway Fabric control plane. + service: + # -- The annotations of the NGINX Gateway Fabric control plane service. + annotations: {} + + # -- The labels of the NGINX Gateway Fabric control plane service. + labels: {} + + # -- The serviceaccount configuration for the NGINX Gateway Fabric control plane. + serviceAccount: + # -- Set of custom annotations for the NGINX Gateway Fabric control plane service account. + annotations: {} + + # -- The name of the service account of the NGINX Gateway Fabric control plane pods. Used for RBAC. + # @default -- Autogenerated if not set or set to "" + name: "" + + # -- The name of the secret containing docker registry credentials for the control plane. + # Secret must exist in the same namespace as the helm release. + imagePullSecret: "" + + # -- A list of secret names containing docker registry credentials for the control plane. + # Secrets must exist in the same namespace as the helm release. + imagePullSecrets: [] + + # -- The number of replicas of the NGINX Gateway Fabric Deployment. + replicas: 1 + + # The configuration for leader election. + leaderElection: + # -- Enable leader election. Leader election is used to avoid multiple replicas of the NGINX Gateway Fabric + # reporting the status of the Gateway API resources. If not enabled, all replicas of NGINX Gateway Fabric + # will update the statuses of the Gateway API resources. + enable: true + + # -- The name of the leader election lock. A Lease object with this name will be created in the same Namespace as + # the controller. + # @default -- Autogenerated if not set or set to "". + lockName: "" + + ## Defines the settings for the control plane readiness probe. This probe returns Ready when the controller + ## has started and is ready to configure NGINX. + readinessProbe: + # -- Enable the /readyz endpoint on the control plane. + enable: true + + # @schema + # type: integer + # minimum: 1 + # maximum: 65535 + # @schema + # -- Port in which the readiness endpoint is exposed. + port: 8081 + + # -- The number of seconds after the Pod has started before the readiness probes are initiated. + initialDelaySeconds: 3 + + # -- The image configuration for the NGINX Gateway Fabric control plane. + image: + # -- The NGINX Gateway Fabric image to use + #repository: ghcr.io/nginx/nginx-gateway-fabric + repository: prodonlinelabcontainerreg.azurecr.io/nginx-gateway-fabric + tag: edge + # @schema + # enum: + # - Always + # - IfNotPresent + # - Never + # @schema + pullPolicy: Always + + productTelemetry: + # -- Enable the collection of product telemetry. + enable: true + + # -- The lifecycle of the nginx-gateway container. + lifecycle: {} + + # -- The resource requests and/or limits of the nginx-gateway container. + #resources: {} + resources: + requests: + cpu: 1000m + memory: 1Gi + + # -- extraVolumes for the NGINX Gateway Fabric control plane pod. Use in conjunction with + # nginxGateway.extraVolumeMounts mount additional volumes to the container. + extraVolumes: [] + + # -- extraVolumeMounts are the additional volume mounts for the nginx-gateway container. + extraVolumeMounts: [] + + # -- The termination grace period of the NGINX Gateway Fabric control plane pod. + terminationGracePeriodSeconds: 30 + + # -- Tolerations for the NGINX Gateway Fabric control plane pod. + tolerations: + - key: "kubernetes.azure.com/scalesetpriority" + operator: "Equal" + value: "spot" + effect: "NoSchedule" + + # -- The nodeSelector of the NGINX Gateway Fabric control plane pod. + nodeSelector: {} + + # -- The affinity of the NGINX Gateway Fabric control plane pod. + affinity: {} + + # -- The topology spread constraints for the NGINX Gateway Fabric control plane pod. + topologySpreadConstraints: [] + + autoscaling: + # Enable or disable Horizontal Pod Autoscaler + enabled: true + annotations: {} + minReplicas: 1 + maxReplicas: 11 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 + behavior: {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + autoscalingTemplate: [] + # Custom or additional autoscaling metrics + # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics + # - type: Pods + # pods: + # metric: + # name: nginx_gateway_fabric_nginx_process_requests_total + # target: + # type: AverageValue + # averageValue: 10000m + + metrics: + # -- Enable exposing metrics in the Prometheus format. + enable: true + + # @schema + # type: integer + # minimum: 1 + # maximum: 65535 + # @schema + # -- Set the port where the Prometheus metrics are exposed. + port: 9113 + + # -- Enable serving metrics via https. By default metrics are served via http. + # Please note that this endpoint will be secured with a self-signed certificate. + secure: false + + gwAPIExperimentalFeatures: + # -- Enable the experimental features of Gateway API which are supported by NGINX Gateway Fabric. Requires the Gateway + # APIs installed from the experimental channel. + enable: false + + snippetsFilters: + # -- Enable SnippetsFilters feature. SnippetsFilters allow inserting NGINX configuration into the generated NGINX + # config for HTTPRoute and GRPCRoute resources. + enable: true + +# -- The nginx section contains the configuration for all NGINX data plane deployments +# installed by the NGINX Gateway Fabric control plane. +nginx: + # @schema + # enum: + # - deployment + # - daemonSet + # @schema + # -- The kind of NGINX deployment. + kind: deployment + + # -- The number of replicas of the NGINX Deployment. + replicas: 1 + + autoscaling: + # Enable or disable Horizontal Pod Autoscaler + enabled: true + hpaAnnotations: + test/owner: "woojin" + minReplicas: 1 + maxReplicas: 11 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 + behavior: + {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + autoscalingTemplate: + [] + # - type: Pods + # pods: + # metric: + # name: nginx_gateway_fabric_nginx_process_requests_total + # target: + # type: AverageValue + # averageValue: 10000m + # Custom or additional autoscaling metrics + # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics + + image: + # -- The NGINX image to use. + #repository: ghcr.io/nginx/nginx-gateway-fabric/nginx + repository: prodonlinelabcontainerreg.azurecr.io/nginx-gateway-fabric/nginx + tag: edge + # @schema + # enum: + # - Always + # - IfNotPresent + # - Never + # @schema + pullPolicy: Always + + # -- Is NGINX Plus image being used. + plus: false + + # -- The name of the secret containing docker registry credentials. + # Secret must exist in the same namespace as the helm release. The control + # plane will copy this secret into any namespace where NGINX is deployed. + imagePullSecret: "" + + # -- A list of secret names containing docker registry credentials. + # Secrets must exist in the same namespace as the helm release. The control + # plane will copy these secrets into any namespace where NGINX is deployed. + imagePullSecrets: [] + + # Configuration for NGINX Plus usage reporting. + usage: + # -- The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace + # that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). + secretName: "nplus-license" + + # -- The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com + endpoint: "" + + # -- The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. + resolver: "" + + # -- Disable client verification of the NGINX Plus usage reporting server certificate. + skipVerify: false + + # -- The name of the Secret containing the NGINX Instance Manager CA certificate. + # Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). + caSecretName: "" + + # -- The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. + # Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). + clientSSLSecretName: "" + + # @schema + # type: object + # properties: + # disableHTTP2: + # description: DisableHTTP2 defines if http2 should be disabled for all servers. + # type: boolean + # ipFamily: + # description: IPFamily specifies the IP family to be used by the NGINX. + # type: string + # enum: + # - ipv4 + # - ipv6 + # - dual + # rewriteClientIP: + # type: object + # description: RewriteClientIP defines configuration for rewriting the client IP to the original client's IP. + # properties: + # mode: + # type: string + # enum: + # - ProxyProtocol + # - XForwardedFor + # setIPRecursively: + # type: boolean + # trustedAddresses: + # type: array + # items: + # properties: + # type: + # type: string + # enum: + # - CIDR + # - IPAddress + # - Hostname + # value: + # type: string + # telemetry: + # type: object + # description: Telemetry specifies the OpenTelemetry configuration. + # properties: + # exporter: + # type: object + # properties: + # endpoint: + # type: string + # pattern: ^(?:http?:\/\/)?[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*(?::\d{1,5})?$ + # interval: + # type: string + # pattern: ^\d{1,4}(ms|s)?$ + # batchSize: + # type: integer + # minimum: 0 + # batchCount: + # type: integer + # minimum: 0 + # serviceName: + # type: string + # pattern: ^[a-zA-Z0-9_-]+$ + # spanAttributes: + # type: array + # items: + # properties: + # key: + # type: string + # pattern: ^([^"$\\]|\\[^$])*$ + # minLength: 1 + # maxLength: 255 + # value: + # type: string + # pattern: ^([^"$\\]|\\[^$])*$ + # minLength: 1 + # maxLength: 255 + # disabledFeatures: + # type: array + # items: + # type: string + # enum: + # - DisableTracing + # metrics: + # type: object + # description: Metrics defines the configuration for Prometheus scraping metrics. + # properties: + # disable: + # type: boolean + # port: + # type: integer + # minimum: 1 + # maximum: 65535 + # logging: + # type: object + # description: Logging defines logging related settings for NGINX. + # properties: + # errorLevel: + # type: string + # enum: + # - debug + # - info + # - notice + # - warn + # - error + # - crit + # - alert + # - emerg + # agentLevel: + # type: string + # enum: + # - debug + # - info + # - error + # - panic + # - fatal + # nginxPlus: + # type: object + # description: NginxPlus specifies NGINX Plus additional settings. + # properties: + # allowedAddresses: + # type: array + # items: + # properties: + # type: + # type: string + # enum: + # - CIDR + # - IPAddress + # value: + # type: string + # @schema + # -- The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways + # managed by this instance of NGINX Gateway Fabric. + config: {} + + # -- The pod configuration for the NGINX data plane pod. This is applied globally to all Gateways managed by this + # instance of NGINX Gateway Fabric. + pod: + # -- The termination grace period of the NGINX data plane pod. + # terminationGracePeriodSeconds: 30 + + # -- Tolerations for the NGINX data plane pod. + tolerations: + - key: "kubernetes.azure.com/scalesetpriority" + operator: "Equal" + value: "spot" + effect: "NoSchedule" + + # -- The nodeSelector of the NGINX data plane pod. + # nodeSelector: {} + + # -- The affinity of the NGINX data plane pod. + # affinity: {} + + # -- The topology spread constraints for the NGINX data plane pod. + # topologySpreadConstraints: [] + + # -- extraVolumes for the NGINX data plane pod. Use in conjunction with + # nginx.container.extraVolumeMounts mount additional volumes to the container. + # extraVolumes: [] + + # -- The container configuration for the NGINX container. This is applied globally to all Gateways managed by this + # instance of NGINX Gateway Fabric. + container: + # -- The resource requirements of the NGINX container. You should be set this value, If you want to use dataplane Autoscaling(HPA). + # resources: {} + resources: + requests: + cpu: 1000m + memory: 1Gi + + # -- The lifecycle of the NGINX container. + # lifecycle: {} + + # -- extraVolumeMounts are the additional volume mounts for the NGINX container. + # extraVolumeMounts: [] + + # -- The service configuration for the NGINX data plane. This is applied globally to all Gateways managed by this + # instance of NGINX Gateway Fabric. + service: + # @schema + # enum: + # - ClusterIP + # - NodePort + # - LoadBalancer + # @schema + # -- The type of service to create for the NGINX data plane. + type: LoadBalancer + + # @schema + # enum: + # - Cluster + # - Local + # @schema + # -- The externalTrafficPolicy of the service. The value Local preserves the client source IP. + externalTrafficPolicy: Local + + # -- The static IP address for the load balancer. Requires nginx.service.type set to LoadBalancer. + loadBalancerIP: "" + + # -- LoadBalancerClass is the class of the load balancer implementation this Service belongs to. + # Requires nginx.service.type set to LoadBalancer. + loadBalancerClass: "" + + # -- The IP ranges (CIDR) that are allowed to access the load balancer. Requires nginx.service.type set to LoadBalancer. + loadBalancerSourceRanges: [] + + # @schema + # type: array + # items: + # type: object + # properties: + # port: + # type: integer + # required: true + # minimum: 1 + # maximum: 65535 + # listenerPort: + # type: integer + # required: true + # minimum: 1 + # maximum: 65535 + # @schema + # -- A list of NodePorts to expose on the NGINX data plane service. Each NodePort MUST map to a Gateway listener port, + # otherwise it will be ignored. The default NodePort range enforced by Kubernetes is 30000-32767. + nodePorts: [] + # - port: 30025 + # listenerPort: 80 + + # -- Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. + debug: false + +# -- The certGenerator section contains the configuration for the cert-generator Job. +certGenerator: + # -- The annotations of the cert-generator Job. + annotations: {} + + # -- The name of the Secret containing TLS CA, certificate, and key for the NGINX Gateway Fabric control plane + # to securely communicate with the NGINX Agent. Must exist in the same namespace that the NGINX Gateway Fabric + # control plane is running in (default namespace: nginx-gateway). + serverTLSSecretName: server-tls + + # -- The name of the base Secret containing TLS CA, certificate, and key for the NGINX Agent to securely + # communicate with the NGINX Gateway Fabric control plane. Must exist in the same namespace that the + # NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). + agentTLSSecretName: agent-tls + + # -- Overwrite existing TLS Secrets on startup. + overwrite: false + + # -- How long to wait after the cert generator job has finished before it is removed by the job controller. + ttlSecondsAfterFinished: 30 + + # -- Tolerations for the cert-generator pod. + tolerations: [] + + # -- The nodeSelector of the cert-generator pod. + nodeSelector: {} + + # -- The affinity of the cert-generator pod. + affinity: {} + + # -- The topology spread constraints for the cert-generator pod. + topologySpreadConstraints: [] + +# -- A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. +gateways: + - name: practice-gateway-test + namespace: practice + annotations: + #cert-manager.io/cluster-issuer: letsencrypt-practice + spec: + gatewayClassName: nginx-test + infrastructure: + # annotations: + # service.beta.kubernetes.io/azure-load-balancer-ipv4: "40.82.159.101" + listeners: + - name: http + port: 80 + protocol: HTTP + hostname: "*.ssafy-practicestg2.com" + - name: https + port: 443 + protocol: HTTPS + hostname: "*.ssafy-practicestg2.com" + # tls: + # mode: Terminate + # certificateRefs: + # - kind: Secret + # name: practice-tls + # - name: env-gateway + # namespace: env + # annotations: + # cert-manager.io/cluster-issuer: letsencrypt-env + # spec: + # gatewayClassName: nginx + # infrastructure: + # annotations: + # service.beta.kubernetes.io/azure-load-balancer-ipv4: "52.141.28.70" + # listeners: + # - name: http-env + # port: 80 + # protocol: HTTP + # hostname: "*.ssafy-execstg.com" + # - name: https-env + # port: 443 + # protocol: HTTPS + # hostname: "*.ssafy-execstg.com" + # tls: + # mode: Terminate + # certificateRefs: + # - kind: Secret + # name: env-tls +# Example gateway object: +# name: nginx-gateway +# namespace: default +# labels: +# key: value +# annotations: +# annotationKey: annotationValue +# spec: +# gatewayClassName: nginx +# infrastructure: +# annotations: +# networking.gke.io/load-balancer-type: Internal +# listeners: +# - name: https +# port: 80 +# protocol: HTTPS +# tls: +# mode: Terminate +# certificateRefs: +# - kind: Secret +# name: my-secret +# namespace: certificate +# allowedRoutes: +# namespaces: +# from: Same diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index 15582052e5..bd86e4d43e 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -98,6 +98,28 @@ "nginx": { "description": "The nginx section contains the configuration for all NGINX data plane deployments\ninstalled by the NGINX Gateway Fabric control plane.", "properties": { + "autoscaling": { + "properties": { + "enabled": { + "default": false, + "description": "Enable or disable Horizontal Pod Autoscaler", + "required": [], + "title": "enabled", + "type": "boolean" + } + }, + "required": [], + "title": "autoscaling", + "type": "object" + }, + "autoscalingTemplate": { + "items": { + "required": [] + }, + "required": [], + "title": "autoscalingTemplate", + "type": "array" + }, "config": { "description": "The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways\nmanaged by this instance of NGINX Gateway Fabric.", "properties": { @@ -325,6 +347,7 @@ "type": "boolean" }, "image": { + "description": "Custom or additional autoscaling metrics\nref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics\n- type: Pods\n pods:\n metric:\n name: nginx_gateway_fabric_nginx_process_requests_total\n target:\n type: AverageValue\n averageValue: 10000m", "properties": { "pullPolicy": { "default": "Always", @@ -540,6 +563,28 @@ "title": "affinity", "type": "object" }, + "autoscaling": { + "properties": { + "enabled": { + "default": false, + "description": "Enable or disable Horizontal Pod Autoscaler", + "required": [], + "title": "enabled", + "type": "boolean" + } + }, + "required": [], + "title": "autoscaling", + "type": "object" + }, + "autoscalingTemplate": { + "items": { + "required": [] + }, + "required": [], + "title": "autoscalingTemplate", + "type": "array" + }, "config": { "description": "The dynamic configuration for the control plane that is contained in the NginxGateway resource.", "properties": { diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index 82c427fd6b..88515bc570 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -159,6 +159,38 @@ nginxGateway: # -- The topology spread constraints for the NGINX Gateway Fabric control plane pod. topologySpreadConstraints: [] + autoscaling: + # Enable or disable Horizontal Pod Autoscaler + enabled: false + # annotations: {} + # minReplicas: 1 + # maxReplicas: 11 + # targetCPUUtilizationPercentage: 50 + # targetMemoryUtilizationPercentage: 50 + # behavior: {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + autoscalingTemplate: [] + # Custom or additional autoscaling metrics + # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics + # - type: Pods + # pods: + # metric: + # name: nginx_gateway_fabric_nginx_process_requests_total + # target: + # type: AverageValue + # averageValue: 10000m + metrics: # -- Enable exposing metrics in the Prometheus format. enable: true @@ -199,6 +231,37 @@ nginx: # -- The number of replicas of the NGINX Deployment. replicas: 1 + autoscaling: + # Enable or disable Horizontal Pod Autoscaler + enabled: false + # hpaAnnotations: {} + # minReplicas: 1 + # maxReplicas: 11 + # targetCPUUtilizationPercentage: 50 + # targetMemoryUtilizationPercentage: 50 + # behavior: {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + autoscalingTemplate: [] + # Custom or additional autoscaling metrics + # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics + # - type: Pods + # pods: + # metric: + # name: nginx_gateway_fabric_nginx_process_requests_total + # target: + # type: AverageValue + # averageValue: 10000m image: # -- The NGINX image to use. repository: ghcr.io/nginx/nginx-gateway-fabric/nginx @@ -403,7 +466,7 @@ nginx: # -- The container configuration for the NGINX container. This is applied globally to all Gateways managed by this # instance of NGINX Gateway Fabric. container: {} - # -- The resource requirements of the NGINX container. + # -- The resource requirements of the NGINX container. You should set this value if you want to use dataplane Autoscaling(HPA). # resources: {} # -- The lifecycle of the NGINX container. diff --git a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml index 9947e34eb7..4ba097de57 100644 --- a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml +++ b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml @@ -3459,6 +3459,693 @@ spec: Deployment is the configuration for the NGINX Deployment. This is the default deployment option. properties: + autoscaling: + description: Autoscaling defines the configuration for Horizontal + Pod Autoscaling. + properties: + autoscalingTemplate: + description: AutoscalingTemplate configures the additional + scaling option. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version + of the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + type: string + required: + - type + type: object + type: array + behavior: + description: |- + Behavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + If not set, the default HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + enabled: + description: Enable or disable Horizontal Pod Autoscaler + type: boolean + hpaAnnotations: + additionalProperties: + type: string + description: |- + Annotation for Horizontal Pod Autoscaler + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + maxReplicas: + description: Maximum number of replicas. + format: int32 + type: integer + minReplicas: + description: Minimum number of replicas. + format: int32 + type: integer + targetCPUUtilizationPercentage: + description: Target cpu utilization percentage of HPA. + format: int32 + type: integer + targetMemoryUtilizationPercentage: + description: Target memory utilization percentage of HPA. + format: int32 + type: integer + required: + - enabled + - maxReplicas + - minReplicas + type: object + x-kubernetes-validations: + - message: at least one metric must be specified when autoscaling + is enabled + rule: '!self.enabled || (has(self.targetCPUUtilizationPercentage) + || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) + && size(self.autoscalingTemplate) > 0))' + - message: minReplicas must be less than or equal to maxReplicas + rule: self.minReplicas <= self.maxReplicas + - message: CPU utilization must be between 1 and 100 + rule: '!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage + >= 1 && self.targetCPUUtilizationPercentage <= 100)' + - message: memory utilization must be between 1 and 100 + rule: '!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage + >= 1 && self.targetMemoryUtilizationPercentage <= 100)' container: description: Container defines container fields for the NGINX container. diff --git a/deploy/crds.yaml b/deploy/crds.yaml index 7517ce1c4a..2b9568aa83 100644 --- a/deploy/crds.yaml +++ b/deploy/crds.yaml @@ -4044,6 +4044,693 @@ spec: Deployment is the configuration for the NGINX Deployment. This is the default deployment option. properties: + autoscaling: + description: Autoscaling defines the configuration for Horizontal + Pod Autoscaling. + properties: + autoscalingTemplate: + description: AutoscalingTemplate configures the additional + scaling option. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version + of the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + type: string + required: + - type + type: object + type: array + behavior: + description: |- + Behavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + If not set, the default HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + enabled: + description: Enable or disable Horizontal Pod Autoscaler + type: boolean + hpaAnnotations: + additionalProperties: + type: string + description: |- + Annotation for Horizontal Pod Autoscaler + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + maxReplicas: + description: Maximum number of replicas. + format: int32 + type: integer + minReplicas: + description: Minimum number of replicas. + format: int32 + type: integer + targetCPUUtilizationPercentage: + description: Target cpu utilization percentage of HPA. + format: int32 + type: integer + targetMemoryUtilizationPercentage: + description: Target memory utilization percentage of HPA. + format: int32 + type: integer + required: + - enabled + - maxReplicas + - minReplicas + type: object + x-kubernetes-validations: + - message: at least one metric must be specified when autoscaling + is enabled + rule: '!self.enabled || (has(self.targetCPUUtilizationPercentage) + || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) + && size(self.autoscalingTemplate) > 0))' + - message: minReplicas must be less than or equal to maxReplicas + rule: self.minReplicas <= self.maxReplicas + - message: CPU utilization must be between 1 and 100 + rule: '!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage + >= 1 && self.targetCPUUtilizationPercentage <= 100)' + - message: memory utilization must be between 1 and 100 + rule: '!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage + >= 1 && self.targetMemoryUtilizationPercentage <= 100)' container: description: Container defines container fields for the NGINX container. diff --git a/internal/controller/manager.go b/internal/controller/manager.go index f7bc0a68e5..bebc0a1eec 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -12,6 +12,7 @@ import ( "google.golang.org/grpc" appsv1 "k8s.io/api/apps/v1" authv1 "k8s.io/api/authentication/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" apiv1 "k8s.io/api/core/v1" discoveryV1 "k8s.io/api/discovery/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -91,6 +92,7 @@ func init() { utilruntime.Must(ngfAPIv1alpha2.AddToScheme(scheme)) utilruntime.Must(apiext.AddToScheme(scheme)) utilruntime.Must(appsv1.AddToScheme(scheme)) + utilruntime.Must(autoscalingv2.AddToScheme(scheme)) utilruntime.Must(authv1.AddToScheme(scheme)) utilruntime.Must(rbacv1.AddToScheme(scheme)) } diff --git a/internal/controller/provisioner/eventloop.go b/internal/controller/provisioner/eventloop.go index 7b85a47780..6f38d75c7c 100644 --- a/internal/controller/provisioner/eventloop.go +++ b/internal/controller/provisioner/eventloop.go @@ -6,6 +6,7 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,6 +74,18 @@ func newEventLoop( ), }, }, + { + objectType: &autoscalingv2.HorizontalPodAutoscaler{}, + options: []controller.Option{ + controller.WithK8sPredicate( + k8spredicate.And( + k8spredicate.GenerationChangedPredicate{}, + nginxResourceLabelPredicate, + predicate.RestartDeploymentAnnotationPredicate{}, + ), + ), + }, + }, { objectType: &appsv1.DaemonSet{}, options: []controller.Option{ @@ -184,6 +197,7 @@ func newEventLoop( // to provision or deprovision any nginx resources. &gatewayv1.GatewayList{}, &appsv1.DeploymentList{}, + &autoscalingv2.HorizontalPodAutoscalerList{}, &corev1.ServiceList{}, &corev1.ServiceAccountList{}, &corev1.ConfigMapList{}, diff --git a/internal/controller/provisioner/handler.go b/internal/controller/provisioner/handler.go index c5f3f2f4a9..f84160ca09 100644 --- a/internal/controller/provisioner/handler.go +++ b/internal/controller/provisioner/handler.go @@ -9,6 +9,7 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -62,7 +63,7 @@ func (h *eventHandler) HandleEventBatch(ctx context.Context, logger logr.Logger, case *gatewayv1.Gateway: h.store.updateGateway(obj) case *appsv1.Deployment, *appsv1.DaemonSet, *corev1.ServiceAccount, - *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding: + *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding, *autoscalingv2.HorizontalPodAutoscaler: objLabels := labels.Set(obj.GetLabels()) if h.labelSelector.Matches(objLabels) { gatewayName := objLabels.Get(controller.GatewayLabel) @@ -118,7 +119,7 @@ func (h *eventHandler) HandleEventBatch(ctx context.Context, logger logr.Logger, } h.store.deleteGateway(e.NamespacedName) case *appsv1.Deployment, *appsv1.DaemonSet, *corev1.Service, *corev1.ServiceAccount, - *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding: + *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding, *autoscalingv2.HorizontalPodAutoscaler: if err := h.reprovisionResources(ctx, e); err != nil { logger.Error(err, "error re-provisioning nginx resources") } diff --git a/internal/controller/provisioner/objects.go b/internal/controller/provisioner/objects.go index 3b43ac8b66..5676144d15 100644 --- a/internal/controller/provisioner/objects.go +++ b/internal/controller/provisioner/objects.go @@ -10,6 +10,7 @@ import ( "time" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -152,6 +153,7 @@ func (p *NginxProvisioner) buildNginxResourceObjects( // role/binding (if openshift) // service // deployment/daemonset + // hpa objects := make([]client.Object, 0, len(configmaps)+len(secrets)+len(openshiftObjs)+3) objects = append(objects, secrets...) @@ -160,11 +162,31 @@ func (p *NginxProvisioner) buildNginxResourceObjects( if p.isOpenshift { objects = append(objects, openshiftObjs...) } + objects = append(objects, service, deployment) + if hpa := p.buildHPA(objectMeta, nProxyCfg); hpa != nil { + objects = append(objects, hpa) + } + return objects, err } +func isAutoscalingEnabled(dep *ngfAPIv1alpha2.DeploymentSpec) bool { + return dep != nil && dep.Autoscaling.Enabled +} + +func (p *NginxProvisioner) buildHPA( + objectMeta metav1.ObjectMeta, + nProxyCfg *graph.EffectiveNginxProxy, +) client.Object { + if nProxyCfg == nil || nProxyCfg.Kubernetes == nil || !isAutoscalingEnabled(nProxyCfg.Kubernetes.Deployment) { + return nil + } + + return buildNginxDeploymentHPA(objectMeta, nProxyCfg.Kubernetes.Deployment.Autoscaling) +} + func (p *NginxProvisioner) buildNginxSecrets( objectMeta metav1.ObjectMeta, agentTLSSecretName string, @@ -895,6 +917,154 @@ func (p *NginxProvisioner) buildImage(nProxyCfg *graph.EffectiveNginxProxy) (str return fmt.Sprintf("%s:%s", image, tag), pullPolicy } +func getMetricTargetByType(target autoscalingv2.MetricTarget) autoscalingv2.MetricTarget { + switch target.Type { + case autoscalingv2.UtilizationMetricType: + return autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: target.AverageUtilization, + } + case autoscalingv2.AverageValueMetricType: + return autoscalingv2.MetricTarget{ + Type: autoscalingv2.AverageValueMetricType, + AverageValue: target.AverageValue, + } + case autoscalingv2.ValueMetricType: + return autoscalingv2.MetricTarget{ + Type: autoscalingv2.ValueMetricType, + Value: target.Value, + } + default: + return autoscalingv2.MetricTarget{} + } +} + +func buildNginxDeploymentHPA( + objectMeta metav1.ObjectMeta, + autoScaling ngfAPIv1alpha2.HPASpec, +) *autoscalingv2.HorizontalPodAutoscaler { + objectMeta.Annotations = autoScaling.HPAAnnotations + + if !autoScaling.Enabled { + return nil + } + var metrics []autoscalingv2.MetricSpec + + cpuUtil := autoScaling.TargetCPUUtilizationPercentage + memUtil := autoScaling.TargetMemoryUtilizationPercentage + autoscalingTemplate := autoScaling.AutoscalingTemplate + + if cpuUtil != nil { + metrics = append(metrics, autoscalingv2.MetricSpec{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: "cpu", + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: cpuUtil, + }, + }, + }) + } + + if memUtil != nil { + metrics = append(metrics, autoscalingv2.MetricSpec{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: "memory", + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: memUtil, + }, + }, + }) + } + + if autoscalingTemplate != nil { + for _, additionalAutoscaling := range *autoscalingTemplate { + metric := buildAdditionalMetric(additionalAutoscaling) + if metric != nil { + metrics = append(metrics, *metric) + } + } + } + + if len(metrics) == 0 { + // No metrics configured, skip HPA creation + return nil + } + + return &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: objectMeta, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: objectMeta.Name, + }, + MinReplicas: &autoScaling.MinReplicas, + MaxReplicas: autoScaling.MaxReplicas, + Metrics: metrics, + Behavior: autoScaling.Behavior, + }, + } +} + +func buildAdditionalMetric(additionalAutoscaling autoscalingv2.MetricSpec) *autoscalingv2.MetricSpec { + switch additionalAutoscaling.Type { + case autoscalingv2.ResourceMetricSourceType: + return &autoscalingv2.MetricSpec{ + Type: additionalAutoscaling.Type, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: additionalAutoscaling.Resource.Name, + Target: getMetricTargetByType(additionalAutoscaling.Resource.Target), + }, + } + case autoscalingv2.PodsMetricSourceType: + return &autoscalingv2.MetricSpec{ + Type: additionalAutoscaling.Type, + Pods: &autoscalingv2.PodsMetricSource{ + Metric: additionalAutoscaling.Pods.Metric, + Target: getMetricTargetByType(additionalAutoscaling.Pods.Target), + }, + } + case autoscalingv2.ContainerResourceMetricSourceType: + return &autoscalingv2.MetricSpec{ + Type: additionalAutoscaling.Type, + ContainerResource: &autoscalingv2.ContainerResourceMetricSource{ + Name: additionalAutoscaling.ContainerResource.Name, + Target: getMetricTargetByType(additionalAutoscaling.ContainerResource.Target), + Container: additionalAutoscaling.ContainerResource.Container, + }, + } + case autoscalingv2.ObjectMetricSourceType: + return &autoscalingv2.MetricSpec{ + Type: additionalAutoscaling.Type, + Object: &autoscalingv2.ObjectMetricSource{ + DescribedObject: additionalAutoscaling.Object.DescribedObject, + Target: getMetricTargetByType(additionalAutoscaling.Object.Target), + Metric: autoscalingv2.MetricIdentifier{ + Name: additionalAutoscaling.Object.Metric.Name, + Selector: additionalAutoscaling.Object.Metric.Selector, + }, + }, + } + case autoscalingv2.ExternalMetricSourceType: + return &autoscalingv2.MetricSpec{ + Type: additionalAutoscaling.Type, + External: &autoscalingv2.ExternalMetricSource{ + Metric: autoscalingv2.MetricIdentifier{ + Name: additionalAutoscaling.External.Metric.Name, + Selector: additionalAutoscaling.External.Metric.Selector, + }, + Target: getMetricTargetByType(additionalAutoscaling.External.Target), + }, + } + default: + return nil + } +} + // TODO(sberman): see about how this can be made more elegant. Maybe create some sort of Object factory // that can better store/build all the objects we need, to reduce the amount of duplicate object lists that we // have everywhere. @@ -906,6 +1076,7 @@ func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName // serviceaccount // configmaps // secrets + // hpa objectMeta := metav1.ObjectMeta{ Name: deploymentNSName.Name, @@ -921,8 +1092,11 @@ func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName service := &corev1.Service{ ObjectMeta: objectMeta, } + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: objectMeta, + } - objects := []client.Object{deployment, daemonSet, service} + objects := []client.Object{deployment, daemonSet, service, hpa} if p.isOpenshift { role := &rbacv1.Role{ diff --git a/internal/controller/provisioner/objects_test.go b/internal/controller/provisioner/objects_test.go index 96710f8902..97ec9f58c0 100644 --- a/internal/controller/provisioner/objects_test.go +++ b/internal/controller/provisioner/objects_test.go @@ -6,6 +6,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -279,6 +280,12 @@ func TestBuildNginxResourceObjects_NginxProxyConfig(t *testing.T) { }, Deployment: &ngfAPIv1alpha2.DeploymentSpec{ Replicas: helpers.GetPointer[int32](3), + Autoscaling: ngfAPIv1alpha2.HPASpec{ + Enabled: true, + MinReplicas: 1, + MaxReplicas: 5, + TargetMemoryUtilizationPercentage: helpers.GetPointer[int32](60), + }, Pod: ngfAPIv1alpha2.PodSpec{ TerminationGracePeriodSeconds: helpers.GetPointer[int64](25), }, @@ -301,7 +308,7 @@ func TestBuildNginxResourceObjects_NginxProxyConfig(t *testing.T) { objects, err := provisioner.buildNginxResourceObjects(resourceName, gateway, nProxyCfg) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(objects).To(HaveLen(6)) + g.Expect(objects).To(HaveLen(7)) cmObj := objects[1] cm, ok := cmObj.(*corev1.ConfigMap) @@ -803,7 +810,7 @@ func TestBuildNginxResourceObjectsForDeletion(t *testing.T) { objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) - g.Expect(objects).To(HaveLen(7)) + g.Expect(objects).To(HaveLen(8)) validateMeta := func(obj client.Object, name string) { g.Expect(obj.GetName()).To(Equal(name)) @@ -825,17 +832,22 @@ func TestBuildNginxResourceObjectsForDeletion(t *testing.T) { g.Expect(ok).To(BeTrue()) validateMeta(svc, deploymentNSName.Name) - svcAcctObj := objects[3] + hpaObj := objects[3] + hpa, ok := hpaObj.(*autoscalingv2.HorizontalPodAutoscaler) + g.Expect(ok).To(BeTrue()) + validateMeta(hpa, deploymentNSName.Name) + + svcAcctObj := objects[4] svcAcct, ok := svcAcctObj.(*corev1.ServiceAccount) g.Expect(ok).To(BeTrue()) validateMeta(svcAcct, deploymentNSName.Name) - cmObj := objects[4] + cmObj := objects[5] cm, ok := cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxIncludesConfigMapNameSuffix)) - cmObj = objects[5] + cmObj = objects[6] cm, ok = cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxAgentConfigMapNameSuffix)) @@ -865,7 +877,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) - g.Expect(objects).To(HaveLen(11)) + g.Expect(objects).To(HaveLen(12)) validateMeta := func(obj client.Object, name string) { g.Expect(obj.GetName()).To(Equal(name)) @@ -887,22 +899,27 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { g.Expect(ok).To(BeTrue()) validateMeta(svc, deploymentNSName.Name) - svcAcctObj := objects[3] + hpaObj := objects[3] + hpa, ok := hpaObj.(*autoscalingv2.HorizontalPodAutoscaler) + g.Expect(ok).To(BeTrue()) + validateMeta(hpa, deploymentNSName.Name) + + svcAcctObj := objects[4] svcAcct, ok := svcAcctObj.(*corev1.ServiceAccount) g.Expect(ok).To(BeTrue()) validateMeta(svcAcct, deploymentNSName.Name) - cmObj := objects[4] + cmObj := objects[5] cm, ok := cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxIncludesConfigMapNameSuffix)) - cmObj = objects[5] + cmObj = objects[6] cm, ok = cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxAgentConfigMapNameSuffix)) - secretObj := objects[6] + secretObj := objects[7] secret, ok := secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -910,7 +927,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { provisioner.cfg.AgentTLSSecretName, )) - secretObj = objects[7] + secretObj = objects[8] secret, ok = secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -918,7 +935,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { provisioner.cfg.NginxDockerSecretNames[0], )) - secretObj = objects[8] + secretObj = objects[9] secret, ok = secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -926,7 +943,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { provisioner.cfg.PlusUsageConfig.CASecretName, )) - secretObj = objects[9] + secretObj = objects[10] secret, ok = secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -948,19 +965,19 @@ func TestBuildNginxResourceObjectsForDeletion_OpenShift(t *testing.T) { objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) - g.Expect(objects).To(HaveLen(9)) + g.Expect(objects).To(HaveLen(10)) validateMeta := func(obj client.Object, name string) { g.Expect(obj.GetName()).To(Equal(name)) g.Expect(obj.GetNamespace()).To(Equal(deploymentNSName.Namespace)) } - roleObj := objects[3] + roleObj := objects[4] role, ok := roleObj.(*rbacv1.Role) g.Expect(ok).To(BeTrue()) validateMeta(role, deploymentNSName.Name) - roleBindingObj := objects[4] + roleBindingObj := objects[5] roleBinding, ok := roleBindingObj.(*rbacv1.RoleBinding) g.Expect(ok).To(BeTrue()) validateMeta(roleBinding, deploymentNSName.Name) diff --git a/internal/controller/provisioner/provisioner_test.go b/internal/controller/provisioner/provisioner_test.go index 9102a8193f..76edf13d5d 100644 --- a/internal/controller/provisioner/provisioner_test.go +++ b/internal/controller/provisioner/provisioner_test.go @@ -7,6 +7,7 @@ import ( "github.com/go-logr/logr" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -43,6 +44,7 @@ func createScheme() *runtime.Scheme { utilruntime.Must(gatewayv1.Install(scheme)) utilruntime.Must(corev1.AddToScheme(scheme)) utilruntime.Must(appsv1.AddToScheme(scheme)) + utilruntime.Must(autoscalingv2.AddToScheme(scheme)) return scheme } diff --git a/internal/controller/provisioner/setter.go b/internal/controller/provisioner/setter.go index 59718440f4..0ffbf53d49 100644 --- a/internal/controller/provisioner/setter.go +++ b/internal/controller/provisioner/setter.go @@ -4,6 +4,7 @@ import ( "maps" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -16,6 +17,8 @@ func objectSpecSetter(object client.Object) controllerutil.MutateFn { switch obj := object.(type) { case *appsv1.Deployment: return deploymentSpecSetter(obj, obj.Spec, obj.ObjectMeta) + case *autoscalingv2.HorizontalPodAutoscaler: + return hpaSpecSetter(obj, obj.Spec, obj.ObjectMeta) case *appsv1.DaemonSet: return daemonSetSpecSetter(obj, obj.Spec, obj.ObjectMeta) case *corev1.Service: @@ -48,6 +51,19 @@ func deploymentSpecSetter( } } +func hpaSpecSetter( + hpa *autoscalingv2.HorizontalPodAutoscaler, + spec autoscalingv2.HorizontalPodAutoscalerSpec, + objectMeta metav1.ObjectMeta, +) controllerutil.MutateFn { + return func() error { + hpa.Labels = objectMeta.Labels + hpa.Annotations = objectMeta.Annotations + hpa.Spec = spec + return nil + } +} + func daemonSetSpecSetter( daemonSet *appsv1.DaemonSet, spec appsv1.DaemonSetSpec, diff --git a/internal/controller/provisioner/store.go b/internal/controller/provisioner/store.go index d089a38f9b..fd095eae5a 100644 --- a/internal/controller/provisioner/store.go +++ b/internal/controller/provisioner/store.go @@ -6,6 +6,7 @@ import ( "sync" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -20,6 +21,7 @@ import ( type NginxResources struct { Gateway *graph.Gateway Deployment metav1.ObjectMeta + HPA metav1.ObjectMeta DaemonSet metav1.ObjectMeta Service metav1.ObjectMeta ServiceAccount metav1.ObjectMeta @@ -133,6 +135,14 @@ func (s *store) registerResourceInGatewayConfig(gatewayNSName types.NamespacedNa } else { cfg.Deployment = obj.ObjectMeta } + case *autoscalingv2.HorizontalPodAutoscaler: + if cfg, ok := s.nginxResources[gatewayNSName]; !ok { + s.nginxResources[gatewayNSName] = &NginxResources{ + HPA: obj.ObjectMeta, + } + } else { + cfg.HPA = obj.ObjectMeta + } case *appsv1.DaemonSet: if cfg, ok := s.nginxResources[gatewayNSName]; !ok { s.nginxResources[gatewayNSName] = &NginxResources{ @@ -301,6 +311,10 @@ func (s *store) gatewayExistsForResource(object client.Object, nsName types.Name if resourceMatches(resources.Deployment, nsName) { return resources.Gateway } + case *autoscalingv2.HorizontalPodAutoscaler: + if resourceMatches(resources.HPA, nsName) { + return resources.Gateway + } case *appsv1.DaemonSet: if resourceMatches(resources.DaemonSet, nsName) { return resources.Gateway @@ -379,6 +393,10 @@ func (s *store) getResourceVersionForObject(gatewayNSName types.NamespacedName, if resources.Deployment.GetName() == obj.GetName() { return resources.Deployment.GetResourceVersion() } + case *autoscalingv2.HorizontalPodAutoscaler: + if resources.HPA.GetName() == obj.GetName() { + return resources.HPA.GetResourceVersion() + } case *appsv1.DaemonSet: if resources.DaemonSet.GetName() == obj.GetName() { return resources.DaemonSet.GetResourceVersion()