Skip to content

Hpa #3492

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Hpa #3492

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions apis/v1alpha2/nginxproxy_types.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package v1alpha2

import (
autoscalingv2 "k8s.io/api/autoscaling/v2"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

Expand Down Expand Up @@ -388,6 +389,11 @@ type DeploymentSpec struct {
// +optional
Replicas *int32 `json:"replicas,omitempty"`

// Horizontal Pod Autoscaling.
//
// +optional
Autoscaling HPASpec `json:"autoscaling"`

// Pod defines Pod-specific fields.
//
// +optional
Expand All @@ -412,6 +418,54 @@ type DaemonSetSpec struct {
Container ContainerSpec `json:"container"`
}

// +kubebuilder:validation:XValidation:message="at least one metric must be specified when autoscaling is enabled",rule="!self.enabled || (has(self.targetCPUUtilizationPercentage) || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) && size(self.autoscalingTemplate) > 0))"
// +kubebuilder:validation:XValidation:message="minReplicas must be less than or equal to maxReplicas",rule="self.minReplicas <= self.maxReplicas"
// +kubebuilder:validation:XValidation:message="CPU utilization must be between 1 and 100",rule="!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage >= 1 && self.targetCPUUtilizationPercentage <= 100)"
// +kubebuilder:validation:XValidation:message="memory utilization must be between 1 and 100",rule="!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage >= 1 && self.targetMemoryUtilizationPercentage <= 100)"
//
//nolint:lll
type HPASpec struct {
// behavior configures the scaling behavior of the target
// in both Up and Down directions (scaleUp and scaleDown fields respectively).
// If not set, the default HPAScalingRules for scale up and scale down are used.
//
// +optional
Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"`

// AutoscalingTemplate configures the additional scaling option.
//
// +optional
AutoscalingTemplate *[]autoscalingv2.MetricSpec `json:"autoscalingTemplate,omitempty"`

// Target cpu utilization percentage of HPA.
//
// +optional
TargetCPUUtilizationPercentage *int32 `json:"targetCPUUtilizationPercentage,omitempty"`

// Target memory utilization percentage of HPA.
//
// +optional
TargetMemoryUtilizationPercentage *int32 `json:"targetMemoryUtilizationPercentage,omitempty"`

// Annotation for Horizontal Pod Autoscaler
// Annotations is an unstructured key value map stored with a resource that may be
// set by external tools to store and retrieve arbitrary metadata. They are not
// queryable and should be preserved when modifying objects.
// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations
//
// +optional
HPAAnnotations map[string]string `json:"hpaAnnotations,omitempty"`

// Minimum number of replicas.
MinReplicas int32 `json:"minReplicas"`

// Maximum number of replicas.
MaxReplicas int32 `json:"maxReplicas"`

// Enable or disable Horizontal Pod Autoscaler
Enabled bool `json:"enabled"`
}

// PodSpec defines Pod-specific fields.
type PodSpec struct {
// TerminationGracePeriodSeconds is the optional duration in seconds the pod needs to terminate gracefully.
Expand Down
50 changes: 50 additions & 0 deletions apis/v1alpha2/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions charts/nginx-gateway-fabric/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
| `certGenerator.ttlSecondsAfterFinished` | How long to wait after the cert generator job has finished before it is removed by the job controller. | int | `30` |
| `clusterDomain` | The DNS cluster domain of your Kubernetes cluster. | string | `"cluster.local"` |
| `gateways` | A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. | list | `[]` |
| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"config":{},"container":{},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` |
| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"autoscaling":{"enabled":false},"autoscalingTemplate":[],"config":{},"container":{},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` |
| `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` |
| `nginx.container` | The container configuration for the NGINX container. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` |
| `nginx.debug` | Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. | bool | `false` |
Expand All @@ -288,7 +288,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
| `nginx.usage.resolver` | The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. | string | `""` |
| `nginx.usage.secretName` | The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"nplus-license"` |
| `nginx.usage.skipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` |
| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` |
| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"autoscaling":{"enabled":false},"autoscalingTemplate":[],"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` |
| `nginxGateway.affinity` | The affinity of the NGINX Gateway Fabric control plane pod. | object | `{}` |
| `nginxGateway.config.logging.level` | Log level. | string | `"info"` |
| `nginxGateway.configAnnotations` | Set of custom annotations for NginxGateway objects. | object | `{}` |
Expand Down
2 changes: 2 additions & 0 deletions charts/nginx-gateway-fabric/templates/clusterrole.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,15 @@ rules:
- apiGroups:
- ""
- apps
- autoscaling
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's only add this if autoscaling is enabled

Suggested change
- autoscaling
{{- if or .Values.nginx.autoscaling.enabled .Values.nginxGateway.autoscaling.enabled }}
- autoscaling
{{- end }}

resources:
- secrets
- configmaps
- serviceaccounts
- services
- deployments
- daemonsets
- horizontalpodautoscalers
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's only add this if autoscaling is enabled

Suggested change
- horizontalpodautoscalers
{{- if or .Values.nginx.autoscaling.enabled .Values.nginxGateway.autoscaling.enabled }}
- horizontalpodautoscalers
{{- end }}

verbs:
- create
- update
Expand Down
46 changes: 46 additions & 0 deletions charts/nginx-gateway-fabric/templates/hpa.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
{{- if and (eq .Values.nginxGateway.kind "deployment") .Values.nginxGateway.autoscaling.enabled -}}
apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }}
kind: HorizontalPodAutoscaler
metadata:
{{- with .Values.nginxGateway.autoscaling.annotations }}
annotations: {{ toYaml . | nindent 4 }}
{{- end }}
labels:
{{- include "nginx-gateway.labels" . | nindent 4 }}
{{- with .Values.nginxGateway.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
name: {{ include "nginx-gateway.fullname" . }}
namespace: {{ .Release.Namespace }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "nginx-gateway.fullname" . }}
minReplicas: {{ .Values.nginxGateway.autoscaling.minReplicas }}
maxReplicas: {{ .Values.nginxGateway.autoscaling.maxReplicas }}
metrics:
{{- with .Values.nginxGateway.autoscaling.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ . }}
{{- end }}
{{- with .Values.nginxGateway.autoscaling.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ . }}
{{- end }}
{{- with .Values.autoscalingTemplate }}
{{- toYaml . | nindent 2 }}
{{- end }}
{{- with .Values.nginxGateway.autoscaling.behavior }}
behavior:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
26 changes: 26 additions & 0 deletions charts/nginx-gateway-fabric/templates/nginxproxy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,33 @@ spec:
kubernetes:
{{- if eq .Values.nginx.kind "deployment" }}
deployment:
{{- if .Values.nginx.replicas }}
replicas: {{ .Values.nginx.replicas }}
{{- end }}
{{- if .Values.nginx.autoscaling.enabled }}
autoscaling:
enabled: {{ .Values.nginx.autoscaling.enabled }}
{{- if .Values.nginx.autoscaling.hpaAnnotations }}
hpaAnnotations:
{{- toYaml .Values.nginx.autoscaling.hpaAnnotations | nindent 10 }}
{{- end }}
minReplicas: {{ .Values.nginx.autoscaling.minReplicas }}
maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }}
{{- if .Values.nginx.autoscaling.targetCPUUtilizationPercentage }}
targetCPUUtilizationPercentage: {{ .Values.nginx.autoscaling.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }}
targetMemoryUtilizationPercentage: {{ .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }}
{{- end }}
{{- if .Values.nginx.autoscaling.behavior }}
behavior:
{{- toYaml .Values.nginx.autoscaling.behavior | nindent 10 }}
{{- end }}
{{- if .Values.nginx.autoscalingTemplate }}
autoscalingTemplate:
{{- toYaml .Values.nginx.autoscalingTemplate | nindent 8 }}
{{- end }}
{{- end }}
{{- if .Values.nginx.pod }}
pod:
{{- toYaml .Values.nginx.pod | nindent 8 }}
Expand Down
45 changes: 45 additions & 0 deletions charts/nginx-gateway-fabric/values.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,28 @@
"nginx": {
"description": "The nginx section contains the configuration for all NGINX data plane deployments\ninstalled by the NGINX Gateway Fabric control plane.",
"properties": {
"autoscaling": {
"properties": {
"enabled": {
"default": false,
"description": "Enable or disable Horizontal Pod Autoscaler",
"required": [],
"title": "enabled",
"type": "boolean"
}
},
"required": [],
"title": "autoscaling",
"type": "object"
},
"autoscalingTemplate": {
"items": {
"required": []
},
"required": [],
"title": "autoscalingTemplate",
"type": "array"
},
"config": {
"description": "The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways\nmanaged by this instance of NGINX Gateway Fabric.",
"properties": {
Expand Down Expand Up @@ -325,6 +347,7 @@
"type": "boolean"
},
"image": {
"description": "Custom or additional autoscaling metrics\nref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics\n- type: Pods\n pods:\n metric:\n name: nginx_gateway_fabric_nginx_process_requests_total\n target:\n type: AverageValue\n averageValue: 10000m",
"properties": {
"pullPolicy": {
"default": "Always",
Expand Down Expand Up @@ -540,6 +563,28 @@
"title": "affinity",
"type": "object"
},
"autoscaling": {
"properties": {
"enabled": {
"default": false,
"description": "Enable or disable Horizontal Pod Autoscaler",
"required": [],
"title": "enabled",
"type": "boolean"
}
},
"required": [],
"title": "autoscaling",
"type": "object"
},
"autoscalingTemplate": {
"items": {
"required": []
},
"required": [],
"title": "autoscalingTemplate",
"type": "array"
},
"config": {
"description": "The dynamic configuration for the control plane that is contained in the NginxGateway resource.",
"properties": {
Expand Down
Loading
Loading