Code Review for status.desiredReplicas

Author: nowjean

apis/v1alpha2/nginxproxy_types.go

@@ -466,12 +466,10 @@ type DaemonSetSpec struct {
 	Patches []Patch `json:"patches,omitempty"`
 }
 
+// HPASpec is the configuration for the Horizontal Pod Autoscaling.
+//
 // +kubebuilder:validation:XValidation:message="at least one metric must be specified when autoscaling is enabled",rule="!self.enabled || (has(self.targetCPUUtilizationPercentage) || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) && size(self.autoscalingTemplate) > 0))"
 // +kubebuilder:validation:XValidation:message="minReplicas must be less than or equal to maxReplicas",rule="self.minReplicas <= self.maxReplicas"
-// +kubebuilder:validation:XValidation:message="CPU utilization must be between 1 and 100",rule="!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage >= 1 && self.targetCPUUtilizationPercentage <= 100)"
-// +kubebuilder:validation:XValidation:message="memory utilization must be between 1 and 100",rule="!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage >= 1 && self.targetMemoryUtilizationPercentage <= 100)"
-//
-// HPASpec is the configuration for the Horizontal Pod Autoscaling.
 //
 //nolint:lll
 type HPASpec struct {
@@ -485,24 +483,30 @@ type HPASpec struct {
 	// AutoscalingTemplate configures the additional scaling option.
 	//
 	// +optional
-	AutoscalingTemplate *[]autoscalingv2.MetricSpec `json:"autoscalingTemplate,omitempty"`
+	AutoscalingTemplate []autoscalingv2.MetricSpec `json:"autoscalingTemplate,omitempty"`
 
 	// Target cpu utilization percentage of HPA.
 	//
 	// +optional
+	// +kubebuilder:validation:Minimum=1
+	// +kubebuilder:validation:Maximum=100
 	TargetCPUUtilizationPercentage *int32 `json:"targetCPUUtilizationPercentage,omitempty"`
 
 	// Target memory utilization percentage of HPA.
 	//
 	// +optional
+	// +kubebuilder:validation:Minimum=1
+	// +kubebuilder:validation:Maximum=100
 	TargetMemoryUtilizationPercentage *int32 `json:"targetMemoryUtilizationPercentage,omitempty"`
 
 	// Minimum number of replicas.
 	//
 	// +optional
+	// +kubebuilder:validation:Minimum=1
 	MinReplicas *int32 `json:"minReplicas,omitempty"`
 
 	// Maximum number of replicas.
+	// +kubebuilder:validation:Minimum=1
 	MaxReplicas int32 `json:"maxReplicas"`
 
 	// Enable or disable Horizontal Pod Autoscaler

apis/v1alpha2/zz_generated.deepcopy.go

@@ -269,7 +269,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `nginx.container` | The container configuration for the NGINX container. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]}` |
 | `nginx.container.hostPorts` | A list of HostPorts to expose on the host. This configuration allows containers to bind to a specific port on the host node, enabling external network traffic to reach the container directly through the host's IP address and port. Use this option when you need to expose container ports on the host for direct access, such as for debugging, legacy integrations, or when NodePort/LoadBalancer services are not suitable. Note: Using hostPort may have security and scheduling implications, as it ties pods to specific nodes and ports. | list | `[]` |
 | `nginx.container.lifecycle` | The lifecycle of the NGINX container. | object | `{}` |
-| `nginx.container.resources` | The resource requirements of the NGINX container. You should be set this value, If you want to use dataplane Autoscaling(HPA). | object | `{}` |
+| `nginx.container.resources` | The resource requirements of the NGINX container. You should set this value if you want to use dataplane Autoscaling(HPA). | object | `{}` |
 | `nginx.container.volumeMounts` | volumeMounts are the additional volume mounts for the NGINX container. | list | `[]` |
 | `nginx.debug` | Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. | bool | `false` |
 | `nginx.image.repository` | The NGINX image to use. | string | `"ghcr.io/nginx/nginx-gateway-fabric/nginx"` |
@@ -278,7 +278,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `nginx.kind` | The kind of NGINX deployment. | string | `"deployment"` |
 | `nginx.plus` | Is NGINX Plus image being used. | bool | `false` |
 | `nginx.pod` | The pod configuration for the NGINX data plane pod. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` |
-| `nginx.replicas` | The number of replicas of the NGINX Deployment. | int | `1` |
+| `nginx.replicas` | The number of replicas of the NGINX Deployment. This value is ignored if autoscaling.enabled is true. | int | `1` |
 | `nginx.service` | The service configuration for the NGINX data plane. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"}` |
 | `nginx.service.externalTrafficPolicy` | The externalTrafficPolicy of the service. The value Local preserves the client source IP. | string | `"Local"` |
 | `nginx.service.loadBalancerClass` | LoadBalancerClass is the class of the load balancer implementation this Service belongs to. Requires nginx.service.type set to LoadBalancer. | string | `""` |
@@ -319,7 +319,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `nginxGateway.readinessProbe.enable` | Enable the /readyz endpoint on the control plane. | bool | `true` |
 | `nginxGateway.readinessProbe.initialDelaySeconds` | The number of seconds after the Pod has started before the readiness probes are initiated. | int | `3` |
 | `nginxGateway.readinessProbe.port` | Port in which the readiness endpoint is exposed. | int | `8081` |
-| `nginxGateway.replicas` | The number of replicas of the NGINX Gateway Fabric Deployment. | int | `1` |
+| `nginxGateway.replicas` | The number of replicas of the NGINX Gateway Fabric Deployment. This value is ignored if autoscaling.enabled is true. | int | `1` |
 | `nginxGateway.resources` | The resource requests and/or limits of the nginx-gateway container. | object | `{}` |
 | `nginxGateway.service` | The service configuration for the NGINX Gateway Fabric control plane. | object | `{"annotations":{},"labels":{}}` |
 | `nginxGateway.service.annotations` | The annotations of the NGINX Gateway Fabric control plane service. | object | `{}` |

charts/nginx-gateway-fabric/README.md

@@ -17,7 +17,9 @@ spec:
     apiVersion: apps/v1
     kind: Deployment
     name: {{ include "nginx-gateway.fullname" . }}
+  {{- if .Values.nginxGateway.autoscaling.minReplicas }}
   minReplicas: {{ .Values.nginxGateway.autoscaling.minReplicas }}
+  {{- end }}
   maxReplicas: {{ .Values.nginxGateway.autoscaling.maxReplicas }}
   metrics:
   {{- with .Values.nginxGateway.autoscaling.targetMemoryUtilizationPercentage }}

charts/nginx-gateway-fabric/templates/hpa.yaml

@@ -18,7 +18,9 @@ spec:
       {{- if .Values.nginx.autoscaling.enabled }}
       autoscaling:
         enabled: {{ .Values.nginx.autoscaling.enabled }}
+        {{- if .Values.nginx.autoscaling.minReplicas }}
         minReplicas: {{ .Values.nginx.autoscaling.minReplicas }}
+        {{- end }}
         maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }}
         {{- if .Values.nginx.autoscaling.targetCPUUtilizationPercentage }}
         targetCPUUtilizationPercentage: {{ .Values.nginx.autoscaling.targetCPUUtilizationPercentage }}

charts/nginx-gateway-fabric/templates/nginxproxy.yaml

@@ -380,7 +380,7 @@
               "type": "object"
             },
             "resources": {
-              "description": "The resource requirements of the NGINX container. You should be set this value, If you want to use dataplane Autoscaling(HPA).",
+              "description": "The resource requirements of the NGINX container. You should set this value if you want to use dataplane Autoscaling(HPA).",
               "required": [],
               "title": "resources",
               "type": "object"
@@ -407,7 +407,7 @@
           "type": "boolean"
         },
         "image": {
-          "description": "Custom or additional autoscaling metrics\nref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics\n- type: Pods\n  pods:\n    metric:\n      name: nginx_gateway_fabric_nginx_process_requests_total\n    target:\n      type: AverageValue\n      averageValue: 10000m",
+          "description": "Custom or additional autoscaling metrics\nref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics\n- type: Pods\n  pods:\n    metric:\n      name: nginx_gateway_fabric_nginx_process_requests_total\n    target:\n      type: nginx_http_requests_total\n      averageValue: 400",
           "properties": {
             "pullPolicy": {
               "default": "Always",
@@ -478,7 +478,7 @@
         },
         "replicas": {
           "default": 1,
-          "description": "The number of replicas of the NGINX Deployment.",
+          "description": "The number of replicas of the NGINX Deployment. This value is ignored if autoscaling.enabled is true.",
           "required": [],
           "title": "replicas",
           "type": "integer"
@@ -897,7 +897,7 @@
         },
         "replicas": {
           "default": 1,
-          "description": "The number of replicas of the NGINX Gateway Fabric Deployment.",
+          "description": "The number of replicas of the NGINX Gateway Fabric Deployment. This value is ignored if autoscaling.enabled is true.",
           "required": [],
           "title": "replicas",
           "type": "integer"

charts/nginx-gateway-fabric/values.schema.json

@@ -80,7 +80,7 @@ nginxGateway:
     # Secrets must exist in the same namespace as the helm release.
     imagePullSecrets: []
 
-  # -- The number of replicas of the NGINX Gateway Fabric Deployment.
+  # -- The number of replicas of the NGINX Gateway Fabric Deployment. This value is ignored if autoscaling.enabled is true.
   replicas: 1
 
   # The configuration for leader election.
@@ -184,10 +184,10 @@ nginxGateway:
   # - type: Pods
   #   pods:
   #     metric:
-  #       name: nginx_gateway_fabric_nginx_process_requests_total
+  #       name: container_memory_usage_bytes
   #     target:
   #       type: AverageValue
-  #       averageValue: 10000m
+  #       averageValue: "400Mi"
 
   metrics:
     # -- Enable exposing metrics in the Prometheus format.
@@ -226,7 +226,7 @@ nginx:
   # -- The kind of NGINX deployment.
   kind: deployment
 
-  # -- The number of replicas of the NGINX Deployment.
+  # -- The number of replicas of the NGINX Deployment. This value is ignored if autoscaling.enabled is true.
   replicas: 1
 
   autoscaling:
@@ -257,8 +257,8 @@ nginx:
   #     metric:
   #       name: nginx_gateway_fabric_nginx_process_requests_total
   #     target:
-  #       type: AverageValue
-  #       averageValue: 10000m
+  #       type: nginx_http_requests_total
+  #       averageValue: 400
   image:
     # -- The NGINX image to use.
     repository: ghcr.io/nginx/nginx-gateway-fabric/nginx
@@ -494,7 +494,7 @@ nginx:
     # - port: 80
     #   containerPort: 80
 
-    # -- The resource requirements of the NGINX container. You should be set this value, If you want to use dataplane Autoscaling(HPA).
+    # -- The resource requirements of the NGINX container. You should set this value if you want to use dataplane Autoscaling(HPA).
     resources: {}
 
     # -- The lifecycle of the NGINX container.

charts/nginx-gateway-fabric/values.yaml

@@ -4175,18 +4175,24 @@ spec:
                           maxReplicas:
                             description: Maximum number of replicas.
                             format: int32
+                            minimum: 1
                             type: integer
                           minReplicas:
                             description: Minimum number of replicas.
                             format: int32
+                            minimum: 1
                             type: integer
                           targetCPUUtilizationPercentage:
                             description: Target cpu utilization percentage of HPA.
                             format: int32
+                            maximum: 100
+                            minimum: 1
                             type: integer
                           targetMemoryUtilizationPercentage:
                             description: Target memory utilization percentage of HPA.
                             format: int32
+                            maximum: 100
+                            minimum: 1
                             type: integer
                         required:
                         - enabled
@@ -4200,12 +4206,6 @@ spec:
                             && size(self.autoscalingTemplate) > 0))'
                         - message: minReplicas must be less than or equal to maxReplicas
                           rule: self.minReplicas <= self.maxReplicas
-                        - message: CPU utilization must be between 1 and 100
-                          rule: '!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage
-                            >= 1 && self.targetCPUUtilizationPercentage <= 100)'
-                        - message: memory utilization must be between 1 and 100
-                          rule: '!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage
-                            >= 1 && self.targetMemoryUtilizationPercentage <= 100)'
                       container:
                         description: Container defines container fields for the NGINX
                           container.

config/crd/bases/gateway.nginx.org_nginxproxies.yaml

@@ -4760,18 +4760,24 @@ spec:
                           maxReplicas:
                             description: Maximum number of replicas.
                             format: int32
+                            minimum: 1
                             type: integer
                           minReplicas:
                             description: Minimum number of replicas.
                             format: int32
+                            minimum: 1
                             type: integer
                           targetCPUUtilizationPercentage:
                             description: Target cpu utilization percentage of HPA.
                             format: int32
+                            maximum: 100
+                            minimum: 1
                             type: integer
                           targetMemoryUtilizationPercentage:
                             description: Target memory utilization percentage of HPA.
                             format: int32
+                            maximum: 100
+                            minimum: 1
                             type: integer
                         required:
                         - enabled
@@ -4785,12 +4791,6 @@ spec:
                             && size(self.autoscalingTemplate) > 0))'
                         - message: minReplicas must be less than or equal to maxReplicas
                           rule: self.minReplicas <= self.maxReplicas
-                        - message: CPU utilization must be between 1 and 100
-                          rule: '!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage
-                            >= 1 && self.targetCPUUtilizationPercentage <= 100)'
-                        - message: memory utilization must be between 1 and 100
-                          rule: '!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage
-                            >= 1 && self.targetMemoryUtilizationPercentage <= 100)'
                       container:
                         description: Container defines container fields for the NGINX
                           container.

deploy/crds.yaml

@@ -614,8 +614,28 @@ func (p *NginxProvisioner) buildNginxDeployment(
 		}
 	}
 
+	var replicas *int32
 	if deploymentCfg.Replicas != nil {
-		deployment.Spec.Replicas = deploymentCfg.Replicas
+		replicas = deploymentCfg.Replicas
+	}
+
+	if isAutoscalingEnabled(&deploymentCfg) {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+
+		hpa := &autoscalingv2.HorizontalPodAutoscaler{}
+		err := p.k8sClient.Get(ctx, types.NamespacedName{
+			Namespace: objectMeta.Namespace,
+			Name:      objectMeta.Name,
+		}, hpa)
+		if err == nil && hpa.Status.DesiredReplicas > 0 {
+			// overwrite with HPA's desiredReplicas
+			replicas = helpers.GetPointer(hpa.Status.DesiredReplicas)
+		}
+	}
+
+	if replicas != nil {
+		deployment.Spec.Replicas = replicas
 	}
 
 	return deployment, nil
@@ -1101,12 +1121,10 @@ func buildNginxDeploymentHPA(
 		})
 	}
 
-	if autoscalingTemplate != nil {
-		for _, additionalAutoscaling := range *autoscalingTemplate {
-			metric := buildAdditionalMetric(additionalAutoscaling)
-			if metric != nil {
-				metrics = append(metrics, *metric)
-			}
+	for _, additionalAutoscaling := range autoscalingTemplate {
+		metric := buildAdditionalMetric(additionalAutoscaling)
+		if metric != nil {
+			metrics = append(metrics, *metric)
 		}
 	}
 
@@ -1193,11 +1211,11 @@ func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName
 	// order to delete:
 	// deployment/daemonset
 	// service
+	// hpa
 	// role/binding (if openshift)
 	// serviceaccount
 	// configmaps
 	// secrets
-	// hpa
 
 	objectMeta := metav1.ObjectMeta{
 		Name:      deploymentNSName.Name,
@@ -1219,12 +1237,6 @@ func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName
 
 	objects := []client.Object{deployment, daemonSet, service, hpa}
 
-	// objects := []client.Object{deployment, daemonSet, service}
-
-	// // if hpa := p.buildHPA(objectMeta, nProxyCfg); hpa != nil {
-	// // 	objects = append(objects, hpa)
-	// // }
-
 	if p.isOpenshift {
 		role := &rbacv1.Role{
 			ObjectMeta: objectMeta,