feat(chart): add relay horizontal autoscaling (#2077)

Signed-off-by: npub1jh9wn95s0472h86ahapupaf7m6kx4v9sx2n0atj2hltcfer8k06s5n3pyf <95cae996907d7cab9f5dbf43c0f53edeac6ab0b032a6feae4abfd784e467b3f5@sprout-oss.stage.blox.sqprod.co>
Co-authored-by: npub1jh9wn95s0472h86ahapupaf7m6kx4v9sx2n0atj2hltcfer8k06s5n3pyf <95cae996907d7cab9f5dbf43c0f53edeac6ab0b032a6feae4abfd784e467b3f5@sprout-oss.stage.blox.sqprod.co>

deploy/charts/buzz/Chart.yaml

@@ -7,7 +7,7 @@ description: |
   PostgreSQL and Redis. Configurable for single-node evaluation
   (subcharts on) and HA production (external services, existingSecret).
 type: application
-version: 0.1.3
+version: 0.1.4
 appVersion: "0.1.0"
 home: https://github.com/block/buzz
 sources:
@@ -24,7 +24,7 @@ maintainers:
 annotations:
   artifacthub.io/changes: |
     - kind: added
-      description: Initial chart for Buzz relay.
+      description: Optional CPU and per-pod WebSocket horizontal autoscaling for relay pods.
   artifacthub.io/license: Apache-2.0
 
 # Optional eval-only subcharts. Production deploys disable both and point

deploy/charts/buzz/README.md

@@ -78,6 +78,28 @@ It does **not** require ReadWriteMany git storage. Git ref/object state is objec
 
 The chart **template-fails** if the Redis invariant is broken at `replicaCount > 1`. No silent degradation.
 
+### Relay autoscaling
+
+The optional HPA scales the relay on the larger recommendation from CPU or
+average active WebSockets per pod:
+
+```yaml
+autoscaling:
+  enabled: true
+  minReplicas: 5
+  maxReplicas: 15
+  targetCPUUtilizationPercentage: 65
+  websocketMetricName: buzz_ws_connections_active
+  targetWebsocketConnections: 5000
+```
+
+CPU scaling requires Kubernetes Metrics Server. WebSocket scaling additionally
+requires a custom-metrics adapter (for example Prometheus Adapter) configured to
+expose the relay's `buzz_ws_connections_active` gauge as a pod metric with the
+name in `websocketMetricName`. The chart creates the HPA but deliberately does
+not install or configure a cluster-wide metrics adapter. Scale-down is gradual
+by default so long-lived WebSocket connections have time to drain.
+
 ## Upgrades
 
 Schema migrations are embedded in the relay binary via `sqlx::migrate!` and run at startup, gated by `BUZZ_AUTO_MIGRATE` (default `true`). Multiple replicas race-safely behind a Postgres advisory lock. `helm upgrade` is the entire upgrade procedure.

deploy/charts/buzz/templates/_helpers.tpl

@@ -103,11 +103,20 @@ secrets.existingSecret, use that. Otherwise use the chart-managed one.
 {{- printf "http://%s.%s.svc.cluster.local:9000" (include "buzz.minioFullname" .) .Release.Namespace -}}
 {{- end -}}
 
+{{/* Minimum number of relay replicas the release can run. */}}
+{{- define "buzz.minimumReplicas" -}}
+{{- if .Values.autoscaling.enabled -}}
+{{- .Values.autoscaling.minReplicas -}}
+{{- else -}}
+{{- .Values.replicaCount -}}
+{{- end -}}
+{{- end -}}
+
 {{/* Effective huddle-audio availability. Nil means safe chart default: on for
      one replica, off for multi-pod until an SFU/shared-room story exists. */}}
 {{- define "buzz.huddleAudioAvailable" -}}
 {{- if kindIs "invalid" .Values.relay.huddleAudioAvailable -}}
-{{- if gt (.Values.replicaCount | int) 1 -}}false{{- else -}}true{{- end -}}
+{{- if gt (include "buzz.minimumReplicas" . | int) 1 -}}false{{- else -}}true{{- end -}}
 {{- else -}}
 {{- .Values.relay.huddleAudioAvailable -}}
 {{- end -}}

deploy/charts/buzz/templates/_validate.tpl

@@ -10,14 +10,15 @@ surface at template time regardless of which manifest helm renders first.
   {{- fail "relayUrl is required: set --set relayUrl=wss://your.domain" -}}
 {{- end -}}
 
-{{/* replicaCount > 1 requires Redis */}}
-{{- if gt (.Values.replicaCount | int) 1 -}}
+{{/* Multiple replicas require Redis, whether fixed or autoscaled. */}}
+{{- $minimumReplicas := include "buzz.minimumReplicas" . | int -}}
+{{- if gt $minimumReplicas 1 -}}
   {{- if and (not .Values.redis.enabled) (not .Values.externalRedis.url) (not .Values.secrets.existingSecret) -}}
-    {{- fail (printf "replicaCount=%d requires Redis for buzz-pubsub. Enable redis.enabled=true, set externalRedis.url, or provide secrets.existingSecret with key REDIS_URL." (.Values.replicaCount | int)) -}}
+    {{- fail (printf "minimum replica count %d requires Redis for buzz-pubsub. Enable redis.enabled=true, set externalRedis.url, or provide secrets.existingSecret with key REDIS_URL." $minimumReplicas) -}}
   {{- end -}}
 {{- end -}}
 
-{{/* replicaCount > 1 does NOT require ReadWriteMany git storage.
+{{/* Multiple replicas do NOT require ReadWriteMany git storage.
 
      Git ref/object state is object-store-backed: every read and write hydrates
      an ephemeral bare repo from S3-compatible storage per request, and writer
@@ -32,6 +33,19 @@ surface at template time regardless of which manifest helm renders first.
      replicas") is no longer true. Redis (validated above) remains the real
      multi-pod requirement for buzz-pubsub. */}}
 
+{{/* Autoscaling bounds must be coherent. */}}
+{{- if .Values.autoscaling.enabled -}}
+  {{- if lt (.Values.autoscaling.minReplicas | int) 1 -}}
+    {{- fail "autoscaling.minReplicas must be at least 1" -}}
+  {{- end -}}
+  {{- if lt (.Values.autoscaling.maxReplicas | int) (.Values.autoscaling.minReplicas | int) -}}
+    {{- fail "autoscaling.maxReplicas must be greater than or equal to autoscaling.minReplicas" -}}
+  {{- end -}}
+  {{- if not .Values.autoscaling.websocketMetricName -}}
+    {{- fail "autoscaling.websocketMetricName is required when autoscaling.enabled=true" -}}
+  {{- end -}}
+{{- end -}}
+
 {{/* Owner pubkey required when requireRelayMembership */}}
 {{- if .Values.relay.requireRelayMembership -}}
   {{- if not .Values.ownerPubkey -}}

deploy/charts/buzz/templates/deployment.yaml

@@ -6,7 +6,9 @@ metadata:
   labels:
     {{- include "buzz.labels" . | nindent 4 }}
 spec:
+  {{- if not .Values.autoscaling.enabled }}
   replicas: {{ .Values.replicaCount }}
+  {{- end }}
   strategy:
     type: RollingUpdate
     rollingUpdate:
@@ -176,7 +178,7 @@ spec:
                 secretKeyRef:
                   name: {{ include "buzz.envSecretName" . }}
                   key: REDIS_URL
-                  optional: {{ and (eq (.Values.replicaCount | int) 1) (not .Values.redis.enabled) (not .Values.externalRedis.url) }}
+                  optional: {{ and (eq (include "buzz.minimumReplicas" . | int) 1) (not .Values.redis.enabled) (not .Values.externalRedis.url) }}
             - name: BUZZ_S3_ACCESS_KEY
               valueFrom:
                 secretKeyRef:

deploy/charts/buzz/templates/hpa.yaml

@@ -0,0 +1,32 @@
+{{- include "buzz.validate" . -}}
+{{- if .Values.autoscaling.enabled }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "buzz.fullname" . }}
+  labels:
+    {{- include "buzz.labels" . | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "buzz.fullname" . }}
+  minReplicas: {{ .Values.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.autoscaling.maxReplicas }}
+  behavior:
+    {{- toYaml .Values.autoscaling.behavior | nindent 4 }}
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Pods
+      pods:
+        metric:
+          name: {{ .Values.autoscaling.websocketMetricName }}
+        target:
+          type: AverageValue
+          averageValue: {{ .Values.autoscaling.targetWebsocketConnections | quote }}
+{{- end }}

deploy/charts/buzz/templates/pdb.yaml

@@ -1,5 +1,5 @@
 {{- include "buzz.validate" . -}}
-{{- if and .Values.podDisruptionBudget.enabled (gt (.Values.replicaCount | int) 1) -}}
+{{- if and .Values.podDisruptionBudget.enabled (gt (include "buzz.minimumReplicas" . | int) 1) -}}
 apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:

deploy/charts/buzz/tests/hpa_test.yaml

@@ -0,0 +1,63 @@
+suite: relay horizontal pod autoscaler
+templates:
+  - templates/hpa.yaml
+  - templates/deployment.yaml
+  - templates/secret-chart.yaml
+set:
+  relayUrl: wss://buzz.example.com
+  ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000"
+  externalPostgresql.url: postgres://u:p@h:5432/d
+  externalRedis.url: redis://h:6379
+  s3.endpoint: http://minio:9000
+  s3.accessKey: a
+  s3.secretKey: s
+tests:
+  - it: omits the HPA by default
+    template: templates/hpa.yaml
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: scales the relay on either CPU or average pod WebSockets
+    template: templates/hpa.yaml
+    set:
+      autoscaling.enabled: true
+    asserts:
+      - isAPIVersion:
+          of: autoscaling/v2
+      - isKind:
+          of: HorizontalPodAutoscaler
+      - equal:
+          path: spec.minReplicas
+          value: 5
+      - equal:
+          path: spec.maxReplicas
+          value: 15
+      - equal:
+          path: spec.scaleTargetRef.name
+          value: RELEASE-NAME-buzz
+      - equal:
+          path: spec.metrics[0].resource.target.averageUtilization
+          value: 65
+      - equal:
+          path: spec.metrics[1].pods.metric.name
+          value: buzz_ws_connections_active
+      - equal:
+          path: spec.metrics[1].pods.target.averageValue
+          value: "5000"
+      - equal:
+          path: spec.behavior.scaleDown.stabilizationWindowSeconds
+          value: 600
+
+  - it: leaves Deployment replicas to the HPA and disables in-process huddle audio
+    template: templates/deployment.yaml
+    set:
+      autoscaling.enabled: true
+    asserts:
+      - notExists:
+          path: spec.replicas
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: BUZZ_HUDDLE_AUDIO_AVAILABLE
+            value: "false"

deploy/charts/buzz/tests/validation_test.yaml

@@ -37,7 +37,7 @@ tests:
       replicaCount: 3
     asserts:
       - failedTemplate:
-          errorPattern: "replicaCount=3 requires Redis"
+          errorPattern: "minimum replica count 3 requires Redis"
 
   - it: fails when ingress and httproute both enabled
     set:

deploy/charts/buzz/values.yaml

@@ -36,6 +36,36 @@ image:
 # in Postgres. Each replica can use its own ReadWriteOnce volume (or none).
 replicaCount: 1
 
+# ── Autoscaling ──────────────────────────────────────────────────────────────
+# Requires Metrics Server for CPU and a custom-metrics adapter exposing the
+# pod-level Prometheus gauge named by websocketMetricName. Kubernetes HPA uses
+# the larger replica recommendation, so either CPU or WebSocket load scales out.
+autoscaling:
+  enabled: false
+  minReplicas: 5
+  maxReplicas: 15
+  targetCPUUtilizationPercentage: 65
+  websocketMetricName: buzz_ws_connections_active
+  targetWebsocketConnections: 5000
+  behavior:
+    scaleUp:
+      stabilizationWindowSeconds: 0
+      policies:
+        - type: Percent
+          value: 100
+          periodSeconds: 60
+        - type: Pods
+          value: 4
+          periodSeconds: 60
+      selectPolicy: Max
+    scaleDown:
+      stabilizationWindowSeconds: 600
+      policies:
+        - type: Pods
+          value: 1
+          periodSeconds: 120
+      selectPolicy: Min
+
 # ── Public URL ───────────────────────────────────────────────────────────────
 # Required. The wss:// URL clients use to connect. Drives:
 #   - RELAY_URL env (relay-side)