mirror of
https://github.com/block/buzz.git
synced 2026-11-07 01:41:06 +00:00
1c006822e4
Signed-off-by: npub12gtutshhh76rx0jx697f32f9tffd4hhp3hx58fp4x6u4uemkm7sqf8f757 <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@sprout-oss.stage.blox.sqprod.co> Co-authored-by: npub12gtutshhh76rx0jx697f32f9tffd4hhp3hx58fp4x6u4uemkm7sqf8f757 <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@sprout-oss.stage.blox.sqprod.co>
90 lines
4.4 KiB
YAML
90 lines
4.4 KiB
YAML
{{- if .Values.prometheusRule.enabled }}
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: {{ include "push.name" . }}
|
|
labels: {{- include "push.labels" . | nindent 4 }}
|
|
{{- with .Values.prometheusRule.labels }}
|
|
{{- toYaml . | nindent 4 }}
|
|
{{- end }}
|
|
spec:
|
|
groups:
|
|
- name: buzz-push-gateway
|
|
rules:
|
|
# Sustained configuration faults mean the provider credential/topic is
|
|
# unhealthy; no endpoint is being invalidated but nothing is delivering.
|
|
- alert: PushGatewayConfigurationFault
|
|
expr: |
|
|
sum(rate(push_gateway_apns_deliveries_total{outcome="configuration_fault"}[5m])) > 0
|
|
for: 10m
|
|
labels: { severity: critical }
|
|
annotations:
|
|
summary: Push gateway APNs configuration faults
|
|
description: >-
|
|
APNs is returning configuration faults (bad/expired provider token
|
|
or topic). Deliveries are failing without invalidating endpoints.
|
|
See runbook: check the APNs .p8 key, key id, team id, and topic.
|
|
# Authority store unavailable at admission = durable dependency is down.
|
|
- alert: PushGatewayAdmissionUnavailable
|
|
expr: |
|
|
sum(rate(push_gateway_admissions_total{result="unavailable"}[5m])) > 0
|
|
for: 5m
|
|
labels: { severity: critical }
|
|
annotations:
|
|
summary: Push gateway authority store unavailable
|
|
description: >-
|
|
authorize_delivery is returning Unavailable — the PostgreSQL
|
|
authority store is unreachable or failing. Check DB connectivity
|
|
and the pod's postgres egress NetworkPolicy.
|
|
# Readiness failing on the authority cause = the pod will be pulled from
|
|
# rotation; alert before all replicas drop out.
|
|
- alert: PushGatewayReadinessAuthorityFailing
|
|
expr: |
|
|
sum(rate(push_gateway_readiness_failures_total{cause="authority"}[5m])) > 0
|
|
for: 5m
|
|
labels: { severity: warning }
|
|
annotations:
|
|
summary: Push gateway readiness failing on authority
|
|
description: >-
|
|
Readiness probes are failing because the authority store check
|
|
fails. Replicas will be removed from the Service. Investigate DB
|
|
health before capacity drops below the PodDisruptionBudget.
|
|
# The retention reaper sweeps expired rows every 5m; a single transient
|
|
# failure self-heals on the next tick. Alert on repeated failure —
|
|
# at least two sweeps failing within ~30m (six ticks) — which grows the
|
|
# bounded crash-before-release window and leaks storage.
|
|
- alert: PushGatewayReaperFailing
|
|
expr: |
|
|
sum(increase(push_gateway_reaper_failures_total[30m])) >= 2
|
|
for: 5m
|
|
labels: { severity: warning }
|
|
annotations:
|
|
summary: Push gateway retention reaper failing
|
|
description: >-
|
|
The retention reaper has failed at least twice within 30m (it runs
|
|
every 5m). Expired delivery reservations are not being swept,
|
|
growing the bounded-until-expiry window. Check DB write availability.
|
|
# High sustained fraction of retryable APNs outcomes indicates APNs
|
|
# throttling or degradation. The ratio is a true fraction over the
|
|
# window (increase = counts, not per-second rate), gated by a minimum
|
|
# sample count so a couple of retries at trivial volume cannot trip it.
|
|
- alert: PushGatewayHighApnsRetryRate
|
|
expr: |
|
|
(
|
|
sum(increase(push_gateway_apns_deliveries_total{outcome="retry"}[10m]))
|
|
/ sum(increase(push_gateway_apns_deliveries_total[10m]))
|
|
> {{ .Values.prometheusRule.apnsRetryRatioThreshold }}
|
|
)
|
|
and
|
|
sum(increase(push_gateway_apns_deliveries_total[10m])) >= {{ .Values.prometheusRule.apnsRetryMinSamples }}
|
|
for: 15m
|
|
labels: { severity: warning }
|
|
annotations:
|
|
summary: Push gateway high APNs retry ratio
|
|
description: >-
|
|
The retryable fraction of APNs attempts over a 10m window
|
|
(429/500/503), above a minimum sample count, has exceeded the
|
|
configured threshold continuously for 15m. APNs may be throttling
|
|
or degraded; deliveries are delayed but not lost.
|
|
{{- end }}
|