feat(mobile): prepare buzz-push-gateway for deployment (#7158)

A few things to prepare for deploying the Buzz push gateway as
implemented in #6269:
- Cut `buzz-push-gateway` chart `0.2.0` from the current config
(certificate auth, dogfood client)
- Datadog-compatible metrics
- Align the desired schema, metrics docs, and production ingress
guidance with the current dogfood deployment contract

## Links

- [BUZZ-17: Publish the MVP-compatible gateway
chart](https://linear.app/squareup/issue/BUZZ-17/publish-the-mvp-compatible-gateway-chart)
- [BUZZ-26: Add Datadog-compatible push-gateway metrics and correct
source
drift](https://linear.app/squareup/issue/BUZZ-26/add-datadog-compatible-push-gateway-metrics-and-correct-source-drift)

---------

Signed-off-by: Tom Brow <tomb@block.xyz>
Co-authored-by: Codex <noreply@openai.com>

crates/buzz-db/src/runtime/migration.rs

@@ -193,8 +193,8 @@ mod postgres_tests {
 
     const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; // sadscan:disable np.postgres.1
 
-    /// Connection parameters parsed out of a `postgres://user:pass@host:port/db`
-    /// URL so the parity test can pass them to the `bin/pgschema` binary, which
+    /// Connection parameters parsed out of a PostgreSQL URL so the parity test
+    /// can pass them to the `bin/pgschema` binary, which
     /// takes discrete `--host/--port/--user/--password/--db` flags rather than a
     /// URL. Only the shapes this test emits (`BUZZ_TEST_DATABASE_URL` /
     /// `DATABASE_URL` / `TEST_DB_URL`) are supported.
@@ -699,7 +699,7 @@ mod postgres_tests {
         let mut migrations: Vec<_> = MIGRATOR.iter().collect();
         migrations.sort_by_key(|migration| migration.version);
 
-        assert_eq!(migrations.len(), 42);
+        assert_eq!(migrations.len(), 43);
         assert_eq!(migrations[0].version, 1);
         assert_eq!(&*migrations[0].description, "initial schema");
         assert!(migrations[0]
@@ -1280,6 +1280,18 @@ mod postgres_tests {
             extract_excluded_table_array(desired_schema),
             "schema.sql exclusion list drifted from migration 0042"
         );
+
+        // Brownfield relay databases created through SQLx still carry the
+        // production/sandbox constraint from 0015. Converge them to the same
+        // dogfood-only authority declared by the desired-state schema.
+        assert_eq!(migrations[42].version, 43);
+        let dogfood_profile = migrations[42].sql.as_str();
+        assert!(dogfood_profile.contains("DELETE FROM push_gateway_delegations"));
+        assert!(dogfood_profile.contains("DELETE FROM push_gateway_installations"));
+        assert!(dogfood_profile
+            .contains("DROP CONSTRAINT push_gateway_installations_app_profile_check"));
+        assert!(dogfood_profile.contains("CHECK (app_profile = 'buzz-ios-dogfood')"));
+        assert!(desired_schema.contains("CHECK (app_profile = 'buzz-ios-dogfood')"));
     }
 
     #[test]

crates/buzz-db/src/runtime/tests.rs

@@ -4,7 +4,7 @@ use buzz_core::CommunityId;
 use sqlx::{Connection, PgPool};
 use uuid::Uuid;
 
-const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz";
+const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; // sadscan:disable np.postgres.1 -- local test-only credentials
 
 async fn setup_db() -> Db {
     let database_url = std::env::var("TEST_DATABASE_URL").unwrap_or_else(|_| TEST_DB_URL.into());
@@ -399,6 +399,98 @@ async fn drop_scratch_db(admin: &PgPool, pool: PgPool, name: &str) {
     .await;
 }
 
+#[tokio::test]
+#[ignore = "requires Postgres"]
+async fn push_gateway_profile_migration_converges_brownfield_authority() {
+    let admin = PgPool::connect(&admin_url().await)
+        .await
+        .expect("connect admin database");
+    let (pool, name) = create_scratch_db_through(&admin, "push_profile", Some(42)).await;
+    let installation_id = Uuid::new_v4();
+    let now = chrono::Utc::now();
+
+    sqlx::query(
+        "INSERT INTO push_gateway_installations(\
+         id, app_attest_key_id, app_attest_public_key, assertion_counter, app_profile, \
+         token_ciphertext, token_fingerprint, endpoint_epoch, expires_at) \
+         VALUES($1, $2, $3, 0, 'buzz-ios-production', $4, $5, 1, $6)",
+    )
+    .bind(installation_id)
+    .bind(vec![1_u8])
+    .bind(vec![2_u8; 33])
+    .bind(vec![3_u8])
+    .bind(vec![4_u8; 32])
+    .bind(now + chrono::Duration::days(1))
+    .execute(&pool)
+    .await
+    .expect("insert legacy production installation");
+    sqlx::query(
+        "INSERT INTO push_gateway_delegations(\
+         id, installation_id, relay_pubkey, endpoint_epoch, generation, not_before, expires_at) \
+         VALUES($1, $2, $3, 1, 1, $4, $5)",
+    )
+    .bind(Uuid::new_v4())
+    .bind(installation_id)
+    .bind(vec![5_u8; 32])
+    .bind(now)
+    .bind(now + chrono::Duration::hours(1))
+    .execute(&pool)
+    .await
+    .expect("insert delegation for legacy installation");
+
+    migration::run_migrations(&pool)
+        .await
+        .expect("apply dogfood-only migration");
+
+    let legacy_installations: i64 =
+        sqlx::query_scalar("SELECT count(*) FROM push_gateway_installations")
+            .fetch_one(&pool)
+            .await
+            .expect("count legacy installations");
+    let legacy_delegations: i64 =
+        sqlx::query_scalar("SELECT count(*) FROM push_gateway_delegations")
+            .fetch_one(&pool)
+            .await
+            .expect("count legacy delegations");
+    assert_eq!(legacy_installations, 0);
+    assert_eq!(legacy_delegations, 0);
+
+    sqlx::query(
+        "INSERT INTO push_gateway_installations(\
+         id, app_attest_key_id, app_attest_public_key, assertion_counter, app_profile, \
+         token_ciphertext, token_fingerprint, endpoint_epoch, expires_at) \
+         VALUES($1, $2, $3, 0, 'buzz-ios-dogfood', $4, $5, 1, $6)",
+    )
+    .bind(Uuid::new_v4())
+    .bind(vec![6_u8])
+    .bind(vec![7_u8; 33])
+    .bind(vec![8_u8])
+    .bind(vec![9_u8; 32])
+    .bind(now + chrono::Duration::days(1))
+    .execute(&pool)
+    .await
+    .expect("dogfood installation is accepted after migration");
+
+    let sandbox = sqlx::query(
+        "INSERT INTO push_gateway_installations(\
+         id, app_attest_key_id, app_attest_public_key, assertion_counter, app_profile, \
+         token_ciphertext, token_fingerprint, endpoint_epoch, expires_at) \
+         VALUES($1, $2, $3, 0, 'buzz-ios-sandbox', $4, $5, 1, $6)",
+    )
+    .bind(Uuid::new_v4())
+    .bind(vec![10_u8])
+    .bind(vec![11_u8; 33])
+    .bind(vec![12_u8])
+    .bind(vec![13_u8; 32])
+    .bind(now + chrono::Duration::days(1))
+    .execute(&pool)
+    .await;
+    assert!(sandbox.is_err(), "legacy sandbox profile must be rejected");
+
+    drop_scratch_db(&admin, pool, &name).await;
+    admin.close().await;
+}
+
 /// Insert identical community + channel rows into a database so the same
 /// (community, channel) ids resolve in both writer and replica.
 async fn seed_community_channel(

deploy/charts/buzz-push-gateway/Chart.yaml

@@ -3,6 +3,6 @@ apiVersion: v2
 # branches (see docs/push-gateway-deployment.md, "Gateway chart release").
 name: buzz-push-gateway
 description: Public capability-gated APNs last-hop gateway for Buzz
-version: 0.1.0
+version: 0.2.0
 appVersion: "0.1.0"
 type: application

deploy/charts/buzz-push-gateway/templates/deployment.yaml

@@ -11,6 +11,9 @@ spec:
   template:
     metadata:
       labels: {{- include "push.runtimeLabels" . | nindent 8 }}
+      {{- with .Values.podAnnotations }}
+      annotations: {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       automountServiceAccountToken: false
       terminationGracePeriodSeconds: 60

deploy/charts/buzz-push-gateway/tests/datadog-values.yaml

@@ -0,0 +1,30 @@
+# Render-only fixture proving Datadog Autodiscovery can scrape the private
+# metrics listener without installing prometheus-operator CRDs. Deployment
+# repositories must replace these illustrative selectors with their agent's
+# actual namespace and pod labels.
+podAnnotations:
+  ad.datadoghq.com/gateway.checks: |
+    {
+      "openmetrics": {
+        "init_config": {},
+        "instances": [
+          {
+            "openmetrics_endpoint": "http://%%host%%:8081/metrics",
+            "service": "buzz-push-gateway",
+            "namespace": "block.buzz_push_gateway",
+            "metrics": ["push_gateway_.*"],
+            "histogram_buckets_as_distributions": true,
+            "send_distribution_buckets": true,
+            "send_monotonic_counter": true,
+            "collect_counters_with_distributions": true
+          }
+        ]
+      }
+    }
+networkPolicy:
+  monitoring:
+    enabled: true
+    namespaceSelector:
+      kubernetes.io/metadata.name: datadog
+    podSelector:
+      app.kubernetes.io/name: datadog-agent

deploy/charts/buzz-push-gateway/tests/release-contract.sh

@@ -3,10 +3,21 @@ set -euo pipefail
 env -u GEM_HOME -u GEM_PATH -u RUBYLIB -u RUBYOPT ruby -ryaml <<'RUBY'
 auto_text = File.read('.github/workflows/auto-tag-on-release-pr-merge.yml')
 publish_text = File.read('.github/workflows/push-gateway-helm-chart.yml')
+deployment_text = File.read('docs/push-gateway-deployment.md')
+chart = YAML.load_file('deploy/charts/buzz-push-gateway/Chart.yaml')
 # Parse first, then pin the tag producer and consumer strings whose agreement
 # makes this a reachable lane rather than an orphan publisher.
 YAML.load(auto_text)
 YAML.load(publish_text)
+version = chart.fetch('version').to_s
+raise "gateway chart version is not semver: #{version}" unless version.match?(/\A\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?\z/)
+workspace_package = File.read('Cargo.toml').match(/\[workspace\.package\](.*?)(?=\n\[|\z)/m)
+raise "workspace package metadata is missing" unless workspace_package
+binary_version = workspace_package[1].match(/^version\s*=\s*"([^"]+)"/)&.[](1)
+raise "workspace package version is missing" unless binary_version
+unless chart.fetch('appVersion').to_s == binary_version
+  raise "gateway chart appVersion does not match packaged binary #{binary_version}"
+end
 [
   'push-chart-release/*)',
   'VERSION="${BRANCH#push-chart-release/}"',
@@ -26,4 +37,14 @@ end
 ].each do |needle|
   raise "missing gateway chart publisher contract: #{needle}" unless publish_text.include?(needle)
 end
+[
+  'inspect and fetch the published chart version',
+  'helm show chart oci://ghcr.io/block/buzz/charts/buzz-push-gateway --version X.Y.Z',
+  'helm pull oci://ghcr.io/block/buzz/charts/buzz-push-gateway --version X.Y.Z',
+].each do |needle|
+  raise "missing gateway chart retrieval guidance: #{needle}" unless deployment_text.include?(needle)
+end
+if deployment_text.include?('verify the immutable chart artifact')
+  raise 'gateway chart retrieval guidance overstates authenticity verification'
+end
 RUBY

deploy/charts/buzz-push-gateway/tests/render.sh

@@ -1,25 +1,32 @@
 #!/usr/bin/env bash
 set -euo pipefail
-out=$(mktemp); production_out=$(mktemp)
-trap 'rm -f "$out" "$production_out"' EXIT
+out=$(mktemp); production_out=$(mktemp); route_out=$(mktemp); datadog_out=$(mktemp)
+trap 'rm -f "$out" "$production_out" "$route_out" "$datadog_out" "${monitoring_out:-}"' EXIT
 
 # Defaults must lint and render without parameter injection.
 helm lint deploy/charts/buzz-push-gateway >/dev/null
 helm template push deploy/charts/buzz-push-gateway >"$out"
-# Production values must attach push.buzz.xyz to an explicit Gateway.
+# Production values support a platform-owned ingress without rendering an
+# HTTPRoute. The environment-owned inputs remain mandatory.
 production_args=(
   -f deploy/charts/buzz-push-gateway/values-production.yaml
   --set 'image.digest=sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
   --set 'profiles.dogfood.appAttestAppId=REALTEAM.xyz.block.buzz.dogfood.mobile'
-  --set 'httpRoute.parentRefs[0].name=production-gateway'
-  --set 'httpRoute.parentRefs[0].namespace=gateway-system'
   --set 'networkPolicy.postgresEgressCidrs[0]=10.42.0.0/16'
 )
 helm lint deploy/charts/buzz-push-gateway "${production_args[@]}" >/dev/null
 helm template push deploy/charts/buzz-push-gateway "${production_args[@]}" >"$production_out"
 
+# Gateway API remains an explicit supported ingress mode when an operator opts
+# in and supplies the environment-owned parent.
+helm template push deploy/charts/buzz-push-gateway \
+  --set httpRoute.enabled=true \
+  --set 'httpRoute.parentRefs[0].name=production-gateway' \
+  --set 'httpRoute.parentRefs[0].namespace=gateway-system' \
+  >"$route_out"
+
 env -u GEM_HOME -u GEM_PATH -u RUBYLIB -u RUBYOPT ruby -ryaml -rset \
-  - "$out" "$production_out" <<'RUBY'
+  - "$out" "$production_out" "$route_out" <<'RUBY'
 def assert!(condition, detail = "assertion failed")
   raise detail unless condition
 end
@@ -38,6 +45,7 @@ migration = runtime.merge("app.kubernetes.io/component" => "migration")
 assert!(svc.dig("spec", "selector") == runtime)
 assert!(d.dig("spec", "selector", "matchLabels") == runtime)
 assert!(d.dig("spec", "template", "metadata", "labels") == runtime)
+assert!(d.dig("spec", "template", "metadata", "annotations").nil?)
 assert!(j.dig("spec", "template", "metadata", "labels") == migration)
 assert!(svc.dig("spec", "selector") != j.dig("spec", "template", "metadata", "labels"))
 jenv = j.dig("spec", "template", "spec", "containers", 0, "env").to_h { |entry| [entry["name"], entry] }
@@ -86,7 +94,11 @@ ingress_ports = np.dig("spec", "ingress")
   .flat_map { |rule| rule.fetch("ports", []) }.map { |port| port["port"] }.to_set
 assert!(ingress_ports == Set[8080], ingress_ports.inspect)
 production = YAML.load_stream(File.read(ARGV[1])).compact
-route = production.find { |x| x["kind"] == "HTTPRoute" }
+assert!(!production.any? { |x| x["kind"] == "HTTPRoute" })
+production_deployment = production.find { |x| x["kind"] == "Deployment" }
+production_image = production_deployment.dig("spec", "template", "spec", "containers", 0, "image")
+assert!(production_image == "ghcr.io/block/buzz-push-gateway@sha256:#{"a" * 64}", production_image.inspect)
+route = YAML.load_stream(File.read(ARGV[2])).compact.find { |x| x["kind"] == "HTTPRoute" }
 assert!(!route.dig("spec", "parentRefs").empty?)
 assert!(route.dig("spec", "hostnames").include?("push.buzz.xyz"))
 RUBY
@@ -107,7 +119,7 @@ if helm template push deploy/charts/buzz-push-gateway --set httpRoute.enabled=tr
 fi
 
 # The checked-in production contract is intentionally undeployable until CI or
-# the release system supplies an immutable digest and environment-owned values.
+# the release system supplies its environment-owned values.
 if helm template push deploy/charts/buzz-push-gateway -f deploy/charts/buzz-push-gateway/values-production.yaml >/dev/null 2>&1; then
   echo 'expected uninjected production values to fail' >&2
   exit 1
@@ -115,7 +127,7 @@ fi
 
 # Enabling observability renders the scrape CRDs and adds a scoped 8081 ingress
 # keyed to the named monitoring source — never a blanket 8081 rule.
-monitoring_out=$(mktemp); trap 'rm -f "$out" "$production_out" "$monitoring_out"' EXIT
+monitoring_out=$(mktemp)
 helm template push deploy/charts/buzz-push-gateway \
   --set podMonitor.enabled=true \
   --set prometheusRule.enabled=true \
@@ -147,6 +159,44 @@ from = monitoring[0].fetch("from")[0]
 assert!(!from.dig("namespaceSelector", "matchLabels").empty? && !from.dig("podSelector", "matchLabels").empty?, from.inspect)
 RUBY
 
+# Datadog discovers the same private endpoint from pod annotations and needs no
+# prometheus-operator CRDs. Its agent ingress remains selector-scoped.
+helm lint deploy/charts/buzz-push-gateway \
+  -f deploy/charts/buzz-push-gateway/tests/datadog-values.yaml >/dev/null
+helm template push deploy/charts/buzz-push-gateway \
+  -f deploy/charts/buzz-push-gateway/tests/datadog-values.yaml \
+  >"$datadog_out"
+
+env -u GEM_HOME -u GEM_PATH -u RUBYLIB -u RUBYOPT ruby -rjson -ryaml -rset \
+  - "$datadog_out" <<'RUBY'
+def assert!(condition, detail = "assertion failed")
+  raise detail unless condition
+end
+
+xs = YAML.load_stream(File.read(ARGV[0])).compact
+assert!(!xs.any? { |x| %w[PodMonitor PrometheusRule].include?(x["kind"]) })
+deployment = xs.find { |x| x["kind"] == "Deployment" }
+raw_check = deployment.dig(
+  "spec", "template", "metadata", "annotations",
+  "ad.datadoghq.com/gateway.checks",
+)
+check = JSON.parse(raw_check)
+instance = check.dig("openmetrics", "instances", 0)
+assert!(instance["openmetrics_endpoint"] == "http://%%host%%:8081/metrics", instance.inspect)
+assert!(instance["metrics"] == ["push_gateway_.*"], instance.inspect)
+
+np = xs.find do |x|
+  x["kind"] == "NetworkPolicy" && x.dig("metadata", "name") == "push-buzz-push-gateway"
+end
+monitoring = np.dig("spec", "ingress").select do |rule|
+  rule.fetch("ports", []).map { |port| port["port"] }.to_set == Set[8081]
+end
+assert!(monitoring.length == 1, "exactly one scoped Datadog 8081 ingress rule")
+from = monitoring[0].fetch("from")[0]
+assert!(!from.dig("namespaceSelector", "matchLabels").empty?, from.inspect)
+assert!(!from.dig("podSelector", "matchLabels").empty?, from.inspect)
+RUBY
+
 # Negative: monitoring enabled with default empty selectors must fail (would
 # otherwise render a blanket 8081 rule matching all namespaces/pods).
 if helm template push deploy/charts/buzz-push-gateway \
@@ -156,9 +206,8 @@ if helm template push deploy/charts/buzz-push-gateway \
   exit 1
 fi
 
-# Negative: scrape flags must be coupled. PodMonitor without ingress = an
-# unreachable scraper; ingress without a PodMonitor = an open hole with no
-# scraper. Both mismatches must fail schema validation.
+# Negative: PodMonitor without ingress is an unreachable scraper and must fail.
+# Scoped ingress without PodMonitor is valid for annotation-discovered agents.
 if helm template push deploy/charts/buzz-push-gateway \
   --set podMonitor.enabled=true \
   --set 'networkPolicy.monitoring.namespaceSelector.kubernetes\.io/metadata\.name=monitoring' \
@@ -167,14 +216,6 @@ if helm template push deploy/charts/buzz-push-gateway \
   echo 'expected podMonitor.enabled without monitoring ingress to fail' >&2
   exit 1
 fi
-if helm template push deploy/charts/buzz-push-gateway \
-  --set networkPolicy.monitoring.enabled=true \
-  --set 'networkPolicy.monitoring.namespaceSelector.kubernetes\.io/metadata\.name=monitoring' \
-  --set 'networkPolicy.monitoring.podSelector.app\.kubernetes\.io/name=prometheus' \
-  >/dev/null 2>&1; then
-  echo 'expected monitoring ingress without podMonitor.enabled to fail' >&2
-  exit 1
-fi
 
 # Negative: retry-ratio threshold is a fraction; a value > 1 must fail schema.
 if helm template push deploy/charts/buzz-push-gateway \

deploy/charts/buzz-push-gateway/values-production.yaml

@@ -7,7 +7,9 @@ profiles:
   dogfood:
     appAttestAppId: ""
 httpRoute:
-  enabled: true
+  # Keep disabled when the platform already routes push.buzz.xyz to this
+  # Service. Gateway API users enable it and inject an explicit parentRef.
+  enabled: false
   parentRefs: []
   hostnames:
     - push.buzz.xyz

deploy/charts/buzz-push-gateway/values.schema.json

@@ -32,6 +32,12 @@
       }
     },
     "apnsKey": false,
+    "podAnnotations": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
     "httpRoute": {
       "type": "object",
       "required": [
@@ -313,7 +319,7 @@
   ],
   "allOf": [
     {
-      "$comment": "Scraping opt-in is coupled: a PodMonitor and its scoped 8081 ingress must be enabled together, so we never render a scraper that cannot reach the port nor an ingress hole with no scraper.",
+      "$comment": "A PodMonitor requires scoped 8081 ingress. External scrapers such as Datadog may enable that ingress without rendering a PodMonitor.",
       "if": {
         "properties": {
           "podMonitor": {
@@ -355,49 +361,6 @@
           "networkPolicy"
         ]
       }
-    },
-    {
-      "if": {
-        "properties": {
-          "networkPolicy": {
-            "properties": {
-              "monitoring": {
-                "properties": {
-                  "enabled": {
-                    "const": true
-                  }
-                },
-                "required": [
-                  "enabled"
-                ]
-              }
-            },
-            "required": [
-              "monitoring"
-            ]
-          }
-        },
-        "required": [
-          "networkPolicy"
-        ]
-      },
-      "then": {
-        "properties": {
-          "podMonitor": {
-            "properties": {
-              "enabled": {
-                "const": true
-              }
-            },
-            "required": [
-              "enabled"
-            ]
-          }
-        },
-        "required": [
-          "podMonitor"
-        ]
-      }
     }
   ]
 }

deploy/charts/buzz-push-gateway/values.yaml

@@ -34,9 +34,11 @@ appAttestRoot:
   secretKey: app-attest-root.pem
 service:
   port: 8080
+podAnnotations: {}
 httpRoute:
   # Disabled by default so a generic install cannot claim an unattached route.
-  # Production enables this with an explicit Gateway parentRef.
+  # Enable only when this chart owns a Gateway API route. Environments with an
+  # existing ingress or service mesh route should keep this disabled.
   enabled: false
   parentRefs: []
   hostnames: [push.buzz.xyz]
@@ -60,8 +62,9 @@ networkPolicy:
     podSelector:
       k8s-app: kube-dns
   # Scoped ingress to the private metrics port (8081). Off by default so 8081
-  # has no pod ingress at all; enable only alongside podMonitor and name the
-  # scraper's namespace/pod so reachability stays narrow.
+  # has no pod ingress at all; enable alongside podMonitor or an external
+  # annotation-discovered scraper and name its namespace/pod so reachability
+  # stays narrow.
   monitoring:
     enabled: false
     namespaceSelector: {}

docs/push-gateway-deployment.md

@@ -67,6 +67,7 @@ The gateway serves Prometheus metrics at `GET /metrics` on the **private health
 
 | Metric | Type | Labels | Meaning |
 |---|---|---|---|
+| `push_gateway_apns_send_attempts_total` | counter | none | Entries into the concrete APNs HTTP send seam. Compare with terminal outcomes to detect work that never reached transport. |
 | `push_gateway_apns_deliveries_total` | counter | `outcome` = `accepted` \| `invalid_endpoint` \| `retry` \| `configuration_fault` \| `permanent_request_fault` | Terminal APNs send outcomes. |
 | `push_gateway_apns_delivery_seconds` | histogram | — | APNs send round-trip latency (seconds). |
 | `push_gateway_admissions_total` | counter | `result` = `admitted` \| `rejected` \| `unavailable` | Outcome at the `authorize_delivery` replay/quota fence. |
@@ -74,9 +75,38 @@ The gateway serves Prometheus metrics at `GET /metrics` on the **private health
 | `push_gateway_reaper_failures_total` | counter | — | Retention reaper sweep failures. |
 | `push_gateway_readiness_failures_total` | counter | `cause` = `not_accepting` \| `authority` | Readiness probe failures by cause. |
 
-`push_gateway_delivery_errors_total` is intentionally **narrow**: it counts only selected exit classes of the `/v1/deliveries/apns` handler — `class` ∈ `invalid_grant` (grant rejected at the admission seam, before a permit is issued), `temporarily_unavailable` (authority unavailable at the admission seam), `profile_mismatch`, `token_custody` (endpoint-token open failure), `finish_failed` (detached disposition/join failure returned as 503). Request/auth/attestation/grant validation on the enrollment, delegation, rotation, and revocation handlers is **not** counted by this metric; it is a delivery-hot-path signal, not a total error rate across the API.
+`push_gateway_delivery_errors_total` is intentionally **narrow**: it counts only selected exit classes of the `/v1/deliveries/apns` handler. `class` ∈ `invalid_grant` (grant rejected at the admission seam, before a permit is issued), `rate_limited`, `temporarily_unavailable` (authority unavailable at the admission seam), `profile_mismatch`, `profile_disabled`, `token_custody` (endpoint-token open failure), `finish_failed` (detached disposition/join failure returned as 503). Request/auth/attestation/grant validation on the enrollment, delegation, rotation, and revocation handlers is **not** counted by this metric; it is a delivery-hot-path signal, not a total error rate across the API.
 
-Scraping is **opt-in** and off by default, so the default chart render is unchanged and `8081` keeps no pod ingress. To enable it, set `podMonitor.enabled=true` (renders a prometheus-operator `PodMonitor` scraping the `health` port `/metrics`) and `networkPolicy.monitoring.enabled=true` with `networkPolicy.monitoring.namespaceSelector` / `podSelector` naming your scraper — this adds a single `8081` ingress rule scoped to that source, never a blanket allowance. Node/kubelet-origin probe traffic remains exempt from NetworkPolicy regardless.
+Scraping is **opt-in** and off by default, so the default chart render is unchanged and `8081` keeps no pod ingress. For prometheus-operator, set `podMonitor.enabled=true` and `networkPolicy.monitoring.enabled=true` with `networkPolicy.monitoring.namespaceSelector` / `podSelector` naming your scraper. For Datadog Autodiscovery, leave `podMonitor.enabled=false`, supply the OpenMetrics check through `podAnnotations`, and enable the same narrowly selected NetworkPolicy ingress:
+
+```yaml
+podAnnotations:
+  ad.datadoghq.com/gateway.checks: |
+    {
+      "openmetrics": {
+        "init_config": {},
+        "instances": [{
+          "openmetrics_endpoint": "http://%%host%%:8081/metrics",
+          "service": "buzz-push-gateway",
+          "namespace": "block.buzz_push_gateway",
+          "metrics": ["push_gateway_.*"],
+          "histogram_buckets_as_distributions": true,
+          "send_distribution_buckets": true,
+          "send_monotonic_counter": true,
+          "collect_counters_with_distributions": true
+        }]
+      }
+    }
+networkPolicy:
+  monitoring:
+    enabled: true
+    namespaceSelector: # replace with the Datadog Agent namespace labels
+      kubernetes.io/metadata.name: datadog
+    podSelector: # replace with the Datadog Agent pod labels
+      app.kubernetes.io/name: datadog-agent
+```
+
+Both modes add one `8081` ingress rule scoped to the configured source, never a blanket allowance. Node/kubelet-origin probe traffic remains exempt from NetworkPolicy regardless. Do not enable `PodMonitor` in clusters without its CRD.
 
 Alerting rules ship as an opt-in prometheus-operator `PrometheusRule` (`prometheusRule.enabled=true`). Thresholds and operator actions:
 
@@ -189,10 +219,22 @@ The chart defaults to the `main` image tag because `.github/workflows/docker.yml
 ```bash
 gh attestation verify \
   oci://ghcr.io/block/buzz-push-gateway@sha256:<64-lowercase-hex> \
-  --owner block
+  --repo block/buzz \
+  --signer-workflow block/buzz/.github/workflows/docker.yml \
+  --source-digest <40-lowercase-hex-source-commit>
 ```
 
-Only after that command succeeds, set the exact digest as `image.digest`; the chart then renders `ghcr.io/block/buzz-push-gateway@sha256:...` and ignores the mutable tag. `values-production.yaml` is an intentionally invalid production-input contract: deployment CI must inject this verified `image.digest`, the provisioned dogfood Apple application identifier, an environment-owned Gateway parent reference, and the actual PostgreSQL network. Schema validation rejects the artifact when any remains empty; the render guard proves both rejection and a fully injected render.
+Only after that command succeeds, inject the exact digest as `image.digest` in
+the environment's GitOps values; the chart then renders
+`ghcr.io/block/buzz-push-gateway@sha256:...` and ignores the mutable tag.
+`values-production.yaml` remains an intentionally invalid production-input
+contract: deployment CI must inject the verified image digest, the provisioned
+dogfood Apple application identifier, and the actual PostgreSQL network. In an
+environment with an existing ingress or service mesh route, keep
+`httpRoute.enabled=false`. If this chart owns a Gateway API route, enable it and
+inject an environment-owned `parentRef`; schema validation rejects an enabled
+route with no parent. The render guard proves both rejection of missing required
+inputs and fully injected renders.
 
 Network policy keeps APNs HTTPS and PostgreSQL egress in separate CIDR lists. APNs currently requires broad TCP/443 reachability; `networkPolicy.postgresEgressCidrs` must be narrowed to the production database network, and the DNS namespace/pod selectors must match the cluster DNS deployment. The sample private CIDR is not a claim about the production topology.
 
@@ -201,8 +243,9 @@ Kubernetes does not restart pods when referenced Secret bytes change. AEAD or AP
 ## Gateway chart release
 
 The gateway chart has a collision-free release lane separate from the main
-`buzz` chart. To publish version `X.Y.Z`, update both `version` and `appVersion`
-in `deploy/charts/buzz-push-gateway/Chart.yaml`, validate the chart, and open a
+`buzz` chart. To publish chart version `X.Y.Z`, update `version` in
+`deploy/charts/buzz-push-gateway/Chart.yaml` and keep `appVersion` equal to the
+gateway binary's workspace package version. Validate the chart, then open a
 same-repository PR whose branch is exactly `push-chart-release/X.Y.Z`:
 
 ```bash
@@ -220,3 +263,10 @@ version. The publisher verifies the checked-out commit is the tag target and the
 chart version equals `X.Y.Z` before pushing
 `oci://ghcr.io/block/buzz/charts/buzz-push-gateway`. A manually pushed
 `push-chart-vX.Y.Z` tag is the documented rescue path and runs the same checks.
+After the publisher succeeds, inspect and fetch the published chart version
+before use:
+
+```bash
+helm show chart oci://ghcr.io/block/buzz/charts/buzz-push-gateway --version X.Y.Z
+helm pull oci://ghcr.io/block/buzz/charts/buzz-push-gateway --version X.Y.Z
+```

migrations/0043_push_gateway_dogfood_profile.sql

@@ -0,0 +1,19 @@
+-- The internal MVP accepts only the dogfood application profile. The legacy
+-- profile names encoded APNs transport environment rather than a verified
+-- application identity, so they cannot be mapped safely to dogfood authority.
+-- Retire their delegations and installations before narrowing the constraint.
+DELETE FROM push_gateway_delegations
+WHERE installation_id IN (
+    SELECT id
+    FROM push_gateway_installations
+    WHERE app_profile <> 'buzz-ios-dogfood'
+);
+
+DELETE FROM push_gateway_installations
+WHERE app_profile <> 'buzz-ios-dogfood';
+
+ALTER TABLE push_gateway_installations
+    DROP CONSTRAINT push_gateway_installations_app_profile_check;
+ALTER TABLE push_gateway_installations
+    ADD CONSTRAINT push_gateway_installations_app_profile_check
+    CHECK (app_profile = 'buzz-ios-dogfood');

schema/schema.sql

@@ -1134,7 +1134,7 @@ CREATE TABLE push_gateway_installations (
     app_attest_key_id BYTEA NOT NULL UNIQUE CHECK (octet_length(app_attest_key_id) BETWEEN 1 AND 128),
     app_attest_public_key BYTEA NOT NULL CHECK (octet_length(app_attest_public_key) BETWEEN 33 AND 256),
     assertion_counter BIGINT NOT NULL CHECK (assertion_counter BETWEEN 0 AND 4294967295),
-    app_profile TEXT NOT NULL CHECK (app_profile IN ('buzz-ios-production','buzz-ios-sandbox')),
+    app_profile TEXT NOT NULL CHECK (app_profile = 'buzz-ios-dogfood'),
     token_ciphertext BYTEA NOT NULL CHECK (octet_length(token_ciphertext) BETWEEN 1 AND 2048),
     token_fingerprint BYTEA NOT NULL CHECK (length(token_fingerprint) = 32),
     endpoint_epoch BIGINT NOT NULL CHECK (endpoint_epoch > 0),