fix(db): disable heartbeat vacuum truncation (#6898)

## Why
`replica_heartbeat` is updated continuously, but autovacuum heap
truncation can take an `ACCESS EXCLUSIVE` lock whose replay cancels
concurrent hot-standby reads. Truncating a fixed, single-row table
provides no meaningful space benefit.

## What
- Add migration 0034 to set `vacuum_truncate = false` on
`replica_heartbeat`
- Keep the desired-state schema aligned for fresh installations
- Extend migration coverage so both paths retain the reloption

## Risk Assessment
Low — this disables only autovacuum's heap-truncation phase for a
single-row table; normal vacuum cleanup remains enabled. The tradeoff is
retaining at most the table's otherwise-truncatable tail pages.

## References
- [Investigation
thread](https://sq-block.slack.com/archives/C0B2WT43BDH/p1787760831375989)
- [PostgreSQL table storage
parameters](https://www.postgresql.org/docs/current/sql-createtable.html#SQL-CREATETABLE-STORAGE-PARAMETERS)

Generated with Codex

---

**Update Aug 27, 2026:** Hardened fresh `pgschema` bootstraps after
review.

- Renamed the shared post-apply script to
`reconcile-schema-after-pgschema.sql` and made it restore the heartbeat
reloption and singleton row idempotently.
- Added live catalog and row assertions that stop bootstrap when the
database does not match the required state.
- Updated every repository `pgschema apply` caller, including the
desktop release smoke path.
- Added a regression test that requires future `pgschema apply` callers
to run reconciliation.
- Documented the `pgschema` DML and storage-parameter limitation in
`AGENTS.md` and at the reconciliation entry point.
- Merged current `main` and renumbered this migration to 0034 after
`main` added migration 0033.

---------

Signed-off-by: Luke Tornquist <tornquist@squareup.com>

.github/workflows/ci.yml

@@ -516,7 +516,7 @@ jobs:
         run: |
           ./bin/pgschema apply --file schema/schema.sql --auto-approve
           docker exec -i -e PGPASSWORD=buzz_dev buzz-postgres \
-            psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/attach-schema-partitions.sql
+            psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/reconcile-schema-after-pgschema.sql
           docker exec -e PGPASSWORD=buzz_dev buzz-postgres \
             psql -U buzz -d buzz -qtA -c "
           INSERT INTO communities (id, host)
@@ -677,7 +677,7 @@ jobs:
         run: |
           ./bin/pgschema apply --file schema/schema.sql --auto-approve
           docker exec -i -e PGPASSWORD=buzz_dev buzz-postgres \
-            psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/attach-schema-partitions.sql
+            psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/reconcile-schema-after-pgschema.sql
           docker exec -e PGPASSWORD=buzz_dev buzz-postgres \
             psql -U buzz -d buzz -qtA -c "
           INSERT INTO communities (id, host)

AGENTS.md

@@ -469,6 +469,7 @@ description. See [PR #803](https://github.com/block/buzz/pull/803).
 4. **Worktrees: `cd` in the same command** — shell CWD doesn't persist between tool calls. Use `cd /path && cargo build` as one command.
 5. **Desktop crate excluded from root workspace** — `cargo test` at repo root does NOT run desktop tests. Use `cargo test --manifest-path desktop/src-tauri/Cargo.toml` explicitly.
 6. **React render perf: `React.memo` is all-or-nothing** — it only skips a re-render when *every* prop is reference-stable; one unstable prop (inline arrow/JSX, or a hook returning a fresh `{}`/`[]`/`Map` each render) defeats it. Two repeat offenders: (a) React Query results (`useMutation`/`useQuery`) are a **new object each render** — depend on the stable method (`mutation.mutateAsync`), not the object; (b) derived `Map`/array state that recomputes on a version bump — wrap in a content-equality ref cache (`shared/hooks/useStableReference.ts`). When chasing interaction lag, **measure with DevTools closed and no perf probes** (an open Web Inspector + per-keystroke `console.log` inflate the numbers), and isolate by removing one suspect at a time rather than guessing.
+7. **`pgschema` omits seed DML and some storage parameters** — Fresh desired-state bootstraps use `./bin/pgschema apply`, which does not execute `INSERT` statements or preserve every table storage parameter from `schema/schema.sql`. Put each unsupported invariant in `scripts/reconcile-schema-after-pgschema.sql` as an idempotent convergence statement plus a live catalog or data assertion. Every `pgschema apply` caller must run that script. A string assertion against `schema.sql` alone does not prove the pgschema-created database has the intended state.
 
 ---
 

crates/buzz-db/src/migration.rs

@@ -175,7 +175,11 @@ async fn reject_legacy_nip_rs_cardinality_ambiguity(conn: &mut PgConnection) ->
 #[cfg(test)]
 mod tests {
     use super::*;
-    use std::collections::BTreeSet;
+    use std::{
+        collections::BTreeSet,
+        fs,
+        path::{Path, PathBuf},
+    };
 
     const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; // sadscan:disable np.postgres.1
 
@@ -645,7 +649,7 @@ mod tests {
         let mut migrations: Vec<_> = MIGRATOR.iter().collect();
         migrations.sort_by_key(|migration| migration.version);
 
-        assert_eq!(migrations.len(), 33);
+        assert_eq!(migrations.len(), 34);
         assert_eq!(migrations[0].version, 1);
         assert_eq!(&*migrations[0].description, "initial schema");
         assert!(migrations[0]
@@ -1098,6 +1102,85 @@ mod tests {
             extract_roster_fence(roster_fence),
             extract_roster_fence(desired_schema)
         );
+
+        // The single-row heartbeat table is updated continuously. Prevent
+        // autovacuum from truncating its heap so standby queries are not
+        // cancelled by the ACCESS EXCLUSIVE truncation lock replay.
+        assert_eq!(migrations[33].version, 34);
+        let heartbeat_vacuum = migrations[33].sql.as_str();
+        assert!(heartbeat_vacuum.contains("ALTER TABLE replica_heartbeat"));
+        assert!(heartbeat_vacuum.contains("vacuum_truncate = false"));
+        assert!(desired_schema.contains("vacuum_truncate = false"));
+
+        // pgschema intentionally reconciles DDL, not seed DML or table storage
+        // parameters. Its post-apply reconciliation must restore and verify
+        // both parts of the live heartbeat contract for fresh bootstraps.
+        let pgschema_reconciliation =
+            include_str!("../../../scripts/reconcile-schema-after-pgschema.sql");
+        assert!(pgschema_reconciliation
+            .contains("ALTER TABLE replica_heartbeat SET (vacuum_truncate = false)"));
+        assert!(pgschema_reconciliation.contains("INSERT INTO replica_heartbeat (id) VALUES (1)"));
+        assert!(pgschema_reconciliation.contains("ON CONFLICT (id) DO NOTHING"));
+        assert!(pgschema_reconciliation.contains("pg_class"));
+        assert!(pgschema_reconciliation.contains("reloptions"));
+    }
+
+    #[test]
+    fn every_pgschema_apply_runs_post_apply_reconciliation() {
+        fn files_under(root: &Path) -> Vec<PathBuf> {
+            let mut pending = vec![root.to_owned()];
+            let mut files = Vec::new();
+
+            while let Some(path) = pending.pop() {
+                for entry in fs::read_dir(&path)
+                    .unwrap_or_else(|error| panic!("could not read {}: {error}", path.display()))
+                {
+                    let path = entry.expect("directory entry").path();
+                    if path.is_dir() {
+                        pending.push(path);
+                    } else {
+                        files.push(path);
+                    }
+                }
+            }
+
+            files
+        }
+
+        let repo_root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
+        let roots = [
+            repo_root.join("scripts"),
+            repo_root.join(".github/workflows"),
+        ];
+        let mut apply_count = 0;
+
+        for path in roots.iter().flat_map(|root| files_under(root)) {
+            let Ok(contents) = fs::read_to_string(&path) else {
+                continue;
+            };
+            let lines: Vec<_> = contents.lines().collect();
+
+            for (index, line) in lines.iter().enumerate() {
+                if !line.contains("./bin/pgschema apply") {
+                    continue;
+                }
+
+                apply_count += 1;
+                let following_lines = &lines[index + 1..(index + 7).min(lines.len())];
+                assert!(
+                    following_lines.iter().any(|line| line.contains(
+                        "scripts/reconcile-schema-after-pgschema.sql"
+                    )),
+                    "{} must run scripts/reconcile-schema-after-pgschema.sql immediately after pgschema apply",
+                    path.display()
+                );
+            }
+        }
+
+        assert!(
+            apply_count > 0,
+            "expected at least one pgschema apply caller"
+        );
     }
 
     #[test]

migrations/0034_replica_heartbeat_vacuum_truncate.sql

@@ -0,0 +1,7 @@
+-- replica_heartbeat is a single-row table updated continuously by every relay
+-- pod. Autovacuum heap truncation briefly takes ACCESS EXCLUSIVE on the table;
+-- replaying that lock on a hot standby can cancel concurrent heartbeat reads.
+-- The table cannot reclaim meaningful disk space by truncating one page, so
+-- disable only the truncation phase while retaining normal autovacuum cleanup.
+
+ALTER TABLE replica_heartbeat SET (vacuum_truncate = false);

schema/schema.sql

@@ -1199,6 +1199,8 @@ CREATE TABLE replica_heartbeat (
     id    smallint PRIMARY KEY CHECK (id = 1),
     epoch uuid     NOT NULL DEFAULT gen_random_uuid(),
     token bigint   NOT NULL DEFAULT 0
+) WITH (
+    vacuum_truncate = false
 );
 
 INSERT INTO replica_heartbeat (id) VALUES (1);

scripts/reconcile-schema-after-pgschema.sql

@@ -1,11 +1,10 @@
--- Attach partition child tables after pgschema apply.
+-- Reconcile schema details that pgschema does not preserve.
 --
--- pgschema currently emits existing partition children as standalone CREATE TABLE
--- statements when applying schema/schema.sql in CI. The tables exist, but they
--- are not attached to their partitioned parents, so inserts into events or
--- delivery_log fail with "no partition of relation ... found for row". Keep this
--- idempotent: raw psql/schema.sql already attaches these partitions, while
--- pgschema-created schemas need this repair step.
+-- pgschema reconciles DDL, but it does not execute seed DML or preserve every
+-- table storage parameter from schema/schema.sql. It also currently emits
+-- partition children as standalone CREATE TABLE statements. Every pgschema
+-- apply caller must run this idempotent script so fresh bootstraps converge on
+-- the same live database contract as migration-managed databases.
 
 DO $$
 BEGIN
@@ -197,3 +196,32 @@ BEGIN
             FOR VALUES FROM ('2026-07-01') TO (MAXVALUE);
     END IF;
 END $$;
+
+-- pgschema reconciles DDL but does not apply seed DML or table storage
+-- parameters from schema/schema.sql. Restore those parts of the desired-state
+-- contract explicitly and fail the bootstrap if the live catalog disagrees.
+ALTER TABLE replica_heartbeat SET (vacuum_truncate = false);
+
+INSERT INTO replica_heartbeat (id) VALUES (1)
+ON CONFLICT (id) DO NOTHING;
+
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1
+        FROM pg_class AS relation
+        JOIN pg_namespace AS namespace ON namespace.oid = relation.relnamespace
+        WHERE namespace.nspname = current_schema()
+          AND relation.relname = 'replica_heartbeat'
+          AND COALESCE(
+              relation.reloptions @> ARRAY['vacuum_truncate=false']::text[],
+              false
+          )
+    ) THEN
+        RAISE EXCEPTION 'replica_heartbeat must disable vacuum truncation after pgschema apply';
+    END IF;
+
+    IF (SELECT count(*) FROM replica_heartbeat WHERE id = 1) <> 1 THEN
+        RAISE EXCEPTION 'replica_heartbeat must contain its singleton row after pgschema apply';
+    END IF;
+END $$;

scripts/run-desktop-release-smoke.sh

@@ -89,7 +89,7 @@ export PGSCHEMA_PLAN_HOST=localhost PGSCHEMA_PLAN_PORT=5432 PGSCHEMA_PLAN_DB="${
 export PGSCHEMA_PLAN_USER=buzz PGSCHEMA_PLAN_PASSWORD=buzz_dev
 ./bin/pgschema apply --file schema/schema.sql --auto-approve
 docker exec -i -e PGPASSWORD=buzz_dev buzz-postgres \
-  psql -U buzz -d "${DB_NAME}" -v ON_ERROR_STOP=1 < scripts/attach-schema-partitions.sql
+  psql -U buzz -d "${DB_NAME}" -v ON_ERROR_STOP=1 < scripts/reconcile-schema-after-pgschema.sql
 BUZZ_DB_NAME="${DB_NAME}" BUZZ_COMMUNITY_HOST="${COMMUNITY_HOST}" ./scripts/setup-desktop-test-data.sh
 docker exec buzz-redis redis-cli -n "${REDIS_DB}" FLUSHDB >/dev/null
 phase database "${phase_start}"

scripts/start-isolated-test-relay.sh

@@ -98,7 +98,7 @@ export PGSCHEMA_PLAN_HOST=localhost PGSCHEMA_PLAN_PORT=${PG_PORT}
 export PGSCHEMA_PLAN_DB=buzz PGSCHEMA_PLAN_USER=buzz PGSCHEMA_PLAN_PASSWORD=buzz_dev
 export PGHOST=localhost PGPORT=${PG_PORT} PGUSER=buzz PGDATABASE=buzz
 ./bin/pgschema apply --file schema/schema.sql --auto-approve
-psql_h < scripts/attach-schema-partitions.sql
+psql_h < scripts/reconcile-schema-after-pgschema.sql
 ok "Schema applied"
 
 # ── Deployment community + channels + members ────────────────────────────────

scripts/start-relay-for-tests.sh

@@ -105,7 +105,7 @@ export PGSCHEMA_PLAN_PASSWORD=buzz_dev
 
 ./bin/pgschema apply --file schema/schema.sql --auto-approve
 docker exec -i -e PGPASSWORD="${PGPASSWORD}" buzz-postgres \
-  psql -U "${PGUSER}" -d "${PGDATABASE}" -v ON_ERROR_STOP=1 < scripts/attach-schema-partitions.sql
+  psql -U "${PGUSER}" -d "${PGDATABASE}" -v ON_ERROR_STOP=1 < scripts/reconcile-schema-after-pgschema.sql
 ok "Schema applied"
 
 # ── Seed the deployment community ────────────────────────────────────────────