From b3baa56ba83d01a0b7292d81aae9370f5616c62b Mon Sep 17 00:00:00 2001 From: Tyler <109685178+tlongwell-block@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:47:35 -0400 Subject: [PATCH] fix(db): exclude kind:30179 ciphertext from brownfield FTS (#6822) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem Kind `30179` (NIP-PMA private managed agent — NIP-44 ciphertext carrying the agent nsec, env vars, prompt) joined `AUTHOR_ONLY_KINDS` in #4593, but only **fresh** installs stopped indexing it: - `migrations/0008` installs the positive FTS allowlist only when `events` is empty; - `migrations/0014` wraps the retained brownfield expression for `30350` alone; - `schema/schema.sql` still carries the legacy negative skip-set without `30179`. So a relay upgraded in place keeps tokenizing `30179` ciphertext into `events.search_tsv`. Found by Wren while reviewing #4999 (which activates publication of 30179 from Desktop). No readable leak: `/query` applies `search_hit_accepted` + `event_visible_to_reader` before serialization and live foreign searches against builderlab returned `[]` — this is the storage-layer privacy invariant (`docs/nips/NIP-PMA.md` deployment step 2) plus wasted FTS work. ## Fix - `migrations/0033_private_managed_agent_fts.sql` — same shape as 0014: capture the current generated expression via `pg_get_expr`, drop/re-add `search_tsv` wrapped with `kind = 30179 → NULL`. Every other kind keeps whatever policy the database already had (fresh allowlist or brownfield skip-set). Rebuilds the GIN index; no heap-wide policy rewrite. - `schema/schema.sql` — add `30179` to the desired-state skip-set. ## Tests - `populated_upgrade_preserves_search_policy_except_for_private_kinds` (renamed from `…_push_leases`): provisions the **legacy negative expression** (migrations 1–7), inserts kind 1 / 30179 / 30350 rows, checkpoints after 0014 (30179 still searchable, 30350 not), then runs to head and asserts 30179 is NULL. **Proven red** with the 0033 file removed: `left: [(1, Some(true)), (30179, Some(true)), (30350, None)]`. - `embedded_migrator_contains_consolidated_initial_schema`: count 32→33, asserts 0033 shape and `schema.sql` parity. - `buzz-search/tests/fts_integration.rs` setup applies 0033 so the FTS tripwires run against the real chain. ## Verification (HEAD 6154cc33d, same shell) - `cargo test -p buzz-db -- --include-ignored --test-threads=1`: 305 passed / 3 failed — the 3 (`create_community_with_owner_enforces_per_owner_limit`, `insert_mentions_indexes_rosters_past_bind_parameter_cap`, `transfer_ownership_returns_limit_reached_for_maxed_transferee`) fail identically on pristine `main` f24971033 with a fresh DB; unrelated to this diff. - `cargo test -p buzz-search -- --include-ignored`: 19 passed (incl. `author_only_kinds_are_storage_level_unsearchable`, `p_gated_persistent_kinds_have_storage_null_tsvector`). - `desired_state_schema_bootstrap_progresses_beyond_fencing` + `run_migrations_applies_consolidated_initial_schema_on_fresh_database`: pass. - `cargo fmt --all --check` clean; `cargo clippy -p buzz-db -p buzz-search --all-targets -D warnings` clean. Independent of #4999 — both main and #4999 are brownfield-exposed today; this lands either order. --------- Signed-off-by: Meli <5aaa86bce934fc3445fc254aab560a40923f10252f92107e665073dede0e04d3@buzz.block.builderlab.xyz> Signed-off-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Co-authored-by: Meli <5aaa86bce934fc3445fc254aab560a40923f10252f92107e665073dede0e04d3@buzz.block.builderlab.xyz> Co-authored-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> --- crates/buzz-db/src/migration.rs | 44 +++++++++++++++--- crates/buzz-search/tests/fts_integration.rs | 5 ++ migrations/0033_private_managed_agent_fts.sql | 46 +++++++++++++++++++ schema/schema.sql | 4 +- 4 files changed, 90 insertions(+), 9 deletions(-) create mode 100644 migrations/0033_private_managed_agent_fts.sql diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index 9df02c8abf..75201afba8 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -645,7 +645,7 @@ mod tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 32); + assert_eq!(migrations.len(), 33); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -844,6 +844,18 @@ mod tests { assert!(migrations[13].sql.as_str().contains("search_tsv")); assert!(!migrations[0].sql.as_str().contains("30350")); + // NIP-PMA kind:30179 FTS exclusion (0033): same wrap-the-existing- + // expression shape as 0014 so brownfield databases stop tokenizing + // private managed-agent ciphertext without a policy rewrite. (The + // migration itself still rewrites the events heap and rebuilds the + // GIN index — see the 0033 header for the operational cost.) + assert_eq!(migrations[32].version, 33); + assert!(migrations[32].sql.as_str().contains("kind = 30179")); + assert!(migrations[32].sql.as_str().contains("search_tsv")); + assert!(!migrations[0].sql.as_str().contains("30179")); + assert!(include_str!("../../../schema/schema.sql") + .contains("kind IN (1059, 30179, 30300, 30350, 30622, 44100, 44101, 44200)")); + // Public push-gateway authority is intentionally deployment-global and // durable: immediate revocation and hostile-relay admission cannot be // honestly provided by a stateless gateway. @@ -1915,7 +1927,7 @@ mod tests { #[tokio::test] #[ignore = "requires Postgres"] - async fn populated_upgrade_preserves_search_policy_except_for_push_leases() { + async fn populated_upgrade_preserves_search_policy_except_for_private_kinds() { let pool = connect_test_pool().await; reset_public_schema(&pool).await; MIGRATOR @@ -1931,7 +1943,7 @@ mod tests { .await .expect("insert community"); - for (marker, kind) in [(1_u8, 1_i32), (2_u8, 30_350_i32)] { + for (marker, kind) in [(1_u8, 1_i32), (2_u8, 30_350_i32), (3_u8, 30_179_i32)] { sqlx::query( "INSERT INTO events \ (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at) \ @@ -1958,19 +1970,37 @@ mod tests { .fetch_all(&pool) .await .expect("read pre-push search behavior"); - assert_eq!(before, vec![(1, true), (30_350, true)]); + assert_eq!(before, vec![(1, true), (30_179, true), (30_350, true)]); + + // 0014 fixes 30350 only. A brownfield database that stopped here still + // tokenized kind:30179 ciphertext — the gap 0033 closes. + MIGRATOR + .run_to(32, &pool) + .await + .expect("apply migrations through 32"); + let pre_0033: Vec<(i32, Option)> = sqlx::query_as( + "SELECT kind, search_tsv @@ plainto_tsquery('simple', 'needle') \ + FROM events ORDER BY kind", + ) + .fetch_all(&pool) + .await + .expect("read pre-0033 search behavior"); + assert_eq!( + pre_0033, + vec![(1, Some(true)), (30_179, Some(true)), (30_350, None)] + ); run_migrations(&pool) .await - .expect("apply push migrations to populated database"); + .expect("apply remaining migrations to populated database"); let after: Vec<(i32, Option)> = sqlx::query_as( "SELECT kind, search_tsv @@ plainto_tsquery('simple', 'needle') \ FROM events ORDER BY kind", ) .fetch_all(&pool) .await - .expect("read post-push search behavior"); - assert_eq!(after, vec![(1, Some(true)), (30_350, None)]); + .expect("read post-upgrade search behavior"); + assert_eq!(after, vec![(1, Some(true)), (30_179, None), (30_350, None)]); } #[tokio::test] diff --git a/crates/buzz-search/tests/fts_integration.rs b/crates/buzz-search/tests/fts_integration.rs index e7c196ee3e..175a01aaaa 100644 --- a/crates/buzz-search/tests/fts_integration.rs +++ b/crates/buzz-search/tests/fts_integration.rs @@ -28,6 +28,8 @@ const MIGRATION_0007_SQL: &str = include_str!("../../../migrations/0007_nip_rs_r const MIGRATION_0008_SQL: &str = include_str!("../../../migrations/0008_fresh_install_search_allowlist.sql"); const MIGRATION_0014_SQL: &str = include_str!("../../../migrations/0014_push_lease_fts.sql"); +const MIGRATION_0033_SQL: &str = + include_str!("../../../migrations/0033_private_managed_agent_fts.sql"); async fn setup() -> (PgPool, String) { let url = std::env::var("BUZZ_TEST_DATABASE_URL").unwrap_or_else(|_| TEST_DB_URL.to_string()); @@ -81,6 +83,9 @@ async fn setup() -> (PgPool, String) { pool.execute(MIGRATION_0014_SQL) .await .expect("apply 0014 migration"); + pool.execute(MIGRATION_0033_SQL) + .await + .expect("apply 0033 migration"); (pool, schema) } diff --git a/migrations/0033_private_managed_agent_fts.sql b/migrations/0033_private_managed_agent_fts.sql new file mode 100644 index 0000000000..d2c2981396 --- /dev/null +++ b/migrations/0033_private_managed_agent_fts.sql @@ -0,0 +1,46 @@ +-- NIP-PMA kind:30179 carries the owner's private managed-agent payload +-- (agent nsec, env vars, prompt) as NIP-44 ciphertext and is author-only. +-- Exclude it from full-text search without changing the search policy of +-- existing installations. Fresh installs are already safe via the positive +-- allowlist from migration 0008; this closes the brownfield gap where a +-- populated database still runs the legacy negative skip-set (0001/0005) +-- and would tokenize the ciphertext into search_tsv. +-- +-- Same shape as 0014 (kind:30350): PostgreSQL cannot alter a generated +-- expression in place, so capture the current expression, drop the column, +-- and re-add it wrapped with the new exclusion. Every other kind keeps +-- whatever policy the database had before. +-- +-- Operational cost: this is not free on large databases. DROP COLUMN + +-- ADD ... GENERATED ... STORED rewrites the entire events heap and then +-- rebuilds the GIN index, all under an ACCESS EXCLUSIVE lock inside the +-- migration transaction (CREATE INDEX CONCURRENTLY is not possible +-- here), with no lock_timeout. Expect relay downtime proportional to +-- the size of events. The index is recreated from the stock definition +-- below; any non-stock indexes or storage parameters on search_tsv are +-- not captured or replayed. 0014 set this precedent on smaller tables; +-- operators with large brownfield databases should schedule a window. +DO $$ +DECLARE + existing_expression TEXT; +BEGIN + SELECT pg_get_expr(d.adbin, d.adrelid) + INTO existing_expression + FROM pg_attrdef d + JOIN pg_attribute a + ON a.attrelid = d.adrelid + AND a.attnum = d.adnum + WHERE d.adrelid = 'events'::regclass + AND a.attname = 'search_tsv'; + + IF existing_expression IS NULL THEN + RAISE EXCEPTION 'events.search_tsv generated expression not found'; + END IF; + + ALTER TABLE events DROP COLUMN search_tsv; + EXECUTE format( + 'ALTER TABLE events ADD COLUMN search_tsv TSVECTOR GENERATED ALWAYS AS (CASE WHEN kind = 30179 THEN NULL::tsvector ELSE (%s) END) STORED', + existing_expression + ); + CREATE INDEX idx_events_search_tsv ON events USING GIN (search_tsv); +END $$; diff --git a/schema/schema.sql b/schema/schema.sql index 6e14e6be1b..c461039485 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -219,9 +219,9 @@ CREATE TABLE events ( -- Privacy: encrypted/private routing wrappers and p-gated membership notices -- must never be discoverable through NIP-50 full-text search. NULL tsvector -- never matches `@@`. - -- Keep in sync with migrations (final state: 0001 + 0005 + 0009). + -- Keep in sync with migrations (final state: 0001 + 0005 + 0014 + 0033). search_tsv TSVECTOR GENERATED ALWAYS AS ( - CASE WHEN kind IN (1059, 30300, 30350, 30622, 44100, 44101, 44200) THEN NULL::tsvector + CASE WHEN kind IN (1059, 30179, 30300, 30350, 30622, 44100, 44101, 44200) THEN NULL::tsvector ELSE to_tsvector('simple', content) END ) STORED,