mirror of
https://github.com/block/buzz.git
synced 2026-10-27 12:40:30 +00:00
fix(db): exclude kind:30179 ciphertext from brownfield FTS
Kind 30179 (NIP-PMA private managed agent) joined AUTHOR_ONLY_KINDS in #4593, but only fresh installs stopped indexing it: migration 0008's positive allowlist never touches populated databases, and 0014 wrapped the retained brownfield expression for 30350 alone. A relay upgraded in place still tokenizes the NIP-44 ciphertext into events.search_tsv. Add migration 0033 in the 0014 shape: capture the current generated expression and re-add search_tsv wrapped with kind = 30179 -> NULL, so every other kind keeps whatever policy the database already had. Mirror the exclusion in schema/schema.sql. Tests: extend the populated-upgrade migration test with a 30179 row and a pre-0033 checkpoint (fails without the new file); assert 0033's shape in the embedded-migrator test; apply 0033 in the buzz-search FTS setup. Co-authored-by: Meli <5aaa86bce934fc3445fc254aab560a40923f10252f92107e665073dede0e04d3@buzz.block.builderlab.xyz> Signed-off-by: Meli <5aaa86bce934fc3445fc254aab560a40923f10252f92107e665073dede0e04d3@buzz.block.builderlab.xyz>
This commit is contained in:
@@ -645,7 +645,7 @@ mod tests {
|
|||||||
let mut migrations: Vec<_> = MIGRATOR.iter().collect();
|
let mut migrations: Vec<_> = MIGRATOR.iter().collect();
|
||||||
migrations.sort_by_key(|migration| migration.version);
|
migrations.sort_by_key(|migration| migration.version);
|
||||||
|
|
||||||
assert_eq!(migrations.len(), 32);
|
assert_eq!(migrations.len(), 33);
|
||||||
assert_eq!(migrations[0].version, 1);
|
assert_eq!(migrations[0].version, 1);
|
||||||
assert_eq!(&*migrations[0].description, "initial schema");
|
assert_eq!(&*migrations[0].description, "initial schema");
|
||||||
assert!(migrations[0]
|
assert!(migrations[0]
|
||||||
@@ -844,6 +844,16 @@ mod tests {
|
|||||||
assert!(migrations[13].sql.as_str().contains("search_tsv"));
|
assert!(migrations[13].sql.as_str().contains("search_tsv"));
|
||||||
assert!(!migrations[0].sql.as_str().contains("30350"));
|
assert!(!migrations[0].sql.as_str().contains("30350"));
|
||||||
|
|
||||||
|
// NIP-PMA kind:30179 FTS exclusion (0033): same wrap-the-existing-
|
||||||
|
// expression shape as 0014 so brownfield databases stop tokenizing
|
||||||
|
// private managed-agent ciphertext without a heap rewrite.
|
||||||
|
assert_eq!(migrations[32].version, 33);
|
||||||
|
assert!(migrations[32].sql.as_str().contains("kind = 30179"));
|
||||||
|
assert!(migrations[32].sql.as_str().contains("search_tsv"));
|
||||||
|
assert!(!migrations[0].sql.as_str().contains("30179"));
|
||||||
|
assert!(include_str!("../../../schema/schema.sql")
|
||||||
|
.contains("kind IN (1059, 30179, 30300, 30350, 30622, 44100, 44101, 44200)"));
|
||||||
|
|
||||||
// Public push-gateway authority is intentionally deployment-global and
|
// Public push-gateway authority is intentionally deployment-global and
|
||||||
// durable: immediate revocation and hostile-relay admission cannot be
|
// durable: immediate revocation and hostile-relay admission cannot be
|
||||||
// honestly provided by a stateless gateway.
|
// honestly provided by a stateless gateway.
|
||||||
@@ -1915,7 +1925,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[ignore = "requires Postgres"]
|
#[ignore = "requires Postgres"]
|
||||||
async fn populated_upgrade_preserves_search_policy_except_for_push_leases() {
|
async fn populated_upgrade_preserves_search_policy_except_for_private_kinds() {
|
||||||
let pool = connect_test_pool().await;
|
let pool = connect_test_pool().await;
|
||||||
reset_public_schema(&pool).await;
|
reset_public_schema(&pool).await;
|
||||||
MIGRATOR
|
MIGRATOR
|
||||||
@@ -1931,7 +1941,7 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.expect("insert community");
|
.expect("insert community");
|
||||||
|
|
||||||
for (marker, kind) in [(1_u8, 1_i32), (2_u8, 30_350_i32)] {
|
for (marker, kind) in [(1_u8, 1_i32), (2_u8, 30_350_i32), (3_u8, 30_179_i32)] {
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
"INSERT INTO events \
|
"INSERT INTO events \
|
||||||
(community_id, id, pubkey, created_at, kind, tags, content, sig, received_at) \
|
(community_id, id, pubkey, created_at, kind, tags, content, sig, received_at) \
|
||||||
@@ -1958,19 +1968,37 @@ mod tests {
|
|||||||
.fetch_all(&pool)
|
.fetch_all(&pool)
|
||||||
.await
|
.await
|
||||||
.expect("read pre-push search behavior");
|
.expect("read pre-push search behavior");
|
||||||
assert_eq!(before, vec![(1, true), (30_350, true)]);
|
assert_eq!(before, vec![(1, true), (30_179, true), (30_350, true)]);
|
||||||
|
|
||||||
|
// 0014 fixes 30350 only. A brownfield database that stopped here still
|
||||||
|
// tokenized kind:30179 ciphertext — the gap 0033 closes.
|
||||||
|
MIGRATOR
|
||||||
|
.run_to(32, &pool)
|
||||||
|
.await
|
||||||
|
.expect("apply migrations through 32");
|
||||||
|
let pre_0033: Vec<(i32, Option<bool>)> = sqlx::query_as(
|
||||||
|
"SELECT kind, search_tsv @@ plainto_tsquery('simple', 'needle') \
|
||||||
|
FROM events ORDER BY kind",
|
||||||
|
)
|
||||||
|
.fetch_all(&pool)
|
||||||
|
.await
|
||||||
|
.expect("read pre-0033 search behavior");
|
||||||
|
assert_eq!(
|
||||||
|
pre_0033,
|
||||||
|
vec![(1, Some(true)), (30_179, Some(true)), (30_350, None)]
|
||||||
|
);
|
||||||
|
|
||||||
run_migrations(&pool)
|
run_migrations(&pool)
|
||||||
.await
|
.await
|
||||||
.expect("apply push migrations to populated database");
|
.expect("apply remaining migrations to populated database");
|
||||||
let after: Vec<(i32, Option<bool>)> = sqlx::query_as(
|
let after: Vec<(i32, Option<bool>)> = sqlx::query_as(
|
||||||
"SELECT kind, search_tsv @@ plainto_tsquery('simple', 'needle') \
|
"SELECT kind, search_tsv @@ plainto_tsquery('simple', 'needle') \
|
||||||
FROM events ORDER BY kind",
|
FROM events ORDER BY kind",
|
||||||
)
|
)
|
||||||
.fetch_all(&pool)
|
.fetch_all(&pool)
|
||||||
.await
|
.await
|
||||||
.expect("read post-push search behavior");
|
.expect("read post-upgrade search behavior");
|
||||||
assert_eq!(after, vec![(1, Some(true)), (30_350, None)]);
|
assert_eq!(after, vec![(1, Some(true)), (30_179, None), (30_350, None)]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -28,6 +28,8 @@ const MIGRATION_0007_SQL: &str = include_str!("../../../migrations/0007_nip_rs_r
|
|||||||
const MIGRATION_0008_SQL: &str =
|
const MIGRATION_0008_SQL: &str =
|
||||||
include_str!("../../../migrations/0008_fresh_install_search_allowlist.sql");
|
include_str!("../../../migrations/0008_fresh_install_search_allowlist.sql");
|
||||||
const MIGRATION_0014_SQL: &str = include_str!("../../../migrations/0014_push_lease_fts.sql");
|
const MIGRATION_0014_SQL: &str = include_str!("../../../migrations/0014_push_lease_fts.sql");
|
||||||
|
const MIGRATION_0033_SQL: &str =
|
||||||
|
include_str!("../../../migrations/0033_private_managed_agent_fts.sql");
|
||||||
|
|
||||||
async fn setup() -> (PgPool, String) {
|
async fn setup() -> (PgPool, String) {
|
||||||
let url = std::env::var("BUZZ_TEST_DATABASE_URL").unwrap_or_else(|_| TEST_DB_URL.to_string());
|
let url = std::env::var("BUZZ_TEST_DATABASE_URL").unwrap_or_else(|_| TEST_DB_URL.to_string());
|
||||||
@@ -81,6 +83,9 @@ async fn setup() -> (PgPool, String) {
|
|||||||
pool.execute(MIGRATION_0014_SQL)
|
pool.execute(MIGRATION_0014_SQL)
|
||||||
.await
|
.await
|
||||||
.expect("apply 0014 migration");
|
.expect("apply 0014 migration");
|
||||||
|
pool.execute(MIGRATION_0033_SQL)
|
||||||
|
.await
|
||||||
|
.expect("apply 0033 migration");
|
||||||
(pool, schema)
|
(pool, schema)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
36
migrations/0033_private_managed_agent_fts.sql
Normal file
36
migrations/0033_private_managed_agent_fts.sql
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
-- NIP-PMA kind:30179 carries the owner's private managed-agent payload
|
||||||
|
-- (agent nsec, env vars, prompt) as NIP-44 ciphertext and is author-only.
|
||||||
|
-- Exclude it from full-text search without changing the search policy of
|
||||||
|
-- existing installations. Fresh installs are already safe via the positive
|
||||||
|
-- allowlist from migration 0008; this closes the brownfield gap where a
|
||||||
|
-- populated database still runs the legacy negative skip-set (0001/0005)
|
||||||
|
-- and would tokenize the ciphertext into search_tsv.
|
||||||
|
--
|
||||||
|
-- Same shape as 0014 (kind:30350): PostgreSQL cannot alter a generated
|
||||||
|
-- expression in place, so capture the current expression, drop the column,
|
||||||
|
-- and re-add it wrapped with the new exclusion. Every other kind keeps
|
||||||
|
-- whatever policy the database had before.
|
||||||
|
DO $$
|
||||||
|
DECLARE
|
||||||
|
existing_expression TEXT;
|
||||||
|
BEGIN
|
||||||
|
SELECT pg_get_expr(d.adbin, d.adrelid)
|
||||||
|
INTO existing_expression
|
||||||
|
FROM pg_attrdef d
|
||||||
|
JOIN pg_attribute a
|
||||||
|
ON a.attrelid = d.adrelid
|
||||||
|
AND a.attnum = d.adnum
|
||||||
|
WHERE d.adrelid = 'events'::regclass
|
||||||
|
AND a.attname = 'search_tsv';
|
||||||
|
|
||||||
|
IF existing_expression IS NULL THEN
|
||||||
|
RAISE EXCEPTION 'events.search_tsv generated expression not found';
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
ALTER TABLE events DROP COLUMN search_tsv;
|
||||||
|
EXECUTE format(
|
||||||
|
'ALTER TABLE events ADD COLUMN search_tsv TSVECTOR GENERATED ALWAYS AS (CASE WHEN kind = 30179 THEN NULL::tsvector ELSE (%s) END) STORED',
|
||||||
|
existing_expression
|
||||||
|
);
|
||||||
|
CREATE INDEX idx_events_search_tsv ON events USING GIN (search_tsv);
|
||||||
|
END $$;
|
||||||
@@ -219,9 +219,9 @@ CREATE TABLE events (
|
|||||||
-- Privacy: encrypted/private routing wrappers and p-gated membership notices
|
-- Privacy: encrypted/private routing wrappers and p-gated membership notices
|
||||||
-- must never be discoverable through NIP-50 full-text search. NULL tsvector
|
-- must never be discoverable through NIP-50 full-text search. NULL tsvector
|
||||||
-- never matches `@@`.
|
-- never matches `@@`.
|
||||||
-- Keep in sync with migrations (final state: 0001 + 0005 + 0009).
|
-- Keep in sync with migrations (final state: 0001 + 0005 + 0014 + 0033).
|
||||||
search_tsv TSVECTOR GENERATED ALWAYS AS (
|
search_tsv TSVECTOR GENERATED ALWAYS AS (
|
||||||
CASE WHEN kind IN (1059, 30300, 30350, 30622, 44100, 44101, 44200) THEN NULL::tsvector
|
CASE WHEN kind IN (1059, 30179, 30300, 30350, 30622, 44100, 44101, 44200) THEN NULL::tsvector
|
||||||
ELSE to_tsvector('simple', content)
|
ELSE to_tsvector('simple', content)
|
||||||
END
|
END
|
||||||
) STORED,
|
) STORED,
|
||||||
|
|||||||
Reference in New Issue
Block a user