diff --git a/.env.example b/.env.example index 6674deec40..2af5dc36bb 100644 --- a/.env.example +++ b/.env.example @@ -372,3 +372,18 @@ RUST_LOG=buzz_relay=debug,buzz_datastore=info,buzz_db=debug,buzz_auth=debug,buzz # BUZZ_TERMS_OF_SERVICE_MARKDOWN="# Terms of Service\n\nFull terms here." # BUZZ_PRIVACY_POLICY_MARKDOWN="# Privacy Policy\n\nFull policy here." # BUZZ_AGE_ATTESTATION_REQUIRED=true + +# ── NIP-FI database community source ───────────────────────────────────────── +# Where NIP-FI communities come from: "env" (BUZZ_NIP_FI_COMMUNITIES, default) +# or "database" (bindings created by provisioning; needs +# BUZZ_NIP_FI_PROVISION_BINDING and must not be combined with +# BUZZ_NIP_FI_COMMUNITIES). See docs/operator-nip-fi-bindings.md. +# BUZZ_NIP_FI_COMMUNITY_SOURCE=env +# Most community-issuer rows one snapshot may hold (1..=10000000). +# BUZZ_NIP_FI_SNAPSHOT_MAX_ROWS=1000000 +# Snapshot load timeout in ms; must be under the 30s refresh interval. +# BUZZ_NIP_FI_SNAPSHOT_REFRESH_TIMEOUT_MS=10000 +# Hosts whose "not found" is cached for 5s (1..=1000000). +# BUZZ_NIP_FI_RESOLVER_NEGATIVE_CACHE_MAX=100000 +# Concurrent live lookups; clamped below the database pool size. +# BUZZ_NIP_FI_RESOLVER_MAX_INFLIGHT=8 diff --git a/Cargo.lock b/Cargo.lock index 57d4a9ff54..d266b8580a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1424,6 +1424,7 @@ name = "buzz-relay" version = "0.2.1" dependencies = [ "anyhow", + "arc-swap", "async-compression", "async-trait", "axum", @@ -1454,6 +1455,7 @@ dependencies = [ "http-body-util", "infer", "jsonwebtoken", + "lru 0.16.4", "metrics", "metrics-exporter-prometheus", "metrics-util", @@ -5182,6 +5184,9 @@ name = "lru" version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown 0.16.1", +] [[package]] name = "lru" diff --git a/Cargo.toml b/Cargo.toml index df77ab7cb3..6207d5a8c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,6 +124,8 @@ zeroize = "1.8" # Concurrent data structures dashmap = "6" moka = { version = "0.12", features = ["sync"] } +arc-swap = "1" +lru = "0.16" # Async stream utilities futures-util = "0.3" diff --git a/Justfile b/Justfile index ca89c5f35b..e479c467f4 100644 --- a/Justfile +++ b/Justfile @@ -522,7 +522,8 @@ test-unit: # unit job either. The REQ subscription-lifecycle tests are picked by # exact name for the same reason: the rest of handlers::req needs a DB. # The third clause adds the NIP-FI HTTP ingress and its router/config - # neighbours: nip_fi_http, nip_fi_config, router, api::parse_query_tests, + # neighbours: nip_fi_http, nip_fi_config, nip_fi_resolver (snapshot + # resolver and its WS/HTTP/audio router tests), router, api::parse_query_tests, # the profile search request bounds (api::profiles), and the Git # transport off_mode_precedence_tests. All are infra-free # and finish in well under a second with no DATABASE_URL, so none wait @@ -536,7 +537,7 @@ test-unit: # are infra-free; config parses env into a struct and never connects. # `^config::` is anchored so it does not also select nip_fi_config. cargo nextest run -p buzz-relay --lib --bin buzz-relay \ - -E 'test(/^api::admin::/) + test(/^push_runtime::/) + test(/^operator_listener::tests::/) + test(/^handlers::channel_authz::/) + test(/^handlers::moderation_authz::/) + test(/^handlers::side_effects::tests::/) + test(/^handlers::system_channels::tests::/) + test(/^nip11::tests::/) + test(/^storage_sweep::tests::/) + test(/^nip_fi_core::tests::/) + test(/^nip_fi_http::tests::/) + test(/^nip_fi_config::tests::/) + test(/^readiness::tests::/) + test(/^router::tests::/) + test(/^api::parse_query_tests::/) + test(/^api::profiles::/) + test(/^api::git::transport::off_mode_precedence_tests::/) + test(/^audio::join::tests::/) + test(/^audio::handler::tests::/) + test(/^nip_fi_gate::tests::/) + test(/^nip_fi_session::tests::/) + test(/^nip_fi_shadow::tests::/) + test(/^nip_fi_shadow_session::tests::/) + test(/^startup_steps::tests::/) + test(/^telemetry::tests::/) + test(/^config::tests::/) + test(=state::tests::neither_a_confirmed_inactive_community_nor_a_failed_lookup_admits_the_socket) + test(=state::tests::periodic_revalidation_disconnects_inside_the_fenced_callback) + test(=state::tests::bare_community_disconnect_fails_closed_when_the_fence_is_unavailable) + test(=connection::tests::send_loop_sends_policy_close_when_community_is_archived) + test(=state::tests::community_disconnect_then_nip_fi_keeps_community_deleted_reason) + test(=state::tests::disconnect_community_wins_reason_losing_nip_fi_does_not_enqueue_frame) + test(=handlers::req::tests::timed_out_historical_read_deregisters_before_closed) + test(=handlers::req::tests::superseded_timeout_leaves_replacement_intact) + test(=handlers::req::tests::search_claim_retires_live_and_yields_to_replacement) + test(=handlers::req::tests::concurrent_claims_and_stale_teardowns_keep_the_last_owner) + test(=handlers::req::tests::timeout_closed_is_emitted_before_a_replacement_can_claim) + test(=handlers::req::tests::revoke_then_replacement_keeps_replacement_whole) + test(=handlers::req::tests::claims_after_connection_cleanup_are_refused) + test(=handlers::req::tests::dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::revoke_dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::count_fallback_covers_filters_that_can_match_private_deletions) + test(=handlers::req::tests::group_state_t_and_creator_tags_reach_sql_only_for_group_state_kinds) + test(=api::bridge::postgres_tests::bridge_search_mode_extension_accepts_prefix_snake_or_camel_case) + test(=api::bridge::postgres_tests::bridge_search_sort_extension_selects_newest_only_when_named) + (kind(bin) & (test(/^tests::/) + test(/^composition_tests::/) + test(/^env_filter_tests::/)) - test(/^tests::postgres_tests::/))' + -E 'test(/^api::admin::/) + test(/^push_runtime::/) + test(/^operator_listener::tests::/) + test(/^handlers::channel_authz::/) + test(/^handlers::moderation_authz::/) + test(/^handlers::side_effects::tests::/) + test(/^handlers::system_channels::tests::/) + test(/^nip11::tests::/) + test(/^storage_sweep::tests::/) + test(/^nip_fi_core::tests::/) + test(/^nip_fi_http::tests::/) + test(/^nip_fi_config::tests::/) + test(/^nip_fi_resolver::tests::/) + test(/^readiness::tests::/) + test(/^router::tests::/) + test(/^api::parse_query_tests::/) + test(/^api::profiles::/) + test(/^api::git::transport::off_mode_precedence_tests::/) + test(/^audio::join::tests::/) + test(/^audio::handler::tests::/) + test(/^nip_fi_gate::tests::/) + test(/^nip_fi_session::tests::/) + test(/^nip_fi_shadow::tests::/) + test(/^nip_fi_shadow_session::tests::/) + test(/^startup_steps::tests::/) + test(/^telemetry::tests::/) + test(/^config::tests::/) + test(=state::tests::neither_a_confirmed_inactive_community_nor_a_failed_lookup_admits_the_socket) + test(=state::tests::periodic_revalidation_disconnects_inside_the_fenced_callback) + test(=state::tests::bare_community_disconnect_fails_closed_when_the_fence_is_unavailable) + test(=connection::tests::send_loop_sends_policy_close_when_community_is_archived) + test(=state::tests::community_disconnect_then_nip_fi_keeps_community_deleted_reason) + test(=state::tests::disconnect_community_wins_reason_losing_nip_fi_does_not_enqueue_frame) + test(=handlers::req::tests::timed_out_historical_read_deregisters_before_closed) + test(=handlers::req::tests::superseded_timeout_leaves_replacement_intact) + test(=handlers::req::tests::search_claim_retires_live_and_yields_to_replacement) + test(=handlers::req::tests::concurrent_claims_and_stale_teardowns_keep_the_last_owner) + test(=handlers::req::tests::timeout_closed_is_emitted_before_a_replacement_can_claim) + test(=handlers::req::tests::revoke_then_replacement_keeps_replacement_whole) + test(=handlers::req::tests::claims_after_connection_cleanup_are_refused) + test(=handlers::req::tests::dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::revoke_dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::count_fallback_covers_filters_that_can_match_private_deletions) + test(=handlers::req::tests::group_state_t_and_creator_tags_reach_sql_only_for_group_state_kinds) + test(=api::bridge::postgres_tests::bridge_search_mode_extension_accepts_prefix_snake_or_camel_case) + test(=api::bridge::postgres_tests::bridge_search_sort_extension_selects_newest_only_when_named) + (kind(bin) & (test(/^tests::/) + test(/^composition_tests::/) + test(/^env_filter_tests::/)) - test(/^tests::postgres_tests::/))' # Note on audio::join::tests scope: the full suite is infra-free (no DB, # no Redis). The infra-free audio/FI regression witnesses — bootstrap # ordering barrier, CommitConfirmed arm, pending-close invisibility, diff --git a/crates/buzz-auth/src/nip_fi/command.rs b/crates/buzz-auth/src/nip_fi/command.rs index 83b864498c..deeae99e75 100644 --- a/crates/buzz-auth/src/nip_fi/command.rs +++ b/crates/buzz-auth/src/nip_fi/command.rs @@ -1214,6 +1214,31 @@ mod tests { ); } + #[test] + fn until_at_maximum_assertion_lifetime_is_accepted_and_one_second_more_is_not() { + // An issuer's `until` is the expiry of the last assertion it issued, so + // a maximum-lifetime assertion minted now (exp = now + max_age) must + // fit the ceiling now + skew + max_age, both at issue time and on a + // re-send just before that deadline. The test policy is skew 30 s, + // maximum assertion age 3600 s. + let cv = test_command_verifier(); + let target = target_key(); + let now = Utc::now(); + for until_offset in [3600, 30 + 3600, 1] { + let token = mint_cmd_jwt(&target, until_offset, serde_json::json!({})); + assert_eq!( + cv.verify_at(&token, METHOD, PATH, &target, now).map(|_| ()), + Ok(()), + "until = now + {until_offset}s must fit the ceiling" + ); + } + let token = mint_cmd_jwt(&target, 30 + 3600 + 1, serde_json::json!({})); + assert_eq!( + cv.verify_at(&token, METHOD, PATH, &target, now).map(|_| ()), + Err(CommandError::UntilExceedsCeiling), + ); + } + // ── jti is the LAST step ───────────────────────────────────────────────── #[test] diff --git a/crates/buzz-auth/src/nip_fi/config.rs b/crates/buzz-auth/src/nip_fi/config.rs index 26aad88115..99231d82d5 100644 --- a/crates/buzz-auth/src/nip_fi/config.rs +++ b/crates/buzz-auth/src/nip_fi/config.rs @@ -368,10 +368,6 @@ pub struct IssuerPolicy { /// Accepted `aud` values for S4 command JWTs only. Assertion `aud` comes /// from the per-community [`super::CommunityBinding`], never from here. command_audiences: Vec, - /// Expected `aud` of every community authorizing this issuer — the - /// issuer's side of the community allowlist (NIP-FI.md:145). Empty until - /// [`IssuerPolicy::authorized_for_communities`] sets it. - community_audiences: Vec, token_class: TokenClass, freshness: FreshnessClass, algorithms: Vec, @@ -488,7 +484,6 @@ impl IssuerPolicy { let mut policy = Self { issuer, command_audiences: canonical_set(command_audiences), - community_audiences: Vec::new(), token_class, freshness, algorithms: canonical_algorithm_set(algorithms), @@ -503,15 +498,18 @@ impl IssuerPolicy { Ok(policy) } - /// Record the expected `aud` of every community that authorizes this - /// issuer and re-derive the policy ID over it, so changing the community - /// allowlist moves the ID (NIP-FI.md:145). Set-valued: order and - /// duplicates do not affect the ID. - #[must_use] - pub fn authorized_for_communities(mut self, community_audiences: Vec) -> Self { - self.community_audiences = canonical_set(community_audiences); - self.id = derive_assertion_policy_id(&self); - self + /// The policy ID an assertion verified for the community expecting + /// `community_aud` carries: `H(base policy contract, community aud)`. The + /// base contract already commits to the issuer. The verifier mints it + /// only after the community's allowlist admitted this issuer, so it moves + /// exactly when this community's association or the base policy changes, + /// never when an unrelated community is added (NIP-FI.md:145). + pub fn id_for_community(&self, community_aud: &str) -> AssertionPolicyId { + let mut hasher = Sha256::new(); + hasher.update(b"buzz:nip-fi:community-assertion-policy:v1\0"); + hasher.update(self.id.0); + hash_field(&mut hasher, community_aud.as_bytes()); + AssertionPolicyId(hasher.finalize().into()) } /// The exact `iss` value this policy is selected by. @@ -524,11 +522,6 @@ impl IssuerPolicy { &self.command_audiences } - /// The expected `aud` of each community authorizing this issuer. - pub fn community_audiences(&self) -> &[String] { - &self.community_audiences - } - /// The single accepted token class. pub fn token_class(&self) -> &TokenClass { &self.token_class @@ -668,7 +661,6 @@ fn derive_assertion_policy_id(policy: &IssuerPolicy) -> AssertionPolicyId { let IssuerPolicy { issuer, command_audiences, - community_audiences, token_class, freshness, algorithms, @@ -699,10 +691,6 @@ fn derive_assertion_policy_id(policy: &IssuerPolicy) -> AssertionPolicyId { } hash_field(&mut hasher, issuer.as_bytes()); hash_seq(&mut hasher, command_audiences.iter().map(String::as_bytes)); - hash_seq( - &mut hasher, - community_audiences.iter().map(String::as_bytes), - ); hash_field(&mut hasher, token_class.discriminant().as_bytes()); match token_class { TokenClass::AccessTokenAtJwt { subject_class } => { @@ -804,7 +792,7 @@ mod jwks_transport_identity_tests { assert_eq!( hex::encode(policy(contract).id().as_bytes()), - "1f904aaaad1c1db3a22ae50bd2d24cdf7181d5d0b8c5006cc1ae9ac1107f9832", + "fae0dd479d79306c5eef135594f678a5cdf3a2ee45770e20a7e83395fc76857f", "public_https mode omits the mesh transport tag" ); } @@ -820,7 +808,7 @@ mod jwks_transport_identity_tests { assert_eq!( hex::encode(policy(contract).id().as_bytes()), - "33e4cf02148ca494c12ffc49cd0bb223e0e007de4084ac5b406f2742bbb91c0a", + "dca3e998388ae75bdea6bb2070c8f28b8d389a8b874de96e7852dd1f71b14bcd", "mesh transport is part of policy identity" ); } diff --git a/crates/buzz-auth/src/nip_fi/verifier.rs b/crates/buzz-auth/src/nip_fi/verifier.rs index 34bd711c28..750c617324 100644 --- a/crates/buzz-auth/src/nip_fi/verifier.rs +++ b/crates/buzz-auth/src/nip_fi/verifier.rs @@ -463,7 +463,7 @@ impl FederatedAssertionVerifier { asserted_key, capabilities, deadlines, - policy.id(), + policy.id_for_community(community.expected_aud()), self.transport_contract_id, RevalidationDependencies::new( header.kid, diff --git a/crates/buzz-auth/src/nip_fi/verifier/tests.rs b/crates/buzz-auth/src/nip_fi/verifier/tests.rs index bdab25370c..77ce87a154 100644 --- a/crates/buzz-auth/src/nip_fi/verifier/tests.rs +++ b/crates/buzz-auth/src/nip_fi/verifier/tests.rs @@ -268,7 +268,10 @@ fn valid_access_token_verifies() { // Spec v2: nostr_pubkey is injected by mint() and unconditionally required. assert!(assertion.asserted_key().is_some()); assert!(!assertion.authority_deadlines().is_empty()); - assert_eq!(assertion.assertion_policy_id(), access_token_policy().id()); + assert_eq!( + assertion.assertion_policy_id(), + access_token_policy().id_for_community(AUDIENCE) + ); } // ---- Token class / typ enforcement, ID-token denial ---------------------- @@ -1943,22 +1946,41 @@ fn aud_array_containing_expected_passes() { } #[test] -fn assertion_policy_id_covers_community_allowlist() { - let base = dedicated_policy(ISSUER); - let a = base - .clone() - .authorized_for_communities(vec![AUDIENCE.to_owned(), COMMUNITY_B_AUD.to_owned()]); - let permuted = base.clone().authorized_for_communities(vec![ - COMMUNITY_B_AUD.to_owned(), - AUDIENCE.to_owned(), - AUDIENCE.to_owned(), - ]); - let fewer = base - .clone() - .authorized_for_communities(vec![AUDIENCE.to_owned()]); - assert_ne!(base.id(), a.id()); - assert_eq!(a.id(), permuted.id()); - assert_ne!(a.id(), fewer.id()); +fn assertion_policy_id_covers_community_audience_and_issuer() { + let policy = dedicated_policy(ISSUER); + let other_issuer = dedicated_policy("https://other-issuer.example"); + let here = policy.id_for_community(AUDIENCE); + // Stable for one (base policy, community, issuer) association. + assert_eq!(here, policy.clone().id_for_community(AUDIENCE)); + // A different community `aud` or issuer moves the ID. + assert_ne!(here, policy.id_for_community(COMMUNITY_B_AUD)); + assert_ne!(here, other_issuer.id_for_community(AUDIENCE)); + // The per-community ID is not the bare base-policy ID. + assert_ne!(here, policy.id()); +} + +#[test] +fn verified_assertion_carries_the_per_community_policy_id() { + let verifier = verifier_with(dedicated_policy(ISSUER)); + let token = mint(Some("nip-fi+jwt"), TEST_KID, json!({ "sub": "user-123" })); + let assertion = verifier + .verify(&token, &community(AUDIENCE, &[ISSUER])) + .expect("verifies"); + // An unrelated community authorizing the same issuer leaves it unchanged. + let unchanged = verifier + .verify( + &token, + &community(AUDIENCE, &[ISSUER, "https://unrelated.example"]), + ) + .expect("verifies"); + assert_eq!( + assertion.assertion_policy_id(), + dedicated_policy(ISSUER).id_for_community(AUDIENCE) + ); + assert_eq!( + assertion.assertion_policy_id(), + unchanged.assertion_policy_id() + ); } #[test] diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 459ddd48ed..e13b06c81d 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -82,7 +82,7 @@ pub use community::{ pub use error::{DbError, Result}; pub use event::{ChannelHeadPrecondition, ChannelHeadWriteStatus}; pub use event::{EventQuery, DEFAULT_MAX_PAGE_LIMIT}; -pub use nip_fi_binding::{NipFiBindingRecord, NipFiUnbind}; +pub use nip_fi_binding::{NipFiBindingRecord, NipFiUnbind, RowCapExceeded}; pub use reaction::ReactionEventInsertOutcome; pub use reminder::DueReminder; pub use usage::UsageMetricsLeader; diff --git a/crates/buzz-db/src/store/nip_fi_binding.rs b/crates/buzz-db/src/store/nip_fi_binding.rs index 3f49f5c875..1e193e2b4c 100644 --- a/crates/buzz-db/src/store/nip_fi_binding.rs +++ b/crates/buzz-db/src/store/nip_fi_binding.rs @@ -311,6 +311,86 @@ impl Db { } } +/// A snapshot load produced more rows than its cap allows. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("row cap of {0} exceeded")] +pub struct RowCapExceeded(pub usize); + +impl Db { + /// Read every binding of an active community to an issuer in `registry`, + /// on the primary, as `(host, issuer)` rows, one per community-issuer + /// pair. It is one `SELECT`, so every row comes from the same statement + /// snapshot. The read stops at the `max_rows + 1`st row and fails. + #[datastore_span(name = "load_nip_fi_binding_snapshot", system = "postgresql")] + pub async fn load_nip_fi_binding_snapshot( + &self, + registry: &[String], + max_rows: usize, + ) -> Result, RowCapExceeded>> { + use futures_util::StreamExt; + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + let mut stream = sqlx::query_as::<_, (String, String)>( + r#" + SELECT c.host, b.issuer + FROM community_nip_fi_issuers b + JOIN communities c ON c.id = b.community_id + WHERE b.issuer = ANY($1) + AND c.archived_at IS NULL + AND c.deletion_state = 'active' + AND c.deleted_at IS NULL + "#, + ) + .bind(registry) + .fetch(&mut *connection); + let mut rows = Vec::new(); + while let Some(row) = stream.next().await { + if rows.len() == max_rows { + return Ok(Err(RowCapExceeded(max_rows))); + } + rows.push(row?); + } + Ok(Ok(rows)) + } + + /// The registry-filtered issuers bound to the active community at + /// `normalized_host`, on the primary. Empty when the host is unknown, + /// inactive, or bound only to unregistered issuers. The primary key + /// `(community_id, issuer)` bounds the result at `registry.len()` rows. + #[datastore_span(name = "lookup_nip_fi_admission_binding", system = "postgresql")] + pub async fn lookup_nip_fi_admission_binding( + &self, + normalized_host: &str, + registry: &[String], + ) -> Result> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + let issuers = sqlx::query_scalar::<_, String>( + r#" + SELECT b.issuer + FROM community_nip_fi_issuers b + JOIN communities c ON c.id = b.community_id + WHERE lower(c.host) = lower($1) + AND b.issuer = ANY($2) + AND c.archived_at IS NULL + AND c.deletion_state = 'active' + AND c.deleted_at IS NULL + "#, + ) + .bind(normalized_host) + .bind(registry) + .fetch_all(&mut *connection) + .await?; + Ok(issuers) + } +} + #[cfg(test)] mod postgres_tests { use super::*; @@ -643,4 +723,129 @@ mod postgres_tests { ); assert_eq!(bound(&db, &unique_host("missing")).await, None); } + + /// Move `host`'s community out of `active` the way the deletion executor + /// does: the tombstone trigger admits the state change only under the + /// executor's community and fence-generation settings. + async fn start_deleting(db: &Db, host: &str) { + let mut tx = db.pool.begin().await.expect("begin"); + let id: Uuid = sqlx::query_scalar("SELECT id FROM communities WHERE host = $1") + .bind(host) + .fetch_one(&mut *tx) + .await + .expect("community id"); + sqlx::query("SELECT set_config('buzz.deletion_executor_community', $1, true), set_config('buzz.deletion_fence_generation', '1', true)") + .bind(id.to_string()) + .execute(&mut *tx) + .await + .expect("executor settings"); + sqlx::query("UPDATE communities SET deletion_state = 'quiescing', deletion_fence_generation = 1 WHERE id = $1") + .bind(id) + .execute(&mut *tx) + .await + .expect("start deletion"); + tx.commit().await.expect("commit"); + } + + async fn snapshot_hosts(db: &Db, registry: &[String]) -> Vec { + db.load_nip_fi_binding_snapshot(registry, 100_000) + .await + .expect("snapshot") + .expect("under cap") + .into_iter() + .map(|(host, _)| host) + .collect() + } + + async fn admission(db: &Db, host: &str, registry: &[String]) -> Vec { + db.lookup_nip_fi_admission_binding(host, registry) + .await + .expect("admission lookup") + } + + // Criterion 14(a): archived and deleting communities drop out of both the + // snapshot load and the admission miss lookup; an active control stays. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn snapshot_and_admission_lookup_skip_archived_and_deleting_communities() { + let db = setup_db().await; + let registry = issuers(); + let [archived, deleting, control] = ["archived", "deleting", "control"].map(unique_host); + for host in [&archived, &deleting, &control] { + db.ensure_configured_community_with_nip_fi_issuers(host, ®istry) + .await + .expect("seed"); + assert!(snapshot_hosts(&db, ®istry).await.contains(host)); + assert_eq!(admission(&db, host, ®istry).await, registry); + } + + sqlx::query("UPDATE communities SET archived_at = now() WHERE host = $1") + .bind(&archived) + .execute(&db.pool) + .await + .expect("archive"); + start_deleting(&db, &deleting).await; + + let hosts = snapshot_hosts(&db, ®istry).await; + for gone in [&archived, &deleting] { + assert!(!hosts.contains(gone), "{gone} must leave the snapshot"); + assert!(admission(&db, gone, ®istry).await.is_empty(), "{gone}"); + } + assert!(hosts.contains(&control)); + assert_eq!(admission(&db, &control, ®istry).await, registry); + } + + // Criterion 17(a)(1): only registered issuers come back, and a host bound + // only to unregistered issuers yields zero rows from the real SQL. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn admission_lookup_returns_only_registered_issuers() { + let db = setup_db().await; + let other = "https://unregistered.example".to_string(); + let mixed = unique_host("mixed"); + let foreign = unique_host("foreign"); + db.ensure_configured_community_with_nip_fi_issuers( + &mixed, + &[ISS.to_string(), other.clone()], + ) + .await + .expect("seed mixed"); + db.ensure_configured_community_with_nip_fi_issuers(&foreign, std::slice::from_ref(&other)) + .await + .expect("seed foreign"); + + assert_eq!( + admission(&db, &mixed.to_uppercase(), &issuers()).await, + issuers() + ); + assert!(admission(&db, &foreign, &issuers()).await.is_empty()); + assert!(!snapshot_hosts(&db, &issuers()).await.contains(&foreign)); + } + + // The load fails at the `max_rows + 1`st row instead of truncating. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn snapshot_load_over_its_row_cap_fails() { + let db = setup_db().await; + let registry = vec![format!("https://issuer-{}.example", uuid::Uuid::new_v4())]; + for prefix in ["cap-a", "cap-b"] { + db.ensure_configured_community_with_nip_fi_issuers(&unique_host(prefix), ®istry) + .await + .expect("seed"); + } + assert_eq!( + db.load_nip_fi_binding_snapshot(®istry, 1) + .await + .expect("load"), + Err(RowCapExceeded(1)) + ); + assert_eq!( + db.load_nip_fi_binding_snapshot(®istry, 2) + .await + .expect("load") + .expect("at cap") + .len(), + 2 + ); + } } diff --git a/crates/buzz-relay/Cargo.toml b/crates/buzz-relay/Cargo.toml index d83c5e4f73..6a4ab379fd 100644 --- a/crates/buzz-relay/Cargo.toml +++ b/crates/buzz-relay/Cargo.toml @@ -78,6 +78,8 @@ rand = { workspace = true } hex = { workspace = true } url = { workspace = true } moka = { workspace = true } +arc-swap = { workspace = true } +lru = { workspace = true } metrics = { workspace = true } metrics-exporter-prometheus = { workspace = true } metrics-util = { workspace = true } diff --git a/crates/buzz-relay/src/api/bridge.rs b/crates/buzz-relay/src/api/bridge.rs index 9bc2940a81..de54b8232c 100644 --- a/crates/buzz-relay/src/api/bridge.rs +++ b/crates/buzz-relay/src/api/bridge.rs @@ -63,7 +63,8 @@ pub(crate) async fn authenticate_member_post( true, true, ), - )?; + ) + .await?; let pubkey = *admission.proven_pubkey(); let (event_id_bytes, signed_created_at) = admission.into_extra(); @@ -1004,7 +1005,8 @@ pub async fn submit_event( // only available through the returned NipFiAdmission. [FI-TRACE-AUTHORITY-UNIFORM] crate::nip_fi_shadow::observe_strict_proof(&state, &headers, "bridge", || { verify_bridge_auth_with_options(&headers, "POST", &url, Some(&body), true, true).map(drop) - }); + }) + .await; let admission = admit_nip_fi_http_on_state(&state, &headers, || { verify_bridge_auth_with_options( &headers, @@ -1016,7 +1018,8 @@ pub async fn submit_event( ) .map(|auth| auth.proof((auth.event_id_bytes, auth.signed_created_at))) .map_err(|e| e.into_response()) - })?; + }) + .await?; let pubkey = *admission.proven_pubkey(); let (event_id_bytes, signed_created_at) = admission.into_extra(); let pubkey_hex = pubkey.to_hex(); @@ -1356,7 +1359,8 @@ pub async fn query_events( // NIP-FI admission. [FI-TRACE-AUTHORITY-UNIFORM] crate::nip_fi_shadow::observe_strict_proof(&state, &headers, "bridge", || { verify_bridge_auth_with_options(&headers, "POST", &url, Some(&body), true, true).map(drop) - }); + }) + .await; let admission = admit_nip_fi_http_on_state(&state, &headers, || { verify_bridge_auth_with_options( &headers, @@ -1368,7 +1372,8 @@ pub async fn query_events( ) .map(|auth| auth.proof((auth.event_id_bytes, auth.signed_created_at))) .map_err(|e| e.into_response()) - })?; + }) + .await?; let pubkey = *admission.proven_pubkey(); let (event_id_bytes, signed_created_at) = admission.into_extra(); let pubkey_hex = pubkey.to_hex(); @@ -2031,7 +2036,8 @@ pub async fn count_events( // NIP-FI admission. [FI-TRACE-AUTHORITY-UNIFORM] crate::nip_fi_shadow::observe_strict_proof(&state, &headers, "bridge", || { verify_bridge_auth_with_options(&headers, "POST", &url, Some(&body), true, true).map(drop) - }); + }) + .await; let admission = admit_nip_fi_http_on_state(&state, &headers, || { verify_bridge_auth_with_options( &headers, @@ -2043,7 +2049,8 @@ pub async fn count_events( ) .map(|auth| auth.proof((auth.event_id_bytes, auth.signed_created_at))) .map_err(|e| e.into_response()) - })?; + }) + .await?; let pubkey = *admission.proven_pubkey(); let (event_id_bytes, signed_created_at) = admission.into_extra(); let pubkey_hex = pubkey.to_hex(); @@ -2873,7 +2880,8 @@ async fn authorize_moderation_read( // NIP-FI admission. [FI-TRACE-AUTHORITY-UNIFORM] crate::nip_fi_shadow::observe_strict_proof(state, headers, "bridge", || { verify_bridge_auth(headers, "GET", &url, None, true).map(drop) - }); + }) + .await; let admission = admit_nip_fi_http_on_state(state, headers, || { verify_bridge_auth( headers, @@ -2884,7 +2892,8 @@ async fn authorize_moderation_read( ) .map(|auth| auth.proof(auth.event_id_bytes)) .map_err(|e| e.into_response()) - })?; + }) + .await?; let pubkey = *admission.proven_pubkey(); let event_id_bytes = admission.into_extra(); diff --git a/crates/buzz-relay/src/api/buzz_v1/auth.rs b/crates/buzz-relay/src/api/buzz_v1/auth.rs index dd65d3dad6..6a3b731f03 100644 --- a/crates/buzz-relay/src/api/buzz_v1/auth.rs +++ b/crates/buzz-relay/src/api/buzz_v1/auth.rs @@ -110,6 +110,7 @@ pub(super) async fn authorize( body.is_some(), ), ) + .await .map_err(|response| { if state.config.nip_fi.mode.restricts() { // Preserve the shared NIP-FI wire contract, not just its status. diff --git a/crates/buzz-relay/src/api/git/settings.rs b/crates/buzz-relay/src/api/git/settings.rs index a4f4edeaf8..3602c8a560 100644 --- a/crates/buzz-relay/src/api/git/settings.rs +++ b/crates/buzz-relay/src/api/git/settings.rs @@ -193,7 +193,8 @@ async fn authenticate( true, // require_auth_token: always required for control-plane require_payload, ), - )?; + ) + .await?; let pubkey = *admission.proven_pubkey(); let (event_id_bytes, signed_created_at) = admission.into_extra(); diff --git a/crates/buzz-relay/src/api/git/transport.rs b/crates/buzz-relay/src/api/git/transport.rs index c3b292d8b4..d1ace7b62d 100644 --- a/crates/buzz-relay/src/api/git/transport.rs +++ b/crates/buzz-relay/src/api/git/transport.rs @@ -105,8 +105,8 @@ impl axum::extract::FromRequestParts> for GitAuth { if !mode.restricts() { if let Err(rejection) = parse_git_auth_header(&parts.headers, method) { // Off returns `rejection` as is; shadow also records the - // enforce verdict for the failed proof, still with no DB work. - crate::nip_fi_http::observe_failed_proof(state, &parts.headers); + // enforce verdict for the failed proof. + crate::nip_fi_http::observe_failed_proof(state, &parts.headers).await; return Err(rejection); } } @@ -195,7 +195,8 @@ impl axum::extract::FromRequestParts> for GitAuth { (event, signed_auth_created_at), )) }, - )?; + ) + .await?; let pubkey = *admission.proven_pubkey(); let (event, signed_auth_created_at) = admission.into_extra(); diff --git a/crates/buzz-relay/src/api/invites.rs b/crates/buzz-relay/src/api/invites.rs index bff1df0fcc..45fa15e1df 100644 --- a/crates/buzz-relay/src/api/invites.rs +++ b/crates/buzz-relay/src/api/invites.rs @@ -317,7 +317,9 @@ async fn mint_invite_checked( true, // invites always require NIP-98; no X-Pubkey dev fallback true, // POST bodies must be covered by a payload tag ), - ) { + ) + .await + { Ok(a) => a, Err(resp) => return resp, }; diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 6864cb15ae..470768e308 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -338,7 +338,8 @@ pub(crate) async fn upload_blob( let strict = BlossomStrictness::Strict; buzz_media::auth::verify_blossom_auth_event(&event, Some(&tenant_host), strict) .map_err(drop) - }); + }) + .await; let headers_clone = headers.clone(); let admission = match admit_nip_fi_http_on_state(&state, &headers, move || { let auth_event = extract_blossom_auth(&headers_clone).map_err(|e| e.into_response())?; @@ -348,7 +349,9 @@ pub(crate) async fn upload_blob( .map_err(|e| e.into_response())?; let pubkey = auth_event.pubkey; Ok(Nip98Proof::new(pubkey, auth_event)) - }) { + }) + .await + { Ok(a) => a, Err(resp) => return resp, }; @@ -818,11 +821,14 @@ pub(crate) async fn get_blob( BlossomStrictness::Strict, ) .map(drop) - }); + }) + .await; let admission = match admit_nip_fi_http_on_state(&state, &req_headers, move || { extract_blossom_read_proof(&headers_clone, &sha256, &tenant_host, strictness) .map_err(|e| e.into_response()) - }) { + }) + .await + { Ok(a) => a, Err(resp) => return Ok(resp), }; @@ -1112,11 +1118,14 @@ pub(crate) async fn head_blob( crate::nip_fi_shadow::observe_strict_proof(&state, &headers, "blossom", || { extract_blossom_read_proof(&headers, &sha256, &tenant_host, BlossomStrictness::Strict) .map(drop) - }); + }) + .await; let admission = match admit_nip_fi_http_on_state(&state, &headers, move || { extract_blossom_read_proof(&headers_clone, &sha256, &tenant_host, strictness) .map_err(|e| e.into_response()) - }) { + }) + .await + { Ok(a) => a, Err(resp) => return Ok(resp), }; diff --git a/crates/buzz-relay/src/api/nip_fi.rs b/crates/buzz-relay/src/api/nip_fi.rs index b3e4267cf6..39a24bbdc3 100644 --- a/crates/buzz-relay/src/api/nip_fi.rs +++ b/crates/buzz-relay/src/api/nip_fi.rs @@ -1930,6 +1930,7 @@ mod route_integration_tests { }, )]; config.nip_fi = NipFiRelayConfig { + source: Default::default(), mode: NipFiMode::Enforce, registry: registry.clone(), jwks_configs: jwks_configs.clone(), @@ -2511,7 +2512,7 @@ mod route_integration_tests { /// for `key`. `Ok(())` = admitted; `Err` = the denial response. // Response is intentionally large (axum's design); see admit_nip_fi_http. #[allow(clippy::result_large_err)] - fn http_admit( + async fn http_admit( state: &crate::state::AppState, iss: &str, key: &nostr::PublicKey, @@ -2526,6 +2527,7 @@ mod route_integration_tests { crate::nip_fi_http::admit_nip_fi_http_on_state(state, &headers, || { Ok(crate::nip_fi_http::Nip98Proof::new(*key, ())) }) + .await .map(|_| ()) } @@ -2551,7 +2553,7 @@ mod route_integration_tests { let key = nostr::Keys::generate().public_key(); let mut state = (*http_enforce_state().await).clone(); assert!( - http_admit(&state, TEST_ISS, &key).is_ok(), + http_admit(&state, TEST_ISS, &key).await.is_ok(), "control: the assertion is admitted on its own community" ); Arc::make_mut(&mut state.config).nip_fi.communities = @@ -2559,7 +2561,9 @@ mod route_integration_tests { "https://relay.example", &[TEST_ISS, OTHER_ISS], ); - let resp = http_admit(&state, TEST_ISS, &key).expect_err("A's aud on B's Host"); + let resp = http_admit(&state, TEST_ISS, &key) + .await + .expect_err("A's aud on B's Host"); assert_eq!(resp.status(), StatusCode::FORBIDDEN); let body = axum::body::to_bytes(resp.into_body(), usize::MAX) .await @@ -2572,13 +2576,14 @@ mod route_integration_tests { let state = http_enforce_state().await; let key = nostr::Keys::generate().public_key(); assert!( - http_admit(&state, TEST_ISS, &key).is_ok(), + http_admit(&state, TEST_ISS, &key).await.is_ok(), "control: (iss-A, k) is admitted before any deny" ); deny_via_route(&state, &key, 300).await; let resp = http_admit(&state, TEST_ISS, &key) + .await .expect_err("(iss-A, k) must be denied at HTTP ingress after the route denies it"); assert_fixed_authorization_denied(resp, "(iss-A, k) HTTP denial").await; } @@ -2590,11 +2595,11 @@ mod route_integration_tests { deny_via_route(&state, &key, 300).await; assert!( - http_admit(&state, TEST_ISS, &key).is_err(), + http_admit(&state, TEST_ISS, &key).await.is_err(), "control: (iss-A, k) is denied" ); assert!( - http_admit(&state, OTHER_ISS, &key).is_ok(), + http_admit(&state, OTHER_ISS, &key).await.is_ok(), "a deny for (iss-A, k) must not block (iss-B, k)" ); } @@ -2624,8 +2629,9 @@ mod route_integration_tests { .expect("router response"); assert_fixed_authorization_denied(ws_resp, "WS admission sees the route's deny").await; - let http_resp = - http_admit(&state, TEST_ISS, &key).expect_err("HTTP admission sees the same deny"); + let http_resp = http_admit(&state, TEST_ISS, &key) + .await + .expect_err("HTTP admission sees the same deny"); assert_fixed_authorization_denied(http_resp, "HTTP admission sees the route's deny").await; } @@ -2656,6 +2662,7 @@ mod route_integration_tests { crate::nip_fi_http::admit_nip_fi_http_on_state(&state, &HeaderMap::new(), || { Ok(crate::nip_fi_http::Nip98Proof::new(key, 7u8)) }) + .await .expect("Off mode admits on NIP-98 success regardless of deny entries"); assert_eq!(admitted.proven_pubkey(), &key); assert!(admitted.assertion().is_none()); @@ -2673,6 +2680,7 @@ mod route_integration_tests { &HeaderMap::new(), || Err(legacy()), ) + .await .expect_err("Off mode propagates the NIP-98 failure"); let expected = legacy(); assert_eq!(resp.status(), expected.status()); @@ -3548,6 +3556,7 @@ mod route_integration_tests { }, )]; config.nip_fi = NipFiRelayConfig { + source: Default::default(), mode: NipFiMode::Shadow, registry: registry.clone(), jwks_configs: vec![test_jwks_config()], @@ -3623,7 +3632,9 @@ mod route_integration_tests { /// Run `state`'s production disconnect subscriber and apply what it /// hears through the production consumer, like `main.rs`. Yields /// the target of each applied message. - fn listen(state: &Arc) -> tokio::sync::mpsc::UnboundedReceiver> { + pub(super) fn listen( + state: &Arc, + ) -> tokio::sync::mpsc::UnboundedReceiver> { let mut rx = state.pubsub.subscribe_nip_fi_disconnect(); let channels = disconnect_subscribe_channels(state.config.nip_fi.mode); tokio::spawn(Arc::clone(&state.pubsub).run_nip_fi_disconnect_subscriber(channels)); @@ -3641,7 +3652,7 @@ mod route_integration_tests { /// Return once a probe that `prober` publishes has been applied via /// `seen`: the subscription is live, and anything published earlier /// on the probe's channel has been applied. - async fn await_probe( + pub(super) async fn await_probe( seen: &mut tokio::sync::mpsc::UnboundedReceiver>, prober: &Arc, ) { @@ -3661,7 +3672,7 @@ mod route_integration_tests { } /// Return once `seen` has applied a message for `key`. - async fn await_target( + pub(super) async fn await_target( seen: &mut tokio::sync::mpsc::UnboundedReceiver>, key: &nostr::PublicKey, ) { @@ -3781,4 +3792,484 @@ mod route_integration_tests { deny_set_full_leaves_command_retryable(redis_guard()).await; } } + + /// Plan criterion 10 over real infrastructure: PostgreSQL holds the two + /// communities, Redis carries the disconnect between pods, and every + /// admission runs the production path end to end. + mod postgres_tests { + use super::external_infra_redis::{await_probe, await_target, listen}; + use super::*; + use futures_util::{SinkExt, StreamExt}; + use tokio_tungstenite::tungstenite::{self, client::IntoClientRequest, Message}; + + const DENY_BODY: &[u8] = b"authorization denied\n"; + + fn redis_url() -> String { + std::env::var("BUZZ_TEST_REDIS_URL") + .or_else(|_| std::env::var("REDIS_URL")) + .unwrap_or_else(|_| "redis://127.0.0.1:6379".into()) + } + + /// An enforce pod serving `hosts` as two exact communities over the + /// shared PostgreSQL and Redis, listening on a real socket and + /// running the production disconnect subscriber. + struct Pod { + state: Arc, + addr: std::net::SocketAddr, + seen: tokio::sync::mpsc::UnboundedReceiver>, + server: tokio::task::JoinHandle<()>, + } + + impl Drop for Pod { + fn drop(&mut self) { + self.server.abort(); + } + } + + async fn pod(hosts: &[String; 2]) -> Pod { + let mut config = crate::config::Config::for_test(); + config.database_url = crate::test_support::database_url(); + config.redis_url = redis_url(); + config.require_relay_membership = false; + config.nip_fi.mode = NipFiMode::Enforce; + config.nip_fi.registry.insert(test_issuer_policy()); + let uris = hosts.clone().map(|host| format!("https://{host}")); + config.nip_fi.communities = crate::nip_fi_config::NipFiCommunities::for_test_many( + &[&uris[0], &uris[1]], + &[TEST_ISS], + ); + let mut state = build_test_app_state(1000, config).await; + state.nip_fi_command_replay = + command_replay_guard(state.redis_pool.clone(), state.config.nip_fi.mode); + let key_source = Arc::new( + ProductionJwksSource::new( + vec![test_jwks_config()], + buzz_auth::HttpJwksFetcher::new(), + ) + .expect("key source"), + ); + key_source + .seed_snapshot_for_test(TEST_ISS, test_jwks()) + .await; + state.nip_fi_verifier = Some(Arc::new(buzz_auth::FederatedAssertionVerifier::new( + state.config.nip_fi.registry.clone(), + key_source, + ))); + let state = Arc::new(state); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let addr = listener.local_addr().expect("addr"); + let app = crate::router::build_router(Arc::clone(&state)) + .into_make_service_with_connect_info::(); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + let seen = listen(&state); + Pod { + state, + addr, + seen, + server, + } + } + + /// Both pods, with each subscription confirmed live. + async fn pair(hosts: &[String; 2]) -> [Pod; 2] { + let mut pods = [pod(hosts).await, pod(hosts).await]; + for p in &mut pods { + await_probe(&mut p.seen, &p.state).await; + } + pods + } + + /// A login assertion for `key` whose audience is `host`'s community. + /// Minted once and replayed as the same bytes in every phase. + fn retained_assertion(key: &nostr::Keys, host: &str) -> String { + use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; + let now = chrono::Utc::now().timestamp(); + let claims = serde_json::json!({ + "iss": TEST_ISS, "aud": format!("https://{host}"), "sub": "restart-test", + "iat": now, "exp": now + 600, "nostr_pubkey": key.public_key().to_hex(), + }); + let mut header = Header::new(Algorithm::ES256); + header.typ = Some("nip-fi+jwt".to_owned()); + header.kid = Some(TEST_KID.to_owned()); + let signing = EncodingKey::from_ec_pem(TEST_PRIVATE_KEY_PEM.as_bytes()).unwrap(); + encode(&header, &claims, &signing).unwrap() + } + + /// A key and its retained assertion for each community. + struct Retained { + keys: nostr::Keys, + assertions: [(String, String); 2], + } + + impl Retained { + fn mint(hosts: &[String; 2]) -> Self { + let keys = nostr::Keys::generate(); + let assertions = hosts.clone().map(|host| { + let token = retained_assertion(&keys, &host); + (host, token) + }); + Self { keys, assertions } + } + } + + /// How one admission ended; allowed cases must reach real admission. + #[derive(Debug, PartialEq, Eq, Clone, Copy)] + enum Outcome { + Admitted, + Denied, + } + use Outcome::{Admitted, Denied}; + + /// A root socket past the upgrade, holding its NIP-42 challenge. + struct RootSession { + socket: tokio_tungstenite::WebSocketStream< + tokio_tungstenite::MaybeTlsStream, + >, + challenge: String, + relay: String, + } + + /// Upgrade the root socket at `host` with `token`. A deny at upgrade + /// is the 403 deny body; anything else that is not an upgrade fails. + async fn open_root(pod: &Pod, host: &str, token: &str) -> Result { + let mut req = format!("ws://{}/", pod.addr) + .into_client_request() + .expect("request"); + req.headers_mut().insert("host", host.parse().unwrap()); + req.headers_mut().insert( + "nostr-federated-identity", + format!("Bearer {token}").parse().unwrap(), + ); + let mut socket = match tokio_tungstenite::connect_async(req).await { + Ok((socket, _)) => socket, + Err(tungstenite::Error::Http(resp)) => { + let body = resp.body().clone().unwrap_or_default(); + assert_eq!( + (resp.status().as_u16(), &body[..]), + (403, DENY_BODY), + "root upgrade refused other than by the deny body" + ); + return Err(Denied); + } + Err(e) => panic!("root upgrade failed: {e}"), + }; + let challenge = loop { + let frame = tokio::time::timeout(std::time::Duration::from_secs(5), socket.next()) + .await + .expect("AUTH challenge in time"); + if let Some(Ok(Message::Text(text))) = frame { + let json: serde_json::Value = serde_json::from_str(&text).unwrap(); + if json[0] == "AUTH" { + break json[1].as_str().unwrap().to_owned(); + } + } else { + panic!("expected the AUTH challenge, got {frame:?}"); + } + }; + let relay = format!( + "{}://{host}", + if pod.state.config.relay_url.starts_with("wss://") { + "wss" + } else { + "ws" + } + ); + Ok(RootSession { + socket, + challenge, + relay, + }) + } + + impl RootSession { + /// Complete NIP-42 as `keys`: `Admitted` on OK true, `Denied` on + /// the authorization-denied frame followed by the close. + async fn authenticate(mut self, keys: &nostr::Keys) -> Outcome { + let auth = nostr::EventBuilder::new(nostr::Kind::Authentication, "") + .tag(nostr::Tag::parse(["relay", &self.relay]).unwrap()) + .tag(nostr::Tag::parse(["challenge", &self.challenge]).unwrap()) + .sign_with_keys(keys) + .unwrap(); + let id = auth.id.to_hex(); + let frame = serde_json::json!(["AUTH", auth]).to_string(); + self.socket.send(Message::Text(frame.into())).await.unwrap(); + let denial = buzz_auth::DenialClass::AuthorizationDenied.nostr_text(); + let mut denied = false; + loop { + let frame = + tokio::time::timeout(std::time::Duration::from_secs(5), self.socket.next()) + .await + .expect("AUTH outcome in time"); + match frame { + Some(Ok(Message::Text(text))) => { + let json: serde_json::Value = serde_json::from_str(&text).unwrap(); + if json[0] == "OK" && json[1] == id.as_str() && json[2] == true { + assert!(!denied, "admitted after a denial frame"); + return Admitted; + } + denied |= text.contains(denial); + } + Some(Ok(Message::Close(_))) | None | Some(Err(_)) => { + assert!(denied, "root closed without the denial frame"); + return Denied; + } + Some(Ok(_)) => {} + } + } + } + } + + /// A full root session: upgrade, then NIP-42. + async fn root(pod: &Pod, keys: &nostr::Keys, host: &str, token: &str) -> Outcome { + match open_root(pod, host, token).await { + Ok(session) => session.authenticate(keys).await, + Err(denied) => denied, + } + } + + /// A signed NIP-98 `POST /query` through the real handler: 200 when + /// admitted, the deny body when denied. + async fn http(pod: &Pod, keys: &nostr::Keys, host: &str, token: &str) -> Outcome { + use base64::Engine as _; + use sha2::Digest as _; + let body = br#"[{"kinds":[1],"limit":1}]"#; + let scheme = if pod.state.config.relay_url.starts_with("wss://") { + "https" + } else { + "http" + }; + let proof = nostr::EventBuilder::new(nostr::Kind::HttpAuth, "") + .tags([ + nostr::Tag::parse(["u", &format!("{scheme}://{host}/query")]).unwrap(), + nostr::Tag::parse(["method", "POST"]).unwrap(), + // A fresh proof per request: the replay guard is shared. + nostr::Tag::parse(["nonce", &uuid::Uuid::new_v4().to_string()]).unwrap(), + nostr::Tag::parse(["payload", &hex::encode(sha2::Sha256::digest(body))]) + .unwrap(), + ]) + .sign_with_keys(keys) + .unwrap(); + let proof = base64::engine::general_purpose::STANDARD + .encode(serde_json::to_vec(&proof).unwrap()); + let request = Request::post("/query") + .header(axum::http::header::HOST, host) + .header(axum::http::header::CONTENT_TYPE, "application/json") + .header(axum::http::header::AUTHORIZATION, format!("Nostr {proof}")) + .header(CLIENT_ATTACHED_HEADER, format!("Bearer {token}")) + .body(Body::from(&body[..])) + .unwrap(); + let resp = crate::router::build_router(Arc::clone(&pod.state)) + .oneshot(request) + .await + .unwrap(); + let status = resp.status(); + let body = axum::body::to_bytes(resp.into_body(), 64 * 1024) + .await + .unwrap(); + match status { + StatusCode::OK => Admitted, + StatusCode::FORBIDDEN if &body[..] == DENY_BODY => Denied, + other => panic!( + "unexpected HTTP outcome {other}: {}", + String::from_utf8_lossy(&body) + ), + } + } + + /// The audio deny-set check after a real NIP-42 session. + async fn audio(pod: &Pod, keys: &nostr::Keys, host: &str, token: &str) -> Outcome { + let binding = pod + .state + .config + .nip_fi + .communities + .resolve(host) + .expect("served"); + let assertion = pod + .state + .nip_fi_verifier + .as_deref() + .expect("verifier") + .verify_assertion(token, binding) + .expect("the retained assertion is still valid"); + if crate::audio::handler::test_support::deny_set_refuses( + Arc::clone(&pod.state), + assertion, + keys, + ) + .await + { + Denied + } else { + Admitted + } + } + + /// Asserts every retained assertion of K (expected `key`) and the + /// control key (always admitted), on both pods and in both + /// communities, at root, audio and HTTP. + async fn check(pods: &[Pod; 2], [k, control]: [&Retained; 2], key: Outcome, phase: &str) { + for (n, pod) in pods.iter().enumerate() { + for (who, retained, expected) in [("K", k, key), ("control", control, Admitted)] { + for (host, token) in &retained.assertions { + let keys = &retained.keys; + assert_eq!( + [ + root(pod, keys, host, token).await, + audio(pod, keys, host, token).await, + http(pod, keys, host, token).await, + ], + [expected; 3], + "{phase}: {who} at {host} on P{} [root, audio, HTTP]", + n + 1 + ); + } + } + } + } + + /// Root sessions for every retained assertion on both pods, upgraded + /// but not yet authenticated. + async fn open_all(pods: &[Pod; 2], retained: &Retained) -> Vec<(String, RootSession)> { + let mut sessions = Vec::new(); + for (n, pod) in pods.iter().enumerate() { + for (host, token) in &retained.assertions { + let session = open_root(pod, host, token) + .await + .unwrap_or_else(|_| panic!("{host} on P{} must upgrade", n + 1)); + sessions.push((format!("{host} on P{}", n + 1), session)); + } + } + sessions + } + + /// Sessions K upgraded before a disconnect landed, now finishing + /// NIP-42: only the post-registration deny-set check refuses them. + /// The control key's open sessions are admitted. + async fn authenticate_all( + phase: &str, + k: (&nostr::Keys, Vec<(String, RootSession)>), + control: (&nostr::Keys, Vec<(String, RootSession)>), + ) { + for (who, (keys, sessions), expected) in + [("K", k, Denied), ("control", control, Admitted)] + { + for (at, session) in sessions { + assert_eq!( + session.authenticate(keys).await, + expected, + "{phase}: {who}'s session upgraded before it, at {at}" + ); + } + } + } + + // Plan criterion 10: restart recovery across pods. K and a control + // key each hold one assertion per community (two exact host bindings + // in PostgreSQL and config), minted before the revocation and replayed + // as the same bytes in every phase, on both pods, through a real root + // NIP-42 session, the audio deny-set check and a signed NIP-98 request + // to a real handler. A disconnect sent to one pod denies K on both. A + // root session K upgraded before the disconnect is refused at NIP-42 + // by the post-registration deny-set check. Restarting both pods + // empties their deny maps, so the retained assertions are admitted + // again until the issuer re-sends; that is the exposure window, + // recorded from restart to restored denial. The control key is + // admitted throughout. Mutations: dropping the post-registration + // check admits K's pre-upgraded sessions; dropping cross-pod + // application leaves K admitted on the other pod. + #[tokio::test] + #[ignore = "requires Postgres and Redis"] + async fn re_sent_disconnect_restores_the_deny_on_every_pod_after_restart() { + let tag = uuid::Uuid::new_v4().simple().to_string(); + let hosts = [ + format!("restart-a-{tag}.example"), + format!("restart-b-{tag}.example"), + ]; + let db = sqlx::PgPool::connect(&crate::test_support::database_url()) + .await + .expect("PostgreSQL must be available"); + for host in &hosts { + sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") + .bind(uuid::Uuid::new_v4()) + .bind(host) + .execute(&db) + .await + .expect("seed community"); + } + let k = Retained::mint(&hosts); + let control = Retained::mint(&hosts); + let minted: Vec = [&k, &control] + .iter() + .flat_map(|r| r.assertions.iter().map(|(_, t)| t.clone())) + .collect(); + let both = [&k, &control]; + let key = k.keys.public_key(); + let until = chrono::Utc::now().timestamp() + 300; + let send = |pod: Arc| async move { + let token = mint_token(&key.to_hex(), 0, serde_json::json!({ "until": until })); + let (status, body) = post_command(&pod, &token, &key.to_hex()).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(&body[..], br#"{"disconnected": true}"#); + drain_publishes(&pod).await; + }; + + let mut pods = pair(&hosts).await; + check(&pods, both, Admitted, "before the disconnect").await; + let (k_open, control_open) = + (open_all(&pods, &k).await, open_all(&pods, &control).await); + send(Arc::clone(&pods[0].state)).await; + await_target(&mut pods[1].seen, &key).await; + authenticate_all( + "after the disconnect", + (&k.keys, k_open), + (&control.keys, control_open), + ) + .await; + check(&pods, both, Denied, "after the disconnect").await; + + // Restart: fresh pods over the same database, empty deny maps. + drop(pods); + let mut pods = pair(&hosts).await; + let restarted = std::time::Instant::now(); + let (first_host, first_token) = &k.assertions[0]; + assert_eq!( + root(&pods[0], &k.keys, first_host, first_token).await, + Admitted, + "restarted: K's retained assertion completes a root session on P1" + ); + let first_admitted = restarted.elapsed(); + check(&pods, both, Admitted, "restarted, before the re-send").await; + + let (k_open, control_open) = + (open_all(&pods, &k).await, open_all(&pods, &control).await); + send(Arc::clone(&pods[1].state)).await; + let [p1, p2] = &mut pods; + await_target(&mut p1.seen, &key).await; + await_target(&mut p2.seen, &key).await; + authenticate_all( + "after the re-send", + (&k.keys, k_open), + (&control.keys, control_open), + ) + .await; + check(&pods, both, Denied, "after the re-send").await; + let restored = restarted.elapsed(); + + let replayed: Vec = [&k, &control] + .iter() + .flat_map(|r| r.assertions.iter().map(|(_, t)| t.clone())) + .collect(); + assert_eq!(replayed, minted, "assertion bytes unchanged"); + eprintln!( + "criterion 10 exposure: first retained root admission {first_admitted:?} after \ + restart; denial restored on every pod {restored:?} after restart" + ); + assert!(first_admitted < restored); + } + } } diff --git a/crates/buzz-relay/src/api/workflows.rs b/crates/buzz-relay/src/api/workflows.rs index 16685f3041..813fe434fc 100644 --- a/crates/buzz-relay/src/api/workflows.rs +++ b/crates/buzz-relay/src/api/workflows.rs @@ -79,7 +79,8 @@ async fn authorize_workflow_read( require_auth, false, ), - )?; + ) + .await?; let pubkey = *admission.proven_pubkey(); let (event_id_bytes, signed_created_at) = admission.into_extra(); diff --git a/crates/buzz-relay/src/audio/handler.rs b/crates/buzz-relay/src/audio/handler.rs index 8115ed9bbd..1a8b251ea2 100644 --- a/crates/buzz-relay/src/audio/handler.rs +++ b/crates/buzz-relay/src/audio/handler.rs @@ -92,14 +92,14 @@ pub async fn ws_audio_handler( req: axum::extract::Request, ) -> impl IntoResponse { // NIP-FI assertion check at upgrade — before tenant lookup and before the - // WebSocket handshake. Running pre-lookup means a denied request pays zero - // DB cost and the gate is reachable in tests without a live community. + // WebSocket handshake. Running pre-lookup means a denied request skips the tenant + // lookup (a resolver miss costs one bounded binding lookup) and the gate is reachable in tests without a live community. // [FI-TRACE-TRANSPORT-CLOSED] [NIP-FI.md §Admission pairing sequence] - let (nip_fi_assertion, shadow_assertion) = { + let (nip_fi_assertion, shadow_assertion, nip_fi_deadline) = { use crate::nip_fi_upgrade::check_nip_fi_at_upgrade; let mode = state.config.nip_fi.mode; let verifier = state.nip_fi_verifier.as_deref(); - let communities = &state.config.nip_fi.communities; + let communities = state.nip_fi_communities(); match check_nip_fi_at_upgrade( crate::nip_fi_session::NipFiWsRoute::Audio, &headers, @@ -107,6 +107,7 @@ pub async fn ws_audio_handler( verifier, mode, ) + .await .into_assertions() { Ok(assertions) => assertions, @@ -134,6 +135,16 @@ pub async fn ws_audio_handler( } }; + // The tenant lookup awaited since the community resolved; a snapshot + // that went stale meanwhile accepts no socket. + if let Err(class) = nip_fi_deadline.check() { + return crate::nip_fi_core::http_denial(class).into_response(); + } + let shadow_assertion = crate::nip_fi_upgrade::observe_deadline( + crate::nip_fi_session::NipFiWsRoute::Audio, + shadow_assertion, + ); + let ws = match WebSocketUpgrade::from_request(req, &state).await { Ok(ws) => ws, Err(e) => return e.into_response(), @@ -158,11 +169,11 @@ pub async fn ws_audio_handler( // so the NIP-FI session partition is rooted at the HTTP handshake, not the // post-community-active-check instant. [FI-TRACE-LEASE-BOUND] let connection_time = chrono::Utc::now(); - let shadow = shadow_assertion.map(|a| { + let shadow = shadow_assertion.map(|(a, community)| { crate::nip_fi_shadow_session::ShadowSession::start( &state, "audio", - &headers, + community, a, connection_time, ) @@ -3533,6 +3544,89 @@ async fn emit_participant_event( } } +/// Drives one audio socket through the production NIP-42 and deny-set path. +#[cfg(test)] +pub(crate) mod test_support { + use tokio_tungstenite::{connect_async, tungstenite::Message}; + + use super::*; + + /// Whether the audio deny-set check refuses a session admitted on + /// `assertion` and authenticated by `key`: `true` on the denial frame, + /// `false` once the handler passes the check. + pub(crate) async fn deny_set_refuses( + state: Arc, + assertion: VerifiedAssertion, + key: &nostr::Keys, + ) -> bool { + let community = buzz_core::tenant::CommunityId::from_uuid(Uuid::new_v4()); + let tenant = TenantContext::resolved(community, "audio-deny.test".to_owned()); + let relay_url = + crate::api::bridge::nip42_expected_relay_url(&state.config.relay_url, &tenant); + let (passed, release) = + crate::nip_fi_test_hooks::audio_after_deny_check_passed_hook::arm(community); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let app = axum::Router::new().route( + "/", + axum::routing::get(move |ws: WebSocketUpgrade| async move { + ws.on_upgrade(move |socket| { + handle_active_audio_connection( + socket, + state, + tenant, + Uuid::new_v4(), + CommunityConnectionControl::new(CancellationToken::new()), + Some(assertion), + chrono::Utc::now(), + None, + ) + }) + }), + ); + let server = tokio::spawn(async move { axum::serve(listener, app).await }); + + let (mut client, _) = connect_async(format!("ws://{addr}/")).await.unwrap(); + let challenge = match client.next().await { + Some(Ok(Message::Text(text))) => { + let json: serde_json::Value = serde_json::from_str(&text).unwrap(); + json["challenge"].as_str().unwrap().to_owned() + } + other => panic!("expected the AUTH challenge, got {other:?}"), + }; + let auth = nostr::EventBuilder::new(nostr::Kind::Authentication, "") + .tag(nostr::Tag::parse(["relay", &relay_url]).unwrap()) + .tag(nostr::Tag::parse(["challenge", &challenge]).unwrap()) + .sign_with_keys(key) + .unwrap(); + let auth = serde_json::json!({"type": "auth", "event": auth}).to_string(); + client.send(Message::Text(auth.into())).await.unwrap(); + + let denial = serde_json::json!({ + "type": "restricted", + "message": buzz_auth::DenialClass::AuthorizationDenied.nostr_text(), + }) + .to_string(); + let refused = tokio::time::timeout(std::time::Duration::from_secs(5), async { + tokio::select! { + frame = client.next() => match frame { + Some(Ok(Message::Text(text))) if text.as_str() == denial => true, + other => panic!("expected the denial frame, got {other:?}"), + }, + arrived = passed => { + arrived.expect("hook"); + false + } + } + }) + .await + .expect("audio session reached neither outcome"); + release.notify_one(); + server.abort(); + refused + } +} + #[cfg(test)] mod tests { use std::sync::Mutex; @@ -5994,7 +6088,7 @@ mod tests { crate::nip_fi_shadow_session::ShadowSession::start( state, "audio", - &axum::http::HeaderMap::new(), + crate::nip_fi_shadow::BOUND.into(), assertion, Utc::now(), ) diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index 129d2792ed..4c9e257318 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -403,6 +403,11 @@ pub struct Config { /// (`BUZZ_NIP_FI_PROVISION_BINDING`). `None` binds nothing at creation. /// Parsed whatever the NIP-FI mode, since creation happens in every mode. pub nip_fi_provision_binding: Option, + + /// Database resolver bounds (`BUZZ_NIP_FI_RESOLVER_*`, + /// `BUZZ_NIP_FI_SNAPSHOT_*`). Validated in every mode, so a bad value + /// fails the deploy that sets it, not the later switch to shadow. + pub nip_fi_resolver_bounds: crate::nip_fi_resolver::ResolverBounds, } fn parse_bind_addr(raw: &str) -> Result { @@ -1377,6 +1382,18 @@ impl Config { }; let nip_fi = crate::nip_fi_config::NipFiRelayConfig::from_env()?; + let nip_fi_provision_binding = crate::nip_fi_config::parse_provision_binding()?; + let nip_fi_resolver_bounds = + crate::nip_fi_resolver::ResolverBounds::from_env(db_pool_size)?; + if nip_fi.source == crate::nip_fi_config::CommunitySource::Database + && nip_fi_provision_binding.is_none() + { + return Err(ConfigError::InvalidValue( + "BUZZ_NIP_FI_COMMUNITY_SOURCE=database requires BUZZ_NIP_FI_PROVISION_BINDING \ + (the suffix and reserved host)" + .into(), + )); + } Ok(Self { bind_addr, database_url, @@ -1442,7 +1459,8 @@ impl Config { admin, web_dir, serve_git_web_gui, - nip_fi_provision_binding: crate::nip_fi_config::parse_provision_binding()?, + nip_fi_provision_binding, + nip_fi_resolver_bounds, nip_fi, }) } @@ -1514,6 +1532,83 @@ mod tests { (fi, cfg) } + // The three `source=database` startup guards. Mutation: dropping any one + // lets that configuration start. + #[test] + fn database_source_config_guards_refuse_startup() { + const VARS: [&str; 6] = [ + "BUZZ_NIP_FI_COMMUNITY_SOURCE", + "BUZZ_NIP_FI_COMMUNITIES", + "BUZZ_NIP_FI_ISSUERS", + "BUZZ_NIP_FI_PROVISION_BINDING", + "BUZZ_NIP_FI_MAX_CONNECTION_LIFETIME_SECS", + "BUZZ_NIP_FI_MODE", + ]; + struct Clear; + impl Drop for Clear { + fn drop(&mut self) { + for v in VARS { + std::env::remove_var(v); + } + } + } + let _guards = env_guards(); + let _clear = Clear; + let refusal = || match Config::from_env() { + Err(ConfigError::InvalidValue(message)) => message, + other => panic!("expected a refusal, got {:?}", other.map(|_| ())), + }; + + std::env::set_var("BUZZ_NIP_FI_COMMUNITY_SOURCE", "ldap"); + assert!(refusal().contains("must be \"env\" or \"database\"")); + + std::env::set_var("BUZZ_NIP_FI_COMMUNITY_SOURCE", "database"); + std::env::set_var("BUZZ_NIP_FI_COMMUNITIES", "[]"); + assert!(refusal().contains("BUZZ_NIP_FI_COMMUNITIES is set")); + + std::env::remove_var("BUZZ_NIP_FI_COMMUNITIES"); + std::env::set_var( + "BUZZ_NIP_FI_ISSUERS", + serde_json::json!([{ + "issuer": "https://idp.example.com", + "command_audiences": ["https://relay.example.com"], + "token_class": "nip-fi+jwt", + "algorithms": ["ES256"], + "maximum_assertion_age_seconds": 3600, + "jwks_uri": "https://idp.example.com/.well-known/jwks.json", + "jwks_refresh_interval_seconds": 300, + "jwks_hard_deadline_seconds": 86400, + "maximum_command_age_seconds": 30, + "authorized_principals": ["https://idp.example.com/operator"] + }]) + .to_string(), + ); + std::env::set_var("BUZZ_NIP_FI_MAX_CONNECTION_LIFETIME_SECS", "3600"); + assert!(refusal().contains("requires BUZZ_NIP_FI_PROVISION_BINDING")); + + std::env::set_var( + "BUZZ_NIP_FI_PROVISION_BINDING", + r#"{"host_suffix":"communities.example.com","authorized_issuers":["https://idp.example.com"]}"#, + ); + let config = Config::from_env().expect("complete database-source config"); + assert!(config.nip_fi_provision_binding.is_some()); + } + + // Mutation: parsing the resolver bounds only when the resolver is built + // lets an off-mode deploy carry a value the switch to shadow refuses. + #[test] + fn resolver_bounds_are_validated_in_off_mode() { + let _guards = env_guards(); + std::env::set_var("BUZZ_NIP_FI_RESOLVER_MAX_INFLIGHT", "0"); + let result = Config::from_env(); + std::env::remove_var("BUZZ_NIP_FI_RESOLVER_MAX_INFLIGHT"); + assert!( + matches!(result, Err(ref e) if e.to_string().contains("BUZZ_NIP_FI_RESOLVER_MAX_INFLIGHT")), + "{:?}", + result.map(|_| ()) + ); + } + /// Look up against a fixed set, standing in for process env. fn env_of<'a>(set: &'a [(&'a str, &'a str)]) -> impl Fn(&str) -> Option + use<'a> { move |name| { diff --git a/crates/buzz-relay/src/handlers/auth.rs b/crates/buzz-relay/src/handlers/auth.rs index 8a94a5827d..f0160ed275 100644 --- a/crates/buzz-relay/src/handlers/auth.rs +++ b/crates/buzz-relay/src/handlers/auth.rs @@ -376,7 +376,7 @@ pub async fn handle_auth(event: nostr::Event, conn: Arc, state: } // NIP-FI key pairing [FI-INV-05]: immediately after successful // verify_auth_event, before community-ban/allowlist/membership gates. - // Pre-DB positioning means a denied caller pays zero DB cost and the + // Running before those gates means a denied caller skips their DB reads and the // production call site is falsifiable without live tenant policy. // [FI-TRACE-DENIAL-ORACLE post-establishment] if crate::nip_fi_session::enforce_nip_fi_key_pairing( @@ -1465,11 +1465,10 @@ mod tests { let t = crate::connection::tests::test_conn(pending, None); let assertion = buzz_auth::VerifiedAssertion::for_test(Some(asserted), vec![Utc::now() + lifetime]); - let headers = axum::http::HeaderMap::new(); let session = crate::nip_fi_shadow_session::ShadowSession::start( state, "ws", - &headers, + crate::nip_fi_shadow::BOUND.into(), assertion, Utc::now(), ); diff --git a/crates/buzz-relay/src/handlers/community_provisioning.rs b/crates/buzz-relay/src/handlers/community_provisioning.rs index b187b223ad..6b258cd084 100644 --- a/crates/buzz-relay/src/handlers/community_provisioning.rs +++ b/crates/buzz-relay/src/handlers/community_provisioning.rs @@ -229,6 +229,18 @@ async fn publish_membership_snapshot_if_required( } } +/// After a committed creation that bound issuers, drop this pod's cached +/// "not found" for the host so the creator's first login looks it up. +pub(crate) fn record_binding_creation( + resolver: Option<&crate::nip_fi_resolver::DatabaseResolver>, + host: &str, + issuers: &[String], +) { + if let (Some(resolver), false) = (resolver, issuers.is_empty()) { + resolver.record_creation(host); + } +} + /// Validate and execute a relay-operator community provisioning request. /// /// The caller is an HTTP operator endpoint, not the Nostr event ingest path. @@ -309,6 +321,11 @@ pub async fn provision_community( owner = %owner_hex, "community created via operator endpoint" ); + record_binding_creation( + state.nip_fi_resolver.as_deref(), + &record.host, + nip_fi_issuers, + ); publish_membership_snapshot_if_required(state, record.id, &record.host).await; return Ok(ProvisionCommunityResponse { community_id: record.id.to_string(), @@ -326,6 +343,13 @@ pub async fn provision_community( .ensure_configured_community_with_nip_fi_issuers(&request.host, nip_fi_issuers) .await .map_err(|e| format!("failed to create community: {e}"))?; + if record.created { + record_binding_creation( + state.nip_fi_resolver.as_deref(), + &record.host, + nip_fi_issuers, + ); + } if let Some(owner_hex) = &initial_owner { match state diff --git a/crates/buzz-relay/src/lib.rs b/crates/buzz-relay/src/lib.rs index e0fd61d1f6..17a7cb49e4 100644 --- a/crates/buzz-relay/src/lib.rs +++ b/crates/buzz-relay/src/lib.rs @@ -52,6 +52,7 @@ pub mod nip_fi_config; /// NIP-FI HTTP ingress enforcement: assertion extraction, verification, /// key-pairing check, and deny-map gate for every protected HTTP surface. pub(crate) mod nip_fi_http; +pub mod nip_fi_resolver; /// Deployment-global operator-listener mention delivery worker. pub mod operator_listener; /// NIP-01 client/relay message parsing. diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index d6a082affa..effda6cc0a 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -856,6 +856,36 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> { )); } } + // NIP-FI database community source. The startup load runs before any + // listener binds, so the pod can never report ready without a snapshot; + // a failed load fails startup and the orchestrator retries. + if config.nip_fi.builds_resolver() { + let step = StepTimer::start(StartupStep::NipFiSnapshotLoad); + // `Config::from_env` refuses the database source without it. + let policy = config + .nip_fi_provision_binding + .clone() + .ok_or_else(|| anyhow::anyhow!("NIP-FI: no provisioning binding policy"))?; + let registry = config + .nip_fi + .registry + .all_policies() + .map(|p| p.issuer().to_owned()) + .collect(); + let resolver = Arc::new(buzz_relay::nip_fi_resolver::DatabaseResolver::new( + Arc::new(app_state.db.clone()), + policy, + registry, + config.nip_fi_resolver_bounds.clone(), + )); + resolver + .refresh() + .await + .map_err(|e| anyhow::anyhow!("NIP-FI: startup binding snapshot load failed: {e:?}"))?; + step.finish(); + info!("NIP-FI: binding snapshot loaded"); + app_state.nip_fi_resolver = Some(resolver); + } let state = Arc::new(app_state); let has_startup_partition_audit = startup_partition_audit.is_some(); if let Some(audit) = startup_partition_audit { @@ -912,6 +942,10 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> { // The refresh task is owned: a `CancellationToken` + `JoinHandle` let the // process cancel it cleanly on shutdown instead of leaking the task. let jwks_refresh_cancel = CancellationToken::new(); + // The binding refresh loop stops with the JWKS loop on shutdown. + if let Some(resolver) = state.nip_fi_resolver.clone() { + tokio::spawn(resolver.run_refresh_loop(jwks_refresh_cancel.child_token())); + } let jwks_refresh_handle = if let Some(jwks_source) = state.nip_fi_jwks_source.clone() { let jwks_configs = state.config.nip_fi.jwks_configs.clone(); info!( diff --git a/crates/buzz-relay/src/nip_fi_config.rs b/crates/buzz-relay/src/nip_fi_config.rs index 6dd846533f..f78f52f225 100644 --- a/crates/buzz-relay/src/nip_fi_config.rs +++ b/crates/buzz-relay/src/nip_fi_config.rs @@ -32,6 +32,7 @@ //! backward-compatible until an operator explicitly enables enforcement. use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; use std::time::Duration; use buzz_auth::nip_fi::provision::canonical_authority; @@ -179,17 +180,17 @@ pub(super) struct CommunityEnvConfig { /// no database access. Empty unless the mode is `enforce`. #[derive(Debug, Clone, Default)] pub struct NipFiCommunities { - by_host: BTreeMap, + by_host: BTreeMap>, /// Test-only: one community served at every Host, for fixtures that /// exercise enforce-mode handlers on per-test unique Hosts. #[cfg(test)] - any_host: Option, + any_host: Option>, } impl NipFiCommunities { /// The community served at `raw_host`, matched after the same /// normalization the tenant binder applies. - pub fn resolve(&self, raw_host: &str) -> Option<&CommunityBinding> { + pub fn resolve(&self, raw_host: &str) -> Option<&Arc> { #[cfg(test)] if let Some(binding) = &self.any_host { return Some(binding); @@ -231,7 +232,7 @@ impl NipFiCommunities { .map_err(|e| format!("community at index {idx}: {e}"))?; // One Host per canonical URI, so a unique Host also means a // unique `aud`. - if by_host.insert(host, binding).is_some() { + if by_host.insert(host, Arc::new(binding)).is_some() { return Err(format!( "community at index {idx}: canonical_uri maps to the same \ Host as an earlier community" @@ -245,13 +246,9 @@ impl NipFiCommunities { }) } - /// Every community `aud` that authorizes `issuer`. - fn audiences_authorizing(&self, issuer: &str) -> Vec { - self.by_host - .values() - .filter(|c| c.authorizes(issuer)) - .map(|c| c.expected_aud().to_owned()) - .collect() + /// Whether any community authorizes `issuer`. + fn authorizes_anywhere(&self, issuer: &str) -> bool { + self.by_host.values().any(|c| c.authorizes(issuer)) } } @@ -274,6 +271,8 @@ pub fn parse_provision_binding() -> Result Result { let mode = parse_mode()?; + let source = parse_community_source()?; + if source == CommunitySource::Database + && !std::env::var("BUZZ_NIP_FI_COMMUNITIES") + .unwrap_or_default() + .trim() + .is_empty() + { + return Err(ConfigError::InvalidValue( + "BUZZ_NIP_FI_COMMUNITY_SOURCE=database but BUZZ_NIP_FI_COMMUNITIES is set; \ + database-sourced communities are read from community_nip_fi_issuers" + .to_string(), + )); + } - if !mode.evaluates() { + // The issuer registry is required whenever admission evaluates, and + // with the database source in every mode: provisioning, bootstrap and + // the resolver's registry filter all read it. + if !mode.evaluates() && source == CommunitySource::Env { return Ok(Self { mode, + source, registry: IssuerRegistry::new(), communities: NipFiCommunities::default(), jwks_configs: Vec::new(), @@ -346,18 +362,21 @@ impl NipFiRelayConfig { // duplicate that could disagree with the per-issuer value is a config-drift // trap — removed in this PR. - let max_connection_lifetime_secs = parse_u64_bounded( + let max_connection_lifetime_secs = match parse_u64_bounded( "BUZZ_NIP_FI_MAX_CONNECTION_LIFETIME_SECS", 1, MAX_CONNECTION_LIFETIME_SECS, - )? - .ok_or_else(|| { - mode_requires( - mode, - "BUZZ_NIP_FI_MAX_CONNECTION_LIFETIME_SECS is not set; \ - every enforce deployment must configure a positive finite value", - ) - })?; + )? { + Some(secs) => secs, + None if !mode.evaluates() => 0, + None => { + return Err(mode_requires( + mode, + "BUZZ_NIP_FI_MAX_CONNECTION_LIFETIME_SECS is not set; \ + every enforce deployment must configure a positive finite value", + )) + } + }; let mut policies = Vec::with_capacity(issuer_entries.len()); let mut jwks_configs = Vec::with_capacity(issuer_entries.len()); @@ -452,30 +471,45 @@ impl NipFiRelayConfig { } } - let configured: BTreeSet<&str> = policies.iter().map(IssuerPolicy::issuer).collect(); - let communities = parse_communities(mode, &configured)?; - // Fold each issuer's community allowlist into its policy ID - // (NIP-FI.md:145), and reject an issuer no community authorizes - // (NIP-FI.md:183). let mut registry = IssuerRegistry::new(); - for (issuer_idx, policy) in policies.into_iter().enumerate() { - let audiences = communities.audiences_authorizing(policy.issuer()); - if audiences.is_empty() { - return Err(ConfigError::InvalidValue(format!( - "BUZZ_NIP_FI_ISSUERS: issuer at index {issuer_idx} is not \ - authorized by any community in BUZZ_NIP_FI_COMMUNITIES" - ))); + let communities = match source { + CommunitySource::Env => { + let configured: BTreeSet<&str> = + policies.iter().map(IssuerPolicy::issuer).collect(); + let communities = parse_communities(mode, &configured)?; + // Reject an issuer no community authorizes (NIP-FI.md:183). + for (issuer_idx, policy) in policies.iter().enumerate() { + if !communities.authorizes_anywhere(policy.issuer()) { + return Err(ConfigError::InvalidValue(format!( + "BUZZ_NIP_FI_ISSUERS: issuer at index {issuer_idx} is not \ + authorized by any community in BUZZ_NIP_FI_COMMUNITIES" + ))); + } + } + communities } - registry.insert(policy.authorized_for_communities(audiences)); + // An issuer no community row references grants nothing: + // authorization still needs a per-community binding. + CommunitySource::Database => NipFiCommunities::default(), + }; + for policy in policies { + registry.insert(policy); } // Delegate final validation to buzz-auth startup gate. - validate_nip_fi_config(mode, ®istry, &jwks_configs).map_err( + // The database source validates the registry in every mode, since + // creation paths bind against it even while admission is off. + let validation_mode = match source { + CommunitySource::Database if !mode.evaluates() => NipFiMode::Enforce, + _ => mode, + }; + validate_nip_fi_config(validation_mode, ®istry, &jwks_configs).map_err( |e: NipFiStartupError| ConfigError::InvalidValue(format!("NIP-FI config invalid: {e}")), )?; Ok(Self { mode, + source, registry, communities, jwks_configs, @@ -512,6 +546,30 @@ fn parse_communities( // ── Helpers ─────────────────────────────────────────────────────────────────── +/// Where the Host → community bindings come from. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum CommunitySource { + /// The static `BUZZ_NIP_FI_COMMUNITIES` map (the default). + #[default] + Env, + /// The `community_nip_fi_issuers` table, via the snapshot resolver. + Database, +} + +fn parse_community_source() -> Result { + match std::env::var("BUZZ_NIP_FI_COMMUNITY_SOURCE") + .ok() + .as_deref() + .map(str::trim) + { + None | Some("") | Some("env") => Ok(CommunitySource::Env), + Some("database") => Ok(CommunitySource::Database), + Some(other) => Err(ConfigError::InvalidValue(format!( + "BUZZ_NIP_FI_COMMUNITY_SOURCE must be \"env\" or \"database\"; got {other:?}" + ))), + } +} + fn parse_mode() -> Result { match std::env::var("BUZZ_NIP_FI_MODE") .ok() @@ -542,7 +600,11 @@ fn mode_requires(mode: NipFiMode, missing: &str) -> ConfigError { /// Parse an optional positive `u64` env var bounded to `[min_val, max_val]`. /// Returns `None` when the variable is absent or empty. -fn parse_u64_bounded(name: &str, min_val: u64, max_val: u64) -> Result, ConfigError> { +pub(crate) fn parse_u64_bounded( + name: &str, + min_val: u64, + max_val: u64, +) -> Result, ConfigError> { match std::env::var(name) { Err(_) => Ok(None), Ok(raw) if raw.trim().is_empty() => Ok(None), @@ -654,6 +716,13 @@ fn build_issuer(entry: &IssuerEnvConfig) -> Result<(IssuerPolicy, IssuerJwksConf // ── Duration helpers ────────────────────────────────────────────────────────── impl NipFiRelayConfig { + /// Whether startup builds the database resolver: only for the `database` + /// source in a mode that evaluates evidence. Off and deny-protected never + /// resolve a community, so they load no snapshot. + pub fn builds_resolver(&self) -> bool { + self.source == CommunitySource::Database && self.mode.evaluates() + } + /// Returns the configured `max_connection_lifetime` as a `Duration`. /// Returns `None` in `Off`/`DenyProtected` mode (sentinel value 0). pub fn max_connection_lifetime(&self) -> Option { @@ -674,12 +743,20 @@ impl NipFiRelayConfig { impl NipFiCommunities { /// One community at `canonical_uri` authorizing `issuers`. pub(crate) fn for_test(canonical_uri: &str, issuers: &[&str]) -> Self { - let entries = vec![CommunityEnvConfig { - canonical_uri: canonical_uri.to_owned(), - authorized_issuers: issuers.iter().map(|i| (*i).to_owned()).collect(), - }]; + Self::for_test_many(&[canonical_uri], issuers) + } + + /// One community per entry of `canonical_uris`, each authorizing `issuers`. + pub(crate) fn for_test_many(canonical_uris: &[&str], issuers: &[&str]) -> Self { + let entries = canonical_uris + .iter() + .map(|uri| CommunityEnvConfig { + canonical_uri: (*uri).to_owned(), + authorized_issuers: issuers.iter().map(|i| (*i).to_owned()).collect(), + }) + .collect(); Self::from_entries(entries, &issuers.iter().copied().collect()) - .expect("valid test community") + .expect("valid test communities") } /// One community, expecting `aud` and authorizing `issuers`, served at @@ -689,7 +766,7 @@ impl NipFiCommunities { CommunityBinding::new(aud.to_owned(), issuers.iter().map(|i| (*i).to_owned())) .expect("valid test community"); Self { - any_host: Some(binding), + any_host: Some(Arc::new(binding)), ..Self::default() } } @@ -916,8 +993,6 @@ mod tests { for policy in [public_policy, mesh_policy] { assert_eq!(policy.command_audiences().len(), 1); assert_eq!(policy.command_audiences()[0], "https://relay.test"); - assert_eq!(policy.community_audiences().len(), 1); - assert_eq!(policy.community_audiences()[0], "https://relay.test"); } assert_eq!(cfg.jwks_configs.len(), 2); assert_eq!(cfg.jwks_configs[0].contract, public_contract); @@ -1894,4 +1969,31 @@ mod tests { ); } } + + // Mutation: building the resolver for the database source in every mode + // loads a snapshot that off and deny-protected never read. + #[test] + fn only_evaluating_modes_with_the_database_source_build_a_resolver() { + let base = { + let _guard = super::NIP_FI_ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _env = EnvGuard::new(NIP_FI_VARS); + NipFiRelayConfig::from_env().expect("off config") + }; + for (mode, source, expected) in [ + (NipFiMode::Off, CommunitySource::Database, false), + (NipFiMode::DenyProtected, CommunitySource::Database, false), + (NipFiMode::Shadow, CommunitySource::Database, true), + (NipFiMode::Enforce, CommunitySource::Database, true), + (NipFiMode::Enforce, CommunitySource::Env, false), + ] { + let config = NipFiRelayConfig { + mode, + source, + ..base.clone() + }; + assert_eq!(config.builds_resolver(), expected, "{mode:?} {source:?}"); + } + } } diff --git a/crates/buzz-relay/src/nip_fi_core.rs b/crates/buzz-relay/src/nip_fi_core.rs index 86d4e776e0..1b1d887df7 100644 --- a/crates/buzz-relay/src/nip_fi_core.rs +++ b/crates/buzz-relay/src/nip_fi_core.rs @@ -47,8 +47,14 @@ impl AssertionRejection { } } -/// Resolve the community served at the request `Host` — enforce-mode -/// admission step 1, before any evidence is examined (NIP-FI.md:266-268). +/// The request `Host`, if present and readable. +fn request_host(headers: &HeaderMap) -> Option<&str> { + headers + .get(axum::http::header::HOST) + .and_then(|v| v.to_str().ok()) +} + +/// Resolve the community served at the request `Host` from a static map. /// /// An absent, unreadable, or unmapped Host is `authorization_unavailable`. /// This makes mapped and unmapped Hosts distinguishable (401/403 vs 503) in @@ -57,14 +63,100 @@ impl AssertionRejection { pub(crate) fn resolve_community<'a>( headers: &HeaderMap, communities: &'a NipFiCommunities, -) -> Result<&'a CommunityBinding, DenialClass> { - headers - .get(axum::http::header::HOST) - .and_then(|v| v.to_str().ok()) +) -> Result<&'a std::sync::Arc, DenialClass> { + request_host(headers) .and_then(|host| communities.resolve(host)) .ok_or(DenialClass::AuthorizationUnavailable) } +/// A community resolved for one enforce admission. +#[derive(Clone)] +pub(crate) struct ResolvedCommunity { + binding: std::sync::Arc, + deadline: AdmissionDeadline, +} + +impl ResolvedCommunity { + /// The deadline admission must re-check after its own awaits. + pub(crate) fn deadline(&self) -> AdmissionDeadline { + self.deadline.clone() + } + + /// The shadow `community` label: the configured URI for an `env` + /// binding, whose set is bounded by config, else `bound`, since the + /// database's host set is unbounded. + pub(crate) fn shadow_label(&self) -> metrics::SharedString { + if self.deadline.0.is_none() { + self.binding.expected_aud().to_owned().into() + } else { + metrics::SharedString::const_str(crate::nip_fi_shadow::BOUND) + } + } +} + +impl std::ops::Deref for ResolvedCommunity { + type Target = CommunityBinding; + fn deref(&self) -> &CommunityBinding { + &self.binding + } +} + +/// The point past which admitting on a resolved community is +/// `authorization_unavailable`: the database snapshot's staleness deadline. +/// A static `env` binding never goes stale. +#[derive(Clone, Default)] +pub(crate) struct AdmissionDeadline(Option); + +impl AdmissionDeadline { + /// Run immediately before admitting. + pub(crate) fn check(&self) -> Result<(), DenialClass> { + self.0.as_ref().map_or(Ok(()), |fresh| fresh.check()) + } +} + +/// The relay's community source (`BUZZ_NIP_FI_COMMUNITY_SOURCE`): the +/// static `env` map, or the database resolver. +#[derive(Clone, Copy)] +pub(crate) enum Communities<'a> { + Env(&'a NipFiCommunities), + Database(&'a std::sync::Arc), + /// The `database` source with no resolver built: every host is + /// `authorization_unavailable`. + Unavailable, +} + +impl Communities<'_> { + /// Resolve the community at the request `Host` — enforce-mode admission + /// step 1, before any evidence is examined (NIP-FI.md:266-268). Called + /// once per admission; the returned binding is what verification uses. + pub(crate) async fn resolve( + self, + headers: &HeaderMap, + ) -> Result { + match self { + Self::Env(map) => resolve_community(headers, map).map(|b| ResolvedCommunity { + binding: std::sync::Arc::clone(b), + deadline: AdmissionDeadline::default(), + }), + Self::Database(resolver) => { + let host = request_host(headers).ok_or(DenialClass::AuthorizationUnavailable)?; + let (binding, fresh) = resolver.resolve(host).await?; + Ok(ResolvedCommunity { + binding, + deadline: AdmissionDeadline(Some(fresh)), + }) + } + Self::Unavailable => Err(DenialClass::AuthorizationUnavailable), + } + } +} + +impl<'a> From<&'a NipFiCommunities> for Communities<'a> { + fn from(map: &'a NipFiCommunities) -> Self { + Self::Env(map) + } +} + /// Extract and verify the attached assertion for `community`. /// /// Transport extraction runs before the verifier-presence check, so a @@ -198,7 +290,7 @@ pub(crate) mod test_support { /// The binding a fixture community expecting `aud` presents. pub(crate) fn binding(aud: &str) -> CommunityBinding { - any_host(aud).resolve(TEST_HOST).expect("served").clone() + (**any_host(aud).resolve(TEST_HOST).expect("served")).clone() } /// Request headers addressed to [`TEST_HOST`]. @@ -220,10 +312,11 @@ pub(crate) mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; fn community() -> CommunityBinding { - communities() - .resolve(super::test_support::TEST_HOST) - .expect("mapped") - .clone() + CommunityBinding::clone( + communities() + .resolve(super::test_support::TEST_HOST) + .expect("mapped"), + ) } /// Verifier returning a fixed result and counting calls. diff --git a/crates/buzz-relay/src/nip_fi_http.rs b/crates/buzz-relay/src/nip_fi_http.rs index eafb68862b..42993bbea3 100644 --- a/crates/buzz-relay/src/nip_fi_http.rs +++ b/crates/buzz-relay/src/nip_fi_http.rs @@ -61,11 +61,40 @@ use chrono::{DateTime, Utc}; use nostr::PublicKey; use std::fmt; -use crate::nip_fi_config::NipFiCommunities; +use crate::nip_fi_core::ResolvedCommunity; use crate::nip_fi_core::{ - asserted_key_matches, evaluate_attached_assertion, http_denial, resolve_community, - AssertionRejection, + asserted_key_matches, evaluate_attached_assertion, http_denial, AssertionRejection, Communities, }; +use buzz_auth::CommunityBinding; + +/// One resolution of the request `Host`, shared by every step of a request. +pub(crate) type Resolution = Result; + +tokio::task_local! { + /// The community the router guard resolved for this request. + static GUARD_COMMUNITY: Resolution; +} + +/// Run `next` with the guard's resolution, so the handler's admission and +/// every shadow record reuse it instead of resolving the Host again. +pub(crate) async fn with_resolved_community( + community: Resolution, + next: F, +) -> F::Output { + GUARD_COMMUNITY.scope(community, next).await +} + +/// Enforce step 1 for one HTTP admission: the guard's resolution when the +/// request passed through the guard, else one resolution (guard-exempt routes). +pub(crate) async fn resolve_for_admission( + communities: Communities<'_>, + headers: &HeaderMap, +) -> Resolution { + match GUARD_COMMUNITY.try_with(Resolution::clone) { + Ok(community) => community, + Err(_) => communities.resolve(headers).await, + } +} use crate::nip_fi_shadow::Stage; // ── Deny-map seam ───────────────────────────────────────────────────────────── @@ -291,10 +320,10 @@ impl NipFiAdmission { // would add allocation without architectural benefit. The large Err variant // is load-bearing: it IS the HTTP response, returned directly by handlers. #[allow(clippy::result_large_err)] -pub(crate) fn admit_nip_fi_http( +pub(crate) async fn admit_nip_fi_http<'a, D, X, F>( headers: &HeaderMap, extract_nip98: F, - communities: &NipFiCommunities, + communities: impl Into>, verifier: Option<&dyn VerifyAssertion>, mode: NipFiMode, deny_map: &D, @@ -309,6 +338,7 @@ where if mode.denies_unconditionally() { return Err(http_denial(DenialClass::AuthorizationUnavailable)); } + let communities = communities.into(); // Off and Shadow: the NIP-98 result passes through unchanged (legacy // behavior); shadow only records the enforce verdict. [FI-INV-15 exemption] @@ -316,7 +346,7 @@ where let proof = extract_nip98(); if mode.observes_only() { let signed = proof.as_ref().ok().filter(|p| p.signed).map(|p| p.pubkey); - observe_shadow_verdict(headers, communities, verifier, deny_map, signed); + observe_shadow_verdict(headers, communities, verifier, deny_map, signed).await; } let proof = proof?; return Ok(NipFiAdmission { @@ -326,20 +356,30 @@ where }); } - let (proof, assertion) = - evaluate_enforce_steps(headers, communities, verifier, deny_map, extract_nip98).map_err( - |(stage, class)| { - let reason = match stage { - Stage::Pairing => Some("nip_fi_http_key_mismatch"), - Stage::DenySet => Some("nip_fi_http_denied_pubkey"), - _ => None, - }; - if let Some(reason) = reason { - metrics::counter!("buzz_auth_failures_total", "reason" => reason).increment(1); - } - http_denial(class) - }, - )?; + let community = resolve_for_admission(communities, headers).await; + let (proof, assertion) = evaluate_enforce_steps( + headers, + community.as_deref(), + verifier, + deny_map, + extract_nip98, + ) + .map_err(|(stage, class)| { + let reason = match stage { + Stage::Pairing => Some("nip_fi_http_key_mismatch"), + Stage::DenySet => Some("nip_fi_http_denied_pubkey"), + _ => None, + }; + if let Some(reason) = reason { + metrics::counter!("buzz_auth_failures_total", "reason" => reason).increment(1); + } + http_denial(class) + })?; + // The handler may have awaited (tenant lookup) since the guard resolved + // the community; a snapshot that went stale meanwhile admits nothing. + if let Ok(community) = &community { + community.deadline().check().map_err(http_denial)?; + } // Step 8: admit. Ok(NipFiAdmission { @@ -351,32 +391,42 @@ where /// Record the verdict enforce would reach for a request whose NIP-98 step /// proved `signed` (`None`: missing, rejected or unsigned). -fn observe_shadow_verdict( +async fn observe_shadow_verdict( headers: &HeaderMap, - communities: &NipFiCommunities, + communities: Communities<'_>, verifier: Option<&dyn VerifyAssertion>, deny_map: &D, signed: Option, ) { - let verdict = evaluate_enforce_steps(headers, communities, verifier, deny_map, || { + let community = resolve_for_admission(communities, headers).await; + let verdict = evaluate_enforce_steps(headers, community.as_deref(), verifier, deny_map, || { signed.map(|key| Nip98Proof::new(key, ())).ok_or(()) + }) + .and_then(|_| match &community { + // Enforce's final re-check, after the handler's awaits. + Ok(community) => community + .deadline() + .check() + .map_err(|class| (Stage::Deadline, class)), + Err(_) => Ok(()), }); - crate::nip_fi_shadow::record("http", headers, communities, verdict.map(drop)); + crate::nip_fi_shadow::record("http", &community, verdict); } /// Shadow only: record the enforce verdict for a request whose NIP-98 proof -/// already failed, without any DB work. Off and enforce are untouched. -pub(crate) fn observe_failed_proof(state: &crate::state::AppState, headers: &HeaderMap) { +/// already failed. Off and enforce are untouched. +pub(crate) async fn observe_failed_proof(state: &crate::state::AppState, headers: &HeaderMap) { let nip_fi = &state.config.nip_fi; if nip_fi.mode.observes_only() { let verifier = state.nip_fi_verifier.as_deref(); observe_shadow_verdict( headers, - &nip_fi.communities, + state.nip_fi_communities(), verifier, &NoDenyMapConfigured, None, - ); + ) + .await; } } @@ -384,7 +434,7 @@ pub(crate) fn observe_failed_proof(state: &crate::state::AppState, headers: &Hea /// (the shadow `stage` label) and its class. fn evaluate_enforce_steps( headers: &HeaderMap, - communities: &NipFiCommunities, + community: Result<&CommunityBinding, &DenialClass>, verifier: Option<&dyn VerifyAssertion>, deny_map: &D, extract_nip98: F, @@ -397,7 +447,7 @@ where // evidence (`Authorization`, NIP-98, assertion) is examined. Explicit // here because some admitted routes are exempt from the router guard. // [NIP-FI.md:266-268] - let community = resolve_community(headers, communities).map_err(|c| (Stage::Community, c))?; + let community = community.map_err(|c| (Stage::Community, *c))?; // Step 2 — cardinality gate: exactly one Authorization field per // NIP-FI.md:695-700. @@ -464,7 +514,7 @@ where /// [FI-TRACE-AUTHORITY-UNIFORM] // Response is intentionally large (axum's design); see admit_nip_fi_http. #[allow(clippy::result_large_err)] -pub(crate) fn admit_nip_fi_http_on_state( +pub(crate) async fn admit_nip_fi_http_on_state( state: &crate::state::AppState, headers: &HeaderMap, extract_nip98: F, @@ -474,24 +524,30 @@ where { let mode = state.config.nip_fi.mode; let verifier = state.nip_fi_verifier.as_deref(); - let communities = &state.config.nip_fi.communities; + let communities = state.nip_fi_communities(); match state.nip_fi_deny_map.as_deref() { - Some(deny_map) => admit_nip_fi_http( - headers, - extract_nip98, - communities, - verifier, - mode, - deny_map, - ), - None => admit_nip_fi_http( - headers, - extract_nip98, - communities, - verifier, - mode, - &NoDenyMapConfigured, - ), + Some(deny_map) => { + admit_nip_fi_http( + headers, + extract_nip98, + communities, + verifier, + mode, + deny_map, + ) + .await + } + None => { + admit_nip_fi_http( + headers, + extract_nip98, + communities, + verifier, + mode, + &NoDenyMapConfigured, + ) + .await + } } } @@ -508,6 +564,7 @@ mod tests { use crate::nip_fi_core::tests::ScriptedVerifier; use axum::http::StatusCode; use buzz_auth::CLIENT_ATTACHED_HEADER; + use futures_util::FutureExt as _; /// Test fixture: a deny map with no entries. struct AlwaysAdmitStubDenyMap; @@ -733,6 +790,8 @@ mod tests { NipFiMode::Enforce, deny_map, ) + .now_or_never() + .expect("env resolution never waits") }; let snapshot = |resp: Response| { let status = resp.status(); @@ -779,7 +838,9 @@ mod tests { None::<&dyn VerifyAssertion>, NipFiMode::Off, &AlwaysAdmitStubDenyMap, - ); + ) + .now_or_never() + .expect("env resolution never waits"); let admission = outcome.expect("Off mode MUST not require NIP-FI assertion — OSS default regression"); assert_eq!(*admission.proven_pubkey(), expected_pubkey); @@ -802,7 +863,9 @@ mod tests { None::<&dyn VerifyAssertion>, NipFiMode::Off, &AlwaysAdmitStubDenyMap, - ); + ) + .now_or_never() + .expect("env resolution never waits"); let resp = outcome.unwrap_err(); assert_eq!(resp.status(), deny_status); } @@ -834,7 +897,9 @@ mod tests { None::<&dyn VerifyAssertion>, NipFiMode::Enforce, &AlwaysAdmitStubDenyMap, - ); + ) + .now_or_never() + .expect("env resolution never waits"); let resp = outcome.unwrap_err(); assert_eq!( resp.status(), @@ -860,7 +925,9 @@ mod tests { None::<&dyn VerifyAssertion>, NipFiMode::Enforce, &AlwaysAdmitStubDenyMap, - ); + ) + .now_or_never() + .expect("env resolution never waits"); let resp = outcome.unwrap_err(); assert_eq!( resp.status(), @@ -884,7 +951,9 @@ mod tests { None::<&dyn VerifyAssertion>, NipFiMode::Off, &AlwaysAdmitStubDenyMap, - ); + ) + .now_or_never() + .expect("env resolution never waits"); let resp = outcome.unwrap_err(); assert_eq!( resp.status(), @@ -922,7 +991,9 @@ mod tests { None::<&dyn VerifyAssertion>, NipFiMode::Off, &AlwaysAdmitStubDenyMap, - ); + ) + .now_or_never() + .expect("env resolution never waits"); let resp = outcome.unwrap_err(); assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); assert_eq!( @@ -1014,7 +1085,9 @@ mod tests { Some(&verifier as &dyn VerifyAssertion), NipFiMode::DenyProtected, &AlwaysAdmitStubDenyMap, - ); + ) + .now_or_never() + .expect("env resolution never waits"); let Err(resp) = outcome else { panic!("{case}: DenyProtected must deny with 503"); }; @@ -1058,7 +1131,9 @@ mod tests { None::<&dyn VerifyAssertion>, NipFiMode::Enforce, &AlwaysAdmitStubDenyMap, - ); + ) + .now_or_never() + .expect("env resolution never waits"); // Missing header → MissingEvidence before verifier check. match outcome { Err(resp) => { @@ -1090,7 +1165,9 @@ mod tests { None::<&dyn VerifyAssertion>, NipFiMode::Enforce, &AlwaysAdmitStubDenyMap, - ); + ) + .now_or_never() + .expect("env resolution never waits"); match outcome { Err(resp) => { assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); @@ -1150,7 +1227,9 @@ mod tests { Some(&verifier as &dyn VerifyAssertion), NipFiMode::Enforce, &AlwaysAdmitStubDenyMap, - ); + ) + .now_or_never() + .expect("env resolution never waits"); match outcome { Err(resp) => { assert_eq!( @@ -1226,7 +1305,9 @@ mod tests { None::<&dyn VerifyAssertion>, NipFiMode::Enforce, &AlwaysAdmitStubDenyMap, - ); + ) + .now_or_never() + .expect("env resolution never waits"); let resp = outcome.unwrap_err(); assert_eq!( resp.status(), @@ -1276,7 +1357,9 @@ mod tests { None::<&dyn VerifyAssertion>, NipFiMode::Off, &AlwaysAdmitStubDenyMap, - ); + ) + .now_or_never() + .expect("env resolution never waits"); let admission = match outcome { Ok(a) => a, Err(resp) => panic!( @@ -1321,6 +1404,8 @@ mod tests { NipFiMode::Enforce, &AlwaysAdmitStubDenyMap, ) + .now_or_never() + .expect("env resolution never waits") } fn denial_parts(outcome: Result, Response>) -> (StatusCode, Vec) { @@ -1434,7 +1519,9 @@ mod tests { Some(&verifier as &dyn VerifyAssertion), NipFiMode::Enforce, &AlwaysAdmitStubDenyMap, - ); + ) + .now_or_never() + .expect("env resolution never waits"); let (status, body) = denial_parts(outcome); assert_eq!(status, StatusCode::UNAUTHORIZED); assert_eq!(body, b"authentication required\n"); @@ -1466,7 +1553,9 @@ mod tests { Some(&verifier as &dyn VerifyAssertion), NipFiMode::Enforce, &AlwaysAdmitStubDenyMap, - ); + ) + .now_or_never() + .expect("env resolution never waits"); let (status, body) = denial_parts(outcome); assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); assert_eq!(body, b"authorization unavailable\n"); @@ -1500,6 +1589,8 @@ mod tests { NipFiMode::Shadow, &AlwaysAdmitStubDenyMap, ) + .now_or_never() + .expect("env resolution never waits") .expect("shadow admits an unmapped Host"); assert_eq!(admission.proven_pubkey, key); assert!(admission.assertion.is_none()); @@ -1537,6 +1628,8 @@ mod tests { NipFiMode::Shadow, &AlwaysAdmitStubDenyMap, ) + .now_or_never() + .expect("env resolution never waits") }) .expect("shadow admits the dev fallback like Off"); assert_eq!(admission.proven_pubkey, key); @@ -1567,6 +1660,8 @@ mod tests { NipFiMode::Shadow, &AlwaysAdmitStubDenyMap, ) + .now_or_never() + .expect("env resolution never waits") .expect("shadow admits"); assert!(admission.assertion.is_none()); assert_eq!(verifier.calls(), 1); @@ -1578,7 +1673,9 @@ mod tests { Some(&verifier as &dyn VerifyAssertion), NipFiMode::Shadow, &AlwaysAdmitStubDenyMap, - ); + ) + .now_or_never() + .expect("env resolution never waits"); let Err(resp) = legacy else { panic!("NIP-98 failure must propagate") }; diff --git a/crates/buzz-relay/src/nip_fi_resolver.rs b/crates/buzz-relay/src/nip_fi_resolver.rs new file mode 100644 index 0000000000..3c1ffb2b9f --- /dev/null +++ b/crates/buzz-relay/src/nip_fi_resolver.rs @@ -0,0 +1,875 @@ +//! Database-sourced NIP-FI community resolution (`BUZZ_NIP_FI_COMMUNITY_SOURCE=database`). +//! +//! Each pod holds a snapshot of every registry-filtered binding of an active +//! community, loaded at startup and refreshed every 30s, start to start. A +//! host found in a fresh snapshot (loaded under two minutes ago) is answered +//! from memory. Any other host, and every host once the snapshot is stale, +//! goes to a bounded, single-flight database lookup whose positive answer is +//! used for that admission only and whose "not found" is cached for 5s. +//! +//! Every answer carries a [`FreshUntil`] two minutes after the read behind +//! it started (the snapshot load, or the lookup), re-checked after every +//! await, so no admission rests on a read older than that. +//! +//! A removed binding stops admitting new connections once a later refresh +//! publishes: within [`REFRESH_INTERVAL`] plus the refresh timeout (40s by +//! default) while refreshes succeed. While they fail, the last snapshot keeps +//! answering until it goes stale, so the bound is [`STALE_AFTER`] (two +//! minutes) from that snapshot's read; after that every host is looked up +//! live. Open WebSocket sessions are not rechecked when a binding is removed; +//! they end at their own session deadline. +//! +//! `buzz_nip_fi_binding_snapshot_loaded_timestamp_seconds` is written on +//! every refresh attempt with the start of the last good load (zero before +//! the first). Alert on `time() - buzz_nip_fi_binding_snapshot_loaded_timestamp_seconds > 120` +//! together with `absent(buzz_nip_fi_binding_snapshot_loaded_timestamp_seconds)` +//! for a database-source pod, since a pod that never exports it cannot age. +//! +//! The snapshot pointer and the negative cache change under one publication +//! lock. Publishing a snapshot and recording a creation bump a sequence +//! number; a "not found" read while the sequence moved is not cached. +//! Snapshot reads are lock-free. + +use std::collections::HashMap; +use std::num::NonZeroUsize; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use arc_swap::ArcSwapOption; +use buzz_auth::{CommunityBinding, DenialClass, ProvisionBindingPolicy}; +use buzz_core::tenant::normalize_host; +use buzz_db::RowCapExceeded; +use futures_util::future::{BoxFuture, FutureExt, Shared}; +use tokio::sync::Semaphore; + +use crate::config::ConfigError; + +#[cfg(test)] +mod tests; + +/// Refreshes start this often, measured start to start. +pub const REFRESH_INTERVAL: Duration = Duration::from_secs(30); +/// The last good snapshot serves for at most this long after its load. +pub const STALE_AFTER: Duration = Duration::from_secs(120); +/// The first retry delay after a failed refresh; it doubles per failure up +/// to [`REFRESH_INTERVAL`]. +pub const RETRY_FIRST: Duration = Duration::from_secs(2); +/// How long a "not found" miss is cached. +pub const NEGATIVE_TTL: Duration = Duration::from_secs(5); + +/// Resolver bounds. Every bound fails closed. Only the first four are read +/// from the environment; the lookup queue limits are fixed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolverBounds { + /// `BUZZ_NIP_FI_RESOLVER_NEGATIVE_CACHE_MAX`, at most 1,000,000. + pub negative_cache_max: usize, + /// `BUZZ_NIP_FI_RESOLVER_MAX_INFLIGHT`, kept below the writer pool size + /// so lookups never take every database connection. + pub max_inflight: usize, + /// `BUZZ_NIP_FI_SNAPSHOT_MAX_ROWS`, at most 10,000,000: + /// community-issuer pairs, not communities. + pub snapshot_max_rows: usize, + /// `BUZZ_NIP_FI_SNAPSHOT_REFRESH_TIMEOUT_MS`, shorter than + /// [`REFRESH_INTERVAL`]. + pub refresh_timeout: Duration, + /// Hosts with a lookup in flight. + pub max_pending_hosts: usize, + /// Callers waiting on those lookups. + pub max_waiters: usize, + /// How long a lookup waits for an inflight permit. + pub permit_wait: Duration, + /// How long a lookup's database read may take. + pub lookup_timeout: Duration, +} + +impl Default for ResolverBounds { + fn default() -> Self { + Self { + negative_cache_max: 100_000, + max_inflight: 8, + snapshot_max_rows: 1_000_000, + refresh_timeout: Duration::from_secs(10), + max_pending_hosts: 1_024, + max_waiters: 4_096, + permit_wait: Duration::from_millis(50), + lookup_timeout: Duration::from_millis(500), + } + } +} + +impl ResolverBounds { + /// Read the tunable bounds from the environment, defaulting unset ones, + /// and keep `max_inflight` below `db_pool_size` (at least 1). + pub fn from_env(db_pool_size: u32) -> Result { + let d = Self::default(); + let bounded = |name: &str, max: u64, default: usize| -> Result { + Ok(crate::nip_fi_config::parse_u64_bounded(name, 1, max)? + .map_or(default, |v| v as usize)) + }; + let below_interval = REFRESH_INTERVAL.as_millis() as u64 - 1; + let refresh_timeout = crate::nip_fi_config::parse_u64_bounded( + "BUZZ_NIP_FI_SNAPSHOT_REFRESH_TIMEOUT_MS", + 1, + below_interval, + )? + .map_or(d.refresh_timeout, Duration::from_millis); + let max_inflight = bounded( + "BUZZ_NIP_FI_RESOLVER_MAX_INFLIGHT", + u32::MAX.into(), + d.max_inflight, + )? + .min(db_pool_size.saturating_sub(1).max(1) as usize); + Ok(Self { + negative_cache_max: bounded( + "BUZZ_NIP_FI_RESOLVER_NEGATIVE_CACHE_MAX", + 1_000_000, + d.negative_cache_max, + )?, + max_inflight, + snapshot_max_rows: bounded( + "BUZZ_NIP_FI_SNAPSHOT_MAX_ROWS", + 10_000_000, + d.snapshot_max_rows, + )?, + refresh_timeout, + ..d + }) + } +} + +/// A snapshot load's `(host, issuer)` rows, or the cap it exceeded; the +/// outer error is a failed read. +pub type SnapshotRows = Result, RowCapExceeded>, String>; + +/// Where bindings are read from. Implemented by [`buzz_db::Db`]; tests +/// substitute counting and failing sources. +pub trait BindingSource: Send + Sync + 'static { + /// One consistent registry-filtered read of every active binding, as + /// `(host, issuer)` rows; more than `max_rows` fails. + fn load<'a>(&'a self, registry: &'a [String], max_rows: usize) -> BoxFuture<'a, SnapshotRows>; + + /// The registry-filtered issuers of the active community at `host`. + fn lookup<'a>( + &'a self, + host: &'a str, + registry: &'a [String], + ) -> BoxFuture<'a, Result, String>>; +} + +impl BindingSource for buzz_db::Db { + fn load<'a>(&'a self, registry: &'a [String], max_rows: usize) -> BoxFuture<'a, SnapshotRows> { + async move { + self.load_nip_fi_binding_snapshot(registry, max_rows) + .await + .map_err(|e| e.to_string()) + } + .boxed() + } + + fn lookup<'a>( + &'a self, + host: &'a str, + registry: &'a [String], + ) -> BoxFuture<'a, Result, String>> { + async move { + self.lookup_nip_fi_admission_binding(host, registry) + .await + .map_err(|e| e.to_string()) + } + .boxed() + } +} + +/// Monotonic time since the resolver started; replaceable in tests. +pub trait Clock: Send + Sync + 'static { + /// Elapsed time on this clock. + fn now(&self) -> Duration; +} + +struct TokioClock(tokio::time::Instant); + +impl Clock for TokioClock { + fn now(&self) -> Duration { + self.0.elapsed() + } +} + +/// The staleness deadline of the read an admission resolved against: two +/// minutes after that snapshot load or lookup started. Admission re-checks +/// it after its own awaits, so a binding resolved just inside the limit +/// cannot be admitted after it. Holds no map. +#[derive(Clone)] +pub struct FreshUntil { + clock: Arc, + deadline: Duration, +} + +impl FreshUntil { + /// `authorization_unavailable` once the deadline has passed. + pub fn check(&self) -> Result<(), DenialClass> { + (self.clock.now() < self.deadline) + .then_some(()) + .ok_or(DenialClass::AuthorizationUnavailable) + } +} + +/// One published full load. +struct Published { + map: HashMap>, + /// Clock time the load started; the staleness deadline counts from here. + loaded_at: Duration, +} + +/// State changed only under the publication lock. +struct Writable { + negative: lru::LruCache, +} + +/// A lookup's binding, if any, with the clock time its read started. +type MissOutcome = Result, Duration)>, ()>; +type PendingLookup = Shared>; + +/// Why a refresh published nothing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LoadFailure { + /// The store failed. + Store(String), + /// More rows than `BUZZ_NIP_FI_SNAPSHOT_MAX_ROWS`. + OverCap, + /// The whole attempt exceeded `BUZZ_NIP_FI_SNAPSHOT_REFRESH_TIMEOUT_MS`. + Timeout, + /// Its read started before the published one's (defensive). + Superseded, + /// An earlier attempt's map build is still running; no new load starts. + BuildOutstanding, +} + +impl LoadFailure { + fn label(&self) -> &'static str { + match self { + Self::Store(_) => "store", + Self::OverCap => "over_cap", + Self::Timeout => "timeout", + Self::Superseded => "superseded", + Self::BuildOutstanding => "build_outstanding", + } + } +} + +/// The database-sourced community resolver. +pub struct DatabaseResolver { + source: Arc, + policy: ProvisionBindingPolicy, + registry: Vec, + bounds: ResolverBounds, + clock: Arc, + published: ArcSwapOption, + publication: Mutex, + sequence: AtomicU64, + pending: Mutex>, + waiters: AtomicUsize, + permits: Arc, + /// Set while an attempt (its load or its map build) is alive. A build + /// abandoned by a timeout keeps it set until the builder thread returns. + building: Arc, + /// Wall-clock seconds of the last successful load's start, as `f64` + /// bits; zero before the first. + loaded_wall_secs: AtomicU64, + /// A uniform draw in `[0, 1)` for refresh jitter; fixed in tests. + pub(crate) jitter: fn() -> f64, + #[cfg(test)] + pub(crate) hooks: Arc, +} + +impl DatabaseResolver { + /// A resolver with no snapshot yet: every admission is unavailable until + /// the first [`DatabaseResolver::refresh`] succeeds. + pub fn new( + source: Arc, + policy: ProvisionBindingPolicy, + registry: Vec, + bounds: ResolverBounds, + ) -> Self { + Self::with_clock( + source, + policy, + registry, + bounds, + Arc::new(TokioClock(tokio::time::Instant::now())), + ) + } + + pub(crate) fn with_clock( + source: Arc, + policy: ProvisionBindingPolicy, + registry: Vec, + bounds: ResolverBounds, + clock: Arc, + ) -> Self { + let negative_cap = + NonZeroUsize::new(bounds.negative_cache_max).unwrap_or(NonZeroUsize::MIN); + Self { + source, + policy, + registry, + permits: Arc::new(Semaphore::new(bounds.max_inflight)), + bounds, + clock, + published: ArcSwapOption::empty(), + publication: Mutex::new(Writable { + negative: lru::LruCache::new(negative_cap), + }), + sequence: AtomicU64::new(0), + pending: Mutex::new(HashMap::new()), + waiters: AtomicUsize::new(0), + building: Arc::default(), + loaded_wall_secs: AtomicU64::new(0), + jitter: rand::random::, + #[cfg(test)] + hooks: Arc::default(), + } + } + + /// Resolve the community at `raw_host` for one admission: from a fresh + /// snapshot when it lists the host, otherwise from a database lookup, + /// each answer judged by its own read's age. + pub async fn resolve( + self: &Arc, + raw_host: &str, + ) -> Result<(Arc, FreshUntil), DenialClass> { + const UNAVAILABLE: DenialClass = DenialClass::AuthorizationUnavailable; + let host = normalize_host(raw_host); + if !self.policy.is_bindable(&host) { + return Err(UNAVAILABLE); + } + if let Some(hit) = self.fresh_snapshot_hit(&host) { + return Ok(hit); + } + if self.negatively_cached(&host) { + return Err(UNAVAILABLE); + } + match self.miss(host).await { + Ok(Some((binding, read_started))) => Ok((binding, self.fresh_until(read_started))), + Ok(None) | Err(()) => Err(UNAVAILABLE), + } + } + + /// Record that this pod just committed a binding for `host`: drop its + /// cached "not found" and bump the sequence, so a lookup that read + /// before the commit cannot cache one, and the creator's first login + /// looks it up. + pub fn record_creation(&self, host: &str) { + let host = normalize_host(host); + let mut writable = self.lock(); + writable.negative.pop(&host); + self.bump(&writable); + } + + /// Run one full load and publish it. A failure publishes nothing and + /// leaves the staleness deadline where it was. + pub async fn refresh(&self) -> Result<(), LoadFailure> { + let started = self.clock.now(); + // The attempt's absolute deadline. The timeout below only fires at an + // await; the synchronous map build and publication after the read + // check this instead, so expired work is never published. + let deadline = started + self.bounds.refresh_timeout; + let started_wall = std::time::SystemTime::now(); + let timer = std::time::Instant::now(); + // Held by the attempt, then by its builder thread: dropping the + // attempt on timeout cannot release it while the build still runs. + let Some(slot) = BuildSlot::claim(&self.building) else { + metrics::counter!("buzz_nip_fi_binding_snapshot_load_failures_total", "reason" => LoadFailure::BuildOutstanding.label()) + .increment(1); + return Err(LoadFailure::BuildOutstanding); + }; + let attempt = async { + let load = self + .source + .load(&self.registry, self.bounds.snapshot_max_rows) + .await + .map_err(LoadFailure::Store)? + .map_err(|_| LoadFailure::OverCap)?; + let rows = load.len(); + // Up to the row cap of grouping and binding construction: run it + // off the runtime worker. + // Set when this attempt is dropped (timeout or cancel); the + // builder checks it, and the deadline, as it goes. + let abandoned = Abandoned::default(); + let stop = { + let flag = Arc::clone(&abandoned.0); + let clock = Arc::clone(&self.clock); + move || flag.load(Ordering::Acquire) || clock.now() >= deadline + }; + #[cfg(test)] + let test_hooks = Arc::clone(&self.hooks); + let map = tokio::task::spawn_blocking(move || { + let _slot = slot; + #[cfg(test)] + test_hooks.hit_sync(hooks::BUILD_START); + build_map(load, stop) + }) + .await + .map_err(|e| LoadFailure::Store(format!("map build failed: {e}")))? + .ok_or(LoadFailure::Timeout)?; + #[cfg(test)] + self.hooks.hit(hooks::REFRESH_BEFORE_PUBLISH).await; + let replaced = self.publish(map, started, deadline)?; + // Freeing up to the row cap of bindings is real work; keep it + // off the runtime worker and outside the publication lock. + if let Some(old) = replaced { + tokio::task::spawn_blocking(move || drop(old)); + } + Ok(rows) + }; + let result = match tokio::time::timeout(self.bounds.refresh_timeout, attempt).await { + Ok(result) => result, + Err(_) => Err(LoadFailure::Timeout), + }; + metrics::histogram!("buzz_nip_fi_binding_snapshot_load_duration_seconds") + .record(timer.elapsed().as_secs_f64()); + if result.is_ok() { + if let Ok(since_epoch) = started_wall.duration_since(std::time::UNIX_EPOCH) { + self.loaded_wall_secs + .store(since_epoch.as_secs_f64().to_bits(), Ordering::Relaxed); + } + } + // Rewritten on every attempt, failed ones included, so the series + // exists from the first attempt and keeps reporting the last good + // load. A timestamp, not an age: `time() - ...` keeps aging at scrape + // time even if the refresh task stops; `absent()` catches a pod that + // never exports it. + let loaded = f64::from_bits(self.loaded_wall_secs.load(Ordering::Relaxed)); + metrics::gauge!("buzz_nip_fi_binding_snapshot_loaded_timestamp_seconds").set(loaded); + metrics::gauge!("buzz_nip_fi_binding_snapshot_max_rows") + .set(self.bounds.snapshot_max_rows as f64); + match result { + Ok(rows) => { + metrics::gauge!("buzz_nip_fi_binding_snapshot_rows").set(rows as f64); + Ok(()) + } + Err(failure) => { + tracing::warn!( + reason = failure.label(), + "NIP-FI binding snapshot load failed" + ); + metrics::counter!("buzz_nip_fi_binding_snapshot_load_failures_total", "reason" => failure.label()) + .increment(1); + Err(failure) + } + } + } + + /// Refresh every [`REFRESH_INTERVAL`], start to start, never + /// overlapping. After a failure the next attempt comes sooner, backing + /// off from [`RETRY_FIRST`] and doubling up to the interval; every delay + /// is jittered down by up to a tenth so pods do not refresh in lockstep. + /// The caller has already done the startup load. + pub async fn run_refresh_loop(self: Arc, cancel: tokio_util::sync::CancellationToken) { + let mut failures = 0u32; + let mut started = tokio::time::Instant::now(); + loop { + let delay = jittered(next_refresh_delay(failures), (self.jitter)()); + tokio::select! { + () = cancel.cancelled() => return, + () = tokio::time::sleep_until(started + delay) => {} + } + started = tokio::time::Instant::now(); + failures = match self.refresh().await { + Ok(()) => 0, + Err(_) => failures.saturating_add(1), + }; + } + } + + // ── Internals ─────────────────────────────────────────────────────────── + + fn lock(&self) -> std::sync::MutexGuard<'_, Writable> { + self.publication + .lock() + .unwrap_or_else(PoisonError::into_inner) + } + + /// Bump the publication sequence; the caller holds the lock. + fn bump(&self, _held: &Writable) { + self.sequence.fetch_add(1, Ordering::AcqRel); + } + + /// The deadline for an answer whose read started at `read_started`. + fn fresh_until(&self, read_started: Duration) -> FreshUntil { + FreshUntil { + clock: Arc::clone(&self.clock), + deadline: read_started + STALE_AFTER, + } + } + + /// `host` in the published snapshot, unless that snapshot is stale (or + /// none is loaded): a stale one may still list a removed binding. Clones + /// only the selected binding. + fn fresh_snapshot_hit(&self, host: &str) -> Option<(Arc, FreshUntil)> { + let published = self.published.load(); + let published = published.as_ref()?; + let fresh = self.fresh_until(published.loaded_at); + fresh.check().ok()?; + Some((Arc::clone(published.map.get(host)?), fresh)) + } + + fn negatively_cached(&self, host: &str) -> bool { + let now = self.clock.now(); + let mut writable = self.lock(); + match writable.negative.get(host) { + Some(expiry) if *expiry > now => true, + Some(_) => { + writable.negative.pop(host); + false + } + None => false, + } + } + + fn publish( + &self, + map: HashMap>, + started: Duration, + deadline: Duration, + ) -> Result>, LoadFailure> { + let mut writable = self.lock(); + if self + .published + .load() + .as_ref() + .is_some_and(|published| started < published.loaded_at) + { + return Err(LoadFailure::Superseded); + } + #[cfg(test)] + self.hooks.hit_sync(hooks::PUBLISH_BEFORE_WRITE); + // Checked under the lock, immediately before the first write: + // waiting for the lock cannot let an expired attempt publish. + if self.clock.now() >= deadline { + return Err(LoadFailure::Timeout); + } + let replaced = self.published.swap(Some(Arc::new(Published { + map, + loaded_at: started, + }))); + writable.negative.clear(); + metrics::gauge!("buzz_nip_fi_resolver_negative_cache_size").set(0.0); + self.bump(&writable); + Ok(replaced) + } + + /// The bounded, single-flight lookup. A positive answer is returned to + /// its waiters only; a "not found" is cached only if no creation or + /// publication happened while it ran. + async fn miss(self: &Arc, host: String) -> MissOutcome { + let lookup = { + let mut pending = self.pending.lock().unwrap_or_else(PoisonError::into_inner); + match pending.get(&host) { + Some(lookup) => lookup.clone(), + None => { + if pending.len() >= self.bounds.max_pending_hosts { + reject("pending_hosts"); + return Err(()); + } + let lookup = self.spawn_lookup(host.clone()); + pending.insert(host, lookup.clone()); + lookup + } + } + }; + let Some(_waiter) = WaiterSlot::take(&self.waiters, self.bounds.max_waiters) else { + reject("waiters"); + return Err(()); + }; + lookup.await + } + + /// Run one lookup on its own task, so it completes (and leaves the + /// pending map) even if every waiter goes away. + fn spawn_lookup(self: &Arc, host: String) -> PendingLookup { + let this = Arc::clone(self); + let task = tokio::spawn(async move { + // Leaves the pending map even if the lookup panics, so the host + // is not stuck behind a dead entry. + struct Leave<'a>(&'a DatabaseResolver, &'a str); + impl Drop for Leave<'_> { + fn drop(&mut self) { + self.0 + .pending + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(self.1); + } + } + let _leave = Leave(&this, &host); + let sequence = this.sequence.load(Ordering::Acquire); + let outcome = this.lookup_once(&host).await; + if matches!(outcome, Ok(None)) { + this.store_negative(&host, sequence); + } + outcome + }); + async move { task.await.unwrap_or(Err(())) } + .boxed() + .shared() + } + + async fn lookup_once(&self, host: &str) -> MissOutcome { + let permit = match tokio::time::timeout( + self.bounds.permit_wait, + Arc::clone(&self.permits).acquire_owned(), + ) + .await + { + Ok(Ok(permit)) => permit, + _ => { + reject("permit"); + return Err(()); + } + }; + // Every waiter on this lookup shares this deadline origin. + let read_started = self.clock.now(); + let timer = std::time::Instant::now(); + let read = tokio::time::timeout( + self.bounds.lookup_timeout, + self.source.lookup(host, &self.registry), + ) + .await; + drop(permit); + metrics::histogram!("buzz_nip_fi_resolver_lookup_duration_seconds") + .record(timer.elapsed().as_secs_f64()); + #[cfg(test)] + self.hooks.hit(hooks::MISS_AFTER_READ).await; + match read { + Err(_) => { + metrics::counter!("buzz_nip_fi_resolver_lookup_timeouts_total").increment(1); + Err(()) + } + Ok(Err(_)) => Err(()), + Ok(Ok(issuers)) if issuers.is_empty() => Ok(None), + Ok(Ok(issuers)) => Ok(binding_for(host, issuers).map(|b| (b, read_started))), + } + } + + fn store_negative(&self, host: &str, sequence: u64) { + let mut writable = self.lock(); + if self.sequence.load(Ordering::Acquire) != sequence { + return; + } + let expiry = self.clock.now() + NEGATIVE_TTL; + if let Some((evicted, _)) = writable.negative.push(host.to_owned(), expiry) { + if evicted != host { + metrics::counter!("buzz_nip_fi_resolver_negative_evictions_total").increment(1); + } + } + metrics::gauge!("buzz_nip_fi_resolver_negative_cache_size") + .set(writable.negative.len() as f64); + } + + #[cfg(test)] + pub(crate) fn negative_len(&self) -> usize { + self.lock().negative.len() + } + + #[cfg(test)] + pub(crate) fn pending_len(&self) -> usize { + self.pending + .lock() + .unwrap_or_else(PoisonError::into_inner) + .len() + } + + #[cfg(test)] + pub(crate) fn waiters_now(&self) -> usize { + self.waiters.load(Ordering::Acquire) + } + + #[cfg(test)] + pub(crate) fn inflight_now(&self) -> usize { + self.bounds.max_inflight - self.permits.available_permits() + } +} + +fn reject(reason: &'static str) { + metrics::counter!("buzz_nip_fi_resolver_miss_rejections_total", "reason" => reason) + .increment(1); +} + +/// One counted caller waiting on a pending lookup. +struct WaiterSlot<'a>(&'a AtomicUsize); + +impl<'a> WaiterSlot<'a> { + fn take(waiters: &'a AtomicUsize, max: usize) -> Option { + waiters + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |n| { + (n < max).then_some(n + 1) + }) + .ok() + .map(|_| Self(waiters)) + } +} + +impl Drop for WaiterSlot<'_> { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } +} + +/// The delay before the next refresh after `failures` consecutive failures. +fn next_refresh_delay(failures: u32) -> Duration { + match failures { + 0 => REFRESH_INTERVAL, + n => RETRY_FIRST + .saturating_mul(1 << (n - 1).min(16)) + .min(REFRESH_INTERVAL), + } +} + +/// `delay` shortened by up to a tenth, by `unit` in `[0, 1)`. +fn jittered(delay: Duration, unit: f64) -> Duration { + delay.mul_f64(1.0 - unit.clamp(0.0, 1.0) / 10.0) +} + +fn binding_for(host: &str, issuers: Vec) -> Option> { + CommunityBinding::new(format!("https://{host}"), issuers) + .ok() + .map(Arc::new) +} + +/// Group rows into bindings, or `None` as soon as `stop` reports the +/// attempt abandoned or expired; checked in both passes. +fn build_map( + rows: Vec<(String, String)>, + stop: impl Fn() -> bool, +) -> Option>> { + let mut grouped: HashMap> = HashMap::new(); + for (host, issuer) in rows { + if stop() { + return None; + } + grouped + .entry(normalize_host(&host)) + .or_default() + .push(issuer); + } + let mut map = HashMap::with_capacity(grouped.len()); + for (host, issuers) in grouped { + if stop() { + return None; + } + if let Some(binding) = binding_for(&host, issuers) { + map.insert(host, binding); + } + } + Some(map) +} + +/// The resolver's single in-flight attempt; released on drop, wherever the +/// owner (the attempt or its builder thread) ends, including a panic. +struct BuildSlot(Arc); + +impl BuildSlot { + fn claim(building: &Arc) -> Option { + building + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .ok() + .map(|_| Self(Arc::clone(building))) + } +} + +impl Drop for BuildSlot { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } +} + +/// Raised when the awaiting attempt is dropped, telling its builder to stop. +#[derive(Default)] +struct Abandoned(Arc); + +impl Drop for Abandoned { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } +} + +#[cfg(test)] +pub(crate) mod hooks { + //! Per-resolver pause points for ordering tests. + use std::collections::HashMap; + use std::sync::{Arc, Mutex}; + + use tokio::sync::Semaphore; + + pub(crate) const REFRESH_BEFORE_PUBLISH: &str = "refresh.before_publish"; + pub(crate) const MISS_AFTER_READ: &str = "miss.after_read"; + /// Synchronous, under the publication lock: see [`Hooks::on_sync`]. + pub(crate) const PUBLISH_BEFORE_WRITE: &str = "publish.before_write"; + /// Synchronous, on the builder thread before the map build. + pub(crate) const BUILD_START: &str = "build.start"; + + /// An armed pause: production signals `reached`, then waits for `release`. + pub(crate) struct Gate { + reached: Semaphore, + release: Semaphore, + } + + impl Gate { + pub(crate) async fn reached(&self) { + self.reached.acquire().await.expect("open").forget(); + } + + pub(crate) fn release(&self) { + self.release.add_permits(1); + } + } + + #[derive(Default)] + pub(crate) struct Hooks( + Mutex>>, + Mutex>, + ); + + type SyncAction = Arc; + + impl Hooks { + /// Pause every later hit of `point` until released, once per release. + pub(crate) fn arm(&self, point: &'static str) -> Arc { + let gate = Arc::new(Gate { + reached: Semaphore::new(0), + release: Semaphore::new(0), + }); + self.0.lock().unwrap().insert(point, Arc::clone(&gate)); + gate + } + + pub(crate) fn disarm(&self, point: &'static str) { + self.0.lock().unwrap().remove(point); + } + + /// Run `action` at every later hit of the synchronous `point`. + pub(crate) fn on_sync( + &self, + point: &'static str, + action: impl Fn() + Send + Sync + 'static, + ) { + self.1.lock().unwrap().insert(point, Arc::new(action)); + } + + pub(crate) fn hit_sync(&self, point: &'static str) { + let action = self.1.lock().unwrap().get(point).cloned(); + if let Some(action) = action { + action(); + } + } + + pub(crate) async fn hit(&self, point: &'static str) { + let gate = self.0.lock().unwrap().get(point).cloned(); + if let Some(gate) = gate { + gate.reached.add_permits(1); + gate.release.acquire().await.expect("open").forget(); + } + } + } +} diff --git a/crates/buzz-relay/src/nip_fi_resolver/tests.rs b/crates/buzz-relay/src/nip_fi_resolver/tests.rs new file mode 100644 index 0000000000..309ca1c310 --- /dev/null +++ b/crates/buzz-relay/src/nip_fi_resolver/tests.rs @@ -0,0 +1,1097 @@ +//! Resolver ordering, staleness, and bounds tests. The store is an in-memory +//! fake that counts every call; the clock is manual. + +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use buzz_auth::{DenialClass, ProvisionBindingPolicy}; +use buzz_db::RowCapExceeded; +use futures_util::future::{BoxFuture, FutureExt}; + +use super::hooks::{BUILD_START, MISS_AFTER_READ, PUBLISH_BEFORE_WRITE, REFRESH_BEFORE_PUBLISH}; +use super::*; + +const ISS: &str = "https://app.builderlab.xyz/api/goose"; +const UNAVAILABLE: Result<(), DenialClass> = Err(DenialClass::AuthorizationUnavailable); + +fn policy() -> ProvisionBindingPolicy { + ProvisionBindingPolicy::parse( + &serde_json::json!({"host_suffix": "communities.buzz.xyz", "authorized_issuers": [ISS]}) + .to_string(), + &serde_json::json!([ + {"issuer": ISS, "command_audiences": ["https://communities.buzz.xyz"], "token_class": "nip-fi+jwt"} + ]) + .to_string(), + ) + .expect("valid policy") +} + +fn host(label: &str) -> String { + format!("{label}.communities.buzz.xyz") +} + +#[derive(Default)] +struct ManualClock(AtomicU64); + +impl ManualClock { + fn set(&self, at: Duration) { + self.0.store(at.as_millis() as u64, Ordering::SeqCst); + } +} + +impl Clock for ManualClock { + fn now(&self) -> Duration { + Duration::from_millis(self.0.load(Ordering::SeqCst)) + } +} + +/// In-memory bindings. +struct FakeStore { + hosts: Mutex>, + fail_loads: std::sync::atomic::AtomicBool, + lookup_fails: std::sync::atomic::AtomicBool, + lookup_panics: std::sync::atomic::AtomicBool, + loads: AtomicUsize, + /// Rows the last load pulled from its stream. + rows_pulled: AtomicUsize, + lookups: Mutex>, + lookup_delay: Mutex, + load_delay: Mutex, +} + +impl FakeStore { + fn new(hosts: &[&str]) -> Arc { + Arc::new(Self { + hosts: Mutex::new(hosts.iter().map(|l| host(l)).collect()), + fail_loads: false.into(), + lookup_fails: false.into(), + lookup_panics: false.into(), + loads: AtomicUsize::new(0), + rows_pulled: AtomicUsize::new(0), + lookups: Mutex::new(Vec::new()), + lookup_delay: Mutex::new(Duration::ZERO), + load_delay: Mutex::new(Duration::ZERO), + }) + } + fn add(&self, label: &str) { + self.hosts.lock().unwrap().push(host(label)); + } + fn remove(&self, label: &str) { + self.hosts.lock().unwrap().retain(|h| h != &host(label)); + } + fn lookups_for(&self, label: &str) -> usize { + let h = host(label); + self.lookups + .lock() + .unwrap() + .iter() + .filter(|l| **l == h) + .count() + } + fn total_lookups(&self) -> usize { + self.lookups.lock().unwrap().len() + } +} + +impl BindingSource for FakeStore { + fn load<'a>(&'a self, _registry: &'a [String], max_rows: usize) -> BoxFuture<'a, SnapshotRows> { + async move { + self.loads.fetch_add(1, Ordering::SeqCst); + // A consistent read: rows as of the read's start, however long + // it then takes. + let hosts = self.hosts.lock().unwrap().clone(); + let delay = *self.load_delay.lock().unwrap(); + if !delay.is_zero() { + tokio::time::sleep(delay).await; + } + if self.fail_loads.load(Ordering::SeqCst) { + return Err("load failed".into()); + } + // Pulled one row at a time and capped as the real store does. + self.rows_pulled.store(0, Ordering::SeqCst); + let mut rows = Vec::new(); + for host in hosts { + self.rows_pulled.fetch_add(1, Ordering::SeqCst); + if rows.len() == max_rows { + return Ok(Err(RowCapExceeded(max_rows))); + } + rows.push((host, ISS.to_owned())); + } + Ok(Ok(rows)) + } + .boxed() + } + + fn lookup<'a>( + &'a self, + host: &'a str, + _registry: &'a [String], + ) -> BoxFuture<'a, Result, String>> { + async move { + self.lookups.lock().unwrap().push(host.to_owned()); + let delay = *self.lookup_delay.lock().unwrap(); + if !delay.is_zero() { + tokio::time::sleep(delay).await; + } + assert!( + !self.lookup_panics.load(Ordering::SeqCst), + "lookup panicked" + ); + if self.lookup_fails.load(Ordering::SeqCst) { + return Err("pool closed".into()); + } + let found = self.hosts.lock().unwrap().iter().any(|h| h == host); + Ok(if found { + vec![ISS.to_owned()] + } else { + Vec::new() + }) + } + .boxed() + } +} + +struct Fixture { + store: Arc, + clock: Arc, + resolver: Arc, +} + +fn fixture_with(hosts: &[&str], bounds: ResolverBounds) -> Fixture { + let store = FakeStore::new(hosts); + let clock = Arc::new(ManualClock::default()); + let resolver = Arc::new(DatabaseResolver::with_clock( + Arc::clone(&store) as Arc, + policy(), + vec![ISS.to_owned()], + bounds, + Arc::clone(&clock) as Arc, + )); + Fixture { + store, + clock, + resolver, + } +} + +fn fixture(hosts: &[&str]) -> Fixture { + fixture_with(hosts, ResolverBounds::default()) +} + +impl Fixture { + async fn resolve(&self, label: &str) -> Result<(), DenialClass> { + self.resolver.resolve(&host(label)).await.map(|_| ()) + } +} + +fn secs(s: f64) -> Duration { + Duration::from_secs_f64(s) +} + +// A lookup that panics still leaves the pending map, so the next admission +// for that host runs a fresh lookup. Mutation: removing the entry only on the +// normal path leaves the host denied by the dead lookup forever. +#[tokio::test] +async fn panicked_lookup_does_not_strand_the_host() { + let f = fixture(&["a"]); + f.store.lookup_panics.store(true, Ordering::SeqCst); + assert!(f.resolve("a").await.is_err()); + f.store.lookup_panics.store(false, Ordering::SeqCst); + assert_eq!(f.resolve("a").await, Ok(())); + assert_eq!(f.store.lookups_for("a"), 2); +} + +// ── Before the first load, and once the snapshot is stale ────────────────── + +// With no snapshot loaded, admissions go to the live lookup; a failed or +// over-cap load publishes nothing. +#[tokio::test] +async fn unloaded_resolver_uses_live_lookups_and_failed_or_over_cap_load_keeps_it_unloaded() { + let f = fixture_with( + &["a", "b"], + ResolverBounds { + snapshot_max_rows: 1, + ..ResolverBounds::default() + }, + ); + assert_eq!(f.resolve("a").await, Ok(())); + assert_eq!(f.resolve("unknown").await, UNAVAILABLE); + assert_eq!(f.resolver.refresh().await, Err(LoadFailure::OverCap)); + f.store.fail_loads.store(true, Ordering::SeqCst); + assert!(matches!( + f.resolver.refresh().await, + Err(LoadFailure::Store(_)) + )); + assert!(f.resolver.published.load().is_none()); + assert_eq!(f.resolve("b").await, Ok(())); + assert_eq!(f.store.lookups_for("a"), 1); + assert_eq!(f.store.lookups_for("b"), 1); +} + +// While refreshes fail, the last snapshot answers for two minutes after its +// load started; after that every host goes to the live lookup, which sees a +// removal at once and fails closed when the database does. Mutation: serving +// a stale snapshot admits "gone"; denying once stale refuses "a". +#[tokio::test] +async fn stale_snapshot_is_skipped_for_live_lookups() { + let f = fixture(&["a", "gone"]); + f.resolver.refresh().await.unwrap(); + f.store.fail_loads.store(true, Ordering::SeqCst); + f.store.remove("gone"); + + f.clock.set(secs(60.0)); + assert!(f.resolver.refresh().await.is_err()); + f.clock.set(secs(119.0)); + assert_eq!(f.resolve("a").await, Ok(())); + assert_eq!( + f.resolve("gone").await, + Ok(()), + "fresh snapshot still lists it" + ); + assert_eq!( + f.store.total_lookups(), + 0, + "fresh snapshot hosts never look up" + ); + + f.clock.set(secs(121.0)); + assert_eq!(f.resolve("a").await, Ok(())); + assert_eq!(f.resolve("gone").await, UNAVAILABLE); + assert_eq!(f.store.lookups_for("a"), 1); + assert_eq!(f.store.lookups_for("gone"), 1); + assert_eq!(f.resolver.negative_len(), 1, "the not-found is cached"); + f.store.lookup_fails.store(true, Ordering::SeqCst); + assert_eq!(f.resolve("a").await, UNAVAILABLE, "database down"); + + f.store.fail_loads.store(false, Ordering::SeqCst); + f.resolver.refresh().await.unwrap(); + assert_eq!(f.resolve("a").await, Ok(())); + assert_eq!(f.store.lookups_for("a"), 2, "fresh snapshot again"); +} + +// A lookup answer is judged by its own read, not the snapshot's: the +// snapshot going stale while the lookup runs still admits, and the answer +// expires two minutes after its read started. Mutation: re-checking the +// snapshot's deadline after the await denies; dating the answer from the +// snapshot load expires it at once. +#[tokio::test] +async fn lookup_answer_carries_its_own_deadline_past_a_stale_snapshot() { + let f = fixture(&["a"]); + f.resolver.refresh().await.unwrap(); + f.store.add("late"); + f.clock.set(secs(119.9)); + let gate = f.resolver.hooks.arm(MISS_AFTER_READ); + let pending = tokio::spawn({ + let resolver = Arc::clone(&f.resolver); + async move { resolver.resolve(&host("late")).await } + }); + gate.reached().await; + f.clock.set(secs(120.1)); + gate.release(); + let (_, fresh) = pending.await.unwrap().expect("admitted"); + assert_eq!(fresh.check(), Ok(())); + f.clock.set(secs(239.8)); + assert_eq!(fresh.check(), Ok(())); + f.clock.set(secs(239.9)); + assert_eq!(fresh.check(), UNAVAILABLE, "read started at 119.9s"); +} + +#[tokio::test] +async fn timed_out_refresh_publishes_nothing_and_keeps_the_deadline() { + let f = fixture_with( + &["a"], + ResolverBounds { + refresh_timeout: Duration::from_millis(20), + ..ResolverBounds::default() + }, + ); + f.resolver.refresh().await.unwrap(); + f.store.remove("a"); + f.store.add("b"); + *f.store.load_delay.lock().unwrap() = Duration::from_millis(200); + f.clock.set(secs(100.0)); + assert_eq!(f.resolver.refresh().await, Err(LoadFailure::Timeout)); + // Retained snapshot unchanged: "a" still resolves without a lookup. + assert_eq!(f.resolve("a").await, Ok(())); + assert_eq!(f.store.lookups_for("a"), 0); + f.clock.set(secs(121.0)); + assert_eq!( + f.resolve("a").await, + UNAVAILABLE, + "the live lookup sees the removal" + ); +} + +// Criterion 3 (publication bound): a deadline that passes during the +// synchronous build/publish tail fails the attempt. The manual clock moves +// while tokio time does not, so only the attempt deadline can catch it. +#[tokio::test] +async fn refresh_whose_deadline_passes_before_publication_publishes_nothing() { + let f = fixture(&["a"]); + f.resolver.refresh().await.unwrap(); + f.store.remove("a"); + f.clock.set(secs(30.0)); + let gate = f.resolver.hooks.arm(REFRESH_BEFORE_PUBLISH); + let refresh = tokio::spawn({ + let resolver = Arc::clone(&f.resolver); + async move { resolver.refresh().await } + }); + gate.reached().await; + f.clock.set(secs(40.0)); + gate.release(); + assert_eq!(refresh.await.unwrap(), Err(LoadFailure::Timeout)); + assert_eq!(f.resolve("a").await, Ok(()), "old map retained"); + f.clock.set(secs(120.0)); + assert_eq!(f.resolve("a").await, UNAVAILABLE, "old deadline retained"); + assert_eq!(f.store.lookups_for("a"), 1); +} + +// The deadline is rechecked under the publication lock, immediately before +// the first write: an attempt that crosses it while waiting publishes +// nothing. Mutation: checking before taking the lock (or not at all) +// publishes the late snapshot. +#[tokio::test] +async fn refresh_crossing_its_deadline_while_preparing_publishes_nothing() { + let f = fixture(&["a"]); + f.resolver.refresh().await.unwrap(); + f.store.remove("a"); + f.clock.set(secs(30.0)); + let clock = Arc::clone(&f.clock); + f.resolver + .hooks + .on_sync(PUBLISH_BEFORE_WRITE, move || clock.set(secs(40.0))); + assert_eq!(f.resolver.refresh().await, Err(LoadFailure::Timeout)); + assert_eq!(f.resolve("a").await, Ok(()), "old map retained"); + assert_eq!(f.store.lookups_for("a"), 0); + f.clock.set(secs(120.0)); + assert_eq!(f.resolve("a").await, UNAVAILABLE, "old deadline retained"); + assert_eq!(f.store.lookups_for("a"), 1); +} + +// A map build abandoned by a timeout keeps running on its blocking thread; +// until it returns, no new load starts, so at most one cap-sized build is +// ever outstanding. Once released it stops cooperatively, publishes nothing, +// and the next tick loads normally. Mutation: releasing the slot with the +// attempt (or not holding one) starts a second build while the first lives. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn timed_out_build_blocks_new_loads_until_it_returns_and_publishes_nothing() { + let f = fixture_with( + &["a"], + ResolverBounds { + refresh_timeout: Duration::from_millis(50), + ..ResolverBounds::default() + }, + ); + f.resolver.refresh().await.unwrap(); + f.store.remove("a"); + f.store.add("b"); + + let outstanding = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let started = Arc::new(AtomicUsize::new(0)); + let release = Arc::new((Mutex::new(false), std::sync::Condvar::new())); + f.resolver.hooks.on_sync(BUILD_START, { + let (outstanding, peak, started, release) = ( + Arc::clone(&outstanding), + Arc::clone(&peak), + Arc::clone(&started), + Arc::clone(&release), + ); + move || { + started.fetch_add(1, Ordering::SeqCst); + let now = outstanding.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(now, Ordering::SeqCst); + let (open, signal) = &*release; + // Bounded, so a regression fails the test instead of hanging it. + let _ = signal + .wait_timeout_while(open.lock().unwrap(), Duration::from_secs(5), |open| !*open) + .unwrap(); + outstanding.fetch_sub(1, Ordering::SeqCst); + } + }); + + assert_eq!(f.resolver.refresh().await, Err(LoadFailure::Timeout)); + let loads = f.store.loads.load(Ordering::SeqCst); + for _ in 0..3 { + assert_eq!( + f.resolver.refresh().await, + Err(LoadFailure::BuildOutstanding) + ); + } + assert_eq!(f.store.loads.load(Ordering::SeqCst), loads, "no new load"); + assert_eq!(started.load(Ordering::SeqCst), 1); + + let (open, signal) = &*release; + *open.lock().unwrap() = true; + signal.notify_all(); + for _ in 0..200 { + if !f.resolver.building.load(Ordering::SeqCst) { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert!( + !f.resolver.building.load(Ordering::SeqCst), + "build released" + ); + assert_eq!(f.resolve("a").await, Ok(()), "nothing published late"); + assert_eq!(f.store.lookups_for("a"), 0); + + f.resolver.refresh().await.unwrap(); + assert_eq!(peak.load(Ordering::SeqCst), 1, "one build at a time"); + assert_eq!(started.load(Ordering::SeqCst), 2); + assert_eq!(f.resolve("b").await, Ok(())); + assert_eq!(f.store.lookups_for("b"), 0, "new snapshot published"); +} + +#[test] +fn build_map_stops_in_either_pass_once_told_to() { + let rows = vec![(host("a"), ISS.to_owned()), (host("b"), ISS.to_owned())]; + assert!(build_map(rows.clone(), || true).is_none(), "grouping pass"); + let checks = AtomicUsize::new(0); + let stop_in_second_pass = || checks.fetch_add(1, Ordering::SeqCst) >= rows.len(); + assert!( + build_map(rows.clone(), stop_in_second_pass).is_none(), + "binding pass" + ); + assert_eq!(build_map(rows, || false).map(|m| m.len()), Some(2)); +} + +// The load stops at cap + 1 rows; a loaded snapshot survives an over-cap +// refresh unchanged. +#[tokio::test] +async fn over_cap_refresh_stops_streaming_and_keeps_the_loaded_snapshot() { + let f = fixture_with( + &["a", "b"], + ResolverBounds { + snapshot_max_rows: 2, + ..ResolverBounds::default() + }, + ); + f.resolver.refresh().await.unwrap(); + for label in ["c", "d", "e", "f"] { + f.store.add(label); + } + f.store.remove("b"); + f.clock.set(secs(60.0)); + assert_eq!(f.resolver.refresh().await, Err(LoadFailure::OverCap)); + assert_eq!(f.store.rows_pulled.load(Ordering::SeqCst), 3, "cap + 1"); + assert_eq!(f.resolve("b").await, Ok(()), "old map retained"); + assert_eq!(f.store.lookups_for("b"), 0); + f.clock.set(secs(120.0)); + assert_eq!(f.resolve("b").await, UNAVAILABLE, "old deadline retained"); +} + +#[tokio::test] +async fn failed_miss_lookup_denies_and_stores_no_negative_entry() { + let f = fixture(&["a"]); + f.resolver.refresh().await.unwrap(); + f.store.lookup_fails.store(true, Ordering::SeqCst); + assert_eq!(f.resolve("x").await, UNAVAILABLE); + assert_eq!(f.resolver.negative_len(), 0); +} + +// ── Criterion 14: refresh removes bindings; 40s removal bound ─────────────── + +#[tokio::test] +async fn refresh_removes_unbound_communities_and_held_positive_miss_is_not_stored() { + let f = fixture(&["a", "b", "control"]); + f.resolver.refresh().await.unwrap(); + f.store.remove("a"); + f.store.remove("b"); + assert_eq!(f.resolve("a").await, Ok(()), "before refresh"); + f.resolver.refresh().await.unwrap(); + assert_eq!(f.resolve("a").await, UNAVAILABLE); + assert_eq!(f.resolve("b").await, UNAVAILABLE); + assert_eq!(f.resolve("control").await, Ok(())); + + // (c) A miss reads "h" active and is held; "h" is removed and a refresh + // publishes; the held answer is returned once and never stored. + f.store.add("h"); + let gate = f.resolver.hooks.arm(MISS_AFTER_READ); + let pending = tokio::spawn({ + let resolver = Arc::clone(&f.resolver); + async move { resolver.resolve(&host("h")).await.map(|_| ()) } + }); + gate.reached().await; + f.resolver.hooks.disarm(MISS_AFTER_READ); + f.store.remove("h"); + f.resolver.refresh().await.unwrap(); + gate.release(); + assert_eq!(pending.await.unwrap(), Ok(())); + assert_eq!(f.resolve("h").await, UNAVAILABLE); +} + +#[tokio::test(start_paused = true)] +async fn unbinding_after_a_snapshot_is_enforced_within_forty_seconds() { + // The production scheduler on the tokio clock: refresh at +30s, its + // read held for just under the 10s limit, published before +40s. + let store = FakeStore::new(&["a", "gone"]); + let mut resolver = DatabaseResolver::new( + Arc::clone(&store) as Arc, + policy(), + vec![ISS.to_owned()], + ResolverBounds::default(), + ); + // No jitter: the worst case, a tick a full interval after the last. + resolver.jitter = || 0.0; + let resolver = Arc::new(resolver); + let resolve = |label: &str| { + let resolver = Arc::clone(&resolver); + let host = host(label); + async move { resolver.resolve(&host).await.map(drop) } + }; + resolver.refresh().await.unwrap(); + store.remove("gone"); + *store.load_delay.lock().unwrap() = Duration::from_millis(9_900); + let cancel = tokio_util::sync::CancellationToken::new(); + let refresh_loop = tokio::spawn(Arc::clone(&resolver).run_refresh_loop(cancel.clone())); + tokio::task::yield_now().await; // the loop arms its schedule at t=0 + + // Step the paused clock so the loop wakes on its +30s tick, and let its + // load start and sleep on the 9.9s read. + tokio::time::advance(REFRESH_INTERVAL).await; + while store.loads.load(Ordering::SeqCst) < 2 { + tokio::task::yield_now().await; + } + tokio::time::advance(Duration::from_millis(9_800)).await; + assert_eq!( + resolve("gone").await, + Ok(()), + "before the refresh publishes" + ); + // +39.9s: the held read returns. The map build runs on a real blocking + // thread the paused clock cannot see, so wait for the publication itself + // (bounded in real time); the paused clock stays at +39.9s meanwhile. + tokio::time::advance(Duration::from_millis(100)).await; + let waited = std::time::Instant::now(); + while resolver + .published + .load() + .as_ref() + .is_none_or(|published| published.loaded_at < REFRESH_INTERVAL) + { + assert!( + waited.elapsed() < Duration::from_secs(30), + "never published" + ); + tokio::task::yield_now().await; + } + assert!(resolver.clock.now() < Duration::from_secs(40)); + assert_eq!(resolve("gone").await, UNAVAILABLE, "removed by +40s"); + assert_eq!(resolve("a").await, Ok(())); + cancel.cancel(); + refresh_loop.await.unwrap(); +} + +// Mutation: retrying a failure only on the next full interval, never +// backing off, or jittering a delay up instead of down. +#[test] +fn refresh_delay_backs_off_after_failures_and_jitters_down() { + let delays: Vec = (0..7).map(|n| next_refresh_delay(n).as_secs()).collect(); + assert_eq!(delays, [30, 2, 4, 8, 16, 30, 30]); + assert_eq!(next_refresh_delay(u32::MAX), REFRESH_INTERVAL); + assert_eq!(jittered(REFRESH_INTERVAL, 0.0), REFRESH_INTERVAL); + assert_eq!(jittered(REFRESH_INTERVAL, 0.5), secs(28.5)); + assert!(jittered(REFRESH_INTERVAL, 0.999_999) > secs(27.0)); +} + +// The loop refreshes every REFRESH_INTERVAL and retries a failure after +// RETRY_FIRST. Mutation: a different interval, or no retry before the next +// interval, changes the load count at each step. +#[tokio::test(start_paused = true)] +async fn refresh_loop_runs_every_interval_and_retries_failures_sooner() { + let store = FakeStore::new(&["a"]); + let mut resolver = DatabaseResolver::new( + Arc::clone(&store) as Arc, + policy(), + vec![ISS.to_owned()], + ResolverBounds::default(), + ); + resolver.jitter = || 0.0; + let resolver = Arc::new(resolver); + let cancel = tokio_util::sync::CancellationToken::new(); + let refresh_loop = tokio::spawn(Arc::clone(&resolver).run_refresh_loop(cancel.clone())); + let loads_after = |step: Duration| { + let store = Arc::clone(&store); + async move { + tokio::time::advance(step).await; + for _ in 0..100 { + tokio::task::yield_now().await; + } + store.loads.load(Ordering::SeqCst) + } + }; + tokio::task::yield_now().await; + assert_eq!(loads_after(REFRESH_INTERVAL - secs(0.1)).await, 0); + store.fail_loads.store(true, Ordering::SeqCst); + assert_eq!( + loads_after(secs(0.1)).await, + 1, + "first tick at the interval" + ); + assert_eq!(loads_after(RETRY_FIRST).await, 2, "failure retried sooner"); + store.fail_loads.store(false, Ordering::SeqCst); + assert_eq!(loads_after(RETRY_FIRST * 2).await, 3, "backed off, then ok"); + // The successful load's map build runs on a real blocking thread the + // paused clock cannot see. Wait for its publication (bounded in real + // time) before moving the clock, or the next advance passes the + // refresh's timeout and fails it. + let succeeded_at = REFRESH_INTERVAL + RETRY_FIRST * 3; + let waited = std::time::Instant::now(); + while resolver + .published + .load() + .as_ref() + .is_none_or(|published| published.loaded_at < succeeded_at) + { + assert!( + waited.elapsed() < Duration::from_secs(30), + "never published" + ); + tokio::task::yield_now().await; + } + assert_eq!(loads_after(REFRESH_INTERVAL - secs(0.1)).await, 3); + assert_eq!(loads_after(secs(0.1)).await, 4, "back on the interval"); + cancel.cancel(); + refresh_loop.await.unwrap(); +} + +// Two slow successful refreshes in a row, the binding removed just after the +// first one's read began: the second refresh starts one interval after the +// first start, so the removal is published within the documented 40s. +// Mutation: timing the interval from when a refresh finishes starts the +// second one at +69.9s and publishes the removal at +79.8s. +#[tokio::test(start_paused = true)] +async fn removal_during_a_slow_refresh_is_published_within_forty_seconds() { + let store = FakeStore::new(&["a", "gone"]); + *store.load_delay.lock().unwrap() = Duration::from_millis(9_900); + let mut resolver = DatabaseResolver::new( + Arc::clone(&store) as Arc, + policy(), + vec![ISS.to_owned()], + ResolverBounds::default(), + ); + resolver.jitter = || 0.0; + let resolver = Arc::new(resolver); + let cancel = tokio_util::sync::CancellationToken::new(); + let refresh_loop = tokio::spawn(Arc::clone(&resolver).run_refresh_loop(cancel.clone())); + tokio::task::yield_now().await; // the loop arms its schedule at t=0 + let started = tokio::time::Instant::now(); + let published_after = |at: Duration| { + let resolver = Arc::clone(&resolver); + async move { + // The map build runs on a blocking thread the paused clock + // cannot see; wait for it in bounded real time. + let waited = std::time::Instant::now(); + while resolver + .published + .load() + .as_ref() + .is_none_or(|published| published.loaded_at < at) + { + assert!( + waited.elapsed() < Duration::from_secs(30), + "never published" + ); + tokio::task::yield_now().await; + } + } + }; + let resolve = |label: &str| { + let resolver = Arc::clone(&resolver); + let host = host(label); + async move { resolver.resolve(&host).await.map(drop) } + }; + + // +30s: the first refresh reads, still seeing "gone"; it is removed now. + tokio::time::advance(REFRESH_INTERVAL).await; + while store.loads.load(Ordering::SeqCst) < 1 { + tokio::task::yield_now().await; + } + store.remove("gone"); + let removed = started.elapsed(); + tokio::time::advance(Duration::from_millis(9_900)).await; + published_after(REFRESH_INTERVAL).await; + assert_eq!( + resolve("gone").await, + Ok(()), + "first snapshot predates removal" + ); + + // The second refresh starts at +60s and publishes at +69.9s. + tokio::time::sleep_until(started + REFRESH_INTERVAL * 2 + Duration::from_millis(9_900)).await; + published_after(REFRESH_INTERVAL * 2).await; + assert!(resolver.clock.now() - removed < Duration::from_secs(40)); + assert_eq!(resolve("gone").await, UNAVAILABLE, "removed within 40s"); + assert_eq!(resolve("a").await, Ok(())); + cancel.cancel(); + refresh_loop.await.unwrap(); +} + +// The freshness timestamp is written on every attempt and keeps the last +// good load through failures. Mutation: setting it only on success leaves +// the series absent after a failed first attempt. +#[tokio::test] +async fn freshness_gauge_is_written_on_every_attempt() { + const GAUGE: &str = "buzz_nip_fi_binding_snapshot_loaded_timestamp_seconds"; + let gauge = |recorder: &metrics_util::debugging::DebuggingRecorder| { + recorder + .snapshotter() + .snapshot() + .into_vec() + .into_iter() + .find(|(key, ..)| key.key().name() == GAUGE) + .map(|(.., value)| match value { + metrics_util::debugging::DebugValue::Gauge(v) => v.into_inner(), + _ => unreachable!(), + }) + }; + let f = fixture(&["a"]); + let recorder = metrics_util::debugging::DebuggingRecorder::new(); + let _guard = metrics::set_default_local_recorder(&recorder); + f.store.fail_loads.store(true, Ordering::SeqCst); + assert!(f.resolver.refresh().await.is_err()); + assert_eq!( + gauge(&recorder), + Some(0.0), + "written by a failed first attempt" + ); + f.store.fail_loads.store(false, Ordering::SeqCst); + f.resolver.refresh().await.unwrap(); + let loaded = gauge(&recorder).unwrap(); + assert!(loaded > 0.0); + f.store.fail_loads.store(true, Ordering::SeqCst); + assert!(f.resolver.refresh().await.is_err()); + assert_eq!(gauge(&recorder), Some(loaded), "kept through a failure"); +} + +// ── Creation and the negative cache ──────────────────────────────────────── + +// A creation on this pod drops the host's cached "not found" (whatever case +// the host arrives in), so the creator's first login looks it up. Mutation: +// not popping the entry denies until the 5s TTL lapses. +#[tokio::test] +async fn record_creation_clears_the_hosts_cached_not_found() { + let f = fixture(&["a"]); + f.resolver.refresh().await.unwrap(); + assert_eq!(f.resolve("h").await, UNAVAILABLE); + f.store.add("h"); + assert_eq!(f.resolve("h").await, UNAVAILABLE, "cached not-found"); + assert_eq!(f.store.lookups_for("h"), 1); + f.resolver.record_creation(&host("H")); + assert_eq!(f.resolver.negative_len(), 0); + assert_eq!(f.resolve("h").await, Ok(())); + assert_eq!(f.store.lookups_for("h"), 2); +} + +// Only a creation or a publication moves the sequence: two unknown hosts +// looked up at once are both cached. Mutation: bumping on a negative store +// leaves the second one uncached. +#[tokio::test] +async fn concurrent_not_found_lookups_are_all_cached() { + let f = fixture(&["a"]); + f.resolver.refresh().await.unwrap(); + let gate = f.resolver.hooks.arm(MISS_AFTER_READ); + let pending: Vec<_> = ["x", "y"] + .into_iter() + .map(|label| { + let resolver = Arc::clone(&f.resolver); + tokio::spawn(async move { resolver.resolve(&host(label)).await.map(drop) }) + }) + .collect(); + gate.reached().await; + gate.reached().await; + gate.release(); + gate.release(); + for task in pending { + assert_eq!(task.await.unwrap(), UNAVAILABLE); + } + assert_eq!(f.resolver.negative_len(), 2); +} + +#[tokio::test] +async fn negative_lookup_racing_a_creation_or_swap_stores_nothing() { + // (d), and criterion 7(f). + for swap in [false, true] { + let f = fixture(&["a"]); + f.resolver.refresh().await.unwrap(); + let gate = f.resolver.hooks.arm(MISS_AFTER_READ); + let pending = tokio::spawn({ + let resolver = Arc::clone(&f.resolver); + async move { resolver.resolve(&host("h")).await.map(|_| ()) } + }); + gate.reached().await; + f.resolver.hooks.disarm(MISS_AFTER_READ); + f.store.add("h"); + if swap { + f.resolver.refresh().await.unwrap(); + } else { + f.resolver.record_creation(&host("h")); + } + gate.release(); + assert_eq!(pending.await.unwrap(), UNAVAILABLE, "answered as read"); + assert_eq!(f.resolver.negative_len(), 0, "swap={swap}"); + assert_eq!(f.resolve("h").await, Ok(()), "swap={swap}"); + } +} + +// The provisioning handler drops the cached "not found" only for a creation +// that bound issuers. Mutation: skipping the resolver call in the handler, or +// calling it for an unbound creation, fails one of the two halves. +#[tokio::test] +async fn provisioning_a_bound_community_clears_its_cached_not_found() { + use crate::handlers::community_provisioning::record_binding_creation; + let f = fixture(&["a"]); + f.resolver.refresh().await.unwrap(); + assert_eq!(f.resolve("h").await, UNAVAILABLE); + f.store.add("h"); + record_binding_creation(Some(&f.resolver), &host("h"), &[]); + assert_eq!( + f.resolve("h").await, + UNAVAILABLE, + "unbound creation keeps it" + ); + record_binding_creation(Some(&f.resolver), &host("h"), &[ISS.to_owned()]); + assert_eq!(f.resolve("h").await, Ok(())); + assert_eq!(f.store.lookups_for("h"), 2); +} + +// One lookup admits at most `max_waiters` admissions; the next is rejected +// rather than queued. Mutation: ignoring the cap lets the third wait. +#[tokio::test(start_paused = true)] +async fn waiters_beyond_the_cap_are_rejected() { + let f = fixture_with( + &["a"], + ResolverBounds { + max_waiters: 2, + ..ResolverBounds::default() + }, + ); + *f.store.lookup_delay.lock().unwrap() = secs(0.5); + let waiters: Vec<_> = (0..2) + .map(|_| { + let resolver = Arc::clone(&f.resolver); + tokio::spawn(async move { resolver.resolve(&host("a")).await.map(drop) }) + }) + .collect(); + tokio::time::sleep(secs(0.1)).await; + assert_eq!(f.resolver.waiters_now(), 2); + assert_eq!(f.resolve("a").await, UNAVAILABLE); + for w in waiters { + assert_eq!(w.await.unwrap(), Ok(())); + } + assert_eq!(f.store.lookups_for("a"), 1); +} + +// ── Criterion 9: miss-path bounds under a flood ───────────────────────────── + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn flood_of_unknown_hosts_respects_every_cap_and_spares_snapshot_hosts() { + let bounds = ResolverBounds { + max_inflight: 4, + max_pending_hosts: 8, + max_waiters: 16, + negative_cache_max: 32, + permit_wait: Duration::from_millis(5), + lookup_timeout: Duration::from_millis(500), + ..ResolverBounds::default() + }; + let f = fixture_with(&["a", "b"], bounds.clone()); + f.resolver.refresh().await.unwrap(); + *f.store.lookup_delay.lock().unwrap() = Duration::from_millis(20); + + let peak = Arc::new(Mutex::new((0usize, 0usize, 0usize, 0usize))); + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let monitor = tokio::spawn({ + let (resolver, peak, stop) = ( + Arc::clone(&f.resolver), + Arc::clone(&peak), + Arc::clone(&stop), + ); + async move { + while !stop.load(Ordering::SeqCst) { + { + let mut p = peak.lock().unwrap(); + p.0 = p.0.max(resolver.inflight_now()); + p.1 = p.1.max(resolver.pending_len()); + p.2 = p.2.max(resolver.waiters_now()); + p.3 = p.3.max(resolver.negative_len()); + } + tokio::task::yield_now().await; + } + } + }); + let flood: Vec<_> = (0..400) + .map(|i| { + let resolver = Arc::clone(&f.resolver); + tokio::spawn(async move { resolver.resolve(&host(&format!("rand{i}"))).await.is_err() }) + }) + .collect(); + for _ in 0..50 { + assert_eq!(f.resolve("a").await, Ok(())); + assert_eq!(f.resolve("b").await, Ok(())); + } + for task in flood { + assert!(task.await.unwrap(), "every made-up host is denied"); + } + stop.store(true, Ordering::SeqCst); + monitor.await.unwrap(); + let p = *peak.lock().unwrap(); + assert!(p.0 <= bounds.max_inflight, "inflight {}", p.0); + assert!(p.1 <= bounds.max_pending_hosts, "pending {}", p.1); + assert!(p.2 <= bounds.max_waiters, "waiters {}", p.2); + assert!(p.3 <= bounds.negative_cache_max, "negative {}", p.3); + assert_eq!(f.store.lookups_for("a") + f.store.lookups_for("b"), 0); +} + +// ── Criterion 17: map lifetime ────────────────────────────────────────────── + +#[tokio::test] +async fn old_snapshot_maps_are_dropped_while_admissions_keep_their_bindings() { + let f = fixture(&["a"]); + f.resolver.refresh().await.unwrap(); + let mut held = Vec::new(); + let mut weak_maps = Vec::new(); + for _ in 0..10 { + held.push(f.resolver.resolve(&host("a")).await.unwrap()); + weak_maps.push(Arc::downgrade( + f.resolver.published.load_full().as_ref().unwrap(), + )); + f.resolver.refresh().await.unwrap(); + } + // Replaced maps are freed on a blocking thread; wait for it (bounded). + let waited = std::time::Instant::now(); + while weak_maps.iter().any(|w| w.strong_count() > 0) { + assert!( + waited.elapsed() < Duration::from_secs(30), + "only the published map is alive" + ); + tokio::task::yield_now().await; + } + assert!(held + .iter() + .all(|b| b.0.expected_aud() == "https://a.communities.buzz.xyz")); +} + +// ── Pre-filter ────────────────────────────────────────────────────────────── + +#[tokio::test] +async fn non_suffix_reserved_and_multi_label_hosts_never_reach_the_store() { + let f = fixture(&["a"]); + f.resolver.refresh().await.unwrap(); + for raw in [ + "communities.buzz.xyz", + "a.b.communities.buzz.xyz", + "evil.example", + "a.communities.buzz.xyz:8443", + ] { + assert!(f.resolver.resolve(raw).await.is_err(), "{raw}"); + } + assert_eq!(f.store.total_lookups(), 0); +} + +/// Unset or blank variables take the defaults, a set value overrides only +/// its own bound, out-of-range or non-numeric values fail startup, and +/// `max_inflight` stays below the writer pool. Mutation: dropping the clamp +/// lets pool 4 run 8 lookups at once. +#[test] +fn resolver_bounds_from_env_defaults_overrides_clamps_and_rejects() { + const VARS: [&str; 4] = [ + "BUZZ_NIP_FI_RESOLVER_NEGATIVE_CACHE_MAX", + "BUZZ_NIP_FI_RESOLVER_MAX_INFLIGHT", + "BUZZ_NIP_FI_SNAPSHOT_MAX_ROWS", + "BUZZ_NIP_FI_SNAPSHOT_REFRESH_TIMEOUT_MS", + ]; + struct Clear; + impl Drop for Clear { + fn drop(&mut self) { + for v in VARS { + std::env::remove_var(v); + } + } + } + let _lock = crate::nip_fi_config::NIP_FI_ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _clear = Clear; + for v in VARS { + std::env::remove_var(v); + } + + assert_eq!( + ResolverBounds::from_env(50).unwrap(), + ResolverBounds::default() + ); + assert_eq!(ResolverBounds::from_env(4).unwrap().max_inflight, 3); + assert_eq!(ResolverBounds::from_env(1).unwrap().max_inflight, 1); + + std::env::set_var("BUZZ_NIP_FI_SNAPSHOT_MAX_ROWS", "250"); + std::env::set_var("BUZZ_NIP_FI_SNAPSHOT_REFRESH_TIMEOUT_MS", "29999"); + std::env::set_var("BUZZ_NIP_FI_RESOLVER_MAX_INFLIGHT", " "); + let bounds = ResolverBounds::from_env(50).unwrap(); + assert_eq!(bounds.snapshot_max_rows, 250); + assert_eq!(bounds.refresh_timeout, Duration::from_millis(29_999)); + assert_eq!(bounds.max_inflight, ResolverBounds::default().max_inflight); + std::env::set_var("BUZZ_NIP_FI_RESOLVER_MAX_INFLIGHT", "40"); + assert_eq!(ResolverBounds::from_env(50).unwrap().max_inflight, 40); + assert_eq!(ResolverBounds::from_env(10).unwrap().max_inflight, 9); + + for (var, bad) in [ + ("BUZZ_NIP_FI_SNAPSHOT_REFRESH_TIMEOUT_MS", "30000"), + ("BUZZ_NIP_FI_RESOLVER_MAX_INFLIGHT", "0"), + ("BUZZ_NIP_FI_SNAPSHOT_MAX_ROWS", "lots"), + ("BUZZ_NIP_FI_SNAPSHOT_MAX_ROWS", "10000001"), + ("BUZZ_NIP_FI_RESOLVER_NEGATIVE_CACHE_MAX", "1000001"), + ] { + let prior = std::env::var(var).ok(); + std::env::set_var(var, bad); + assert!( + ResolverBounds::from_env(50).is_err(), + "{var}={bad} must fail startup" + ); + match prior { + Some(p) => std::env::set_var(var, p), + None => std::env::remove_var(var), + } + } +} + +mod router_tests; + +// A database-source community is labelled `bound` in shadow, never by its +// host, whose set is unbounded. Mutation: labelling every resolved community +// by its URI, as the `env` source is, names the host. +#[tokio::test] +async fn database_source_shadow_label_never_names_the_host() { + let f = fixture(&["a"]); + let mut headers = axum::http::HeaderMap::new(); + headers.insert(axum::http::header::HOST, host("a").parse().unwrap()); + let community = crate::nip_fi_core::Communities::Database(&f.resolver) + .resolve(&headers) + .await + .expect("bound"); + assert_eq!( + community.shadow_label(), + metrics::SharedString::from("bound") + ); +} + +// Every step of one request reuses the router guard's resolution, a failed +// one included, so shadow resolves the Host once. Mutation: reusing only a +// successful resolution sends the denied host back to the live lookup. +#[tokio::test] +async fn admission_reuses_the_guards_resolution_without_a_lookup() { + let f = fixture(&["a"]); + let mut headers = axum::http::HeaderMap::new(); + headers.insert(axum::http::header::HOST, host("a").parse().unwrap()); + let communities = crate::nip_fi_core::Communities::Database(&f.resolver); + let denied = Err(DenialClass::AuthorizationUnavailable); + let reused = crate::nip_fi_http::with_resolved_community(denied, async { + crate::nip_fi_http::resolve_for_admission(communities, &headers).await + }) + .await; + assert_eq!(reused.map(|_| ()), UNAVAILABLE); + assert_eq!(f.store.lookups_for("a"), 0); +} diff --git a/crates/buzz-relay/src/nip_fi_resolver/tests/router_tests.rs b/crates/buzz-relay/src/nip_fi_resolver/tests/router_tests.rs new file mode 100644 index 0000000000..ab00467cf4 --- /dev/null +++ b/crates/buzz-relay/src/nip_fi_resolver/tests/router_tests.rs @@ -0,0 +1,817 @@ +//! Database-resolver admission through the real router (plan criteria 1, 2, +//! 3, 4, 5, 6f, 7e, 8). Root and audio upgrades and the protected-HTTP guard all +//! run against a resolver over an in-memory store that counts lookups; the +//! assertion verifier is the production one with a seeded key snapshot. +//! The relay database is unreachable, so a request that clears NIP-FI ends +//! at the next step: the upgrades' tenant lookup or the route's extractor. + +use std::collections::HashMap; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use buzz_auth::{ + FederatedAssertionVerifier, FreshnessClass, HttpJwksFetcher, IssuerJwksConfig, IssuerPolicy, + IssuerRegistry, JwksSourceContract, NipFiMode, ProductionJwksSource, TokenClass, +}; +use tower::ServiceExt; + +use super::*; +use crate::state::AppState; + +/// A second registered issuer. Never seeded with keys, so any verification +/// that reaches its key source is `authorization unavailable`. +const ISS_B: &str = "https://issuer-b.example"; +const KID: &str = "resolver-router-test-key"; +// Same public ES256 fixture as the router and command tests. +const PRIVATE_KEY_PEM: &str = + "-----BEGIN PRIVATE KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgcnxDM4EiirH9dHUE\nWZc759TX4s5PAn8kO5ovXSnGxCWhRANCAARFb6ZnsfkqOOXyEhj3KBQphGKF4vTa\nzhebbavbZ1ZoklqkF1cGg+jTO7rONAVEzXvXUWtV6CdDV+rybiVmFP2w\n-----END PRIVATE KEY-----\n"; // sadscan:disable np.pem.1 kingfisher.privkey.2 -- public ES256 test fixture + +const UNAVAILABLE_BODY: &[u8] = b"authorization unavailable\n"; +const REJECTED_BODY: &[u8] = b"evidence rejected\n"; + +/// Host → bound issuers, counting every per-host lookup. +#[derive(Default)] +struct Bindings { + rows: Mutex>>, + lookups: Mutex>, + lookup_fails: std::sync::atomic::AtomicBool, +} + +impl Bindings { + fn bind(&self, label: &str, issuers: &[&str]) { + let issuers = issuers.iter().map(|i| (*i).to_owned()).collect(); + self.rows.lock().unwrap().insert(host(label), issuers); + } + fn unbind(&self, label: &str) { + self.rows.lock().unwrap().remove(&host(label)); + } + fn lookups_for(&self, raw_host: &str) -> usize { + let lookups = self.lookups.lock().unwrap(); + lookups.iter().filter(|h| *h == raw_host).count() + } +} + +impl BindingSource for Bindings { + fn load<'a>( + &'a self, + _registry: &'a [String], + _max_rows: usize, + ) -> BoxFuture<'a, Result, RowCapExceeded>, String>> { + async move { + let rows = + self.rows + .lock() + .unwrap() + .iter() + .fold(Vec::new(), |mut acc, (h, issuers)| { + acc.extend(issuers.iter().map(|i| (h.clone(), i.clone()))); + acc + }); + Ok(Ok(rows)) + } + .boxed() + } + + fn lookup<'a>( + &'a self, + host: &'a str, + _registry: &'a [String], + ) -> BoxFuture<'a, Result, String>> { + async move { + self.lookups.lock().unwrap().push(host.to_owned()); + if self.lookup_fails.load(Ordering::SeqCst) { + return Err("pool closed".into()); + } + Ok(self + .rows + .lock() + .unwrap() + .get(host) + .cloned() + .unwrap_or_default()) + } + .boxed() + } +} + +fn jwks_contract(issuer: &str) -> JwksSourceContract { + JwksSourceContract::new(format!("{issuer}/.well-known/jwks.json"), 300, 86_400) + .expect("valid JWKS contract") +} + +fn issuer_policy(issuer: &str) -> IssuerPolicy { + IssuerPolicy::new( + issuer.to_owned(), + vec!["https://communities.buzz.xyz".to_owned()], + TokenClass::DedicatedNipFi, + FreshnessClass::OfflineJwt, + vec![jsonwebtoken::Algorithm::ES256], + 30, + 3600, + None, + jwks_contract(issuer), + ) + .expect("valid issuer policy") +} + +/// A valid assertion from `iss` for `https://`, signed by the +/// fixture key. +fn mint(iss: &str, raw_host: &str) -> String { + mint_for(iss, raw_host, &nostr::Keys::generate().public_key()) +} + +/// [`mint`] asserting `key`. +fn mint_for(iss: &str, raw_host: &str, key: &nostr::PublicKey) -> String { + use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; + let now = chrono::Utc::now().timestamp(); + let claims = serde_json::json!({ + "iss": iss, + "aud": format!("https://{raw_host}"), + "sub": "resolver-router-test", + "iat": now, + "exp": now + 600, + "nostr_pubkey": key.to_hex(), + }); + let mut header = Header::new(Algorithm::ES256); + header.typ = Some("nip-fi+jwt".to_owned()); + header.kid = Some(KID.to_owned()); + let key = EncodingKey::from_ec_pem(PRIVATE_KEY_PEM.as_bytes()).expect("test EC key"); + encode(&header, &claims, &key).expect("sign assertion") +} + +/// One relay pod: router state with the database resolver installed. +struct Pod { + resolver: Arc, + state: Arc, +} + +async fn pod(store: &Arc, clock: &Arc, mode: NipFiMode) -> Pod { + let base = + crate::state::tests::test_state_with_database_url("postgres://127.0.0.1:1/none").await; + pod_on(base, store, clock, mode).await +} + +/// [`pod`] over a caller-built base state (its tenant store). +async fn pod_on( + base: Arc, + store: &Arc, + clock: &Arc, + mode: NipFiMode, +) -> Pod { + let resolver = Arc::new(DatabaseResolver::with_clock( + Arc::clone(store) as Arc, + policy(), + vec![ISS.to_owned(), ISS_B.to_owned()], + ResolverBounds::default(), + Arc::clone(clock) as Arc, + )); + resolver.refresh().await.expect("startup load"); + + let mut state = (*base).clone(); + let config = Arc::make_mut(&mut state.config); + config.nip_fi.mode = mode; + config.nip_fi.source = crate::nip_fi_config::CommunitySource::Database; + config.nip_fi.registry = { + let mut registry = IssuerRegistry::new(); + registry.insert(issuer_policy(ISS)); + registry.insert(issuer_policy(ISS_B)); + registry + }; + let key_source = Arc::new( + ProductionJwksSource::new( + [ISS, ISS_B] + .map(|issuer| IssuerJwksConfig { + issuer: issuer.to_owned(), + contract: jwks_contract(issuer), + }) + .into(), + HttpJwksFetcher::new(), + ) + .expect("key source"), + ); + let jwk: jsonwebtoken::jwk::Jwk = serde_json::from_value(serde_json::json!({ + "kty": "EC", "crv": "P-256", "alg": "ES256", "use": "sig", "kid": KID, + "x": "RW-mZ7H5Kjjl8hIY9ygUKYRiheL02s4Xm22r22dWaJI", + "y": "WqQXVwaD6NM7us40BUTNe9dRa1XoJ0NX6vJuJWYU_bA", + })) + .expect("test JWK"); + key_source + .seed_snapshot_for_test(ISS, jsonwebtoken::jwk::JwkSet { keys: vec![jwk] }) + .await; + state.nip_fi_verifier = Some(Arc::new(FederatedAssertionVerifier::new( + state.config.nip_fi.registry.clone(), + Arc::clone(&key_source), + ))); + state.nip_fi_jwks_source = Some(key_source); + state.nip_fi_resolver = Some(Arc::clone(&resolver)); + Pod { + resolver, + state: Arc::new(state), + } +} + +/// The three NIP-FI admission sites: root upgrade, audio upgrade, protected +/// HTTP (the router guard). +fn sites() -> [String; 3] { + [ + "/".to_owned(), + format!("/huddle/{}/audio", uuid::Uuid::new_v4()), + "/workflows/wf/runs".to_owned(), + ] +} + +impl Pod { + async fn send(&self, path: &str, raw_host: &str, token: Option<&str>) -> (StatusCode, Vec) { + self.send_signed(path, raw_host, token, None).await + } + + /// [`Pod::send`] with an optional `Authorization` (NIP-98) header. + async fn send_signed( + &self, + path: &str, + raw_host: &str, + token: Option<&str>, + authorization: Option, + ) -> (StatusCode, Vec) { + let mut builder = Request::get(path) + .header(axum::http::header::HOST, raw_host) + .header("Upgrade", "websocket") + .header("Connection", "Upgrade") + .header("Sec-WebSocket-Version", "13") + .header("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ=="); + if let Some(token) = token { + builder = builder.header("Nostr-Federated-Identity", format!("Bearer {token}")); + } + if let Some(authorization) = authorization { + builder = builder.header(axum::http::header::AUTHORIZATION, authorization); + } + let response = crate::router::build_router(Arc::clone(&self.state)) + .oneshot(builder.body(Body::empty()).expect("request")) + .await + .expect("router response"); + let status = response.status(); + let body = axum::body::to_bytes(response.into_body(), 64 * 1024) + .await + .expect("body"); + (status, body.to_vec()) + } + + /// Every site's outcome for a token from `iss` presented at `label`. + async fn every_site(&self, iss: &str, label: &str) -> Vec<(StatusCode, Vec)> { + let token = mint(iss, &host(label)); + let mut outcomes = Vec::new(); + for path in sites() { + outcomes.push(self.send(&path, &host(label), Some(&token)).await); + } + outcomes + } +} + +fn unavailable() -> (StatusCode, Vec) { + (StatusCode::SERVICE_UNAVAILABLE, UNAVAILABLE_BODY.to_vec()) +} + +fn rejected() -> (StatusCode, Vec) { + (StatusCode::FORBIDDEN, REJECTED_BODY.to_vec()) +} + +/// The request cleared NIP-FI admission and was answered by the next step: +/// the upgrades' tenant lookup (no reachable database) or the protected +/// route's path extractor (`wf` is not a UUID). +fn cleared_nip_fi(outcomes: &[(StatusCode, Vec)]) -> bool { + outcomes.iter().all(|(status, body)| match *status { + StatusCode::NOT_FOUND => body == b"relay: no community is configured for this host", + StatusCode::BAD_REQUEST => body.starts_with(b"Invalid URL: Cannot parse `workflow_id`"), + _ => false, + }) +} + +fn setup() -> (Arc, Arc) { + ( + Arc::new(Bindings::default()), + Arc::new(ManualClock::default()), + ) +} + +// ── Criterion 1: one resolution per admission, no suffix fallback ─────────── + +#[tokio::test] +async fn each_admission_resolves_its_host_exactly_once() { + let (store, clock) = setup(); + let p = pod(&store, &clock, NipFiMode::Enforce).await; + // Bound after the startup load: every admission takes the miss lookup, + // and a found binding is never cached, so lookups count resolutions. + store.bind("fresh", &[ISS]); + let token = mint(ISS, &host("fresh")); + for (n, path) in sites().iter().enumerate() { + let outcome = p.send(path, &host("fresh"), Some(&token)).await; + assert!( + cleared_nip_fi(std::slice::from_ref(&outcome)), + "{path}: {outcome:?}" + ); + assert_eq!(store.lookups_for(&host("fresh")), n + 1, "{path}"); + } +} + +#[tokio::test] +async fn one_label_suffix_host_with_no_rows_is_unavailable_at_every_site() { + let (store, clock) = setup(); + let p = pod(&store, &clock, NipFiMode::Enforce).await; + assert_eq!(p.every_site(ISS, "nobody").await, vec![unavailable(); 3]); +} + +// ── Criterion 2: an issuer the community does not list never reaches keys ─── + +#[tokio::test] +async fn unlisted_issuer_is_rejected_before_its_key_source() { + let (store, clock) = setup(); + store.bind("only-a", &[ISS]); + store.bind("lists-b", &[ISS_B]); + let p = pod(&store, &clock, NipFiMode::Enforce).await; + // Issuer B has no key snapshot. Where B is listed, verification reaches + // that key source and is unavailable; where it is not, the allowlist + // rejects the same kind of token first, so the missing keys never show. + assert_eq!(p.every_site(ISS_B, "lists-b").await, vec![unavailable(); 3]); + assert_eq!(p.every_site(ISS_B, "only-a").await, vec![rejected(); 3]); + assert!(cleared_nip_fi(&p.every_site(ISS, "only-a").await)); +} + +// ── Criterion 3: removing the community's issuer revokes its tokens ───────── + +#[tokio::test] +async fn removing_the_issuer_denies_a_previously_valid_token() { + let (store, clock) = setup(); + store.bind("shared", &[ISS, ISS_B]); + let p = pod(&store, &clock, NipFiMode::Enforce).await; + assert!(cleared_nip_fi(&p.every_site(ISS, "shared").await)); + + store.bind("shared", &[ISS_B]); + p.resolver.refresh().await.expect("refresh"); + assert_eq!(p.every_site(ISS, "shared").await, vec![rejected(); 3]); +} + +// ── Criterion 4: full admission per request; failed miss stores nothing ───── + +#[tokio::test] +async fn second_request_after_unbind_and_refresh_is_denied() { + let (store, clock) = setup(); + store.bind("gone", &[ISS]); + let p = pod(&store, &clock, NipFiMode::Enforce).await; + let token = mint(ISS, &host("gone")); + let path = "/workflows/wf/runs"; + assert!(cleared_nip_fi(&[p + .send(path, &host("gone"), Some(&token)) + .await])); + + store.unbind("gone"); + p.resolver.refresh().await.expect("refresh"); + assert_eq!( + p.send(path, &host("gone"), Some(&token)).await, + unavailable() + ); +} + +#[tokio::test] +async fn closed_pool_miss_is_unavailable_everywhere_and_caches_nothing() { + let (store, clock) = setup(); + let p = pod(&store, &clock, NipFiMode::Enforce).await; + store.bind("late", &[ISS]); + store.lookup_fails.store(true, Ordering::SeqCst); + assert_eq!(p.every_site(ISS, "late").await, vec![unavailable(); 3]); + assert_eq!(store.lookups_for(&host("late")), 3, "no negative entry"); + assert!(!p.resolver.negatively_cached(&host("late"))); + + store.lookup_fails.store(false, Ordering::SeqCst); + assert!(cleared_nip_fi(&p.every_site(ISS, "late").await)); +} + +// ── Criterion 6(f): the reserved command-audience host never admits ───────── + +#[tokio::test] +async fn reserved_apex_is_unavailable_without_a_lookup() { + let (store, clock) = setup(); + store + .rows + .lock() + .unwrap() + .insert("communities.buzz.xyz".to_owned(), vec![ISS.to_owned()]); + let p = pod(&store, &clock, NipFiMode::Enforce).await; + let token = mint(ISS, "communities.buzz.xyz"); + for path in sites() { + let outcome = p.send(&path, "communities.buzz.xyz", Some(&token)).await; + assert_eq!(outcome, unavailable(), "{path}"); + } + assert_eq!(store.lookups_for("communities.buzz.xyz"), 0); +} + +// ── Criterion 7(e): negative cache per pod; creating pod admits at once ───── + +#[tokio::test] +async fn creation_admits_on_its_pod_now_and_elsewhere_after_negative_ttl() { + let (store, clock) = setup(); + let p1 = pod(&store, &clock, NipFiMode::Enforce).await; + let p2 = pod(&store, &clock, NipFiMode::Enforce).await; + let path = "/workflows/wf/runs"; + let token = mint(ISS, &host("newco")); + let h = host("newco"); + for p in [&p1, &p2] { + assert_eq!(p.send(path, &h, Some(&token)).await, unavailable()); + } + assert_eq!(store.lookups_for(&h), 2); + + // Created through P1, after both published their snapshots. + store.bind("newco", &[ISS]); + p1.resolver.record_creation(&h); + assert!(cleared_nip_fi(&[p1.send(path, &h, Some(&token)).await])); + assert_eq!(p2.send(path, &h, Some(&token)).await, unavailable()); + assert_eq!(store.lookups_for(&h), 3, "P1 looks up, P2 from cache"); + + clock.set(secs(5.1)); + assert!(cleared_nip_fi(&[p2.send(path, &h, Some(&token)).await])); + assert_eq!(store.lookups_for(&h), 4); +} + +// ── Criterion 8: mode off never resolves or gates ─────────────────────────── + +#[tokio::test] +async fn off_mode_with_database_source_neither_gates_nor_resolves() { + let (store, clock) = setup(); + store.bind("quiet", &[ISS]); + let p = pod(&store, &clock, NipFiMode::Off).await; + for path in sites() { + let outcome = p.send(&path, &host("quiet"), None).await; + assert!( + cleared_nip_fi(std::slice::from_ref(&outcome)), + "{path}: {outcome:?}" + ); + } + assert!(store.lookups.lock().unwrap().is_empty()); +} + +// Shadow resolves as enforce does: a community bound after the startup load +// is found by the live lookup, so shadow records no `community` would-deny. +// Mutation: resolving shadow from memory only takes no lookup and records a +// `community`-stage denial at every site. +#[tokio::test] +async fn shadow_resolves_through_the_database_lookup_like_enforce() { + let (store, clock) = setup(); + let p = pod(&store, &clock, NipFiMode::Shadow).await; + store.bind("late", &[ISS]); + let token = mint(ISS, &host("late")); + let recorder = metrics_util::debugging::DebuggingRecorder::new(); + { + let _guard = metrics::set_default_local_recorder(&recorder); + for path in sites() { + let outcome = p.send(&path, &host("late"), Some(&token)).await; + assert!( + cleared_nip_fi(std::slice::from_ref(&outcome)), + "{path}: {outcome:?}" + ); + } + } + assert_eq!(store.lookups_for(&host("late")), sites().len()); + let community_denials = recorder + .snapshotter() + .snapshot() + .into_vec() + .into_iter() + .filter(|(key, ..)| { + let key = key.key(); + key.name() == "buzz_nip_fi_shadow_total" + && key + .labels() + .any(|l| l.key() == "stage" && l.value() == "community") + }) + .count(); + assert_eq!(community_denials, 0); +} + +// ── Criterion 5: a disconnect covers communities created after it ────────── + +#[tokio::test] +#[allow(clippy::result_large_err)] // the NIP-98 closure returns the production `Response` error type +async fn key_disconnected_before_its_community_exists_is_denied_there() { + let recorder = metrics_util::debugging::DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _metrics = metrics::set_default_local_recorder(&recorder); + + let (store, clock) = setup(); + let p = pod(&store, &clock, NipFiMode::Enforce).await; + let mut state = (*p.state).clone(); + let deny_map = Arc::new(buzz_auth::NipFiDenyMap::new( + 16, + vec![buzz_auth::IssuerCapacity { + issuer: ISS.to_owned(), + capacity: 16, + }], + )); + state.nip_fi_deny_map = Some(Arc::clone(&deny_map)); + let p = Pod { + resolver: p.resolver, + state: Arc::new(state), + }; + + // Disconnect K while H does not exist anywhere. + let (k_keys, control_keys) = (nostr::Keys::generate(), nostr::Keys::generate()); + let (k, control) = (k_keys.public_key(), control_keys.public_key()); + let now = chrono::Utc::now(); + let merged = deny_map.merge_cross_pod_deny(ISS, &k, now + chrono::Duration::hours(1), now); + assert!(matches!(merged, buzz_auth::CrossPodMergeResult::Merged)); + let h = host("after"); + assert_eq!( + p.send("/", &h, Some(&mint_for(ISS, &h, &k))).await, + unavailable() + ); + + // Provision H through this pod. + store.bind("after", &[ISS]); + p.resolver.record_creation(&h); + + let denied = (StatusCode::FORBIDDEN, b"authorization denied\n".to_vec()); + assert_eq!(p.send("/", &h, Some(&mint_for(ISS, &h, &k))).await, denied); + assert!(cleared_nip_fi(&[p + .send("/", &h, Some(&mint_for(ISS, &h, &control))) + .await])); + + // Audio resolves H at upgrade and denies after NIP-42, at its deny-set + // check, on an assertion verified against H's resolved binding. + let audio = format!("/huddle/{}/audio", uuid::Uuid::new_v4()); + let (binding, _) = p.resolver.resolve(&h).await.expect("H resolves"); + let verifier = p.state.nip_fi_verifier.as_deref().expect("verifier"); + for (keys, refused) in [(&k_keys, true), (&control_keys, false)] { + let token = mint_for(ISS, &h, &keys.public_key()); + assert!(cleared_nip_fi(&[p.send(&audio, &h, Some(&token)).await])); + let assertion = verifier.verify_assertion(&token, &binding).expect("valid"); + let outcome = crate::audio::handler::test_support::deny_set_refuses( + Arc::clone(&p.state), + assertion, + keys, + ) + .await; + assert_eq!(outcome, refused, "audio deny-set check"); + } + + let http_admit = |key: nostr::PublicKey| { + let state = Arc::clone(&p.state); + let h = h.clone(); + async move { + let mut headers = axum::http::HeaderMap::new(); + headers.insert(axum::http::header::HOST, h.parse().unwrap()); + headers.insert( + "Nostr-Federated-Identity", + format!("Bearer {}", mint_for(ISS, &h, &key)) + .parse() + .unwrap(), + ); + crate::nip_fi_http::admit_nip_fi_http_on_state(&state, &headers, || { + Ok(crate::nip_fi_http::Nip98Proof::new(key, ())) + }) + .await + .map(drop) + } + }; + let resp = http_admit(k) + .await + .expect_err("K is denied at HTTP admission"); + let status = resp.status(); + let body = axum::body::to_bytes(resp.into_body(), 4096).await.unwrap(); + assert_eq!((status, body.to_vec()), denied); + assert!(http_admit(control).await.is_ok(), "control key is admitted"); + + let http_denials: u64 = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter(|(key, ..)| { + key.key().name() == "buzz_auth_failures_total" + && key + .key() + .labels() + .any(|l| l.value() == "nip_fi_http_denied_pubkey") + }) + .map(|(.., value)| match value { + metrics_util::debugging::DebugValue::Counter(n) => n, + _ => 0, + }) + .sum(); + assert_eq!(http_denials, 1); +} + +// ── Reachable handlers over a real tenant store ───────────────────────────── + +/// Requests reach their handlers: the tenant store is real PostgreSQL with a +/// row for each test host, and protected HTTP uses a valid workflow path with +/// a NIP-98 proof, so the handler itself calls `admit_nip_fi_http_on_state`. +/// The first tenant-store connection a request takes can move the resolver +/// clock, which places a deadline crossing inside the tenant-lookup await. +mod postgres_tests { + use super::*; + + /// Pod over a one-connection pool whose next acquire, once armed, moves + /// the clock to the armed time. + struct TenantPod { + pod: Pod, + clock: Arc, + advance_on_acquire: Arc>>, + label: String, + } + + async fn tenant_pod( + store: &Arc, + clock: &Arc, + mode: NipFiMode, + ) -> TenantPod { + let label = format!("t{}", uuid::Uuid::new_v4().simple()); + let advance_on_acquire = Arc::new(Mutex::new(None::)); + let (hook, hook_clock) = (Arc::clone(&advance_on_acquire), Arc::clone(clock)); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .before_acquire(move |_, _| { + if let Some(at) = hook.lock().unwrap().take() { + hook_clock.set(at); + } + Box::pin(async { Ok(true) }) + }) + .connect(&crate::test_support::database_url()) + .await + .expect("PostgreSQL must be available"); + // Seeding leaves the pool's only connection idle, so every request + // acquire reuses it and runs the hook. + sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") + .bind(uuid::Uuid::new_v4()) + .bind(host(&label)) + .execute(&pool) + .await + .expect("seed community"); + store.bind(&label, &[ISS]); + let base = crate::state::tests::test_state_with_database_pool(pool).await; + let pod = pod_on(base, store, clock, mode).await; + TenantPod { + pod, + clock: Arc::clone(clock), + advance_on_acquire, + label, + } + } + + fn nip98(keys: &nostr::Keys, url: &str) -> String { + use base64::Engine as _; + let event = nostr::EventBuilder::new(nostr::Kind::HttpAuth, "") + .tags([ + nostr::Tag::parse(["u", url]).unwrap(), + nostr::Tag::parse(["method", "GET"]).unwrap(), + ]) + .sign_with_keys(keys) + .unwrap(); + format!( + "Nostr {}", + base64::engine::general_purpose::STANDARD.encode(serde_json::to_vec(&event).unwrap()) + ) + } + + impl TenantPod { + /// Each admission site's outcome for one fresh key: root and audio + /// upgrades, then the workflow-runs handler with a matching proof. + /// With `crossing = Some((start, at))` every request resolves at + /// `start` and its tenant lookup moves the clock to `at`. + async fn every_site( + &self, + crossing: Option<(Duration, Duration)>, + ) -> Vec<(StatusCode, Vec)> { + let raw_host = host(&self.label); + let keys = nostr::Keys::generate(); + let token = mint_for(ISS, &raw_host, &keys.public_key()); + let scheme = if self.pod.state.config.relay_url.starts_with("wss://") { + "https" + } else { + "http" + }; + let workflow = format!("/workflows/{}/runs", uuid::Uuid::new_v4()); + let requests = [ + ("/".to_owned(), None), + (format!("/huddle/{}/audio", uuid::Uuid::new_v4()), None), + ( + workflow.clone(), + Some(nip98(&keys, &format!("{scheme}://{raw_host}{workflow}"))), + ), + ]; + let mut outcomes = Vec::new(); + for (path, proof) in requests { + if let Some((start, at)) = crossing { + self.clock.set(start); + *self.advance_on_acquire.lock().unwrap() = Some(at); + } + outcomes.push( + self.pod + .send_signed(&path, &raw_host, Some(&token), proof) + .await, + ); + } + outcomes + } + } + + /// Every site cleared NIP-FI and its handler went on past tenant + /// binding: a oneshot request cannot upgrade, so root falls back to its + /// NIP-11 document and audio refuses the upgrade, and the workflow handler + /// passed admission to its rate limiter, whose Redis is unreachable. + fn reached_handlers(outcomes: &[(StatusCode, Vec)]) -> bool { + let body = |o: &(StatusCode, Vec)| String::from_utf8_lossy(&o.1).into_owned(); + match outcomes { + [root, audio, http] => { + root.0 == StatusCode::OK + && body(root).contains("\"federated_identity\"") + && audio.0 == StatusCode::UPGRADE_REQUIRED + && http.0 == StatusCode::SERVICE_UNAVAILABLE + && body(http).contains("shared admission unavailable") + } + _ => false, + } + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn every_reachable_handler_resolves_its_host_exactly_once() { + let (store, clock) = setup(); + let p = tenant_pod(&store, &clock, NipFiMode::Enforce).await; + // Absent from the published map, so every resolution is a counted miss. + let raw_host = host(&p.label); + store.unbind(&p.label); + p.pod.resolver.refresh().await.expect("refresh"); + store.bind(&p.label, &[ISS]); + + let outcomes = p.every_site(None).await; + assert!(reached_handlers(&outcomes), "{outcomes:?}"); + assert_eq!(store.lookups_for(&raw_host), 3, "one lookup per admission"); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn snapshot_going_stale_during_the_tenant_lookup_admits_nowhere() { + let (store, clock) = setup(); + let p = tenant_pod(&store, &clock, NipFiMode::Enforce).await; + let inside = (secs(119.0), secs(119.9)); + assert!( + reached_handlers(&p.every_site(Some(inside)).await), + "control" + ); + + // Startup loaded at 0 s: fresh at resolution, stale at admission. + let crossing = (secs(119.9), secs(120.1)); + assert_eq!(p.every_site(Some(crossing)).await, vec![unavailable(); 3]); + } + + /// Each `buzz_nip_fi_shadow_total` series recorded while `run` ran, as + /// sorted `route/stage/outcome` labels. + async fn shadow_records(run: F) -> (F::Output, Vec) { + let recorder = metrics_util::debugging::DebuggingRecorder::new(); + let output = { + let _guard = metrics::set_default_local_recorder(&recorder); + run.await + }; + let mut records: Vec = recorder + .snapshotter() + .snapshot() + .into_vec() + .into_iter() + .filter(|(key, ..)| key.key().name() == "buzz_nip_fi_shadow_total") + .map(|(key, ..)| { + let label = |name: &str| { + let label = key.key().labels().find(|l| l.key() == name); + label.map(|l| l.value().to_owned()).unwrap_or_default() + }; + format!("{}/{}/{}", label("route"), label("stage"), label("outcome")) + }) + .collect(); + records.sort(); + (output, records) + } + + // Shadow re-checks the binding's deadline where enforce does, after the + // tenant lookup, and records enforce's refusal without changing any + // response. Mutation: dropping the upgrade's observed deadline, or + // resolving HTTP afresh instead of reusing the guard's binding, records + // no `deadline` would-deny at that site. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn shadow_records_a_snapshot_going_stale_during_the_tenant_lookup() { + let (store, clock) = setup(); + let p = tenant_pod(&store, &clock, NipFiMode::Shadow).await; + let inside = (secs(119.0), secs(119.9)); + let (outcomes, records) = shadow_records(p.every_site(Some(inside))).await; + assert!(reached_handlers(&outcomes), "control: {outcomes:?}"); + assert_eq!(records, ["http/admit/admit"], "control"); + + let crossing = (secs(119.9), secs(120.1)); + let (outcomes, records) = shadow_records(p.every_site(Some(crossing))).await; + assert!(reached_handlers(&outcomes), "{outcomes:?}"); + assert_eq!( + records, + [ + "audio/deadline/unavailable", + "http/deadline/unavailable", + "ws/deadline/unavailable" + ] + ); + } +} diff --git a/crates/buzz-relay/src/nip_fi_shadow.rs b/crates/buzz-relay/src/nip_fi_shadow.rs index 5f78fb5d7f..7a924f1161 100644 --- a/crates/buzz-relay/src/nip_fi_shadow.rs +++ b/crates/buzz-relay/src/nip_fi_shadow.rs @@ -1,15 +1,18 @@ //! Shadow-mode recording of what enforce would have decided. The verdict -//! never reaches admission, a connection, or a response. Labels are bounded: -//! the community is its configured URI or `unmapped`, never the raw `Host`, -//! and no token, claim, or issuer is logged. The one key logged is the NIP-98 -//! proof key, in the debug line of the key-pairing step shadow shares with +//! never reaches admission, a connection, or a response. Shadow resolves the +//! community exactly as enforce does, database lookup and freshness included, +//! and once per HTTP request. Labels are bounded and never the raw `Host`: the +//! community label is the configured URI for the `env` source (a set bounded +//! by config), `bound` for the `database` source, and `unmapped` when the +//! community did not resolve. No token, claim, or issuer is logged. The one +//! key logged is the NIP-98 proof key, in the debug line of the key-pairing step shadow shares with //! enforce. [FI-TRACE-PRIVACY-NONPUBLIC] //! //! # Metrics //! //! | Series | Labels | One increment per | //! |---|---|---| -//! | `buzz_nip_fi_shadow_total` | `route` (`http`, `ws`, `audio`), `stage`, `outcome`, `community` | NIP-FI decision enforce would make: one per HTTP request that reaches admission, one per WebSocket upgrade enforce would refuse, and one per observed session: at the AUTH where enforce would admit or deny it, or `deadline` when its deadline passes first | +//! | `buzz_nip_fi_shadow_total` | `route` (`http`, `ws`, `audio`), `stage`, `outcome`, `community` | NIP-FI decision enforce would make: one per HTTP request that reaches admission, one per WebSocket upgrade enforce would refuse, and one per observed session: at the AUTH where enforce would admit or deny it, or `deadline` when its deadline passes first. A request or upgrade whose community binding goes stale before enforce's final re-check also records `deadline` | //! | `buzz_nip_fi_shadow_strict_proof_total` | `route` (`bridge`, `blossom`), `outcome` (`pass`, `rejected`), `community` | strict NIP-98 side check on a route whose proof is stricter in enforce (payload tag, Blossom proof); not every request runs one | //! | `buzz_nip_fi_shadow_session_end_total` | `route` (`ws`, `audio`), `reason` (`expired`, `revoked`), `community` | admitted shadow session enforce would have ended; a session never admitted records none | //! | `buzz_nip_fi_shadow_disconnect_total` | `route` (`admin`, `cross_pod`), `outcome` | disconnect-path event that moves a real disconnect counter in enforce: an accepted admin command, a capacity rejection, a subscriber lag, or a cross-pod capacity or poison failsafe. An admin command refused for its header, body, proof, replay, or a dependency error records nothing | @@ -24,7 +27,9 @@ use axum::http::HeaderMap; use buzz_auth::DenialClass; -use crate::nip_fi_config::NipFiCommunities; +use metrics::SharedString; + +use crate::nip_fi_http::Resolution; /// Which enforce step would have denied, and with which class. pub(crate) type WouldDeny = (Stage, DenialClass); @@ -55,18 +60,30 @@ impl Stage { } } -/// Record one would-be enforce verdict for a request at `route`. -pub(crate) fn record( - route: &'static str, - headers: &HeaderMap, - communities: &NipFiCommunities, - verdict: Result<(), WouldDeny>, -) { - record_for(route, community_label(headers, communities), verdict); +/// The `community` label of a `database`-source community that resolved. +pub(crate) const BOUND: &str = "bound"; +/// The `community` label of a verdict whose community did not resolve. +const UNMAPPED: &str = "unmapped"; + +/// The `community` label of `community`. +pub(crate) fn label(community: &Resolution) -> SharedString { + community + .as_ref() + .map_or(SharedString::const_str(UNMAPPED), |c| c.shadow_label()) +} + +/// Record one would-be enforce verdict for a request at `route` whose Host +/// resolved to `community`. +pub(crate) fn record(route: &'static str, community: &Resolution, verdict: Result<(), WouldDeny>) { + record_for(route, label(community), verdict); } /// [`record`] for a caller that resolved its community label earlier. -pub(crate) fn record_for(route: &'static str, community: String, verdict: Result<(), WouldDeny>) { +pub(crate) fn record_for( + route: &'static str, + community: SharedString, + verdict: Result<(), WouldDeny>, +) { let (stage, outcome) = match verdict { Ok(()) => ("admit", "admit"), Err((stage, class)) => (stage.label(), class_label(class)), @@ -74,11 +91,10 @@ pub(crate) fn record_for(route: &'static str, community: String, verdict: Result emit(route, community, stage, outcome); } -/// The bounded community label for `headers`: its configured URI or `unmapped`. -pub(crate) fn community_label(headers: &HeaderMap, communities: &NipFiCommunities) -> String { - crate::nip_fi_core::resolve_community(headers, communities) - .map_or("unmapped", |binding| binding.expected_aud()) - .to_owned() +/// The request's community, from the router guard's resolution when there is +/// one. +async fn resolution(state: &crate::state::AppState, headers: &HeaderMap) -> Resolution { + crate::nip_fi_http::resolve_for_admission(state.nip_fi_communities(), headers).await } tokio::task_local! { @@ -90,14 +106,13 @@ tokio::task_local! { /// verdict. Enforce never lets a guard-denied request reach a handler, so /// none of that request's handler-side records are emitted. pub(crate) async fn observe_guard( - state: &crate::state::AppState, - headers: &HeaderMap, + community: SharedString, verdict: Result<(), WouldDeny>, run: F, ) -> F::Output { let denied = verdict.is_err(); if denied { - record("http", headers, &state.config.nip_fi.communities, verdict); + record_for("http", community, verdict); } GUARD_DENIED.scope(denied, run).await } @@ -107,7 +122,7 @@ fn guard_denied() -> Option { GUARD_DENIED.try_with(|denied| *denied).ok() } -fn emit(route: &'static str, community: String, stage: &'static str, outcome: &'static str) { +fn emit(route: &'static str, community: SharedString, stage: &'static str, outcome: &'static str) { if guard_denied() == Some(true) { return; } @@ -121,7 +136,7 @@ fn emit(route: &'static str, community: String, stage: &'static str, outcome: &' .increment(1); // The counter is the record; only a would-deny earns a (debug) line. if stage != "admit" { - tracing::debug!(route, stage, outcome, community, "nip-fi shadow would-deny"); + tracing::debug!(route, stage, outcome, %community, "nip-fi shadow would-deny"); } } @@ -136,7 +151,7 @@ const fn class_label(class: DenialClass) -> &'static str { /// In shadow mode only, record whether `strict` (a pure check) would reject, /// on its own counter: a pass is one proof, not an admission verdict. -pub(crate) fn observe_strict_proof( +pub(crate) async fn observe_strict_proof( state: &crate::state::AppState, headers: &HeaderMap, route: &'static str, @@ -145,11 +160,12 @@ pub(crate) fn observe_strict_proof( let nip_fi = &state.config.nip_fi; if nip_fi.mode.observes_only() && guard_denied() != Some(true) { let outcome = strict().map_or(class_label(DenialClass::EvidenceRejected), |()| "pass"); + let community = label(&resolution(state, headers).await); metrics::counter!( "buzz_nip_fi_shadow_strict_proof_total", "route" => route, "outcome" => outcome, - "community" => community_label(headers, &nip_fi.communities) + "community" => community ) .increment(1); } @@ -170,7 +186,7 @@ pub(crate) async fn bind_tenant( .await .ok(); if tenant.is_none() { - observe_unbound(state, headers); + observe_unbound(state, headers).await; } tenant } @@ -179,15 +195,16 @@ pub(crate) async fn bind_tenant( /// Host bound no tenant: `community` when the Host maps to no configured /// community; an admit when enforce's guard passed it, since the handler's /// own rejection is no NIP-FI denial; nothing outside a guarded route. -fn observe_unbound(state: &crate::state::AppState, headers: &HeaderMap) { +async fn observe_unbound(state: &crate::state::AppState, headers: &HeaderMap) { let nip_fi = &state.config.nip_fi; if nip_fi.mode.observes_only() { - let verdict = match crate::nip_fi_core::resolve_community(headers, &nip_fi.communities) { - Err(class) => Err((Stage::Community, class)), + let community = resolution(state, headers).await; + let verdict = match &community { + Err(class) => Err((Stage::Community, *class)), Ok(_) if guard_denied() == Some(false) => Ok(()), Ok(_) => return, }; - record("http", headers, &nip_fi.communities, verdict); + record("http", &community, verdict); } } @@ -241,15 +258,19 @@ mod tests { headers } - // Mutation: labelling by the raw Host leaks `attacker.example`. - #[test] - fn community_label_is_configured_uri_or_unmapped() { + // Mutation: labelling by the raw Host leaks `attacker.example`; labelling + // an `env` community `bound` drops its per-community series. + #[tokio::test(flavor = "current_thread")] + async fn env_label_is_configured_uri_or_unmapped() { + let map = communities(); let recorder = DebuggingRecorder::new(); - metrics::with_local_recorder(&recorder, || { - for host in [Some(TEST_HOST), Some("attacker.example"), None] { - super::record("http", &with_host(host), &communities(), Ok(())); - } - }); + let _guard = metrics::set_default_local_recorder(&recorder); + for host in [Some(TEST_HOST), Some("attacker.example"), None] { + let community = crate::nip_fi_core::Communities::from(&map) + .resolve(&with_host(host)) + .await; + super::record("http", &community, Ok(())); + } let labels: Vec = shadow_counts(&recorder) .into_iter() .map(|(c, _, _)| c) @@ -311,7 +332,8 @@ mod tests { state.config = Arc::new(config); state.nip98_replay = guard.clone(); let recorder = DebuggingRecorder::new(); - metrics::with_local_recorder(&recorder, || { + { + let _guard = metrics::set_default_local_recorder(&recorder); super::observe_strict_proof(&state, &headers, "bridge", || { crate::api::bridge::verify_bridge_auth_with_options( &headers, @@ -322,8 +344,9 @@ mod tests { true, ) .map(drop) - }); - }); + }) + .await; + } let counts = counts_of(&recorder, STRICT); assert_eq!(counts.len(), expected, "{mode:?}"); assert!( @@ -336,13 +359,15 @@ mod tests { // Both outcomes stay on the proof counter, never an admit // that could be summed with whole-request verdicts. let recorder = DebuggingRecorder::new(); - metrics::with_local_recorder(&recorder, || { - super::observe_strict_proof(&state, &headers, "bridge", || Ok::<_, ()>(())); - super::observe_strict_proof(&state, &headers, "bridge", || Err(())); - }); + { + let _guard = metrics::set_default_local_recorder(&recorder); + super::observe_strict_proof(&state, &headers, "bridge", || Ok::<_, ()>(())) + .await; + super::observe_strict_proof(&state, &headers, "bridge", || Err(())).await; + } let label = |outcome: &str| { ( - "https://relay.example".to_owned(), + TEST_COMMUNITY_URI.to_owned(), "-".to_owned(), outcome.to_owned(), ) diff --git a/crates/buzz-relay/src/nip_fi_shadow_session.rs b/crates/buzz-relay/src/nip_fi_shadow_session.rs index e841737f82..b6fb243882 100644 --- a/crates/buzz-relay/src/nip_fi_shadow_session.rs +++ b/crates/buzz-relay/src/nip_fi_shadow_session.rs @@ -14,7 +14,6 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock, PoisonError, Weak}; -use axum::http::HeaderMap; use buzz_auth::{DenialClass, VerifiedAssertion}; use nostr::PublicKey; use tokio_util::sync::CancellationToken; @@ -69,10 +68,12 @@ impl ShadowSessions { } } +/// A shadow session exists only for an assertion verified against a resolved +/// community, whose label all its records carry. pub(crate) struct ShadowSession { id: u64, route: &'static str, - community: String, + community: metrics::SharedString, assertion: VerifiedAssertion, phase: Mutex, /// The socket's cancellation, read only to stop recording. @@ -87,7 +88,7 @@ impl ShadowSession { pub(crate) fn start( state: &AppState, route: &'static str, - headers: &HeaderMap, + community: metrics::SharedString, assertion: VerifiedAssertion, connection_time: chrono::DateTime, ) -> Arc { @@ -101,7 +102,7 @@ impl ShadowSession { let session = Arc::new(Self { id: sessions.next_id.fetch_add(1, Ordering::Relaxed), route, - community: crate::nip_fi_shadow::community_label(headers, &nip_fi.communities), + community, assertion, phase: Mutex::new(Phase::Pending { revoked: false }), cancel: OnceLock::new(), @@ -263,13 +264,13 @@ impl Drop for ShadowSession { /// returns `true` when enforce would have refused the upgrade here. pub(crate) fn upgrade_denied( state: &AppState, - headers: &HeaderMap, + community: &metrics::SharedString, assertion: &VerifiedAssertion, ) -> bool { let denied = deny_listed(state, assertion); if denied { let verdict = Err((Stage::DenySet, DenialClass::AuthorizationDenied)); - crate::nip_fi_shadow::record("ws", headers, &state.config.nip_fi.communities, verdict); + crate::nip_fi_shadow::record_for("ws", community.clone(), verdict); } denied } @@ -383,7 +384,13 @@ pub(crate) mod tests { lifetime: Duration, ) -> Arc { let assertion = VerifiedAssertion::for_test(key, vec![Utc::now() + lifetime]); - ShadowSession::start(state, "ws", &HeaderMap::new(), assertion, Utc::now()) + ShadowSession::start( + state, + "ws", + crate::nip_fi_shadow::BOUND.into(), + assertion, + Utc::now(), + ) } fn auth(session: &Arc, state: &AppState, pubkey: PublicKey) { @@ -629,8 +636,16 @@ pub(crate) mod tests { VerifiedAssertion::for_test(Some(key), vec![Utc::now() + Duration::hours(1)]); let clean = VerifiedAssertion::for_test(Some(Keys::generate().public_key()), vec![Utc::now()]); - assert!(upgrade_denied(&state, &HeaderMap::new(), &listed)); - assert!(!upgrade_denied(&state, &HeaderMap::new(), &clean)); + assert!(upgrade_denied( + &state, + &crate::nip_fi_shadow::BOUND.into(), + &listed + )); + assert!(!upgrade_denied( + &state, + &crate::nip_fi_shadow::BOUND.into(), + &clean + )); }); assert_eq!(records, ["deny_set/denied"]); } diff --git a/crates/buzz-relay/src/nip_fi_upgrade.rs b/crates/buzz-relay/src/nip_fi_upgrade.rs index 7edabfa99c..5c5e864469 100644 --- a/crates/buzz-relay/src/nip_fi_upgrade.rs +++ b/crates/buzz-relay/src/nip_fi_upgrade.rs @@ -15,34 +15,67 @@ use axum::body::Body; use axum::http::{HeaderMap, Response}; use buzz_auth::{DenialClass, NipFiMode, VerifiedAssertion, VerifyAssertion}; -use crate::nip_fi_config::NipFiCommunities; use crate::nip_fi_core::{ - evaluate_attached_assertion, http_denial, resolve_community, AssertionRejection, + evaluate_attached_assertion, http_denial, AdmissionDeadline, AssertionRejection, Communities, + ResolvedCommunity, }; /// Outcome of NIP-FI assertion validation at upgrade time. pub(crate) enum NipFiUpgradeOutcome { - /// Assertion validated successfully. Carry the result into the connection. - Admitted(VerifiedAssertion), + /// Assertion validated successfully. Carry the result into the connection, + /// with the deadline the upgrade re-checks after its tenant lookup. + Admitted(VerifiedAssertion, AdmissionDeadline), /// Off, or Shadow without a passing assertion — none is required. NotRequired, - /// Shadow: the assertion passed and is only observed from here on. - Observed(VerifiedAssertion), + /// Shadow: the assertion passed and is only observed from here on, with + /// its community, whose deadline enforce would re-check after the tenant + /// lookup. + Observed(VerifiedAssertion, ResolvedCommunity), /// Enforcement active but assertion absent/rejected — return the HTTP /// denial response. Denied(Response), } -/// The enforced and the shadow-observed assertion slots of an admitted upgrade. -pub(crate) type UpgradeAssertions = (Option, Option); +/// The enforced and the shadow-observed assertion slots of an admitted upgrade, +/// and the deadline to re-check immediately before accepting the socket. The +/// observed assertion carries its own community, for [`observe_deadline`]. +pub(crate) type UpgradeAssertions = ( + Option, + Option<(VerifiedAssertion, ResolvedCommunity)>, + AdmissionDeadline, +); + +/// Shadow's re-check at the point enforce re-checks its deadline: a binding +/// that went stale during the tenant lookup records enforce's refusal and +/// stops observing the session. The socket itself is still accepted. A +/// session still observed keeps its community label. +pub(crate) fn observe_deadline( + route: crate::nip_fi_session::NipFiWsRoute, + observed: Option<(VerifiedAssertion, ResolvedCommunity)>, +) -> Option<(VerifiedAssertion, metrics::SharedString)> { + let (assertion, community) = observed?; + let label = community.shadow_label(); + match community.deadline().check() { + Ok(()) => Some((assertion, label)), + Err(class) => { + let verdict = Err((crate::nip_fi_shadow::Stage::Deadline, class)); + crate::nip_fi_shadow::record_for(route.shadow_label(), label, verdict); + None + } + } +} impl NipFiUpgradeOutcome { /// The enforced and the shadow-observed assertion, or the denial response. pub(crate) fn into_assertions(self) -> Result>> { match self { - Self::NotRequired => Ok((None, None)), - Self::Admitted(assertion) => Ok((Some(assertion), None)), - Self::Observed(assertion) => Ok((None, Some(assertion))), + Self::NotRequired => Ok((None, None, AdmissionDeadline::default())), + Self::Admitted(assertion, deadline) => Ok((Some(assertion), None, deadline)), + Self::Observed(assertion, community) => Ok(( + None, + Some((assertion, community)), + AdmissionDeadline::default(), + )), Self::Denied(response) => Err(Box::new(response)), } } @@ -61,32 +94,37 @@ impl NipFiUpgradeOutcome { /// be valid but authorization is temporarily unavailable — so "authorization /// denied" would be false. "authorization unavailable, retry after repair" is /// the accurate and correct signal. [FI-TRACE-DENIAL-ORACLE] -pub(crate) fn check_nip_fi_at_upgrade( +pub(crate) async fn check_nip_fi_at_upgrade<'a>( route: crate::nip_fi_session::NipFiWsRoute, headers: &HeaderMap, - communities: &NipFiCommunities, + communities: impl Into>, verifier: Option<&dyn VerifyAssertion>, mode: NipFiMode, ) -> NipFiUpgradeOutcome { + let communities = communities.into(); if !mode.restricts() { // Shadow records an upgrade would-deny; a passing assertion is // observed until AUTH, where its admission verdict is recorded. if mode.observes_only() { - let verdict = resolve_community(headers, communities) - .map_err(|class| (crate::nip_fi_shadow::Stage::Community, class)) - .and_then(|community| { - evaluate_attached_assertion(headers, community, verifier).map_err(|rejection| { + let community = communities.resolve(headers).await; + let verdict = match &community { + Err(class) => Err((crate::nip_fi_shadow::Stage::Community, *class)), + Ok(binding) => { + evaluate_attached_assertion(headers, binding, verifier).map_err(|rejection| { ( crate::nip_fi_shadow::Stage::Assertion, rejection.denial_class(), ) }) - }); - match verdict { - Ok(assertion) => return NipFiUpgradeOutcome::Observed(assertion), - Err(deny) => { - let route = route.shadow_label(); - crate::nip_fi_shadow::record(route, headers, communities, Err(deny)); + } + }; + match (verdict, community) { + (Ok(assertion), Ok(community)) => { + return NipFiUpgradeOutcome::Observed(assertion, community) + } + (verdict, community) => { + let verdict = verdict.map(drop); + crate::nip_fi_shadow::record(route.shadow_label(), &community, verdict); } } } @@ -98,12 +136,12 @@ pub(crate) fn check_nip_fi_at_upgrade( } // Enforce mode: resolve the Host's community, then validate the assertion. - let community = match resolve_community(headers, communities) { + let community = match communities.resolve(headers).await { Ok(community) => community, Err(class) => return NipFiUpgradeOutcome::Denied(http_denial(class)), }; - match evaluate_attached_assertion(headers, community, verifier) { - Ok(assertion) => NipFiUpgradeOutcome::Admitted(assertion), + match evaluate_attached_assertion(headers, &community, verifier) { + Ok(assertion) => NipFiUpgradeOutcome::Admitted(assertion, community.deadline()), Err(rejection) => { if let AssertionRejection::Verifier(err) = rejection { tracing::debug!(code = err.code(), "nip-fi assertion denied at upgrade"); @@ -116,6 +154,7 @@ pub(crate) fn check_nip_fi_at_upgrade( #[cfg(test)] mod tests { use super::*; + use futures_util::FutureExt as _; const ROOT: crate::nip_fi_session::NipFiWsRoute = crate::nip_fi_session::NipFiWsRoute::Root; use crate::nip_fi_core::extract_bearer_token; @@ -393,7 +432,9 @@ mod tests { &communities(), None, buzz_auth::NipFiMode::Enforce, - ); + ) + .now_or_never() + .expect("env resolution never waits"); match outcome { NipFiUpgradeOutcome::Denied(resp) => { assert_eq!(resp.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE); @@ -413,7 +454,9 @@ mod tests { &communities(), None, buzz_auth::NipFiMode::Enforce, - ); + ) + .now_or_never() + .expect("env resolution never waits"); // Missing header → MissingEvidence; but None verifier fires first. // Correct behavior: extract_bearer_token is called before verifier check, // so missing header → 401 (MissingEvidence) before reaching the None verifier path. @@ -443,7 +486,9 @@ mod tests { &communities(), None, buzz_auth::NipFiMode::Off, - ); + ) + .now_or_never() + .expect("env resolution never waits"); assert!( matches!(outcome, NipFiUpgradeOutcome::NotRequired), "Off mode must not require assertion — OSS default must not regress" @@ -471,7 +516,9 @@ mod tests { &communities(), None, buzz_auth::NipFiMode::DenyProtected, - ); + ) + .now_or_never() + .expect("env resolution never waits"); match outcome { NipFiUpgradeOutcome::Denied(resp) => { assert_eq!( @@ -519,13 +566,17 @@ mod tests { ]; for (err, status, body) in rows { let verifier = ScriptedVerifier::new(Err(err)); - let (got_status, got_body) = denied_parts(check_nip_fi_at_upgrade( - ROOT, - &headers_with("Bearer a.b.c"), - &communities(), - Some(&verifier), - NipFiMode::Enforce, - )); + let (got_status, got_body) = denied_parts( + check_nip_fi_at_upgrade( + ROOT, + &headers_with("Bearer a.b.c"), + &communities(), + Some(&verifier), + NipFiMode::Enforce, + ) + .now_or_never() + .expect("env resolution never waits"), + ); assert_eq!(got_status, status, "{err:?}"); assert_eq!(got_body, body, "{err:?}"); } @@ -535,13 +586,17 @@ mod tests { // Mutation: checking the verifier first turns this 403 into 503. #[test] fn characterize_upgrade_transport_precedes_verifier_presence() { - let (status, body) = denied_parts(check_nip_fi_at_upgrade( - ROOT, - &headers_with("junk"), - &communities(), - None, - NipFiMode::Enforce, - )); + let (status, body) = denied_parts( + check_nip_fi_at_upgrade( + ROOT, + &headers_with("junk"), + &communities(), + None, + NipFiMode::Enforce, + ) + .now_or_never() + .expect("env resolution never waits"), + ); assert_eq!(status, StatusCode::FORBIDDEN); assert_eq!(body, b"evidence rejected\n"); } @@ -558,8 +613,11 @@ mod tests { &communities(), Some(&verifier), NipFiMode::Enforce, - ) { - NipFiUpgradeOutcome::Admitted(assertion) => { + ) + .now_or_never() + .expect("env resolution never waits") + { + NipFiUpgradeOutcome::Admitted(assertion, _) => { assert_eq!(assertion.asserted_key(), None); } _ => panic!("claimless assertion must be admitted at upgrade"), @@ -581,16 +639,22 @@ mod tests { &communities(), Some(&verifier), NipFiMode::Off - ), + ) + .now_or_never() + .expect("env resolution never waits"), NipFiUpgradeOutcome::NotRequired )); - let (status, _) = denied_parts(check_nip_fi_at_upgrade( - ROOT, - &headers, - &communities(), - Some(&verifier), - NipFiMode::DenyProtected, - )); + let (status, _) = denied_parts( + check_nip_fi_at_upgrade( + ROOT, + &headers, + &communities(), + Some(&verifier), + NipFiMode::DenyProtected, + ) + .now_or_never() + .expect("env resolution never waits"), + ); assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); assert_eq!(verifier.calls(), 0); } @@ -611,18 +675,22 @@ mod tests { Some(&verifier), NipFiMode::Shadow, ) + .now_or_never() + .expect("env resolution never waits") }; - let Ok((None, Some(_))) = check().into_assertions() else { + let Ok((None, Some(_), _)) = check().into_assertions() else { panic!("a passing shadow assertion is observed, never enforced"); }; assert_eq!(verifier.calls(), 1, "shadow must evaluate the assertion"); - let Ok((None, None)) = check_nip_fi_at_upgrade( + let Ok((None, None, _)) = check_nip_fi_at_upgrade( ROOT, &host_headers(), &communities(), Some(&verifier), NipFiMode::Shadow, ) + .now_or_never() + .expect("env resolution never waits") .into_assertions() else { panic!("a missing shadow assertion admits as Off"); }; @@ -647,7 +715,9 @@ mod tests { &communities(), verifier, NipFiMode::Shadow, - ); + ) + .now_or_never() + .expect("env resolution never waits"); assert!(matches!(outcome, NipFiUpgradeOutcome::NotRequired)); } } diff --git a/crates/buzz-relay/src/router.rs b/crates/buzz-relay/src/router.rs index e7194c8f72..5970f26f6e 100644 --- a/crates/buzz-relay/src/router.rs +++ b/crates/buzz-relay/src/router.rs @@ -189,17 +189,24 @@ async fn nip_fi_assertion_guard( return http_denial(buzz_auth::DenialClass::AuthorizationUnavailable); } - let verdict = nip_fi_guard_steps(&state, request.headers()); + // Shadow resolves exactly as enforce does; enforce hands the binding to + // the handler's admission, so a request costs one resolution. + let community = state.nip_fi_communities().resolve(request.headers()).await; + let verdict = nip_fi_guard_steps(&state, request.headers(), community.as_deref()); // Shadow: transparent, but the guard's verdict is recorded in enforce's // order, ahead of any handler step. + // The resolved binding rides along so the handler's observed admission + // rechecks the same deadline enforce does. if mode.observes_only() { - let headers = request.headers().clone(); - return crate::nip_fi_shadow::observe_guard(&state, &headers, verdict, next.run(request)) - .await; + let label = crate::nip_fi_shadow::label(&community); + let run = crate::nip_fi_http::with_resolved_community(community, next.run(request)); + return crate::nip_fi_shadow::observe_guard(label, verdict, run).await; } - match verdict { - Ok(()) => next.run(request).await, - Err((_, class)) => http_denial(class), + match (verdict, community) { + (Ok(()), Ok(community)) => { + crate::nip_fi_http::with_resolved_community(Ok(community), next.run(request)).await + } + (Err((_, class)), _) | (Ok(()), Err(class)) => http_denial(class), } } @@ -213,10 +220,9 @@ async fn nip_fi_assertion_guard( fn nip_fi_guard_steps( state: &AppState, headers: &axum::http::HeaderMap, + community: Result<&buzz_auth::CommunityBinding, &buzz_auth::DenialClass>, ) -> Result<(), crate::nip_fi_shadow::WouldDeny> { - let community = - crate::nip_fi_core::resolve_community(headers, &state.config.nip_fi.communities) - .map_err(|class| (crate::nip_fi_shadow::Stage::Community, class))?; + let community = community.map_err(|class| (crate::nip_fi_shadow::Stage::Community, *class))?; crate::nip_fi_core::evaluate_attached_assertion( headers, community, @@ -631,15 +637,16 @@ async fn nip11_or_ws_handler( // accepts is also gated — no hand-rolled predicate can diverge from the // extractor's accepted shapes when `http2` is eventually enabled. // - // Zero DB cost invariant: the active HTTP/1.1 fire-point runs before - // `bind_community`, so denied h1 upgrades pay zero DB cost + // No-tenant-lookup invariant: the active HTTP/1.1 fire-point runs before + // `bind_community`, so denied h1 upgrades skip the tenant lookup (a + // resolver cache miss still costs one bounded binding lookup) // [FI-TRACE-TRANSPORT-CLOSED], and tests that assert 401/503 are not // pre-empted by a 404 from an unseeded DB. The `Ok(ws)` backstop runs after // `bind_community`; it is unreachable until `http2` is enabled. // // Keying on the header pair (not on `Accept`) means an HTML Accept header // on a real WS upgrade is still gated correctly. - let (upgrade_checked, nip_fi_assertion, shadow_assertion) = { + let (upgrade_checked, nip_fi_assertion, shadow_assertion, nip_fi_deadline) = { let is_h1_ws_upgrade = headers .get(axum::http::header::UPGRADE) .and_then(|v| v.to_str().ok()) @@ -666,18 +673,19 @@ async fn nip11_or_ws_handler( use crate::nip_fi_upgrade::check_nip_fi_at_upgrade; let mode = state.config.nip_fi.mode; let verifier = state.nip_fi_verifier.as_deref(); - let communities = &state.config.nip_fi.communities; + let communities = state.nip_fi_communities(); match check_nip_fi_at_upgrade(Root, &headers, communities, verifier, mode) + .await .into_assertions() { - Ok((assertion, shadow)) => (true, assertion, shadow), + Ok((assertion, shadow, deadline)) => (true, assertion, shadow, deadline), Err(resp) => return (*resp).into_response(), } } else { // Not a WS upgrade shape — a NIP-11 request or a plain browser GET. // The `Ok(ws)` arm below backstops any extractor-accepted shape this // predicate misses. [F3-H2-GATE] - (false, None, None) + (false, None, None, Default::default()) } }; @@ -689,7 +697,7 @@ async fn nip11_or_ws_handler( // That normative sequence runs in handlers/auth.rs after set_authenticated_pubkey. // // This pre-upgrade check provides a cheap bounce for keys already in the deny - // map before the connection is upgraded — pays zero DB cost and rejects before + // map before the connection is upgraded — needs no DB read and rejects before // tungstenite hands the socket to the application. It is NOT race-free against // a concurrent disconnect (the session isn't registered yet), which is why the // normative post-registration check in auth.rs is the correctness gate. @@ -707,8 +715,9 @@ async fn nip11_or_ws_handler( } } } - let shadow_assertion = shadow_assertion - .filter(|a| !crate::nip_fi_shadow_session::upgrade_denied(&state, &headers, a)); + let shadow_assertion = shadow_assertion.filter(|(a, c)| { + !crate::nip_fi_shadow_session::upgrade_denied(&state, &c.shadow_label(), a) + }); // Row zero: bind the connection to its community from the request host // BEFORE the WebSocket upgrade, so no frame is ever read on an unbound @@ -719,7 +728,7 @@ async fn nip11_or_ws_handler( // `icon` simply absent), so the doc cannot leak which hosts are mapped. // // The NIP-FI upgrade gate runs above (before bind_community) so denied - // upgrades pay zero DB cost; the `Ok(ws)` backstop runs below. + // upgrades skip the tenant lookup; the `Ok(ws)` backstop runs below. let tenant = match crate::tenant::bind_community(&state.db, raw_host).await { Ok(ctx) => ctx, Err(_) => { @@ -733,6 +742,15 @@ async fn nip11_or_ws_handler( .into_response(); } }; + // The tenant lookup awaited since the community resolved; a snapshot + // that went stale meanwhile accepts no socket. + if let Err(class) = nip_fi_deadline.check() { + return http_denial(class).into_response(); + } + let shadow_assertion = crate::nip_fi_upgrade::observe_deadline( + crate::nip_fi_session::NipFiWsRoute::Root, + shadow_assertion, + ); let max_frame_bytes = state.config.max_frame_bytes; @@ -754,11 +772,15 @@ async fn nip11_or_ws_handler( use crate::nip_fi_upgrade::check_nip_fi_at_upgrade; let mode = state.config.nip_fi.mode; let verifier = state.nip_fi_verifier.as_deref(); - let communities = &state.config.nip_fi.communities; + let communities = state.nip_fi_communities(); + // Resolved after every await of this upgrade: nothing to re-check. match check_nip_fi_at_upgrade(Root, &headers, communities, verifier, mode) + .await .into_assertions() { - Ok(assertions) => assertions, + Ok((assertion, shadow, _deadline)) => { + (assertion, shadow.map(|(a, c)| (a, c.shadow_label()))) + } Err(resp) => return (*resp).into_response(), } } else { @@ -779,11 +801,11 @@ async fn nip11_or_ws_handler( // handshake, not the post-community-active-check instant. // [FI-TRACE-LEASE-BOUND] let connection_time = chrono::Utc::now(); - let shadow = shadow_assertion.map(|a| { + let shadow = shadow_assertion.map(|(a, community)| { crate::nip_fi_shadow_session::ShadowSession::start( &state, "ws", - &headers, + community, a, connection_time, ) @@ -2144,6 +2166,7 @@ mod tests { // Enforce that is the startup-race condition that must return 503 for // a token-carrying request. config.nip_fi = NipFiRelayConfig { + source: Default::default(), mode, registry: IssuerRegistry::new(), jwks_configs: vec![], @@ -2349,7 +2372,7 @@ mod tests { // (gate denial codes), which holds regardless of host resolution state. // // Fix 6: corrected the DB-free comment (bind_community is reached by plain - // GETs; only WS-upgrade requests pay zero DB cost via the pre-gate path). + // GETs; only WS-upgrade requests skip the tenant lookup via the pre-gate path). // // Mutation evidence: // A) Move the NIP-FI gate to fire on plain GETs too → response becomes @@ -3926,6 +3949,7 @@ mod tests { }; config.nip_fi = NipFiRelayConfig { + source: Default::default(), mode: NipFiMode::Enforce, registry: { let mut r = IssuerRegistry::new(); diff --git a/crates/buzz-relay/src/startup_steps.rs b/crates/buzz-relay/src/startup_steps.rs index 3980b6b86a..189731fd6b 100644 --- a/crates/buzz-relay/src/startup_steps.rs +++ b/crates/buzz-relay/src/startup_steps.rs @@ -61,6 +61,8 @@ pub enum StartupStep { SearchConnect, /// Warm NIP-FI JWKS snapshots (only when NIP-FI is configured). NipFiJwksWarm, + /// Load the first NIP-FI binding snapshot (only for the `database` source). + NipFiSnapshotLoad, /// Start the inter-relay mesh (no-op when the mesh is off). MeshBoot, /// Run the git object-store conformance probe (when enabled). @@ -79,7 +81,7 @@ pub enum StartupStep { impl StartupStep { /// The complete `phase` label vocabulary. - pub const ALL: [Self; 18] = [ + pub const ALL: [Self; 19] = [ Self::DbConnect, Self::DbMigrate, Self::PartitionEnsure, @@ -91,6 +93,7 @@ impl StartupStep { Self::RedisConnect, Self::SearchConnect, Self::NipFiJwksWarm, + Self::NipFiSnapshotLoad, Self::MeshBoot, Self::GitConformanceProbe, Self::ChannelRosterFence, @@ -114,6 +117,7 @@ impl StartupStep { Self::RedisConnect => "redis_connect", Self::SearchConnect => "search_connect", Self::NipFiJwksWarm => "nip_fi_jwks_warm", + Self::NipFiSnapshotLoad => "nip_fi_snapshot_load", Self::MeshBoot => "mesh_boot", Self::GitConformanceProbe => "git_conformance_probe", Self::ChannelRosterFence => "channel_roster_fence", diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index 1e88eb8652..1a20aa6bc4 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -1379,6 +1379,10 @@ pub struct AppState { /// 4) and HTTP admission (`nip_fi_http`): one `Arc`, never a second map. pub nip_fi_deny_map: Option>, + /// The database community resolver (`BUZZ_NIP_FI_COMMUNITY_SOURCE=database`), + /// installed by `main.rs`; `None` resolves from `config.nip_fi.communities`. + pub nip_fi_resolver: Option>, + /// Command JWT verifier for the NIP-FI admin disconnect endpoint. /// /// `None` when mode is `Off` (no command API is reachable). When @@ -1399,6 +1403,17 @@ pub struct AppState { } impl AppState { + /// The NIP-FI community source admission resolves against. + pub(crate) fn nip_fi_communities(&self) -> crate::nip_fi_core::Communities<'_> { + use crate::nip_fi_config::CommunitySource; + use crate::nip_fi_core::Communities; + match (self.config.nip_fi.source, &self.nip_fi_resolver) { + (CommunitySource::Env, _) => Communities::Env(&self.config.nip_fi.communities), + (CommunitySource::Database, Some(resolver)) => Communities::Database(resolver), + (CommunitySource::Database, None) => Communities::Unavailable, + } + } + /// Constructs `AppState` from its component services. /// /// Returns `(state, audit_shutdown)`. The caller should call @@ -1600,6 +1615,7 @@ impl AppState { // `main.rs` after startup validation. `None` is safe before that // call: the endpoint returns 503 when the verifier is absent. nip_fi_deny_map: None, + nip_fi_resolver: None, nip_fi_command_verifier: None, nip_fi_command_replay, nip_fi_publish_tasks: tokio_util::task::TaskTracker::new(), diff --git a/crates/buzz-relay/tests/boot_lifecycle.rs b/crates/buzz-relay/tests/boot_lifecycle.rs index 027fc4d1c2..5d7dd2491f 100644 --- a/crates/buzz-relay/tests/boot_lifecycle.rs +++ b/crates/buzz-relay/tests/boot_lifecycle.rs @@ -674,7 +674,10 @@ mod postgres_tests { /// passes. Unlike `run_relay`, a relay that keeps serving is a result to /// assert on rather than a panic, which is the whole point here. fn run_until_exit(environment: &[(&str, &str)], timeout: Duration) -> (bool, Output) { - let mut process = RelayProcess::spawn(environment); + wait_for_exit(RelayProcess::spawn(environment), timeout) + } + + fn wait_for_exit(mut process: RelayProcess, timeout: Duration) -> (bool, Output) { let deadline = Instant::now() + timeout; while Instant::now() < deadline { if process.try_wait().is_some() { @@ -868,4 +871,187 @@ mod postgres_tests { "the sampler must be owned by a relay that finished booting: {logs}" ); } + + const NIP_FI_ISSUERS: &str = r#"[{ + "issuer": "https://idp.example.com", + "command_audiences": ["https://relay.example.com"], + "token_class": "nip-fi+jwt", + "algorithms": ["ES256"], + "maximum_assertion_age_seconds": 3600, + "jwks_uri": "https://idp.example.com/.well-known/jwks.json", + "jwks_refresh_interval_seconds": 300, + "jwks_hard_deadline_seconds": 86400, + "maximum_command_age_seconds": 30, + "authorized_principals": ["https://idp.example.com/operator"] + }]"#; + /// A suffix that never covers the relay's own host, so deployment + /// bootstrap writes no binding and the startup load is the first reader + /// of `community_nip_fi_issuers`. + const NIP_FI_PROVISION_BINDING: &str = r#"{"host_suffix":"communities.example.com","authorized_issuers":["https://idp.example.com"]}"#; + + /// Boots the relay with the database community source; `extra` adds or + /// overrides variables. Returns the health port alongside the process. + fn spawn_database_source(extra: &[(&str, &str)]) -> (RelayProcess, u16) { + let database_url = std::env::var("DATABASE_URL") + .expect("postgres lane provides DATABASE_URL for each test process"); + let redis_url = std::env::var("REDIS_URL") + .expect("the postgres lane runs alongside Redis and exports REDIS_URL"); + let metrics_port = reserve_closed_port().to_string(); + let health_port = reserve_closed_port(); + let health_port_value = health_port.to_string(); + let bind_addr = format!("127.0.0.1:{}", reserve_closed_port()); + let mut environment = vec![ + ("BUZZ_RELAY_PRIVATE_KEY", VALID_RELAY_PRIVATE_KEY), + ("BUZZ_METRICS_PORT", metrics_port.as_str()), + ("BUZZ_HEALTH_PORT", health_port_value.as_str()), + ("BUZZ_BIND_ADDR", bind_addr.as_str()), + ("DATABASE_URL", database_url.as_str()), + ("REDIS_URL", redis_url.as_str()), + ("BUZZ_GIT_CONFORMANCE_PROBE", "false"), + // Shadow: off builds no resolver, so it loads no snapshot. + ("BUZZ_NIP_FI_MODE", "shadow"), + ("BUZZ_NIP_FI_COMMUNITY_SOURCE", "database"), + ("BUZZ_NIP_FI_ISSUERS", NIP_FI_ISSUERS), + ("BUZZ_NIP_FI_PROVISION_BINDING", NIP_FI_PROVISION_BINDING), + ("BUZZ_NIP_FI_MAX_CONNECTION_LIFETIME_SECS", "3600"), + ]; + environment.extend_from_slice(extra); + (RelayProcess::spawn(&environment), health_port) + } + + /// Binds `count` active communities under the provision suffix to the + /// registered issuer, on the test's own database. + fn seed_bindings(count: u32) { + use sqlx::Connection as _; + let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL"); + let runtime = tokio::runtime::Runtime::new().expect("tokio runtime"); + runtime.block_on(async { + let mut connection = sqlx::PgConnection::connect(&database_url) + .await + .expect("connect to the test database"); + // One statement per thousand rows: each insert takes a write-fence + // advisory lock per community, held to the end of its transaction. + for start in (1..=count).step_by(1000) { + sqlx::query( + "WITH c AS ( + INSERT INTO communities (id, host) + SELECT gen_random_uuid(), 'c' || n || '.communities.example.com' + FROM generate_series($1, $2) AS n + RETURNING id + ) + INSERT INTO community_nip_fi_issuers (community_id, issuer) + SELECT id, 'https://idp.example.com' FROM c", + ) + .bind(i64::from(start)) + .bind(i64::from(count.min(start + 999))) + .execute(&mut connection) + .await + .expect("seed NIP-FI bindings"); + } + }); + } + + fn assert_failed_before_listeners( + exited: bool, + output: &Output, + health_port: u16, + failure: &str, + ) { + let logs = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + exited && !output.status.success(), + "a failed startup binding load must be startup-fatal: {logs}" + ); + assert!( + logs.contains(&format!("startup binding snapshot load failed: {failure}")), + "the failure must name the snapshot load and why it failed: {logs}" + ); + assert!( + !logs.contains("Health probe listener started"), + "no listener may bind before the snapshot loads: {logs}" + ); + assert!( + TcpListener::bind(("0.0.0.0", health_port)).is_ok(), + "the health port must never have been bound" + ); + } + + /// The database source must never let a pod serve without a snapshot: a + /// load that fails exits non-zero before any listener binds, so the + /// orchestrator restarts the pod instead of routing to it. Two bindings + /// over a one-row cap make the load itself fail after every earlier + /// startup step has read the same table successfully. + #[test] + #[ignore = "requires PostgreSQL"] + fn database_source_failed_snapshot_stops_boot_before_listeners_bind() { + seed_bindings(2); + let (process, health_port) = + spawn_database_source(&[("BUZZ_NIP_FI_SNAPSHOT_MAX_ROWS", "1")]); + let (exited, output) = wait_for_exit(process, CHILD_TIMEOUT); + assert_failed_before_listeners(exited, &output, health_port, "OverCap"); + } + + /// A load that cannot finish in time must hit the refresh timeout and fail + /// the same way, rather than leaving the pod waiting unbound. A + /// millisecond budget cannot cover a scan of this many rows. + #[test] + #[ignore = "requires PostgreSQL"] + fn database_source_slow_snapshot_times_out_before_listeners_bind() { + seed_bindings(20_000); + let (process, health_port) = + spawn_database_source(&[("BUZZ_NIP_FI_SNAPSHOT_REFRESH_TIMEOUT_MS", "1")]); + let (exited, output) = wait_for_exit(process, CHILD_TIMEOUT); + assert_failed_before_listeners(exited, &output, health_port, "Timeout"); + } + + /// With a loadable snapshot the same configuration boots and readiness + /// turns ready, so the two failure tests above fail for the load alone. One + /// seeded binding proves the snapshot reads rows, not just an empty table. + #[test] + #[ignore = "requires PostgreSQL"] + fn database_source_snapshot_load_lets_readiness_turn_ready() { + seed_bindings(1); + let (mut process, health_port) = spawn_database_source(&[]); + let deadline = Instant::now() + CHILD_TIMEOUT; + let ready = loop { + assert!( + process.try_wait().is_none(), + "relay exited: {}", + String::from_utf8_lossy(&process.terminate().stdout) + ); + let response = TcpStream::connect(("127.0.0.1", health_port)) + .and_then(|mut stream| { + stream.set_read_timeout(Some(Duration::from_secs(1)))?; + stream.write_all( + b"GET /_readiness HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + )?; + let mut response = String::new(); + stream.read_to_string(&mut response)?; + Ok(response) + }) + .unwrap_or_default(); + if response.starts_with("HTTP/1.1 200") || Instant::now() >= deadline { + break response; + } + thread::sleep(Duration::from_millis(50)); + }; + let output = process.terminate(); + let logs = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + ready.starts_with("HTTP/1.1 200"), + "readiness never turned ready: {ready}: {logs}" + ); + assert!( + logs.contains("NIP-FI: binding snapshot loaded"), + "readiness must follow the snapshot load: {logs}" + ); + } } diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index 5704a7622b..f2ef677c9e 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -195,6 +195,15 @@ gating the probe on them removed the whole deployment from the load balancer at once and left a reconnect burst with nowhere to land. There is no separate "starting" state — the health listener does not bind until the database, migrations, Redis, and pub/sub are up, so a process that can answer has booted. +With `BUZZ_NIP_FI_COMMUNITY_SOURCE=database` and a mode other than `off`, the +first NIP-FI binding snapshot also loads before any listener binds (startup +phase `nip_fi_snapshot_load`); a failed load exits the process, so a pod never +reports ready without one. Later refresh failures do not affect readiness — +watch `buzz_nip_fi_binding_snapshot_loaded_timestamp_seconds` instead. The +source's env vars (`BUZZ_NIP_FI_COMMUNITY_SOURCE`, +`BUZZ_NIP_FI_SNAPSHOT_MAX_ROWS`, `BUZZ_NIP_FI_SNAPSHOT_REFRESH_TIMEOUT_MS`, +`BUZZ_NIP_FI_RESOLVER_NEGATIVE_CACHE_MAX`, `BUZZ_NIP_FI_RESOLVER_MAX_INFLIGHT`) +and alerts are in `docs/operator-nip-fi-bindings.md`. Shared-dependency health moved to **`/_status`** on the same private health listener, under a `dependencies` object carrying the `postgres`, `redis`, diff --git a/docs/operator-nip-fi-bindings.md b/docs/operator-nip-fi-bindings.md index f5c77be0a0..f63c73e755 100644 --- a/docs/operator-nip-fi-bindings.md +++ b/docs/operator-nip-fi-bindings.md @@ -36,6 +36,29 @@ binds nothing. relay's environment: `BUZZ_NIP_FI_PROVISION_BINDING`, `BUZZ_NIP_FI_ISSUERS`, `BUZZ_NIP_FI_COMMUNITIES`, and `DATABASE_URL`. +### Database community source + +`BUZZ_NIP_FI_COMMUNITY_SOURCE=database` resolves communities from these +bindings instead of `BUZZ_NIP_FI_COMMUNITIES`. The relay refuses to start if +`BUZZ_NIP_FI_PROVISION_BINDING` is unset or `BUZZ_NIP_FI_COMMUNITIES` is set. +In `off` mode no snapshot is loaded. + +| Variable | Default | Bound | +|----------|---------|-------| +| `BUZZ_NIP_FI_COMMUNITY_SOURCE` | `env` | `env` or `database` | +| `BUZZ_NIP_FI_SNAPSHOT_MAX_ROWS` | 1,000,000 | 1 to 10,000,000 community-issuer rows; a larger load fails | +| `BUZZ_NIP_FI_SNAPSHOT_REFRESH_TIMEOUT_MS` | 10,000 | under the 30s refresh interval | +| `BUZZ_NIP_FI_RESOLVER_NEGATIVE_CACHE_MAX` | 100,000 | 1 to 1,000,000 hosts | +| `BUZZ_NIP_FI_RESOLVER_MAX_INFLIGHT` | 8 | at least 1, clamped below the database pool size | + +Each pod loads a snapshot before it binds any listener, and refreshes it +every 30s. A removed binding stops admitting new connections within 30s plus +the configured refresh timeout (40s by default) while refreshes succeed, and within two minutes of the last good load while they +fail. Open WebSocket sessions are not rechecked when a binding is removed; +they end at their own session deadline. Alert on +`time() - buzz_nip_fi_binding_snapshot_loaded_timestamp_seconds > 120` and on +`absent(buzz_nip_fi_binding_snapshot_loaded_timestamp_seconds)`. + ## Commands ```sh @@ -95,3 +118,98 @@ contract: it exits 0. 3. Do not turn on the database binding source in enforce mode while any bindable community is unbound: an unbound community fails closed. + +## Database source runbook + +`BUZZ_NIP_FI_COMMUNITY_SOURCE=database` reads communities from these bindings +instead of `BUZZ_NIP_FI_COMMUNITIES`. Move one mode at a time, and only move +forward when the step's go signals hold. + +### Off + +Set `BUZZ_NIP_FI_COMMUNITY_SOURCE=database` with `BUZZ_NIP_FI_MODE=off`. No +snapshot loads in `off`, but every `BUZZ_NIP_FI_RESOLVER_*` and +`BUZZ_NIP_FI_SNAPSHOT_*` value is validated, so a bad one fails this deploy. + +- Go: the rollout completes, and `buzz-admin nip-fi backfill` exits 0. +- No-go: a pod fails config validation. Fix the named variable. + +### Shadow + +Set `BUZZ_NIP_FI_MODE=shadow`. Each pod now loads the snapshot before any +listener binds, refreshes it every 30 s, and records what enforce would decide +without refusing anything. Shadow labels database-source records +`community="bound"` or `"unmapped"`, never a host. + +- Go, held for at least a day of normal traffic: + - `time() - buzz_nip_fi_binding_snapshot_loaded_timestamp_seconds` stays + under 60 on every pod. + - `buzz_nip_fi_binding_snapshot_load_failures_total` is flat. + - `buzz_nip_fi_binding_snapshot_rows / buzz_nip_fi_binding_snapshot_max_rows` + is under 0.5. + - `buzz_nip_fi_resolver_miss_rejections_total` is flat. + - The non-`admit` share of `buzz_nip_fi_shadow_total` is only what you expect + to deny; `community="unmapped"` comes only from hosts you do not serve. +- No-go: any of these fails. Return to `off` and investigate. + +### Enforce + +Set `BUZZ_NIP_FI_MODE=enforce`. Watch the same signals, plus the rate of 503 +responses on login. + +- Go: the shadow signals keep holding, and 503s stay at their shadow baseline. +- No-go: logins return 503 or are refused unexpectedly. Set `mode=off` (below). + +### Alerts + +- Stale snapshot: `time() - buzz_nip_fi_binding_snapshot_loaded_timestamp_seconds > 120`, + together with `absent(buzz_nip_fi_binding_snapshot_loaded_timestamp_seconds)` + for a database-source pod. +- Near the row cap: `buzz_nip_fi_binding_snapshot_rows / buzz_nip_fi_binding_snapshot_max_rows > 0.8`. + `buzz_nip_fi_binding_snapshot_rows` is the size of the last good load. +- Over the cap or too slow: any increase of + `buzz_nip_fi_binding_snapshot_load_failures_total{reason=~"over_cap|timeout"}`. +- Live lookups shed: any increase of `buzz_nip_fi_resolver_miss_rejections_total`. + +### When the snapshot is too big or too slow + +A load fails when the snapshot has more rows than +`BUZZ_NIP_FI_SNAPSHOT_MAX_ROWS` or takes longer than +`BUZZ_NIP_FI_SNAPSHOT_REFRESH_TIMEOUT_MS`. Both get worse as communities are +added, so they usually arrive during normal growth, not at a deploy. + +- New pods: the startup load fails and the process exits, so new pods + crash-loop and a rollout stalls. Boot already fails when the database is + unreachable; what this source adds is that snapshot size and load time can + also fail it. +- Running pods: their refreshes fail but the last good snapshot keeps + answering. Once it is 120 s old it is stale, and every login's host goes to + a live lookup. Live lookups share `BUZZ_NIP_FI_RESOLVER_MAX_INFLIGHT` slots + (8 by default). Concurrent logins for the same host share one lookup, and a + login waits up to 50 ms for a free slot; how many are shed depends on login + load across distinct hosts and on lookup latency. In enforce, a shed login + gets 503 (`authorization_unavailable`); in shadow it + is only recorded as a would-deny. + +Ways out, in order: + +1. Raise `BUZZ_NIP_FI_SNAPSHOT_MAX_ROWS` (up to 10,000,000) or + `BUZZ_NIP_FI_SNAPSHOT_REFRESH_TIMEOUT_MS` (under 30,000), whichever the + failure `reason` names, and roll out. +2. If that cannot land quickly, set `BUZZ_NIP_FI_MODE=off`. `off` loads no + snapshot, so pods boot and admit as they did before NIP-FI. + +## Assertion policy ID change + +This release changes every verified assertion's policy ID once, on purpose. +The base ID (`derive_assertion_policy_id` in `crates/buzz-auth/src/nip_fi/config.rs`) +no longer hashes the configured community audiences. The ID carried on a +verified assertion is now `IssuerPolicy::id_for_community`: a hash of that base +and the one community audience the request resolved to. The issuer is still +committed through the base. + +Policy IDs from earlier releases therefore do not match. Inside the relay the ID +is only carried on the verified assertion and never compared with a stored ID, so +admission is unaffected. Anything outside the relay that recorded old policy IDs, +such as dashboards, log queries or allow-lists, should expect new values after +the upgrade. diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 13b5c73f7c..bd7961124d 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -248,6 +248,9 @@ run_unit_tests() { run_test_step "buzz-relay NIP-FI config tests" \ cargo test -p buzz-relay --lib nip_fi_config:: -- --nocapture + run_test_step "buzz-relay NIP-FI resolver tests" \ + cargo test -p buzz-relay --lib nip_fi_resolver:: -- --nocapture + # Infra-free REQ subscription-lifecycle tests, by exact name: the rest of # handlers::req needs a database. run_test_step "buzz-relay REQ subscription lifecycle tests" \