diff --git a/Justfile b/Justfile index 9f8736bbfe..cb186312df 100644 --- a/Justfile +++ b/Justfile @@ -638,11 +638,12 @@ test-unit: + test(/^api::nip_fi::/)' # Invitation drain/cutoff boundary contracts are infra-free. cargo nextest run -p buzz-relay --lib -E 'test(/^invite_token::/) + test(/^config::invite_cutoff_tests::/)' - # Relay read caches: the admitted-connection index, open-set - # single-flight, tenant host cache and the ban disconnect's - # restriction-cache drop. Infra-free; the database URL - # refuses at once. Keep scripts/run-tests.sh in step. - cargo nextest run -p buzz-relay --lib -E 'test(=state::tests::admitted_index_follows_every_key_change) + test(=state::tests::concurrent_open_channel_misses_share_the_in_flight_read) + test(=tenant::tests::app_state_resolver_serves_fresh_cached_hosts_only) + test(=state::tests::disconnect_drops_the_cached_restriction_row)' + # Relay read caches: the admitted-connection index, the shared-read + # gates (open set, restriction, tenant host), tenant host cache, the + # overlong-host rejection and the ban disconnect's restriction-cache + # drop. Infra-free; the + # database URL refuses at once. Keep scripts/run-tests.sh in step. + cargo nextest run -p buzz-relay --lib -E 'test(=state::tests::admitted_index_follows_every_key_change) + test(/^shared_reads::tests::/) + test(=state::tests::concurrent_open_channel_misses_share_the_in_flight_read) + test(=state::tests::concurrent_restriction_and_host_misses_share_the_in_flight_read) + test(=tenant::tests::app_state_resolver_serves_fresh_cached_hosts_only) + test(=tenant::tests::an_overlong_host_fails_closed_before_the_resolver) + test(=state::tests::disconnect_drops_the_cached_restriction_row)' # Retain partially consumed mesh receives across housekeeping ticks. cargo nextest run -p buzz-relay --lib -E 'test(=mesh_boot::tests::demo_echo_retains_pending_receive_across_housekeeping_ticks)' # boot_lifecycle spawns the real relay binary and asserts its startup diff --git a/crates/buzz-admin/src/communities.rs b/crates/buzz-admin/src/communities.rs index f8bcd7c015..4830d7d9ef 100644 --- a/crates/buzz-admin/src/communities.rs +++ b/crates/buzz-admin/src/communities.rs @@ -1,7 +1,7 @@ //! Reversible whole-community lifecycle commands. use anyhow::{Context, Result}; -use buzz_core::tenant::{normalize_host, relay_url_authority, TenantContext}; +use buzz_core::tenant::{normalize_host, relay_url_authority, TenantContext, MAX_HOST_LEN}; use buzz_db::{ArchivedCommunityRecord, UnarchivedCommunityRecord}; use buzz_pubsub::conn_control::ConnControl; use buzz_pubsub::PubSubManager; @@ -9,8 +9,6 @@ use clap::Subcommand; use serde_json::{json, Value}; use url::{Host, Url}; -const MAX_HOST_LEN: usize = 255; - /// Reversible community lifecycle actions. #[derive(Debug, Subcommand)] pub enum CommunitiesCommand { diff --git a/crates/buzz-core/src/tenant.rs b/crates/buzz-core/src/tenant.rs index 46c7e71c56..d7a0d50df0 100644 --- a/crates/buzz-core/src/tenant.rs +++ b/crates/buzz-core/src/tenant.rs @@ -97,6 +97,14 @@ impl TenantContext { } } +/// Longest host a community can have, in characters: `communities.host` is +/// `VARCHAR(255)`. A longer host maps to no community. +/// +/// Provisioning checks count bytes instead, which is stricter for non-ASCII +/// input and identical for the ASCII authorities they accept. Keep both: the +/// lookup side must not refuse any host the column can store. +pub const MAX_HOST_LEN: usize = 255; + /// Normalize a connection `Host` into the canonical form used as the community /// lookup key. /// diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index f7046a7f60..6fd74d5f32 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -111,6 +111,21 @@ pub use reminder::DueReminder; pub use store::{ArtifactSnapshot, SnapshotDenied}; pub use usage::SessionLeader; +/// A value read from the database, with when its reading connection was +/// checked out of the pool. +/// +/// The statement that produced `value` started after `read_at`, so its +/// snapshot is no older than `read_at`. A cache that serves `value` for a +/// bound after `read_at` is never staler than that bound, however long the +/// read waited for a connection. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReadAt { + /// The value as read. + pub value: T, + /// When the connection that read `value` was checked out. + pub read_at: std::time::Instant, +} + use buzz_core::CommunityId; /// Durable NIP-FI verified-profile projection state. diff --git a/crates/buzz-db/src/store/channel_members.rs b/crates/buzz-db/src/store/channel_members.rs index 96501f0227..936e2b9078 100644 --- a/crates/buzz-db/src/store/channel_members.rs +++ b/crates/buzz-db/src/store/channel_members.rs @@ -9,7 +9,7 @@ use uuid::Uuid; use crate::channel::{row_to_channel_record, ChannelRecord}; use crate::error::{DbError, Result}; -use crate::{AdmittedTx, Db}; +use crate::{AdmittedTx, Db, ReadAt}; use buzz_core::CommunityId; use buzz_datastore_tracing::datastore_span; @@ -1550,11 +1550,22 @@ pub async fn get_listed_open_channel_ids( pool: &PgPool, community_id: CommunityId, ) -> Result> { + Ok(get_listed_open_channel_ids_read_at(pool, community_id) + .await? + .value) +} + +/// [`get_listed_open_channel_ids`], with when its connection was checked out. +pub async fn get_listed_open_channel_ids_read_at( + pool: &PgPool, + community_id: CommunityId, +) -> Result>> { let mut connection = crate::observability::acquire_writer( pool, crate::observability::WriterOperation::Authorization, ) .await?; + let read_at = std::time::Instant::now(); let sql = format!( r#" SELECT id @@ -1565,11 +1576,11 @@ pub async fn get_listed_open_channel_ids( open_types = crate::store::channel::legacy_channel_type_sql("channel_type"), ); // Built only from the constant type list. - sqlx::query_scalar(sqlx::AssertSqlSafe(sql)) + let value = sqlx::query_scalar(sqlx::AssertSqlSafe(sql)) .bind(community_id.as_uuid()) .fetch_all(&mut *connection) - .await - .map_err(Into::into) + .await?; + Ok(ReadAt { value, read_at }) } /// Appends the channels of `more` that `ids` lacks. @@ -2399,12 +2410,24 @@ impl Db { } /// A community's listed open channel IDs. See [`get_listed_open_channel_ids`]. - #[datastore_span(name = "get_listed_open_channel_ids", system = "postgresql")] pub async fn get_listed_open_channel_ids( &self, community_id: CommunityId, ) -> Result> { - get_listed_open_channel_ids(&self.pool, community_id).await + Ok(self + .get_listed_open_channel_ids_read_at(community_id) + .await? + .value) + } + + /// [`Self::get_listed_open_channel_ids`], with when its connection was + /// checked out, for caches that bound staleness from the read. + #[datastore_span(name = "get_listed_open_channel_ids", system = "postgresql")] + pub async fn get_listed_open_channel_ids_read_at( + &self, + community_id: CommunityId, + ) -> Result>> { + get_listed_open_channel_ids_read_at(&self.pool, community_id).await } /// The channels of non-legacy types a pubkey can read (member, or open diff --git a/crates/buzz-db/src/store/community.rs b/crates/buzz-db/src/store/community.rs index 1df5bea021..0a2b1d457b 100644 --- a/crates/buzz-db/src/store/community.rs +++ b/crates/buzz-db/src/store/community.rs @@ -6,7 +6,7 @@ use chrono::{DateTime, Utc}; use sqlx::Row; use uuid::Uuid; -use crate::{relay_members, Db, DbError, Result}; +use crate::{relay_members, Db, DbError, ReadAt, Result}; /// Community host-map row returned by [`Db::lookup_community_by_host`]. #[derive(Debug, Clone, PartialEq, Eq)] @@ -110,16 +110,29 @@ impl Db { /// /// The caller owns host normalization and turns `None` into the fail-closed /// request/connection error. buzz-db only reads the durable host map. - #[datastore_span(name = "lookup_community_by_host", system = "postgresql")] pub async fn lookup_community_by_host( &self, normalized_host: &str, ) -> Result> { + Ok(self + .lookup_community_by_host_read_at(normalized_host) + .await? + .value) + } + + /// [`Self::lookup_community_by_host`], with when its connection was + /// checked out, for caches that bound staleness from the read. + #[datastore_span(name = "lookup_community_by_host", system = "postgresql")] + pub async fn lookup_community_by_host_read_at( + &self, + normalized_host: &str, + ) -> Result>> { let mut connection = crate::observability::acquire_writer( &self.pool, crate::observability::WriterOperation::TenantResolution, ) .await?; + let read_at = std::time::Instant::now(); let row = sqlx::query( r#" SELECT id, host @@ -134,16 +147,18 @@ impl Db { .fetch_optional(&mut *connection) .await?; - row.map(|row| { - let id: Uuid = row.try_get("id")?; - let host: String = row.try_get("host")?; + let value = row + .map(|row| { + let id: Uuid = row.try_get("id")?; + let host: String = row.try_get("host")?; - Ok(CommunityRecord { - id: CommunityId::from_uuid(id), - host, + Ok::<_, DbError>(CommunityRecord { + id: CommunityId::from_uuid(id), + host, + }) }) - }) - .transpose() + .transpose()?; + Ok(ReadAt { value, read_at }) } /// Returns whether a community id still exists in the active lifecycle state. diff --git a/crates/buzz-db/src/store/moderation.rs b/crates/buzz-db/src/store/moderation.rs index f47cc083c8..6720fcb0eb 100644 --- a/crates/buzz-db/src/store/moderation.rs +++ b/crates/buzz-db/src/store/moderation.rs @@ -20,7 +20,7 @@ use sqlx::{PgPool, Row as _}; use uuid::Uuid; use crate::error::Result; -use crate::{CommunityId, Db}; +use crate::{CommunityId, Db, ReadAt}; /// What a report points at. Exactly one target class per report row. #[derive(Debug, Clone, PartialEq, Eq)] @@ -631,11 +631,23 @@ pub async fn restriction_state( community: CommunityId, pubkey: &[u8], ) -> Result { + Ok(restriction_state_read_at(pool, community, pubkey) + .await? + .value) +} + +/// [`restriction_state`], with when its connection was checked out. +pub async fn restriction_state_read_at( + pool: &PgPool, + community: CommunityId, + pubkey: &[u8], +) -> Result> { let mut connection = crate::observability::acquire_writer( pool, crate::observability::WriterOperation::Authorization, ) .await?; + let read_at = std::time::Instant::now(); let row = sqlx::query( r#" SELECT @@ -659,10 +671,13 @@ pub async fn restriction_state( .fetch_one(&mut *connection) .await?; - Ok(RestrictionState { - banned: row.try_get("banned")?, - own_banned: row.try_get("own_banned")?, - muted_until: row.try_get("muted_until")?, + Ok(ReadAt { + value: RestrictionState { + banned: row.try_get("banned")?, + own_banned: row.try_get("own_banned")?, + muted_until: row.try_get("muted_until")?, + }, + read_at, }) } @@ -1025,13 +1040,26 @@ impl Db { /// Fetch the effective (own or agent-owner) ban/timeout restriction state /// for enforcement hot paths. - #[datastore_span(name = "moderation_restriction_state", system = "postgresql")] pub async fn moderation_restriction_state( &self, community: CommunityId, pubkey: &[u8], ) -> Result { - restriction_state(&self.pool, community, pubkey).await + Ok(self + .moderation_restriction_state_read_at(community, pubkey) + .await? + .value) + } + + /// [`Self::moderation_restriction_state`], with when its connection was + /// checked out, for caches that bound staleness from the read. + #[datastore_span(name = "moderation_restriction_state", system = "postgresql")] + pub async fn moderation_restriction_state_read_at( + &self, + community: CommunityId, + pubkey: &[u8], + ) -> Result> { + restriction_state_read_at(&self.pool, community, pubkey).await } /// Fetch the full ban/timeout row for a member pubkey. diff --git a/crates/buzz-relay/src/handlers/community_provisioning.rs b/crates/buzz-relay/src/handlers/community_provisioning.rs index 6c45c5800e..533f2eacbb 100644 --- a/crates/buzz-relay/src/handlers/community_provisioning.rs +++ b/crates/buzz-relay/src/handlers/community_provisioning.rs @@ -30,14 +30,11 @@ use std::sync::Arc; use serde::{Deserialize, Serialize}; use tracing::info; -use buzz_core::tenant::{normalize_host, TenantContext}; +use buzz_core::tenant::{normalize_host, TenantContext, MAX_HOST_LEN}; use url::{Host, Url}; use crate::state::AppState; -/// Maximum accepted authority length. Matches `communities.host VARCHAR(255)`. -const MAX_HOST_LEN: usize = 255; - /// JSON body for `POST /operator/communities`. #[derive(Debug, Deserialize)] pub struct ProvisionCommunityRequest { diff --git a/crates/buzz-relay/src/lib.rs b/crates/buzz-relay/src/lib.rs index f21d0181af..8b8fa0807b 100644 --- a/crates/buzz-relay/src/lib.rs +++ b/crates/buzz-relay/src/lib.rs @@ -65,6 +65,8 @@ pub mod push_runtime; pub mod readiness; /// Axum router construction. pub mod router; +/// Per-key gates that let concurrent cache misses share one database read. +mod shared_reads; /// Timing for startup work between metrics bind and serving. pub mod startup_steps; /// Shared application state. diff --git a/crates/buzz-relay/src/shared_reads.rs b/crates/buzz-relay/src/shared_reads.rs new file mode 100644 index 0000000000..fe384b74fa --- /dev/null +++ b/crates/buzz-relay/src/shared_reads.rs @@ -0,0 +1,108 @@ +//! Per-key gates that let concurrent misses on a short-lived read cache share +//! one database read. +//! +//! A reconnect storm sends many requests for the same key at once (every +//! socket on one host resolves the same tenant; a desktop client checks its +//! own restriction state on every request). Without a gate each miss issues +//! its own read, and the reads queue for the same pool connections. With one, +//! the first miss reads and the rest wait for it and use its cache entry. + +use std::hash::Hash; +use std::sync::Arc; +use std::time::Duration; + +use tokio::sync::{Mutex, OwnedMutexGuard}; + +/// One gate per key, held by the caller that is reading that key. +pub(crate) struct SharedReads { + gates: moka::sync::Cache>>, + wait: Duration, +} + +impl SharedReads +where + K: Hash + Eq + Send + Sync + 'static, +{ + /// Gates for up to `max_capacity` keys, each waited on for at most + /// `wait`; an idle gate is dropped after a minute. + /// + /// `wait` must cover the holder's wait for a pool connection, which is + /// what grows in a storm, plus its statement: a waiter that gives up + /// while the holder still queues for a connection only queues behind it + /// with a duplicate read. + pub(crate) fn new(max_capacity: u64, wait: Duration) -> Self { + Self { + gates: moka::sync::Cache::builder() + .max_capacity(max_capacity) + .time_to_idle(Duration::from_secs(60)) + .build(), + wait, + } + } + + /// Takes `key`'s gate, or waits for its holder to finish. + /// + /// Returns the guard when no read of `key` is in flight: the caller reads + /// and holds the guard until its cache entry is written. Otherwise waits + /// for the holder, at most this gate's `wait`, and returns `None`; the + /// caller then rechecks its cache and, if the holder failed, was + /// cancelled or is still reading, reads on its own without the gate. A + /// read that hangs therefore stalls each waiter for at most `wait`. + pub(crate) async fn turn(&self, key: K) -> Option> { + let gate = self.gates.get_with(key, || Arc::new(Mutex::new(()))); + match Arc::clone(&gate).try_lock_owned() { + Ok(guard) => Some(guard), + Err(_) => { + drop(tokio::time::timeout(self.wait, gate.lock()).await); + None + } + } + } + + /// The gate for `key`, if one exists. + #[cfg(test)] + pub(crate) fn gate(&self, key: &K) -> Option>> { + self.gates.get(key) + } +} + +#[cfg(test)] +mod tests { + use super::SharedReads; + use std::sync::Arc; + use std::time::Duration; + + #[tokio::test] + async fn one_caller_reads_while_the_rest_wait_for_it() { + let reads = Arc::new(SharedReads::::new(16, Duration::from_secs(5))); + let leader = reads.turn(1).await; + assert!(leader.is_some(), "the first caller takes the gate"); + + // A second key has its own gate. + assert!(reads.turn(2).await.is_some()); + + let waiter = { + let reads = Arc::clone(&reads); + tokio::spawn(async move { reads.turn(1).await }) + }; + tokio::task::yield_now().await; + assert!(!waiter.is_finished(), "a waiter blocks while the read runs"); + drop(leader); + assert!( + waiter.await.expect("join").is_none(), + "a waiter rechecks the cache instead of becoming a reader" + ); + + // Once the gate is free the next caller reads again. + assert!(reads.turn(1).await.is_some()); + } + + #[tokio::test(start_paused = true)] + async fn a_waiter_stops_waiting_for_a_hung_read() { + let reads = SharedReads::::new(16, Duration::from_secs(4)); + let _hung = reads.turn(1).await; + let started = tokio::time::Instant::now(); + assert!(reads.turn(1).await.is_none()); + assert_eq!(started.elapsed(), Duration::from_secs(4)); + } +} diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index a807b4a53e..11abba1596 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -1334,35 +1334,46 @@ impl Default for ConnectionManager { } /// How long a pod uses a community's cached open set after the database read -/// that produced it began. Cross-pod invalidation is best-effort, so this -/// bounds how long a channel that left the open set elsewhere stays granted -/// here. +/// that produced it began (its connection checkout, see [`buzz_db::ReadAt`]). +/// Cross-pod invalidation is best-effort, so this bounds how long a channel +/// that left the open set elsewhere stays granted here. pub const OPEN_CHANNELS_MAX_AGE: std::time::Duration = std::time::Duration::from_secs(1); /// How long a pod serves a cached ban/timeout row after the database read -/// that produced it began (see [`AppState::restriction_cache`]). +/// that produced it began, at its connection checkout (see +/// [`AppState::restriction_cache`]). pub const RESTRICTION_CACHE_MAX_AGE: std::time::Duration = std::time::Duration::from_secs(1); /// How long a pod serves a cached host → community mapping after the -/// database read that produced it began (see [`AppState::tenant_host_cache`]). +/// database read that produced it began, at its connection checkout (see +/// [`AppState::tenant_host_cache`]). pub const TENANT_HOST_CACHE_MAX_AGE: std::time::Duration = std::time::Duration::from_secs(1); -/// A cached restriction row and when the read that produced it began. +/// How long a cache miss waits for a shared read's statement, beyond the +/// writer pool's acquire timeout that its holder may spend waiting for a +/// connection (see [`crate::shared_reads::SharedReads::new`]). A holder still +/// waiting for a connection either gets one or times out within the acquire +/// timeout, so only a statement that runs longer than this makes waiters +/// give up and read on their own. +pub const SHARED_READ_STATEMENT_WAIT: std::time::Duration = std::time::Duration::from_secs(1); + +/// A cached restriction row and when the read that produced it checked out +/// its connection. #[derive(Debug, Clone)] pub struct CachedRestriction { /// The restriction row as read. pub state: buzz_db::moderation::RestrictionState, - /// When the database read that produced it began. + /// When the database read that produced it checked out its connection. pub read_at: Instant, } /// A community's listed open channel IDs and when the read that listed them -/// began. +/// checked out its connection. #[derive(Debug)] pub struct OpenChannels { /// The community's listed open channel IDs. pub ids: Vec, - /// When the database read that listed them began. + /// When the database read that listed them checked out its connection. pub read_at: Instant, } @@ -1454,12 +1465,10 @@ pub struct AppState { /// Listed open channel IDs per community, the half of every reader's /// accessible set that does not depend on the reader. Invalidated with /// the community's accessible channels, and used only for - /// [`OPEN_CHANNELS_MAX_AGE`] after its database read began. + /// [`OPEN_CHANNELS_MAX_AGE`] after its read's connection checkout. pub open_channels_cache: Arc>>, - /// Per-community gate held while a pod reads a community's open set, so - /// concurrent misses wait for that read instead of each issuing their own - /// (see [`AppState::get_accessible_channel_ids_cached`]). - pub open_channels_reads: Arc>>>, + /// Lets concurrent misses on [`Self::open_channels_cache`] share one read. + pub(crate) open_channels_reads: Arc>, /// Per-community channel visibility string, used to gate the private-channel fan-out /// access check so open channels stay zero-cost. Invalidated on a flip. pub channel_visibility_cache: Arc>, @@ -1553,22 +1562,27 @@ pub struct AppState { /// lifts the moment `muted_until` passes. (`banned` is computed at read /// time, so an expiring ban can outlive its expiry by up to /// [`RESTRICTION_CACHE_MAX_AGE`].) Key: (community, pubkey bytes). An entry - /// is served only for [`RESTRICTION_CACHE_MAX_AGE`] after its read began, - /// which is the only staleness bound: on every pod and for every - /// restriction writer, a cached reader sees a ban/timeout change within - /// that long. As a best effort, signed moderation commands (9040–9043) - /// also drop the target's own entry on the pod that handles them; owned - /// agents' entries, other pods, and the admin/report paths just age out. + /// is served only for [`RESTRICTION_CACHE_MAX_AGE`] after its read's + /// connection checkout, which is the only staleness bound: on every pod + /// and for every restriction writer, a cached reader sees a ban/timeout + /// change within that long. As a best effort, signed moderation commands + /// (9040–9043) also drop the target's own entry on the pod that handles + /// them; owned agents' entries, other pods, and the admin/report paths + /// just age out. /// Socket admission and moderation-command actors read fresh: a grant that /// outlives the bound must not rest on a cached verdict. #[allow(clippy::type_complexity)] pub restriction_cache: Arc), CachedRestriction>>, + /// Lets concurrent misses on [`Self::restriction_cache`] share one read. + pub(crate) restriction_reads: Arc)>>, /// Normalized request host → community, from the row-zero host binding /// ([`crate::tenant::bind_community`]). Only mapped hosts are cached, and - /// an entry is served only for [`TENANT_HOST_CACHE_MAX_AGE`] after its read - /// began: a host remapped, archived or deleted elsewhere resolves to its - /// old community for at most that long. + /// an entry is served only for [`TENANT_HOST_CACHE_MAX_AGE`] after its + /// read's connection checkout: a host remapped, archived or deleted + /// elsewhere resolves to its old community for at most that long. pub tenant_host_cache: Arc>, + /// Lets concurrent misses on [`Self::tenant_host_cache`] share one read. + pub(crate) tenant_host_reads: Arc>, /// Runtime conformance tracer. Production binds [`crate::conformance::NoopTracer`] /// (zero cost). Conformance tests bind [`crate::conformance::JsonlTracer`] to @@ -1736,6 +1750,8 @@ impl AppState { let audit_enabled = audit_arc.is_some(); // Build NIP-FI components before moving config into the state Arc. let (nip_fi_verifier, nip_fi_jwks_source) = build_nip_fi_components(&config); + let shared_read_wait = + db.pool().options().get_acquire_timeout() + SHARED_READ_STATEMENT_WAIT; let state = Self { config: Arc::new(config), db, @@ -1789,12 +1805,10 @@ impl AppState { .time_to_live(std::time::Duration::from_secs(10)) .build(), ), - open_channels_reads: Arc::new( - moka::sync::Cache::builder() - .max_capacity(10_000) - .time_to_idle(std::time::Duration::from_secs(60)) - .build(), - ), + open_channels_reads: Arc::new(crate::shared_reads::SharedReads::new( + 10_000, + shared_read_wait, + )), channel_visibility_cache: Arc::new( moka::sync::Cache::builder() .max_capacity(10_000) @@ -1860,12 +1874,20 @@ impl AppState { .time_to_live(std::time::Duration::from_secs(10)) .build(), ), + restriction_reads: Arc::new(crate::shared_reads::SharedReads::new( + 100_000, + shared_read_wait, + )), tenant_host_cache: Arc::new( moka::sync::Cache::builder() .max_capacity(10_000) .time_to_live(std::time::Duration::from_secs(10)) .build(), ), + tenant_host_reads: Arc::new(crate::shared_reads::SharedReads::new( + 10_000, + shared_read_wait, + )), // Default to NoopTracer: production builds pay zero cost. // Conformance tests overwrite this with a JsonlTracer after // construction (see test helpers in @@ -1990,34 +2012,45 @@ impl AppState { /// `moderation_restriction_state` served from [`Self::restriction_cache`] /// for [`RESTRICTION_CACHE_MAX_AGE`] after the read that produced it - /// began. For per-request gates (HTTP requests, publishes, ephemeral - /// pulses) whose grant ends with the request. Errors are not cached. + /// checked out its connection. For per-request gates (HTTP requests, + /// publishes, ephemeral pulses) whose grant ends with the request. Errors + /// are not cached. + /// + /// Concurrent misses for one pubkey share a read through + /// [`Self::restriction_reads`]. pub async fn restriction_state_cached( &self, community_id: CommunityId, pubkey: &[u8], ) -> Result { let key = (community_id, pubkey.to_vec()); - if let Some(cached) = self.restriction_cache.get(&key) { - if cached.read_at.elapsed() < RESTRICTION_CACHE_MAX_AGE { - metrics::counter!("buzz_restriction_cache_hits_total").increment(1); - return Ok(cached.state); - } + let fresh = || { + self.restriction_cache + .get(&key) + .filter(|cached| cached.read_at.elapsed() < RESTRICTION_CACHE_MAX_AGE) + }; + if let Some(cached) = fresh() { + metrics::counter!("buzz_restriction_cache_hits_total").increment(1); + return Ok(cached.state); + } + let _turn = self.restriction_reads.turn(key.clone()).await; + if let Some(cached) = fresh() { + metrics::counter!("buzz_restriction_cache_hits_total").increment(1); + return Ok(cached.state); } metrics::counter!("buzz_restriction_cache_misses_total").increment(1); - let read_at = Instant::now(); - let state = self + let read = self .db - .moderation_restriction_state(community_id, pubkey) + .moderation_restriction_state_read_at(community_id, pubkey) .await?; self.restriction_cache.insert( key, CachedRestriction { - state: state.clone(), - read_at, + state: read.value.clone(), + read_at: read.read_at, }, ); - Ok(state) + Ok(read.value) } /// Best-effort drop of a pubkey's cached restriction row on this pod. @@ -2532,10 +2565,10 @@ impl AppState { /// /// The open half is shared by every reader in the community and its /// cross-pod invalidation is best-effort, so a cached set is served only - /// for [`OPEN_CHANNELS_MAX_AGE`] after its read began: a channel that left - /// the open set on another pod is granted from the cache for at most that - /// long. A caller whose own read straddles a revocation gets what that - /// read saw, as with any uncached read. + /// for [`OPEN_CHANNELS_MAX_AGE`] after its read's connection checkout: a + /// channel that left the open set on another pod is granted from the + /// cache for at most that long. A caller whose own read straddles a + /// revocation gets what that read saw, as with any uncached read. pub async fn get_accessible_channel_ids_cached( &self, community_id: CommunityId, @@ -2566,9 +2599,9 @@ impl AppState { /// gate in [`Self::open_channels_reads`] and reads; the rest wait for it /// to finish and use its set. A waiter whose leader failed or was /// cancelled reads on its own rather than queueing behind another leader. - /// A waiter also stops waiting after [`OPEN_CHANNELS_MAX_AGE`] and reads - /// on its own: the leader's set would be stale on arrival by then, so a - /// leader whose read hangs stalls only its own request. + /// A waiter also stops waiting once the leader has had the writer pool's + /// acquire timeout plus [`SHARED_READ_STATEMENT_WAIT`], and reads on its + /// own, so a leader whose read hangs stalls its waiters at most that long. async fn open_channels( &self, community_id: CommunityId, @@ -2582,29 +2615,25 @@ impl AppState { metrics::counter!("buzz_open_channels_cache_hits_total").increment(1); return Ok(open); } - let gate = self - .open_channels_reads - .get_with(community_id, || Arc::new(tokio::sync::Mutex::new(()))); - let _leader = match gate.try_lock() { - Ok(guard) => Some(guard), - Err(_) => { - drop(tokio::time::timeout(OPEN_CHANNELS_MAX_AGE, gate.lock()).await); - if let Some(open) = fresh() { - metrics::counter!("buzz_open_channels_cache_hits_total").increment(1); - return Ok(open); - } - None - } - }; + let _turn = self.open_channels_reads.turn(community_id).await; + if let Some(open) = fresh() { + metrics::counter!("buzz_open_channels_cache_hits_total").increment(1); + return Ok(open); + } let reason = if self.open_channels_cache.contains_key(&community_id) { "expired" } else { "absent" }; metrics::counter!("buzz_open_channels_cache_misses_total", "reason" => reason).increment(1); - let read_at = Instant::now(); - let ids = self.db.get_listed_open_channel_ids(community_id).await?; - let open = Arc::new(OpenChannels { ids, read_at }); + let read = self + .db + .get_listed_open_channel_ids_read_at(community_id) + .await?; + let open = Arc::new(OpenChannels { + ids: read.value, + read_at: read.read_at, + }); self.open_channels_cache.insert(community_id, open.clone()); Ok(open) } @@ -3686,13 +3715,149 @@ pub(crate) mod tests { use super::*; use buzz_core::channel::{ChannelType, ChannelVisibility}; + /// Starts `misses` concurrent runs of `read` while the pool's only + /// connection is held for `wait`, checks that every one is still + /// waiting, then releases the connection. + async fn behind_held_connection( + pool: &sqlx::PgPool, + wait: std::time::Duration, + misses: usize, + read: impl Fn() -> F, + ) -> Vec + where + T: Send + 'static, + F: std::future::Future + Send + 'static, + { + let held = pool.acquire().await.unwrap(); + let reads = (0..misses) + .map(|_| tokio::spawn(read())) + .collect::>(); + tokio::time::sleep(wait).await; + assert!( + reads.iter().all(|read| !read.is_finished()), + "every miss waits for the connection" + ); + drop(held); + let mut results = Vec::with_capacity(misses); + for read in reads { + results.push(read.await.expect("join")); + } + results + } + + /// The sum of every counter named `name` in `snapshotter`. + fn counted(snapshotter: &metrics_util::debugging::Snapshotter, name: &str) -> u64 { + snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter(|(key, ..)| key.key().name() == name) + .map(|(.., value)| match value { + metrics_util::debugging::DebugValue::Counter(n) => n, + _ => 0, + }) + .sum() + } + + /// Concurrent misses on each cache that bounds staleness from its + /// read, queued behind a held pool connection for longer than the + /// cache's max age, share one read, and its entry still lands fresh: + /// waiters wait out the leader's pool wait, and the entry is stamped + /// at connection checkout, not when the miss began. + /// Mutations: stamp any of the three entries before the pool wait, + /// or let waiters give up after the cache's max age → RED. + #[tokio::test(flavor = "current_thread")] + #[ignore = "requires Postgres"] + async fn cache_misses_behind_a_held_pool_share_one_fresh_read() { + use crate::tenant::HostResolver as _; + + let recorder = metrics_util::debugging::DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _recorder = metrics::set_default_local_recorder(&recorder); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(std::time::Duration::from_secs(30)) + .connect(&crate::test_support::database_url()) + .await + .unwrap(); + let state = test_state_with_database_pool(pool.clone()).await; + let host = format!("pool-wait-{}.local", Uuid::new_v4()); + let community = state + .db + .ensure_configured_community(&host) + .await + .unwrap() + .id; + let pubkey = vec![6u8; 32]; + let wait = RESTRICTION_CACHE_MAX_AGE + .max(TENANT_HOST_CACHE_MAX_AGE) + .max(OPEN_CHANNELS_MAX_AGE) + .max(SHARED_READ_STATEMENT_WAIT) + + std::time::Duration::from_millis(500); + let misses = 8; + + behind_held_connection(&pool, wait, misses, || { + let (reader, key) = (Arc::clone(&state), pubkey.clone()); + async move { reader.restriction_state_cached(community, &key).await } + }) + .await + .into_iter() + .for_each(|read| { + read.expect("restriction read"); + }); + assert_eq!( + counted(&snapshotter, "buzz_restriction_cache_misses_total"), + 1 + ); + let restriction = state + .restriction_cache + .get(&(community, pubkey)) + .expect("restriction entry"); + assert!(restriction.read_at.elapsed() < RESTRICTION_CACHE_MAX_AGE); + + let mapped = behind_held_connection(&pool, wait, misses, || { + let (reader, key) = (Arc::clone(&state), host.clone()); + async move { reader.resolve_host(&key).await } + }) + .await; + assert!(mapped + .into_iter() + .all(|mapped| mapped.expect("host read") == Some(community))); + assert_eq!( + counted(&snapshotter, "buzz_tenant_host_cache_misses_total"), + 1 + ); + let (_, read_at) = state.tenant_host_cache.get(&host).expect("host entry"); + assert!(read_at.elapsed() < TENANT_HOST_CACHE_MAX_AGE); + + behind_held_connection(&pool, wait, misses, || { + let reader = Arc::clone(&state); + async move { reader.open_channels(community).await } + }) + .await + .into_iter() + .for_each(|read| { + read.expect("open-set read"); + }); + assert_eq!( + counted(&snapshotter, "buzz_open_channels_cache_misses_total"), + 1 + ); + let open = state + .open_channels_cache + .get(&community) + .expect("open-set entry"); + assert!(open.read_at.elapsed() < OPEN_CHANNELS_MAX_AGE); + } + /// Open-set reads park on a lock on `channels`, where the test counts /// them; waiters are counted as they arrive by the clones of the /// community's gate they hold. A leader's eight waiters share its /// set. When the leader is cancelled, its orphans each pass the gate /// and read, none holding it across its own read. When the leader's - /// read hangs, its waiters stop waiting after - /// [`OPEN_CHANNELS_MAX_AGE`] and each read on their own. + /// read hangs, its waiters stop waiting after the pool's acquire + /// timeout plus [`SHARED_READ_STATEMENT_WAIT`] and each read on their + /// own. /// Mutations: drop the waiter path (distinct sets in the first /// phase), hold the gate across a waiter's own read (the gate never /// frees in the second), or wait on the gate with no limit (the third @@ -3703,7 +3868,10 @@ pub(crate) mod tests { let database_url = crate::test_support::database_url(); let state = test_state_with_database_url_and_pool_options( &database_url, - sqlx::postgres::PgPoolOptions::new().max_connections(16), + // Bounds how long the hung leader's waiters wait for it. + sqlx::postgres::PgPoolOptions::new() + .max_connections(16) + .acquire_timeout(std::time::Duration::from_secs(1)), ) .await; let admin = sqlx::PgPool::connect(&database_url).await.unwrap(); @@ -3765,7 +3933,7 @@ pub(crate) mod tests { parks(1).await; let gate = state .open_channels_reads - .get(&community) + .gate(&community) .expect("the leader's gate"); let base = Arc::strong_count(&gate); let waiters = misses(8); @@ -4241,10 +4409,11 @@ pub(crate) mod tests { .collect::>() }; - let gate = state + let leader = state .open_channels_reads - .get_with(community, || Arc::new(tokio::sync::Mutex::new(()))); - let leader = gate.lock().await; + .turn(community) + .await + .expect("no read in flight"); let shared = waiters(&state); tokio::task::yield_now().await; state.open_channels_cache.insert( @@ -4261,6 +4430,98 @@ pub(crate) mod tests { } } + /// Restriction and tenant-host misses that arrive while a read of the + /// same key is in flight wait for it and use its entry. The database is + /// unreachable, so any read of a waiter's own fails. + /// Mutation: drop either waiter path → RED. + #[tokio::test] + async fn concurrent_restriction_and_host_misses_share_the_in_flight_read() { + use crate::tenant::HostResolver as _; + + let state = + test_state_with_unreachable_database_url("postgres://buzz@127.0.0.1:1/buzz").await; + let community = CommunityId::from_uuid(Uuid::from_u128(0xAAAA)); + let pubkey = vec![5u8; 32]; + let host = "shared-read.local".to_owned(); + let muted = buzz_db::moderation::RestrictionState { + banned: false, + own_banned: false, + muted_until: Some(chrono::Utc::now() + chrono::Duration::hours(1)), + }; + // On this current-thread runtime a waiter's first poll runs without + // interruption from its count to its first await, the wait on the + // leader's gate. Once all have counted, all are waiting at the gate, + // so none can take the initial cache-hit path when the cache is + // filled. (The gate's `Arc` count is no proof: the cache may hold a + // transient clone of its own.) + async fn enrolled(started: &std::sync::atomic::AtomicUsize, waiters: usize) { + let start = Instant::now(); + while started.load(std::sync::atomic::Ordering::SeqCst) < waiters { + // A watchdog only; timing is not part of what is checked. + assert!( + start.elapsed() < std::time::Duration::from_secs(10), + "waiters did not reach the gate" + ); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + } + fn spawn_waiters( + started: &Arc, + read: impl Fn() -> F, + ) -> Vec> + where + F: std::future::Future + Send + 'static, + { + (0..8) + .map(|_| { + let (started, read) = (Arc::clone(started), read()); + tokio::spawn(async move { + started.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + read.await + }) + }) + .collect() + } + + let leader = state + .restriction_reads + .turn((community, pubkey.clone())) + .await + .expect("no read in flight"); + let started = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let waiters = spawn_waiters(&started, || { + let (state, pubkey) = (Arc::clone(&state), pubkey.clone()); + async move { state.restriction_state_cached(community, &pubkey).await } + }); + enrolled(&started, waiters.len()).await; + state.seed_restriction((community, pubkey.clone()), muted.clone()); + drop(leader); + for waiter in waiters { + let read = waiter.await.expect("join").expect("the leader's row"); + assert_eq!(read, muted); + } + + let leader = state + .tenant_host_reads + .turn(host.clone()) + .await + .expect("no read in flight"); + let started = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let waiters = spawn_waiters(&started, || { + let (state, host) = (Arc::clone(&state), host.clone()); + async move { state.resolve_host(&host).await } + }); + enrolled(&started, waiters.len()).await; + state + .tenant_host_cache + .insert(host.clone(), (community, Instant::now())); + drop(leader); + for waiter in waiters { + let read = waiter.await.expect("join").expect("the leader's mapping"); + assert_eq!(read, Some(community)); + } + } + #[tokio::test] async fn channel_deleted_invalidation_is_scoped_to_community() { let state = test_state().await; diff --git a/crates/buzz-relay/src/tenant.rs b/crates/buzz-relay/src/tenant.rs index 8038ec3583..b3b52ee931 100644 --- a/crates/buzz-relay/src/tenant.rs +++ b/crates/buzz-relay/src/tenant.rs @@ -14,7 +14,7 @@ //! (`Db::resolve_host`); the relay depends on the trait, not the query, so the //! binding is testable without a database. -use buzz_core::tenant::{normalize_host, CommunityId, TenantContext}; +use buzz_core::tenant::{normalize_host, CommunityId, TenantContext, MAX_HOST_LEN}; /// Resolves a normalized connection host to its community, or `None` when the /// host maps to no community on this deployment. @@ -81,7 +81,10 @@ pub async fn bind_community( // bind to a misconfigured empty-host community. Reuse `UnmappedHost` (not a // distinct variant) so the rejection is byte-identical to any other unmapped // host — an unauthenticated caller cannot probe for an empty-host row. - if host.is_empty() { + // A host longer than any `communities.host` can hold maps to nothing + // either; rejecting it here keeps an unauthenticated caller's arbitrarily + // long host out of the resolver's caches and off the database. + if host.is_empty() || host.chars().count() > MAX_HOST_LEN { return Err(BindError::UnmappedHost); } match resolver.resolve_host(&host).await { @@ -149,8 +152,9 @@ impl HostResolver for buzz_db::Db { /// [`crate::state::AppState::tenant_host_cache`]. A connection's handshake and /// its follow-up HTTP requests resolve the same host many times a second, so /// a mapped host is reread at most once per -/// [`crate::state::TENANT_HOST_CACHE_MAX_AGE`]. Unmapped hosts and lookup -/// errors are never cached. +/// [`crate::state::TENANT_HOST_CACHE_MAX_AGE`], and concurrent misses share +/// one read through [`crate::state::AppState::tenant_host_reads`]. Unmapped +/// hosts and lookup errors are never cached. impl HostResolver for crate::state::AppState { type Error = buzz_db::DbError; @@ -158,18 +162,33 @@ impl HostResolver for crate::state::AppState { &self, normalized_host: &str, ) -> Result, Self::Error> { - if let Some((community, read_at)) = self.tenant_host_cache.get(normalized_host) { - if read_at.elapsed() < crate::state::TENANT_HOST_CACHE_MAX_AGE { - metrics::counter!("buzz_tenant_host_cache_hits_total").increment(1); - return Ok(Some(community)); - } + let fresh = || { + self.tenant_host_cache + .get(normalized_host) + .filter(|(_, read_at)| read_at.elapsed() < crate::state::TENANT_HOST_CACHE_MAX_AGE) + .map(|(community, _)| community) + }; + if let Some(community) = fresh() { + metrics::counter!("buzz_tenant_host_cache_hits_total").increment(1); + return Ok(Some(community)); + } + let _turn = self + .tenant_host_reads + .turn(normalized_host.to_owned()) + .await; + if let Some(community) = fresh() { + metrics::counter!("buzz_tenant_host_cache_hits_total").increment(1); + return Ok(Some(community)); } metrics::counter!("buzz_tenant_host_cache_misses_total").increment(1); - let read_at = std::time::Instant::now(); - let community = self.db.resolve_host(normalized_host).await?; + let read = self + .db + .lookup_community_by_host_read_at(normalized_host) + .await?; + let community = read.value.map(|record| record.id); if let Some(community) = community { self.tenant_host_cache - .insert(normalized_host.to_owned(), (community, read_at)); + .insert(normalized_host.to_owned(), (community, read.read_at)); } Ok(community) } @@ -361,6 +380,34 @@ mod tests { } } + /// A host longer than `communities.host` can hold fails closed as + /// unmapped before the request-path resolver sees it: no cache or gate + /// entry and no lookup. The database is unreachable, so a lookup would + /// fail as `Lookup`, as it does for a host at the limit. + /// Mutation: drop the length check, or count bytes past the column's + /// characters → RED. + #[tokio::test] + async fn an_overlong_host_fails_closed_before_the_resolver() { + let state = crate::state::tests::test_state_with_unreachable_database_url( + "postgres://buzz@127.0.0.1:1/buzz", + ) + .await; + let overlong = "a".repeat(MAX_HOST_LEN + 1); + assert!(matches!( + bind_community(&*state, &overlong).await, + Err(BindError::UnmappedHost) + )); + assert!(state.tenant_host_reads.gate(&overlong).is_none()); + + // At the limit, even in multibyte characters, the host is looked up. + for longest in ["a".repeat(MAX_HOST_LEN), "é".repeat(MAX_HOST_LEN)] { + assert!(matches!( + bind_community(&*state, &longest).await, + Err(BindError::Lookup(_)) + )); + } + } + /// The request-path resolver serves a mapped host from the cache only /// while the entry is younger than `TENANT_HOST_CACHE_MAX_AGE`; past /// that it rereads, and a failed reread fails closed. The database is diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index a8a4aea48d..92951b95a5 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -378,8 +378,12 @@ run_unit_tests() { run_test_step "buzz-relay read cache tests" \ cargo test -p buzz-relay --lib -- --exact --nocapture \ state::tests::admitted_index_follows_every_key_change \ + shared_reads::tests::one_caller_reads_while_the_rest_wait_for_it \ + shared_reads::tests::a_waiter_stops_waiting_for_a_hung_read \ state::tests::concurrent_open_channel_misses_share_the_in_flight_read \ + state::tests::concurrent_restriction_and_host_misses_share_the_in_flight_read \ tenant::tests::app_state_resolver_serves_fresh_cached_hosts_only \ + tenant::tests::an_overlong_host_fails_closed_before_the_resolver \ state::tests::disconnect_drops_the_cached_restriction_row run_test_step "buzz-relay router tests" \