From d21f0c0fdbec9534b9396552b2550ab306d218df Mon Sep 17 00:00:00 2001 From: Luke Tornquist Date: Wed, 4 Nov 2026 09:52:21 -0500 Subject: [PATCH] perf(relay): keep 1 s read caches warm when the pool is saturated (#9012) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What The relay's 1-second read caches (restriction state, tenant host, open channels) now date each entry from when the read checked out its pool connection, not from when the cache miss began. Concurrent misses on the restriction and tenant-host caches now share one database read. The open-channels cache already did this. ## Why Each of these caches serves an entry for 1 s after "the read that produced it began". Today that start time is taken before waiting for a pool connection. Once pool waits reach about a second (for example, many clients reconnecting at once), every entry is already expired by the time it lands. The cache hit rate falls toward zero exactly when the pool is saturated, so every request does its own read. Those reads queue for the same connections, which stretches the waits further. Requests that time out in the pool then fail closed (`internal error checking restriction state`, `community active check failed; refusing the socket`), and their clients reconnect and add more load. In a local synthetic reconnect load test with a 50-connection pool: - the restriction cache served fewer than one request in five from cache; - the tenant-host cache, which has a single key for the whole deployment, missed more often than it hit; - the open-channels cache missed hundreds of thousands of times on expired entries. ## How - **Stamp at checkout.** `buzz_db::ReadAt` carries a value plus the instant its connection was checked out. The statement's snapshot is no older than that instant, so serving the value for 1 s after it keeps the documented staleness bound, however long the read queued. Three `*_read_at` methods return it: `moderation_restriction_state_read_at`, `lookup_community_by_host_read_at` and `get_listed_open_channel_ids_read_at`. The existing methods now call these and drop the value's timestamp. Each operation keeps one datastore span under its existing name, carried by the `*_read_at` method. - **Share concurrent misses.** `SharedReads` is the per-key gate the open-channels cache already used, moved into its own module. The open-channels cache, the restriction cache and the tenant-host cache now all use it. The first miss reads; the others wait for it, then recheck the cache. A waiter waits for at most the writer pool's acquire timeout plus `SHARED_READ_STATEMENT_WAIT` (1 s). That covers the leader's own wait for a connection, which is what grows in a storm, so waiters keep sharing its read instead of queueing duplicate reads behind it. A failed or cancelled leader releases its waiters at once. A leader whose statement hangs stalls them for at most 1 s past the acquire timeout, never the statement timeout. - **Reject hosts no community can have.** `bind_community` rejects a host longer than `communities.host VARCHAR(255)` can hold (`buzz_core::tenant::MAX_HOST_LEN`, counted in characters) as unmapped, next to the empty-host check. An unauthenticated caller's arbitrarily long host therefore never reaches a resolver, cache, gate or the database. No staleness bound changes: every entry is still served for at most 1 s after its read's snapshot. Invalidation paths are unchanged. A read in flight at an invalidation can still refill the old row for at most that bound, as documented today. ## What got simpler The open-channels gate's hand-rolled `try_lock`/`timeout` block is replaced by `SharedReads::turn`. One gate implementation now serves three caches instead of one implementation that only one cache had. The gate's wait is set once per gate rather than passed at each call site. Provisioning and host binding share one `MAX_HOST_LEN` instead of a private constant. ## Tests - `shared_reads::tests`: one caller reads while the rest wait; a waiter gives up on a hung read after its bound. - `concurrent_restriction_and_host_misses_share_the_in_flight_read`: waiters on both caches use the leader's entry; with an unreachable database, any waiter that read on its own would fail. - `postgres_tests::cache_misses_behind_a_held_pool_share_one_fresh_read`: with a one-connection pool held for longer than every max age, eight concurrent misses on each of the three caches cost one database read, and each entry still lands fresh. - `tenant::tests::an_overlong_host_fails_closed_before_the_resolver`: a host over the limit fails as unmapped with no gate entry and no lookup; a host at the limit, including one in multibyte characters, is still looked up. - Mutation-checked: each of the following turns these tests red: stamping any of the three entries before the pool wait, dropping either new waiter path, letting waiters give up after the cache's max age, dropping the host length check, or counting the host length in bytes. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Signed-off-by: Luke Tornquist Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-authored-by: Claude Opus 5.5 --- Justfile | 11 +- crates/buzz-admin/src/communities.rs | 4 +- crates/buzz-core/src/tenant.rs | 8 + crates/buzz-db/src/lib.rs | 15 + crates/buzz-db/src/store/channel_members.rs | 35 +- crates/buzz-db/src/store/community.rs | 35 +- crates/buzz-db/src/store/moderation.rs | 42 +- .../src/handlers/community_provisioning.rs | 5 +- crates/buzz-relay/src/lib.rs | 2 + crates/buzz-relay/src/shared_reads.rs | 108 +++++ crates/buzz-relay/src/state.rs | 407 ++++++++++++++---- crates/buzz-relay/src/tenant.rs | 71 ++- scripts/run-tests.sh | 4 + 13 files changed, 627 insertions(+), 120 deletions(-) create mode 100644 crates/buzz-relay/src/shared_reads.rs diff --git a/Justfile b/Justfile index 9f8736bbfe..cb186312df 100644 --- a/Justfile +++ b/Justfile @@ -638,11 +638,12 @@ test-unit: + test(/^api::nip_fi::/)' # Invitation drain/cutoff boundary contracts are infra-free. cargo nextest run -p buzz-relay --lib -E 'test(/^invite_token::/) + test(/^config::invite_cutoff_tests::/)' - # Relay read caches: the admitted-connection index, open-set - # single-flight, tenant host cache and the ban disconnect's - # restriction-cache drop. Infra-free; the database URL - # refuses at once. Keep scripts/run-tests.sh in step. - cargo nextest run -p buzz-relay --lib -E 'test(=state::tests::admitted_index_follows_every_key_change) + test(=state::tests::concurrent_open_channel_misses_share_the_in_flight_read) + test(=tenant::tests::app_state_resolver_serves_fresh_cached_hosts_only) + test(=state::tests::disconnect_drops_the_cached_restriction_row)' + # Relay read caches: the admitted-connection index, the shared-read + # gates (open set, restriction, tenant host), tenant host cache, the + # overlong-host rejection and the ban disconnect's restriction-cache + # drop. Infra-free; the + # database URL refuses at once. Keep scripts/run-tests.sh in step. + cargo nextest run -p buzz-relay --lib -E 'test(=state::tests::admitted_index_follows_every_key_change) + test(/^shared_reads::tests::/) + test(=state::tests::concurrent_open_channel_misses_share_the_in_flight_read) + test(=state::tests::concurrent_restriction_and_host_misses_share_the_in_flight_read) + test(=tenant::tests::app_state_resolver_serves_fresh_cached_hosts_only) + test(=tenant::tests::an_overlong_host_fails_closed_before_the_resolver) + test(=state::tests::disconnect_drops_the_cached_restriction_row)' # Retain partially consumed mesh receives across housekeeping ticks. cargo nextest run -p buzz-relay --lib -E 'test(=mesh_boot::tests::demo_echo_retains_pending_receive_across_housekeeping_ticks)' # boot_lifecycle spawns the real relay binary and asserts its startup diff --git a/crates/buzz-admin/src/communities.rs b/crates/buzz-admin/src/communities.rs index f8bcd7c015..4830d7d9ef 100644 --- a/crates/buzz-admin/src/communities.rs +++ b/crates/buzz-admin/src/communities.rs @@ -1,7 +1,7 @@ //! Reversible whole-community lifecycle commands. use anyhow::{Context, Result}; -use buzz_core::tenant::{normalize_host, relay_url_authority, TenantContext}; +use buzz_core::tenant::{normalize_host, relay_url_authority, TenantContext, MAX_HOST_LEN}; use buzz_db::{ArchivedCommunityRecord, UnarchivedCommunityRecord}; use buzz_pubsub::conn_control::ConnControl; use buzz_pubsub::PubSubManager; @@ -9,8 +9,6 @@ use clap::Subcommand; use serde_json::{json, Value}; use url::{Host, Url}; -const MAX_HOST_LEN: usize = 255; - /// Reversible community lifecycle actions. #[derive(Debug, Subcommand)] pub enum CommunitiesCommand { diff --git a/crates/buzz-core/src/tenant.rs b/crates/buzz-core/src/tenant.rs index 46c7e71c56..d7a0d50df0 100644 --- a/crates/buzz-core/src/tenant.rs +++ b/crates/buzz-core/src/tenant.rs @@ -97,6 +97,14 @@ impl TenantContext { } } +/// Longest host a community can have, in characters: `communities.host` is +/// `VARCHAR(255)`. A longer host maps to no community. +/// +/// Provisioning checks count bytes instead, which is stricter for non-ASCII +/// input and identical for the ASCII authorities they accept. Keep both: the +/// lookup side must not refuse any host the column can store. +pub const MAX_HOST_LEN: usize = 255; + /// Normalize a connection `Host` into the canonical form used as the community /// lookup key. /// diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index f7046a7f60..6fd74d5f32 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -111,6 +111,21 @@ pub use reminder::DueReminder; pub use store::{ArtifactSnapshot, SnapshotDenied}; pub use usage::SessionLeader; +/// A value read from the database, with when its reading connection was +/// checked out of the pool. +/// +/// The statement that produced `value` started after `read_at`, so its +/// snapshot is no older than `read_at`. A cache that serves `value` for a +/// bound after `read_at` is never staler than that bound, however long the +/// read waited for a connection. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReadAt { + /// The value as read. + pub value: T, + /// When the connection that read `value` was checked out. + pub read_at: std::time::Instant, +} + use buzz_core::CommunityId; /// Durable NIP-FI verified-profile projection state. diff --git a/crates/buzz-db/src/store/channel_members.rs b/crates/buzz-db/src/store/channel_members.rs index 96501f0227..936e2b9078 100644 --- a/crates/buzz-db/src/store/channel_members.rs +++ b/crates/buzz-db/src/store/channel_members.rs @@ -9,7 +9,7 @@ use uuid::Uuid; use crate::channel::{row_to_channel_record, ChannelRecord}; use crate::error::{DbError, Result}; -use crate::{AdmittedTx, Db}; +use crate::{AdmittedTx, Db, ReadAt}; use buzz_core::CommunityId; use buzz_datastore_tracing::datastore_span; @@ -1550,11 +1550,22 @@ pub async fn get_listed_open_channel_ids( pool: &PgPool, community_id: CommunityId, ) -> Result> { + Ok(get_listed_open_channel_ids_read_at(pool, community_id) + .await? + .value) +} + +/// [`get_listed_open_channel_ids`], with when its connection was checked out. +pub async fn get_listed_open_channel_ids_read_at( + pool: &PgPool, + community_id: CommunityId, +) -> Result>> { let mut connection = crate::observability::acquire_writer( pool, crate::observability::WriterOperation::Authorization, ) .await?; + let read_at = std::time::Instant::now(); let sql = format!( r#" SELECT id @@ -1565,11 +1576,11 @@ pub async fn get_listed_open_channel_ids( open_types = crate::store::channel::legacy_channel_type_sql("channel_type"), ); // Built only from the constant type list. - sqlx::query_scalar(sqlx::AssertSqlSafe(sql)) + let value = sqlx::query_scalar(sqlx::AssertSqlSafe(sql)) .bind(community_id.as_uuid()) .fetch_all(&mut *connection) - .await - .map_err(Into::into) + .await?; + Ok(ReadAt { value, read_at }) } /// Appends the channels of `more` that `ids` lacks. @@ -2399,12 +2410,24 @@ impl Db { } /// A community's listed open channel IDs. See [`get_listed_open_channel_ids`]. - #[datastore_span(name = "get_listed_open_channel_ids", system = "postgresql")] pub async fn get_listed_open_channel_ids( &self, community_id: CommunityId, ) -> Result> { - get_listed_open_channel_ids(&self.pool, community_id).await + Ok(self + .get_listed_open_channel_ids_read_at(community_id) + .await? + .value) + } + + /// [`Self::get_listed_open_channel_ids`], with when its connection was + /// checked out, for caches that bound staleness from the read. + #[datastore_span(name = "get_listed_open_channel_ids", system = "postgresql")] + pub async fn get_listed_open_channel_ids_read_at( + &self, + community_id: CommunityId, + ) -> Result>> { + get_listed_open_channel_ids_read_at(&self.pool, community_id).await } /// The channels of non-legacy types a pubkey can read (member, or open diff --git a/crates/buzz-db/src/store/community.rs b/crates/buzz-db/src/store/community.rs index 1df5bea021..0a2b1d457b 100644 --- a/crates/buzz-db/src/store/community.rs +++ b/crates/buzz-db/src/store/community.rs @@ -6,7 +6,7 @@ use chrono::{DateTime, Utc}; use sqlx::Row; use uuid::Uuid; -use crate::{relay_members, Db, DbError, Result}; +use crate::{relay_members, Db, DbError, ReadAt, Result}; /// Community host-map row returned by [`Db::lookup_community_by_host`]. #[derive(Debug, Clone, PartialEq, Eq)] @@ -110,16 +110,29 @@ impl Db { /// /// The caller owns host normalization and turns `None` into the fail-closed /// request/connection error. buzz-db only reads the durable host map. - #[datastore_span(name = "lookup_community_by_host", system = "postgresql")] pub async fn lookup_community_by_host( &self, normalized_host: &str, ) -> Result> { + Ok(self + .lookup_community_by_host_read_at(normalized_host) + .await? + .value) + } + + /// [`Self::lookup_community_by_host`], with when its connection was + /// checked out, for caches that bound staleness from the read. + #[datastore_span(name = "lookup_community_by_host", system = "postgresql")] + pub async fn lookup_community_by_host_read_at( + &self, + normalized_host: &str, + ) -> Result>> { let mut connection = crate::observability::acquire_writer( &self.pool, crate::observability::WriterOperation::TenantResolution, ) .await?; + let read_at = std::time::Instant::now(); let row = sqlx::query( r#" SELECT id, host @@ -134,16 +147,18 @@ impl Db { .fetch_optional(&mut *connection) .await?; - row.map(|row| { - let id: Uuid = row.try_get("id")?; - let host: String = row.try_get("host")?; + let value = row + .map(|row| { + let id: Uuid = row.try_get("id")?; + let host: String = row.try_get("host")?; - Ok(CommunityRecord { - id: CommunityId::from_uuid(id), - host, + Ok::<_, DbError>(CommunityRecord { + id: CommunityId::from_uuid(id), + host, + }) }) - }) - .transpose() + .transpose()?; + Ok(ReadAt { value, read_at }) } /// Returns whether a community id still exists in the active lifecycle state. diff --git a/crates/buzz-db/src/store/moderation.rs b/crates/buzz-db/src/store/moderation.rs index f47cc083c8..6720fcb0eb 100644 --- a/crates/buzz-db/src/store/moderation.rs +++ b/crates/buzz-db/src/store/moderation.rs @@ -20,7 +20,7 @@ use sqlx::{PgPool, Row as _}; use uuid::Uuid; use crate::error::Result; -use crate::{CommunityId, Db}; +use crate::{CommunityId, Db, ReadAt}; /// What a report points at. Exactly one target class per report row. #[derive(Debug, Clone, PartialEq, Eq)] @@ -631,11 +631,23 @@ pub async fn restriction_state( community: CommunityId, pubkey: &[u8], ) -> Result { + Ok(restriction_state_read_at(pool, community, pubkey) + .await? + .value) +} + +/// [`restriction_state`], with when its connection was checked out. +pub async fn restriction_state_read_at( + pool: &PgPool, + community: CommunityId, + pubkey: &[u8], +) -> Result> { let mut connection = crate::observability::acquire_writer( pool, crate::observability::WriterOperation::Authorization, ) .await?; + let read_at = std::time::Instant::now(); let row = sqlx::query( r#" SELECT @@ -659,10 +671,13 @@ pub async fn restriction_state( .fetch_one(&mut *connection) .await?; - Ok(RestrictionState { - banned: row.try_get("banned")?, - own_banned: row.try_get("own_banned")?, - muted_until: row.try_get("muted_until")?, + Ok(ReadAt { + value: RestrictionState { + banned: row.try_get("banned")?, + own_banned: row.try_get("own_banned")?, + muted_until: row.try_get("muted_until")?, + }, + read_at, }) } @@ -1025,13 +1040,26 @@ impl Db { /// Fetch the effective (own or agent-owner) ban/timeout restriction state /// for enforcement hot paths. - #[datastore_span(name = "moderation_restriction_state", system = "postgresql")] pub async fn moderation_restriction_state( &self, community: CommunityId, pubkey: &[u8], ) -> Result { - restriction_state(&self.pool, community, pubkey).await + Ok(self + .moderation_restriction_state_read_at(community, pubkey) + .await? + .value) + } + + /// [`Self::moderation_restriction_state`], with when its connection was + /// checked out, for caches that bound staleness from the read. + #[datastore_span(name = "moderation_restriction_state", system = "postgresql")] + pub async fn moderation_restriction_state_read_at( + &self, + community: CommunityId, + pubkey: &[u8], + ) -> Result> { + restriction_state_read_at(&self.pool, community, pubkey).await } /// Fetch the full ban/timeout row for a member pubkey. diff --git a/crates/buzz-relay/src/handlers/community_provisioning.rs b/crates/buzz-relay/src/handlers/community_provisioning.rs index 6c45c5800e..533f2eacbb 100644 --- a/crates/buzz-relay/src/handlers/community_provisioning.rs +++ b/crates/buzz-relay/src/handlers/community_provisioning.rs @@ -30,14 +30,11 @@ use std::sync::Arc; use serde::{Deserialize, Serialize}; use tracing::info; -use buzz_core::tenant::{normalize_host, TenantContext}; +use buzz_core::tenant::{normalize_host, TenantContext, MAX_HOST_LEN}; use url::{Host, Url}; use crate::state::AppState; -/// Maximum accepted authority length. Matches `communities.host VARCHAR(255)`. -const MAX_HOST_LEN: usize = 255; - /// JSON body for `POST /operator/communities`. #[derive(Debug, Deserialize)] pub struct ProvisionCommunityRequest { diff --git a/crates/buzz-relay/src/lib.rs b/crates/buzz-relay/src/lib.rs index f21d0181af..8b8fa0807b 100644 --- a/crates/buzz-relay/src/lib.rs +++ b/crates/buzz-relay/src/lib.rs @@ -65,6 +65,8 @@ pub mod push_runtime; pub mod readiness; /// Axum router construction. pub mod router; +/// Per-key gates that let concurrent cache misses share one database read. +mod shared_reads; /// Timing for startup work between metrics bind and serving. pub mod startup_steps; /// Shared application state. diff --git a/crates/buzz-relay/src/shared_reads.rs b/crates/buzz-relay/src/shared_reads.rs new file mode 100644 index 0000000000..fe384b74fa --- /dev/null +++ b/crates/buzz-relay/src/shared_reads.rs @@ -0,0 +1,108 @@ +//! Per-key gates that let concurrent misses on a short-lived read cache share +//! one database read. +//! +//! A reconnect storm sends many requests for the same key at once (every +//! socket on one host resolves the same tenant; a desktop client checks its +//! own restriction state on every request). Without a gate each miss issues +//! its own read, and the reads queue for the same pool connections. With one, +//! the first miss reads and the rest wait for it and use its cache entry. + +use std::hash::Hash; +use std::sync::Arc; +use std::time::Duration; + +use tokio::sync::{Mutex, OwnedMutexGuard}; + +/// One gate per key, held by the caller that is reading that key. +pub(crate) struct SharedReads { + gates: moka::sync::Cache>>, + wait: Duration, +} + +impl SharedReads +where + K: Hash + Eq + Send + Sync + 'static, +{ + /// Gates for up to `max_capacity` keys, each waited on for at most + /// `wait`; an idle gate is dropped after a minute. + /// + /// `wait` must cover the holder's wait for a pool connection, which is + /// what grows in a storm, plus its statement: a waiter that gives up + /// while the holder still queues for a connection only queues behind it + /// with a duplicate read. + pub(crate) fn new(max_capacity: u64, wait: Duration) -> Self { + Self { + gates: moka::sync::Cache::builder() + .max_capacity(max_capacity) + .time_to_idle(Duration::from_secs(60)) + .build(), + wait, + } + } + + /// Takes `key`'s gate, or waits for its holder to finish. + /// + /// Returns the guard when no read of `key` is in flight: the caller reads + /// and holds the guard until its cache entry is written. Otherwise waits + /// for the holder, at most this gate's `wait`, and returns `None`; the + /// caller then rechecks its cache and, if the holder failed, was + /// cancelled or is still reading, reads on its own without the gate. A + /// read that hangs therefore stalls each waiter for at most `wait`. + pub(crate) async fn turn(&self, key: K) -> Option> { + let gate = self.gates.get_with(key, || Arc::new(Mutex::new(()))); + match Arc::clone(&gate).try_lock_owned() { + Ok(guard) => Some(guard), + Err(_) => { + drop(tokio::time::timeout(self.wait, gate.lock()).await); + None + } + } + } + + /// The gate for `key`, if one exists. + #[cfg(test)] + pub(crate) fn gate(&self, key: &K) -> Option>> { + self.gates.get(key) + } +} + +#[cfg(test)] +mod tests { + use super::SharedReads; + use std::sync::Arc; + use std::time::Duration; + + #[tokio::test] + async fn one_caller_reads_while_the_rest_wait_for_it() { + let reads = Arc::new(SharedReads::::new(16, Duration::from_secs(5))); + let leader = reads.turn(1).await; + assert!(leader.is_some(), "the first caller takes the gate"); + + // A second key has its own gate. + assert!(reads.turn(2).await.is_some()); + + let waiter = { + let reads = Arc::clone(&reads); + tokio::spawn(async move { reads.turn(1).await }) + }; + tokio::task::yield_now().await; + assert!(!waiter.is_finished(), "a waiter blocks while the read runs"); + drop(leader); + assert!( + waiter.await.expect("join").is_none(), + "a waiter rechecks the cache instead of becoming a reader" + ); + + // Once the gate is free the next caller reads again. + assert!(reads.turn(1).await.is_some()); + } + + #[tokio::test(start_paused = true)] + async fn a_waiter_stops_waiting_for_a_hung_read() { + let reads = SharedReads::::new(16, Duration::from_secs(4)); + let _hung = reads.turn(1).await; + let started = tokio::time::Instant::now(); + assert!(reads.turn(1).await.is_none()); + assert_eq!(started.elapsed(), Duration::from_secs(4)); + } +} diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index a807b4a53e..11abba1596 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -1334,35 +1334,46 @@ impl Default for ConnectionManager { } /// How long a pod uses a community's cached open set after the database read -/// that produced it began. Cross-pod invalidation is best-effort, so this -/// bounds how long a channel that left the open set elsewhere stays granted -/// here. +/// that produced it began (its connection checkout, see [`buzz_db::ReadAt`]). +/// Cross-pod invalidation is best-effort, so this bounds how long a channel +/// that left the open set elsewhere stays granted here. pub const OPEN_CHANNELS_MAX_AGE: std::time::Duration = std::time::Duration::from_secs(1); /// How long a pod serves a cached ban/timeout row after the database read -/// that produced it began (see [`AppState::restriction_cache`]). +/// that produced it began, at its connection checkout (see +/// [`AppState::restriction_cache`]). pub const RESTRICTION_CACHE_MAX_AGE: std::time::Duration = std::time::Duration::from_secs(1); /// How long a pod serves a cached host → community mapping after the -/// database read that produced it began (see [`AppState::tenant_host_cache`]). +/// database read that produced it began, at its connection checkout (see +/// [`AppState::tenant_host_cache`]). pub const TENANT_HOST_CACHE_MAX_AGE: std::time::Duration = std::time::Duration::from_secs(1); -/// A cached restriction row and when the read that produced it began. +/// How long a cache miss waits for a shared read's statement, beyond the +/// writer pool's acquire timeout that its holder may spend waiting for a +/// connection (see [`crate::shared_reads::SharedReads::new`]). A holder still +/// waiting for a connection either gets one or times out within the acquire +/// timeout, so only a statement that runs longer than this makes waiters +/// give up and read on their own. +pub const SHARED_READ_STATEMENT_WAIT: std::time::Duration = std::time::Duration::from_secs(1); + +/// A cached restriction row and when the read that produced it checked out +/// its connection. #[derive(Debug, Clone)] pub struct CachedRestriction { /// The restriction row as read. pub state: buzz_db::moderation::RestrictionState, - /// When the database read that produced it began. + /// When the database read that produced it checked out its connection. pub read_at: Instant, } /// A community's listed open channel IDs and when the read that listed them -/// began. +/// checked out its connection. #[derive(Debug)] pub struct OpenChannels { /// The community's listed open channel IDs. pub ids: Vec, - /// When the database read that listed them began. + /// When the database read that listed them checked out its connection. pub read_at: Instant, } @@ -1454,12 +1465,10 @@ pub struct AppState { /// Listed open channel IDs per community, the half of every reader's /// accessible set that does not depend on the reader. Invalidated with /// the community's accessible channels, and used only for - /// [`OPEN_CHANNELS_MAX_AGE`] after its database read began. + /// [`OPEN_CHANNELS_MAX_AGE`] after its read's connection checkout. pub open_channels_cache: Arc>>, - /// Per-community gate held while a pod reads a community's open set, so - /// concurrent misses wait for that read instead of each issuing their own - /// (see [`AppState::get_accessible_channel_ids_cached`]). - pub open_channels_reads: Arc>>>, + /// Lets concurrent misses on [`Self::open_channels_cache`] share one read. + pub(crate) open_channels_reads: Arc>, /// Per-community channel visibility string, used to gate the private-channel fan-out /// access check so open channels stay zero-cost. Invalidated on a flip. pub channel_visibility_cache: Arc>, @@ -1553,22 +1562,27 @@ pub struct AppState { /// lifts the moment `muted_until` passes. (`banned` is computed at read /// time, so an expiring ban can outlive its expiry by up to /// [`RESTRICTION_CACHE_MAX_AGE`].) Key: (community, pubkey bytes). An entry - /// is served only for [`RESTRICTION_CACHE_MAX_AGE`] after its read began, - /// which is the only staleness bound: on every pod and for every - /// restriction writer, a cached reader sees a ban/timeout change within - /// that long. As a best effort, signed moderation commands (9040–9043) - /// also drop the target's own entry on the pod that handles them; owned - /// agents' entries, other pods, and the admin/report paths just age out. + /// is served only for [`RESTRICTION_CACHE_MAX_AGE`] after its read's + /// connection checkout, which is the only staleness bound: on every pod + /// and for every restriction writer, a cached reader sees a ban/timeout + /// change within that long. As a best effort, signed moderation commands + /// (9040–9043) also drop the target's own entry on the pod that handles + /// them; owned agents' entries, other pods, and the admin/report paths + /// just age out. /// Socket admission and moderation-command actors read fresh: a grant that /// outlives the bound must not rest on a cached verdict. #[allow(clippy::type_complexity)] pub restriction_cache: Arc), CachedRestriction>>, + /// Lets concurrent misses on [`Self::restriction_cache`] share one read. + pub(crate) restriction_reads: Arc)>>, /// Normalized request host → community, from the row-zero host binding /// ([`crate::tenant::bind_community`]). Only mapped hosts are cached, and - /// an entry is served only for [`TENANT_HOST_CACHE_MAX_AGE`] after its read - /// began: a host remapped, archived or deleted elsewhere resolves to its - /// old community for at most that long. + /// an entry is served only for [`TENANT_HOST_CACHE_MAX_AGE`] after its + /// read's connection checkout: a host remapped, archived or deleted + /// elsewhere resolves to its old community for at most that long. pub tenant_host_cache: Arc>, + /// Lets concurrent misses on [`Self::tenant_host_cache`] share one read. + pub(crate) tenant_host_reads: Arc>, /// Runtime conformance tracer. Production binds [`crate::conformance::NoopTracer`] /// (zero cost). Conformance tests bind [`crate::conformance::JsonlTracer`] to @@ -1736,6 +1750,8 @@ impl AppState { let audit_enabled = audit_arc.is_some(); // Build NIP-FI components before moving config into the state Arc. let (nip_fi_verifier, nip_fi_jwks_source) = build_nip_fi_components(&config); + let shared_read_wait = + db.pool().options().get_acquire_timeout() + SHARED_READ_STATEMENT_WAIT; let state = Self { config: Arc::new(config), db, @@ -1789,12 +1805,10 @@ impl AppState { .time_to_live(std::time::Duration::from_secs(10)) .build(), ), - open_channels_reads: Arc::new( - moka::sync::Cache::builder() - .max_capacity(10_000) - .time_to_idle(std::time::Duration::from_secs(60)) - .build(), - ), + open_channels_reads: Arc::new(crate::shared_reads::SharedReads::new( + 10_000, + shared_read_wait, + )), channel_visibility_cache: Arc::new( moka::sync::Cache::builder() .max_capacity(10_000) @@ -1860,12 +1874,20 @@ impl AppState { .time_to_live(std::time::Duration::from_secs(10)) .build(), ), + restriction_reads: Arc::new(crate::shared_reads::SharedReads::new( + 100_000, + shared_read_wait, + )), tenant_host_cache: Arc::new( moka::sync::Cache::builder() .max_capacity(10_000) .time_to_live(std::time::Duration::from_secs(10)) .build(), ), + tenant_host_reads: Arc::new(crate::shared_reads::SharedReads::new( + 10_000, + shared_read_wait, + )), // Default to NoopTracer: production builds pay zero cost. // Conformance tests overwrite this with a JsonlTracer after // construction (see test helpers in @@ -1990,34 +2012,45 @@ impl AppState { /// `moderation_restriction_state` served from [`Self::restriction_cache`] /// for [`RESTRICTION_CACHE_MAX_AGE`] after the read that produced it - /// began. For per-request gates (HTTP requests, publishes, ephemeral - /// pulses) whose grant ends with the request. Errors are not cached. + /// checked out its connection. For per-request gates (HTTP requests, + /// publishes, ephemeral pulses) whose grant ends with the request. Errors + /// are not cached. + /// + /// Concurrent misses for one pubkey share a read through + /// [`Self::restriction_reads`]. pub async fn restriction_state_cached( &self, community_id: CommunityId, pubkey: &[u8], ) -> Result { let key = (community_id, pubkey.to_vec()); - if let Some(cached) = self.restriction_cache.get(&key) { - if cached.read_at.elapsed() < RESTRICTION_CACHE_MAX_AGE { - metrics::counter!("buzz_restriction_cache_hits_total").increment(1); - return Ok(cached.state); - } + let fresh = || { + self.restriction_cache + .get(&key) + .filter(|cached| cached.read_at.elapsed() < RESTRICTION_CACHE_MAX_AGE) + }; + if let Some(cached) = fresh() { + metrics::counter!("buzz_restriction_cache_hits_total").increment(1); + return Ok(cached.state); + } + let _turn = self.restriction_reads.turn(key.clone()).await; + if let Some(cached) = fresh() { + metrics::counter!("buzz_restriction_cache_hits_total").increment(1); + return Ok(cached.state); } metrics::counter!("buzz_restriction_cache_misses_total").increment(1); - let read_at = Instant::now(); - let state = self + let read = self .db - .moderation_restriction_state(community_id, pubkey) + .moderation_restriction_state_read_at(community_id, pubkey) .await?; self.restriction_cache.insert( key, CachedRestriction { - state: state.clone(), - read_at, + state: read.value.clone(), + read_at: read.read_at, }, ); - Ok(state) + Ok(read.value) } /// Best-effort drop of a pubkey's cached restriction row on this pod. @@ -2532,10 +2565,10 @@ impl AppState { /// /// The open half is shared by every reader in the community and its /// cross-pod invalidation is best-effort, so a cached set is served only - /// for [`OPEN_CHANNELS_MAX_AGE`] after its read began: a channel that left - /// the open set on another pod is granted from the cache for at most that - /// long. A caller whose own read straddles a revocation gets what that - /// read saw, as with any uncached read. + /// for [`OPEN_CHANNELS_MAX_AGE`] after its read's connection checkout: a + /// channel that left the open set on another pod is granted from the + /// cache for at most that long. A caller whose own read straddles a + /// revocation gets what that read saw, as with any uncached read. pub async fn get_accessible_channel_ids_cached( &self, community_id: CommunityId, @@ -2566,9 +2599,9 @@ impl AppState { /// gate in [`Self::open_channels_reads`] and reads; the rest wait for it /// to finish and use its set. A waiter whose leader failed or was /// cancelled reads on its own rather than queueing behind another leader. - /// A waiter also stops waiting after [`OPEN_CHANNELS_MAX_AGE`] and reads - /// on its own: the leader's set would be stale on arrival by then, so a - /// leader whose read hangs stalls only its own request. + /// A waiter also stops waiting once the leader has had the writer pool's + /// acquire timeout plus [`SHARED_READ_STATEMENT_WAIT`], and reads on its + /// own, so a leader whose read hangs stalls its waiters at most that long. async fn open_channels( &self, community_id: CommunityId, @@ -2582,29 +2615,25 @@ impl AppState { metrics::counter!("buzz_open_channels_cache_hits_total").increment(1); return Ok(open); } - let gate = self - .open_channels_reads - .get_with(community_id, || Arc::new(tokio::sync::Mutex::new(()))); - let _leader = match gate.try_lock() { - Ok(guard) => Some(guard), - Err(_) => { - drop(tokio::time::timeout(OPEN_CHANNELS_MAX_AGE, gate.lock()).await); - if let Some(open) = fresh() { - metrics::counter!("buzz_open_channels_cache_hits_total").increment(1); - return Ok(open); - } - None - } - }; + let _turn = self.open_channels_reads.turn(community_id).await; + if let Some(open) = fresh() { + metrics::counter!("buzz_open_channels_cache_hits_total").increment(1); + return Ok(open); + } let reason = if self.open_channels_cache.contains_key(&community_id) { "expired" } else { "absent" }; metrics::counter!("buzz_open_channels_cache_misses_total", "reason" => reason).increment(1); - let read_at = Instant::now(); - let ids = self.db.get_listed_open_channel_ids(community_id).await?; - let open = Arc::new(OpenChannels { ids, read_at }); + let read = self + .db + .get_listed_open_channel_ids_read_at(community_id) + .await?; + let open = Arc::new(OpenChannels { + ids: read.value, + read_at: read.read_at, + }); self.open_channels_cache.insert(community_id, open.clone()); Ok(open) } @@ -3686,13 +3715,149 @@ pub(crate) mod tests { use super::*; use buzz_core::channel::{ChannelType, ChannelVisibility}; + /// Starts `misses` concurrent runs of `read` while the pool's only + /// connection is held for `wait`, checks that every one is still + /// waiting, then releases the connection. + async fn behind_held_connection( + pool: &sqlx::PgPool, + wait: std::time::Duration, + misses: usize, + read: impl Fn() -> F, + ) -> Vec + where + T: Send + 'static, + F: std::future::Future + Send + 'static, + { + let held = pool.acquire().await.unwrap(); + let reads = (0..misses) + .map(|_| tokio::spawn(read())) + .collect::>(); + tokio::time::sleep(wait).await; + assert!( + reads.iter().all(|read| !read.is_finished()), + "every miss waits for the connection" + ); + drop(held); + let mut results = Vec::with_capacity(misses); + for read in reads { + results.push(read.await.expect("join")); + } + results + } + + /// The sum of every counter named `name` in `snapshotter`. + fn counted(snapshotter: &metrics_util::debugging::Snapshotter, name: &str) -> u64 { + snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter(|(key, ..)| key.key().name() == name) + .map(|(.., value)| match value { + metrics_util::debugging::DebugValue::Counter(n) => n, + _ => 0, + }) + .sum() + } + + /// Concurrent misses on each cache that bounds staleness from its + /// read, queued behind a held pool connection for longer than the + /// cache's max age, share one read, and its entry still lands fresh: + /// waiters wait out the leader's pool wait, and the entry is stamped + /// at connection checkout, not when the miss began. + /// Mutations: stamp any of the three entries before the pool wait, + /// or let waiters give up after the cache's max age → RED. + #[tokio::test(flavor = "current_thread")] + #[ignore = "requires Postgres"] + async fn cache_misses_behind_a_held_pool_share_one_fresh_read() { + use crate::tenant::HostResolver as _; + + let recorder = metrics_util::debugging::DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _recorder = metrics::set_default_local_recorder(&recorder); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(std::time::Duration::from_secs(30)) + .connect(&crate::test_support::database_url()) + .await + .unwrap(); + let state = test_state_with_database_pool(pool.clone()).await; + let host = format!("pool-wait-{}.local", Uuid::new_v4()); + let community = state + .db + .ensure_configured_community(&host) + .await + .unwrap() + .id; + let pubkey = vec![6u8; 32]; + let wait = RESTRICTION_CACHE_MAX_AGE + .max(TENANT_HOST_CACHE_MAX_AGE) + .max(OPEN_CHANNELS_MAX_AGE) + .max(SHARED_READ_STATEMENT_WAIT) + + std::time::Duration::from_millis(500); + let misses = 8; + + behind_held_connection(&pool, wait, misses, || { + let (reader, key) = (Arc::clone(&state), pubkey.clone()); + async move { reader.restriction_state_cached(community, &key).await } + }) + .await + .into_iter() + .for_each(|read| { + read.expect("restriction read"); + }); + assert_eq!( + counted(&snapshotter, "buzz_restriction_cache_misses_total"), + 1 + ); + let restriction = state + .restriction_cache + .get(&(community, pubkey)) + .expect("restriction entry"); + assert!(restriction.read_at.elapsed() < RESTRICTION_CACHE_MAX_AGE); + + let mapped = behind_held_connection(&pool, wait, misses, || { + let (reader, key) = (Arc::clone(&state), host.clone()); + async move { reader.resolve_host(&key).await } + }) + .await; + assert!(mapped + .into_iter() + .all(|mapped| mapped.expect("host read") == Some(community))); + assert_eq!( + counted(&snapshotter, "buzz_tenant_host_cache_misses_total"), + 1 + ); + let (_, read_at) = state.tenant_host_cache.get(&host).expect("host entry"); + assert!(read_at.elapsed() < TENANT_HOST_CACHE_MAX_AGE); + + behind_held_connection(&pool, wait, misses, || { + let reader = Arc::clone(&state); + async move { reader.open_channels(community).await } + }) + .await + .into_iter() + .for_each(|read| { + read.expect("open-set read"); + }); + assert_eq!( + counted(&snapshotter, "buzz_open_channels_cache_misses_total"), + 1 + ); + let open = state + .open_channels_cache + .get(&community) + .expect("open-set entry"); + assert!(open.read_at.elapsed() < OPEN_CHANNELS_MAX_AGE); + } + /// Open-set reads park on a lock on `channels`, where the test counts /// them; waiters are counted as they arrive by the clones of the /// community's gate they hold. A leader's eight waiters share its /// set. When the leader is cancelled, its orphans each pass the gate /// and read, none holding it across its own read. When the leader's - /// read hangs, its waiters stop waiting after - /// [`OPEN_CHANNELS_MAX_AGE`] and each read on their own. + /// read hangs, its waiters stop waiting after the pool's acquire + /// timeout plus [`SHARED_READ_STATEMENT_WAIT`] and each read on their + /// own. /// Mutations: drop the waiter path (distinct sets in the first /// phase), hold the gate across a waiter's own read (the gate never /// frees in the second), or wait on the gate with no limit (the third @@ -3703,7 +3868,10 @@ pub(crate) mod tests { let database_url = crate::test_support::database_url(); let state = test_state_with_database_url_and_pool_options( &database_url, - sqlx::postgres::PgPoolOptions::new().max_connections(16), + // Bounds how long the hung leader's waiters wait for it. + sqlx::postgres::PgPoolOptions::new() + .max_connections(16) + .acquire_timeout(std::time::Duration::from_secs(1)), ) .await; let admin = sqlx::PgPool::connect(&database_url).await.unwrap(); @@ -3765,7 +3933,7 @@ pub(crate) mod tests { parks(1).await; let gate = state .open_channels_reads - .get(&community) + .gate(&community) .expect("the leader's gate"); let base = Arc::strong_count(&gate); let waiters = misses(8); @@ -4241,10 +4409,11 @@ pub(crate) mod tests { .collect::>() }; - let gate = state + let leader = state .open_channels_reads - .get_with(community, || Arc::new(tokio::sync::Mutex::new(()))); - let leader = gate.lock().await; + .turn(community) + .await + .expect("no read in flight"); let shared = waiters(&state); tokio::task::yield_now().await; state.open_channels_cache.insert( @@ -4261,6 +4430,98 @@ pub(crate) mod tests { } } + /// Restriction and tenant-host misses that arrive while a read of the + /// same key is in flight wait for it and use its entry. The database is + /// unreachable, so any read of a waiter's own fails. + /// Mutation: drop either waiter path → RED. + #[tokio::test] + async fn concurrent_restriction_and_host_misses_share_the_in_flight_read() { + use crate::tenant::HostResolver as _; + + let state = + test_state_with_unreachable_database_url("postgres://buzz@127.0.0.1:1/buzz").await; + let community = CommunityId::from_uuid(Uuid::from_u128(0xAAAA)); + let pubkey = vec![5u8; 32]; + let host = "shared-read.local".to_owned(); + let muted = buzz_db::moderation::RestrictionState { + banned: false, + own_banned: false, + muted_until: Some(chrono::Utc::now() + chrono::Duration::hours(1)), + }; + // On this current-thread runtime a waiter's first poll runs without + // interruption from its count to its first await, the wait on the + // leader's gate. Once all have counted, all are waiting at the gate, + // so none can take the initial cache-hit path when the cache is + // filled. (The gate's `Arc` count is no proof: the cache may hold a + // transient clone of its own.) + async fn enrolled(started: &std::sync::atomic::AtomicUsize, waiters: usize) { + let start = Instant::now(); + while started.load(std::sync::atomic::Ordering::SeqCst) < waiters { + // A watchdog only; timing is not part of what is checked. + assert!( + start.elapsed() < std::time::Duration::from_secs(10), + "waiters did not reach the gate" + ); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + } + fn spawn_waiters( + started: &Arc, + read: impl Fn() -> F, + ) -> Vec> + where + F: std::future::Future + Send + 'static, + { + (0..8) + .map(|_| { + let (started, read) = (Arc::clone(started), read()); + tokio::spawn(async move { + started.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + read.await + }) + }) + .collect() + } + + let leader = state + .restriction_reads + .turn((community, pubkey.clone())) + .await + .expect("no read in flight"); + let started = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let waiters = spawn_waiters(&started, || { + let (state, pubkey) = (Arc::clone(&state), pubkey.clone()); + async move { state.restriction_state_cached(community, &pubkey).await } + }); + enrolled(&started, waiters.len()).await; + state.seed_restriction((community, pubkey.clone()), muted.clone()); + drop(leader); + for waiter in waiters { + let read = waiter.await.expect("join").expect("the leader's row"); + assert_eq!(read, muted); + } + + let leader = state + .tenant_host_reads + .turn(host.clone()) + .await + .expect("no read in flight"); + let started = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let waiters = spawn_waiters(&started, || { + let (state, host) = (Arc::clone(&state), host.clone()); + async move { state.resolve_host(&host).await } + }); + enrolled(&started, waiters.len()).await; + state + .tenant_host_cache + .insert(host.clone(), (community, Instant::now())); + drop(leader); + for waiter in waiters { + let read = waiter.await.expect("join").expect("the leader's mapping"); + assert_eq!(read, Some(community)); + } + } + #[tokio::test] async fn channel_deleted_invalidation_is_scoped_to_community() { let state = test_state().await; diff --git a/crates/buzz-relay/src/tenant.rs b/crates/buzz-relay/src/tenant.rs index 8038ec3583..b3b52ee931 100644 --- a/crates/buzz-relay/src/tenant.rs +++ b/crates/buzz-relay/src/tenant.rs @@ -14,7 +14,7 @@ //! (`Db::resolve_host`); the relay depends on the trait, not the query, so the //! binding is testable without a database. -use buzz_core::tenant::{normalize_host, CommunityId, TenantContext}; +use buzz_core::tenant::{normalize_host, CommunityId, TenantContext, MAX_HOST_LEN}; /// Resolves a normalized connection host to its community, or `None` when the /// host maps to no community on this deployment. @@ -81,7 +81,10 @@ pub async fn bind_community( // bind to a misconfigured empty-host community. Reuse `UnmappedHost` (not a // distinct variant) so the rejection is byte-identical to any other unmapped // host — an unauthenticated caller cannot probe for an empty-host row. - if host.is_empty() { + // A host longer than any `communities.host` can hold maps to nothing + // either; rejecting it here keeps an unauthenticated caller's arbitrarily + // long host out of the resolver's caches and off the database. + if host.is_empty() || host.chars().count() > MAX_HOST_LEN { return Err(BindError::UnmappedHost); } match resolver.resolve_host(&host).await { @@ -149,8 +152,9 @@ impl HostResolver for buzz_db::Db { /// [`crate::state::AppState::tenant_host_cache`]. A connection's handshake and /// its follow-up HTTP requests resolve the same host many times a second, so /// a mapped host is reread at most once per -/// [`crate::state::TENANT_HOST_CACHE_MAX_AGE`]. Unmapped hosts and lookup -/// errors are never cached. +/// [`crate::state::TENANT_HOST_CACHE_MAX_AGE`], and concurrent misses share +/// one read through [`crate::state::AppState::tenant_host_reads`]. Unmapped +/// hosts and lookup errors are never cached. impl HostResolver for crate::state::AppState { type Error = buzz_db::DbError; @@ -158,18 +162,33 @@ impl HostResolver for crate::state::AppState { &self, normalized_host: &str, ) -> Result, Self::Error> { - if let Some((community, read_at)) = self.tenant_host_cache.get(normalized_host) { - if read_at.elapsed() < crate::state::TENANT_HOST_CACHE_MAX_AGE { - metrics::counter!("buzz_tenant_host_cache_hits_total").increment(1); - return Ok(Some(community)); - } + let fresh = || { + self.tenant_host_cache + .get(normalized_host) + .filter(|(_, read_at)| read_at.elapsed() < crate::state::TENANT_HOST_CACHE_MAX_AGE) + .map(|(community, _)| community) + }; + if let Some(community) = fresh() { + metrics::counter!("buzz_tenant_host_cache_hits_total").increment(1); + return Ok(Some(community)); + } + let _turn = self + .tenant_host_reads + .turn(normalized_host.to_owned()) + .await; + if let Some(community) = fresh() { + metrics::counter!("buzz_tenant_host_cache_hits_total").increment(1); + return Ok(Some(community)); } metrics::counter!("buzz_tenant_host_cache_misses_total").increment(1); - let read_at = std::time::Instant::now(); - let community = self.db.resolve_host(normalized_host).await?; + let read = self + .db + .lookup_community_by_host_read_at(normalized_host) + .await?; + let community = read.value.map(|record| record.id); if let Some(community) = community { self.tenant_host_cache - .insert(normalized_host.to_owned(), (community, read_at)); + .insert(normalized_host.to_owned(), (community, read.read_at)); } Ok(community) } @@ -361,6 +380,34 @@ mod tests { } } + /// A host longer than `communities.host` can hold fails closed as + /// unmapped before the request-path resolver sees it: no cache or gate + /// entry and no lookup. The database is unreachable, so a lookup would + /// fail as `Lookup`, as it does for a host at the limit. + /// Mutation: drop the length check, or count bytes past the column's + /// characters → RED. + #[tokio::test] + async fn an_overlong_host_fails_closed_before_the_resolver() { + let state = crate::state::tests::test_state_with_unreachable_database_url( + "postgres://buzz@127.0.0.1:1/buzz", + ) + .await; + let overlong = "a".repeat(MAX_HOST_LEN + 1); + assert!(matches!( + bind_community(&*state, &overlong).await, + Err(BindError::UnmappedHost) + )); + assert!(state.tenant_host_reads.gate(&overlong).is_none()); + + // At the limit, even in multibyte characters, the host is looked up. + for longest in ["a".repeat(MAX_HOST_LEN), "é".repeat(MAX_HOST_LEN)] { + assert!(matches!( + bind_community(&*state, &longest).await, + Err(BindError::Lookup(_)) + )); + } + } + /// The request-path resolver serves a mapped host from the cache only /// while the entry is younger than `TENANT_HOST_CACHE_MAX_AGE`; past /// that it rereads, and a failed reread fails closed. The database is diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index a8a4aea48d..92951b95a5 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -378,8 +378,12 @@ run_unit_tests() { run_test_step "buzz-relay read cache tests" \ cargo test -p buzz-relay --lib -- --exact --nocapture \ state::tests::admitted_index_follows_every_key_change \ + shared_reads::tests::one_caller_reads_while_the_rest_wait_for_it \ + shared_reads::tests::a_waiter_stops_waiting_for_a_hung_read \ state::tests::concurrent_open_channel_misses_share_the_in_flight_read \ + state::tests::concurrent_restriction_and_host_misses_share_the_in_flight_read \ tenant::tests::app_state_resolver_serves_fresh_cached_hosts_only \ + tenant::tests::an_overlong_host_fails_closed_before_the_resolver \ state::tests::disconnect_drops_the_cached_restriction_row run_test_step "buzz-relay router tests" \