//! Prometheus metrics: recorder setup, upkeep task, and HTTP middleware. //! //! ```text //! ┌──────────────────────────────────────────────────────────┐ //! │ metrics-rs facade (metrics::counter!, histogram!, etc.) │ //! │ ↓ │ //! │ PrometheusBuilder → HTTP listener on :9102 │ //! │ ↓ │ //! │ GET /metrics → Prometheus text format │ //! └──────────────────────────────────────────────────────────┘ //! ``` //! //! Framework metrics (`http_requests_total`, `http_request_latency_ms`) are //! recorded by [`track_metrics`] middleware on the app router. Buzz-specific //! metrics are recorded inline at their call sites. use std::time::{Duration, Instant}; use axum::{ extract::{MatchedPath, Request}, middleware::Next, response::Response, }; use metrics_exporter_prometheus::{BuildError, Matcher, PrometheusBuilder}; use metrics_util::MetricKindMask; /// The fixed method label for the WebSocket NIP-42 challenge lifecycle. pub(crate) const AUTH_METHOD_NIP42: &str = "nip42"; /// Bounded terminal results for one WebSocket authentication attempt. /// /// Keep this enum free of user-controlled or error-string values. Rollout /// analysis relies on these labels having a small, stable cardinality. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum AuthOutcome { Success, Invalid, Banned, BanCheckError, AllowlistCheckError, AllowlistDenied, RelayMembershipCheckError, NotRelayMember, /// NIP-FI key pairing mismatch: the NIP-42 key differs from the asserted key. PairingMismatch, Timeout, Disconnect, Shutdown, } impl AuthOutcome { pub(crate) const ALL: [Self; 12] = [ Self::Success, Self::Invalid, Self::Banned, Self::BanCheckError, Self::AllowlistCheckError, Self::AllowlistDenied, Self::RelayMembershipCheckError, Self::NotRelayMember, Self::PairingMismatch, Self::Timeout, Self::Disconnect, Self::Shutdown, ]; pub(crate) const fn as_str(self) -> &'static str { match self { Self::Success => "success", Self::Invalid => "invalid", Self::Banned => "banned", Self::BanCheckError => "ban_check_error", Self::AllowlistCheckError => "allowlist_check_error", Self::AllowlistDenied => "allowlist_denied", Self::RelayMembershipCheckError => "relay_membership_check_error", Self::NotRelayMember => "not_relay_member", Self::PairingMismatch => "pairing_mismatch", Self::Timeout => "timeout", Self::Disconnect => "disconnect", Self::Shutdown => "shutdown", } } } /// Bounded state of an AUTH frame received after the issued challenge had /// already reached a terminal. These frames are protocol misuse, not recovery /// lifecycles, and therefore stay outside rollout-gating attempts/outcomes. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum AuthPostTerminalState { Authenticated, Failed, } impl AuthPostTerminalState { pub(crate) const ALL: [Self; 2] = [Self::Authenticated, Self::Failed]; pub(crate) const fn as_str(self) -> &'static str { match self { Self::Authenticated => "authenticated", Self::Failed => "failed", } } } /// HTTP latency buckets (milliseconds) — only for `http_request_latency_ms`. const LATENCY_BUCKETS_MS: [f64; 11] = [ 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, 5000.0, 10000.0, ]; /// Seconds-scale buckets for internal processing histograms (event, search, audit). const DURATION_BUCKETS_S: [f64; 10] = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0]; /// Readiness buckets concentrate resolution near the two-second failure budget. const READINESS_DURATION_BUCKETS_S: [f64; 15] = [ 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, ]; /// Pool checkout buckets: dense around normal sub-100ms waits, with explicit /// coverage of the reader's 150ms and writer's default three-second budgets. const DB_POOL_ACQUIRE_DURATION_BUCKETS_S: [f64; 9] = [0.001, 0.005, 0.01, 0.025, 0.05, 0.15, 0.5, 1.0, 3.0]; const DB_POOL_ACQUIRE_DURATION_UNIT: metrics::Unit = metrics::Unit::Seconds; /// Writer pool/session buckets preserve sub-millisecond setup while retaining /// seconds-scale failures in the final bucket. const DB_CONNECTION_STEP_DURATION_BUCKETS_S: [f64; 10] = [ 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.15, 0.5, 1.0, 3.0, ]; /// Seconds-scale buckets for Git hydration and pack streams. const GIT_DURATION_BUCKETS_S: [f64; 13] = [ 0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, ]; /// Mesh dial and reconcile-pass buckets. A pass dials peers serially, so a /// large or stale peer table can stretch it to minutes; the default /// five-second ceiling would hide exactly the stall these exist to show. const MESH_DURATION_BUCKETS_S: [f64; 14] = [ 0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, ]; /// Byte buckets for hydrated repositories and streamed clone/fetch responses. const GIT_BYTES_BUCKETS: [f64; 9] = [ 0.0, 64.0 * 1024.0, 1024.0 * 1024.0, 10.0 * 1024.0 * 1024.0, 50.0 * 1024.0 * 1024.0, 100.0 * 1024.0 * 1024.0, 250.0 * 1024.0 * 1024.0, 500.0 * 1024.0 * 1024.0, 1024.0 * 1024.0 * 1024.0, ]; /// Pack-count buckets bounded by the manifest's maximum pack count. const GIT_PACK_BUCKETS: [f64; 9] = [0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0]; /// Integer-count buckets for fan-out recipient histograms. const FANOUT_BUCKETS: [f64; 9] = [0.0, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 500.0, 1000.0]; fn configured_prometheus_builder(gauge_idle_timeout_secs: u64) -> PrometheusBuilder { PrometheusBuilder::new() // Remove gauge series that the relay intentionally stops emitting. .idle_timeout( MetricKindMask::GAUGE, Some(Duration::from_secs(gauge_idle_timeout_secs)), ) // Per-metric buckets: ms for HTTP latency, seconds for internal processing. .set_buckets_for_metric( Matcher::Full("http_request_latency_ms".to_owned()), &LATENCY_BUCKETS_MS, ) .expect("valid ms bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_git_hydrate_seconds".to_owned()), &GIT_DURATION_BUCKETS_S, ) .expect("valid git hydration duration bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_git_upload_pack_stream_seconds".to_owned()), &GIT_DURATION_BUCKETS_S, ) .expect("valid git stream duration bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_git_pack_cache_populate_seconds".to_owned()), &GIT_DURATION_BUCKETS_S, ) .expect("valid git cache population duration bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_git_pack_cache_population_wait_seconds".to_owned()), &GIT_DURATION_BUCKETS_S, ) .expect("valid git cache population wait bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_git_pack_compaction_seconds".to_owned()), &GIT_DURATION_BUCKETS_S, ) .expect("valid git compaction duration bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_readiness_check_duration_seconds".to_owned()), &READINESS_DURATION_BUCKETS_S, ) .expect("valid readiness duration bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_db_pool_acquire_duration_seconds".to_owned()), &DB_POOL_ACQUIRE_DURATION_BUCKETS_S, ) .expect("valid DB pool acquisition duration bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_db_connection_step_duration_seconds".to_owned()), &DB_CONNECTION_STEP_DURATION_BUCKETS_S, ) .expect("valid DB connection-step duration bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_git_hydrate_bytes".to_owned()), &GIT_BYTES_BUCKETS, ) .expect("valid git hydration byte bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_git_upload_pack_stream_bytes".to_owned()), &GIT_BYTES_BUCKETS, ) .expect("valid git stream byte bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_git_pack_compaction_bytes".to_owned()), &GIT_BYTES_BUCKETS, ) .expect("valid git compaction byte bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_git_hydrate_packs".to_owned()), &GIT_PACK_BUCKETS, ) .expect("valid git pack-count bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_git_pack_compaction_packs_before".to_owned()), &GIT_PACK_BUCKETS, ) .expect("valid git compaction input pack-count bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_git_pack_compaction_packs_after".to_owned()), &GIT_PACK_BUCKETS, ) .expect("valid git compaction output pack-count bucket boundaries") .set_buckets_for_metric( Matcher::Full(buzz_relay_mesh::telemetry::DIAL_DURATION_SECONDS.to_owned()), &MESH_DURATION_BUCKETS_S, ) .expect("valid mesh dial bucket boundaries") .set_buckets_for_metric( Matcher::Full(buzz_relay_mesh::telemetry::RECONCILE_PASS_SECONDS.to_owned()), &MESH_DURATION_BUCKETS_S, ) .expect("valid mesh reconcile pass bucket boundaries") .set_buckets_for_metric(Matcher::Suffix("_seconds".to_owned()), &DURATION_BUCKETS_S) .expect("valid seconds bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_fanout_recipients".to_owned()), &FANOUT_BUCKETS, ) .expect("valid fanout bucket boundaries") } /// A bounded class of metrics installation failure. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum MetricsInstallFailure { /// The Prometheus listener could not bind. Bind, /// Another component already installed a global recorder. RecorderConflict, /// The exporter could not be built for another reason. ExporterBuild, } /// An error returned while installing Prometheus metrics. #[derive(Debug, thiserror::Error)] pub enum MetricsInstallError { /// Prometheus exporter construction failed. #[error("failed to build Prometheus exporter: {0}")] Build(#[source] BuildError), /// Another component already installed the process-global recorder. #[error("the global metrics recorder is already installed")] RecorderConflict, } impl MetricsInstallError { /// Return the secret-safe lifecycle classification. pub const fn failure(&self) -> MetricsInstallFailure { match self { Self::Build(BuildError::FailedToCreateHTTPListener(_)) => MetricsInstallFailure::Bind, Self::Build(_) => MetricsInstallFailure::ExporterBuild, Self::RecorderConflict => MetricsInstallFailure::RecorderConflict, } } } /// Try to install the global metrics recorder and spawn the Prometheus HTTP exporter. /// /// `build()` returns the recorder + exporter future and internally spawns /// the upkeep task, so no separate upkeep call is needed. /// /// Must be called from within a Tokio runtime. /// Listener and global-recorder failures are returned rather than panicking. /// A later exporter exit remains detached from relay service; external scrape /// coverage is authoritative for exporter availability. pub fn try_install(port: u16, gauge_idle_timeout_secs: u64) -> Result<(), MetricsInstallError> { let (recorder, exporter) = configured_prometheus_builder(gauge_idle_timeout_secs) .with_http_listener(([0, 0, 0, 0], port)) .build() .map_err(MetricsInstallError::Build)?; metrics::set_global_recorder(recorder) .map_err(|_error| MetricsInstallError::RecorderConflict)?; describe_readiness_metrics(); describe_community_admission_metrics(); describe_db_pool_metrics(); describe_auth_metrics(); crate::startup_steps::describe_metrics(); buzz_relay_mesh::telemetry::describe_metrics(); initialize_auth_metric_series(); initialize_partition_audit_metric_series(); tokio::spawn(exporter); Ok(()) } /// Install the global metrics recorder and spawn the Prometheus HTTP exporter. /// /// This compatibility entry point preserves the original panic-on-failure API. /// New startup code should use [`try_install`] to report typed failures. pub fn install(port: u16, gauge_idle_timeout_secs: u64) { try_install(port, gauge_idle_timeout_secs) .unwrap_or_else(|error| panic!("metrics exporter must install exactly once: {error}")); } /// Register the frozen readiness and dependency-diagnostic metric descriptions. /// /// The two `buzz_readiness_*` probe families describe local process lifecycle. /// The dependency families keep their names for dashboard continuity but are /// published by the per-pod dependency sampler, not by the Kubernetes probe or /// by an `/_status` request — a shared-dependency failure no longer deroutes /// the pod, and nobody has to read the endpoint for the metrics to move. pub(crate) fn describe_readiness_metrics() { metrics::describe_counter!( "buzz_readiness_checks_total", "Kubernetes health-listener readiness probes by lifecycle reason (ready, shutting_down)" ); metrics::describe_counter!( "buzz_readiness_dependency_checks_total", "Completed dependency-sampler attempts by dependency and bounded outcome" ); metrics::describe_histogram!( "buzz_readiness_check_duration_seconds", metrics::Unit::Seconds, "Completed dependency-sampler check duration without outcome label multiplication" ); metrics::describe_gauge!( "buzz_readiness_state", "Latest private readiness-probe observation, where 1 is ready and 0 is shutting down" ); metrics::describe_gauge!( "buzz_readiness_dependency_sample_completed_timestamp_seconds", "Unix time the cached /_status dependency report completed, absent until the first sample completes; sampler completion advances it and the publisher re-emits it" ); } /// Register the bounded community-admission contract. pub(crate) fn describe_community_admission_metrics() { metrics::describe_counter!( "buzz_community_admission_checks_total", "Durable community-active checks at socket admission by bounded outcome" ); } /// Register the frozen operation-aware pool-acquisition contract. pub(crate) fn describe_db_pool_metrics() { metrics::describe_counter!( "buzz_db_pool_acquire_started_total", "Database pool checkout starts by valid pool role and operation" ); metrics::describe_histogram!( "buzz_db_pool_acquire_duration_seconds", DB_POOL_ACQUIRE_DURATION_UNIT, "Database pool checkout duration by valid pool role and operation" ); metrics::describe_counter!( "buzz_db_pool_acquire_attempts_total", "Database pool checkout terminals by valid pool role, operation, and outcome" ); metrics::describe_gauge!( "buzz_db_pool_waiters", "Current tracked-operation database pool checkout attempts in progress by valid pool role and operation" ); metrics::describe_counter!( "buzz_db_connection_step_started_total", "Writer connection setup phase starts by fixed pool role and step" ); metrics::describe_counter!( "buzz_db_connection_step_attempts_total", "Writer connection setup terminals by fixed pool role, step, and outcome" ); metrics::describe_histogram!( "buzz_db_connection_step_duration_seconds", metrics::Unit::Seconds, "Writer connection setup phase duration by fixed pool role and step" ); metrics::describe_gauge!( "buzz_db_pool_connections", "Current Postgres pool connections by physical pool role and bounded utilization state" ); metrics::describe_gauge!( "buzz_db_pool_max_connections", "Maximum Postgres pool connections by physical pool role" ); metrics::describe_gauge!( "buzz_db_pool_configured", "Whether a physical Postgres pool role is configured, where 1 is configured and 0 is not configured" ); } /// Named utilization snapshots for every physical Postgres pool role. pub struct DbPoolMetricsInput { /// Primary writer pool utilization. pub writer: buzz_db::DbPoolStats, /// Optional read-replica pool utilization. pub reader: Option, /// Optional audit pool utilization. pub audit: Option, /// Search pool utilization. pub search: buzz_db::DbPoolStats, } /// Record fixed-cardinality utilization for every physical Postgres pool role. /// /// Optional roles emit zero-valued utilization when absent so the unified /// scrape always contains four roles by two states, four maximum-capacity /// series, and four configured series. The original writer and reader gauges /// are also emitted unchanged for dashboard compatibility. pub fn record_db_pool_metrics(input: DbPoolMetricsInput) { let DbPoolMetricsInput { writer, reader, audit, search, } = input; let pools = [Some(writer), reader, audit, Some(search)]; for (role, stats) in buzz_db::DbPoolRole::ALL.into_iter().zip(pools) { let role = role.as_str(); let configured = stats.is_some(); let stats = stats.unwrap_or(buzz_db::DbPoolStats { size: 0, idle: 0, max: 0, }); metrics::gauge!("buzz_db_pool_configured", "pool_role" => role).set(if configured { 1.0 } else { 0.0 }); for (state, value) in [("idle", stats.idle), ("active", stats.active())] { metrics::gauge!( "buzz_db_pool_connections", "pool_role" => role, "state" => state, ) .set(value as f64); } metrics::gauge!("buzz_db_pool_max_connections", "pool_role" => role).set(stats.max as f64); } metrics::gauge!("buzz_db_pool_size").set(writer.size as f64); metrics::gauge!("buzz_db_pool_idle").set(writer.idle as f64); metrics::gauge!("buzz_db_pool_active").set(writer.active() as f64); metrics::gauge!("buzz_db_pool_max").set(writer.max as f64); if let Some(reader) = reader { metrics::gauge!("buzz_db_read_pool_size").set(reader.size as f64); metrics::gauge!("buzz_db_read_pool_idle").set(reader.idle as f64); metrics::gauge!("buzz_db_read_pool_active").set(reader.active() as f64); metrics::gauge!("buzz_db_read_pool_max").set(reader.max as f64); } } /// Register the bounded WebSocket authentication metric contract. pub(crate) fn describe_auth_metrics() { metrics::describe_counter!( "buzz_auth_attempts_total", "NIP-42 challenge lifecycles issued to WebSocket clients by bounded method" ); metrics::describe_counter!( "buzz_auth_outcomes_total", "WebSocket authentication attempts by bounded method and terminal outcome" ); metrics::describe_histogram!( "buzz_auth_duration_seconds", metrics::Unit::Seconds, "WebSocket authentication attempt duration by bounded method and terminal outcome" ); metrics::describe_gauge!( "buzz_ws_authenticated_connections_active", "Current WebSocket connections that completed authentication on this pod" ); metrics::describe_counter!( "buzz_auth_post_terminal_frames_total", "AUTH frames received after the issued challenge reached a terminal, by bounded state" ); } /// Publish zero-valued authentication series at process start. /// /// Counters are never idle-evicted by the configured recorder. The active /// gauge is refreshed alongside the other lifecycle-relative gauges in /// `main.rs`, so a quiet healthy pod remains distinguishable from a missing /// scrape target without racing its increments and decrements. pub(crate) fn initialize_auth_metric_series() { metrics::counter!( "buzz_auth_attempts_total", "method" => AUTH_METHOD_NIP42 ) .increment(0); for outcome in AuthOutcome::ALL { metrics::counter!( "buzz_auth_outcomes_total", "method" => AUTH_METHOD_NIP42, "outcome" => outcome.as_str() ) .increment(0); // Register the fixed-label histogram without recording a fake sample. // Prometheus then exports zero buckets, `_sum`, and `_count` at boot. let _ = metrics::histogram!( "buzz_auth_duration_seconds", "method" => AUTH_METHOD_NIP42, "outcome" => outcome.as_str() ); } for state in AuthPostTerminalState::ALL { metrics::counter!( "buzz_auth_post_terminal_frames_total", "state" => state.as_str() ) .increment(0); } metrics::gauge!("buzz_ws_authenticated_connections_active").set(0.0); } /// Export the partition audit counters at zero before the first audit, so /// `increase()`-based alerts count the first failure and every new outcome. pub(crate) fn initialize_partition_audit_metric_series() { buzz_db::partition::initialize_audit_metric_series(); metrics::counter!("buzz_partition_audit_failures_total").increment(0); } /// Record one issued NIP-42 challenge after it enters the connection writer. /// /// Before this lifecycle contract, `buzz_auth_attempts_total` incremented only /// when an AUTH frame reached the pending handler. It now uses one consistent, /// recovery-oriented unit: challenges successfully queued to the connection. pub(crate) fn record_auth_attempt_started() { metrics::counter!( "buzz_auth_attempts_total", "method" => AUTH_METHOD_NIP42 ) .increment(1); } /// Record exactly one bounded terminal for an authentication attempt. pub(crate) fn record_auth_outcome(outcome: AuthOutcome, duration: Duration) { metrics::counter!( "buzz_auth_outcomes_total", "method" => AUTH_METHOD_NIP42, "outcome" => outcome.as_str() ) .increment(1); metrics::histogram!( "buzz_auth_duration_seconds", "method" => AUTH_METHOD_NIP42, "outcome" => outcome.as_str() ) .record(duration.as_secs_f64()); } /// Record protocol noise after the issued challenge already terminalized. pub(crate) fn record_post_terminal_auth_frame(state: AuthPostTerminalState) { metrics::counter!( "buzz_auth_post_terminal_frames_total", "state" => state.as_str() ) .increment(1); } #[cfg(test)] pub(crate) fn readiness_test_recorder() -> ( metrics_exporter_prometheus::PrometheusRecorder, metrics_exporter_prometheus::PrometheusHandle, ) { readiness_test_recorder_with_idle_timeout(300) } #[cfg(test)] pub(crate) fn readiness_test_recorder_with_idle_timeout( gauge_idle_timeout_secs: u64, ) -> ( metrics_exporter_prometheus::PrometheusRecorder, metrics_exporter_prometheus::PrometheusHandle, ) { let recorder = configured_prometheus_builder(gauge_idle_timeout_secs).build_recorder(); let handle = recorder.handle(); (recorder, handle) } /// Axum middleware that records CAKE framework HTTP metrics. /// /// Emits: /// - `http_requests_total{code, caller, action}` — counter /// - `http_request_latency_ms{code, caller, action}` — histogram /// /// Skips health/metrics paths (`/_*`, `/health`) to avoid polluting dashboards. /// /// Labels: /// - `code`: exact HTTP status code (e.g. "200", "404") /// - `caller`: upstream service from Istio `x-envoy-downstream-service-cluster` header /// - `action`: matched route pattern (e.g. `/api/channels/{channel_id}`) pub async fn track_metrics(req: Request, next: Next) -> Response { // Use the route pattern (e.g. "/api/channels/{channel_id}"), NOT the raw URI. // Falling back to raw URI on 404s would create unbounded cardinality from scanners. let path = req .extensions() .get::() .map(|p| p.as_str().to_owned()); // Skip health probes, metrics endpoint, and unmatched paths (404 scanners). match path.as_deref() { Some(p) if p.starts_with("/_") || p == "/health" || p == "/metrics" => { return next.run(req).await; } None => { // No matched route — 404/scanner traffic. Skip to avoid cardinality bomb. return next.run(req).await; } _ => {} } let action = path.unwrap(); // safe: None case returned above // Caller from Istio header. In CAKE, this is set by the mesh (trusted). // On the public TCP listener it's client-controlled, so validate format: // only accept short alphanumeric-with-hyphens service names. let caller = req .headers() .get("x-envoy-downstream-service-cluster") .and_then(|v| v.to_str().ok()) .filter(|s| { s.len() <= 64 && s.bytes() .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') }) .unwrap_or("unknown") .to_owned(); let start = Instant::now(); let response = next.run(req).await; let status = response.status().as_u16().to_string(); let latency_ms = start.elapsed().as_secs_f64() * 1000.0; let labels = [("code", status), ("caller", caller), ("action", action)]; metrics::counter!("http_requests_total", &labels).increment(1); metrics::histogram!("http_request_latency_ms", &labels).record(latency_ms); response } #[cfg(test)] mod contract_tests { use std::collections::BTreeSet; use std::time::Duration; const OUTCOMES: [&str; 4] = ["success", "timeout", "error", "cancelled"]; fn label_keys(line: &str) -> BTreeSet<&str> { line.split_once('{') .and_then(|(_, rest)| rest.split_once('}')) .map(|(labels, _)| { labels .split(',') .filter_map(|label| label.split_once('=').map(|(key, _)| key)) .collect() }) .unwrap_or_default() } fn sample_value<'a>(scrape: &'a str, metric: &str, labels: &[(&str, &str)]) -> &'a str { scrape .lines() .find(|line| { line.starts_with(metric) && labels .iter() .all(|(key, value)| line.contains(&format!(r#"{key}="{value}""#))) }) .and_then(|line| line.rsplit_once(' ').map(|(_, value)| value)) .unwrap_or_else(|| panic!("missing {metric} with {labels:?} in scrape:\n{scrape}")) } #[test] fn production_pool_sampler_has_fixed_roles_utilization_and_compatibility_metrics() { let (recorder, handle) = super::readiness_test_recorder(); metrics::with_local_recorder(&recorder, || { super::describe_db_pool_metrics(); super::record_db_pool_metrics(super::DbPoolMetricsInput { writer: buzz_db::DbPoolStats { size: 12, idle: 5, max: 20, }, reader: None, audit: None, search: buzz_db::DbPoolStats { size: 4, idle: 1, max: 10, }, }); }); let scrape = handle.render(); let connections = scrape .lines() .filter(|line| line.starts_with("buzz_db_pool_connections{")) .collect::>(); let max_connections = scrape .lines() .filter(|line| line.starts_with("buzz_db_pool_max_connections{")) .collect::>(); let configured = scrape .lines() .filter(|line| line.starts_with("buzz_db_pool_configured{")) .collect::>(); assert_eq!(connections.len(), 8, "unexpected scrape:\n{scrape}"); assert_eq!(max_connections.len(), 4, "unexpected scrape:\n{scrape}"); assert_eq!(configured.len(), 4, "unexpected scrape:\n{scrape}"); assert_eq!( connections.len() + max_connections.len() + configured.len(), 16, "physical pool contract must stay fixed at 16 raw series:\n{scrape}" ); let roles = connections .iter() .filter_map(|line| { ["writer", "reader", "audit", "search"] .into_iter() .find(|role| line.contains(&format!(r#"pool_role="{role}""#))) }) .collect::>(); assert_eq!( roles, BTreeSet::from(["writer", "reader", "audit", "search"]) ); assert!(connections .iter() .all(|line| label_keys(line) == BTreeSet::from(["pool_role", "state"]))); assert!(max_connections .iter() .all(|line| label_keys(line) == BTreeSet::from(["pool_role"]))); assert!(connections.iter().all(|line| { line.contains(r#"state="idle""#) || line.contains(r#"state="active""#) })); assert!(!connections .iter() .any(|line| line.contains(r#"state="size""#) || line.contains(r#"state="max""#))); assert_eq!( sample_value( &scrape, "buzz_db_pool_connections{", &[("pool_role", "writer"), ("state", "active")] ), "7" ); for state in ["idle", "active"] { assert_eq!( sample_value( &scrape, "buzz_db_pool_connections{", &[("pool_role", "reader"), ("state", state)] ), "0" ); } assert_eq!( sample_value( &scrape, "buzz_db_pool_max_connections{", &[("pool_role", "writer")] ), "20" ); assert_eq!( sample_value( &scrape, "buzz_db_pool_max_connections{", &[("pool_role", "reader")] ), "0" ); assert_eq!( sample_value( &scrape, "buzz_db_pool_configured{", &[("pool_role", "reader")] ), "0" ); assert_eq!(sample_value(&scrape, "buzz_db_pool_size ", &[]), "12"); assert_eq!(sample_value(&scrape, "buzz_db_pool_idle ", &[]), "5"); assert_eq!(sample_value(&scrape, "buzz_db_pool_active ", &[]), "7"); assert_eq!(sample_value(&scrape, "buzz_db_pool_max ", &[]), "20"); assert!(!scrape.contains("buzz_db_read_pool_size")); } #[test] fn production_pool_sampler_reports_configured_reader_and_legacy_reader_family() { let (recorder, handle) = super::readiness_test_recorder(); metrics::with_local_recorder(&recorder, || { super::record_db_pool_metrics(super::DbPoolMetricsInput { writer: buzz_db::DbPoolStats { size: 1, idle: 1, max: 50, }, reader: Some(buzz_db::DbPoolStats { size: 9, idle: 2, max: 15, }), audit: None, search: buzz_db::DbPoolStats { size: 1, idle: 1, max: 10, }, }); }); let scrape = handle.render(); assert_eq!( sample_value( &scrape, "buzz_db_pool_configured{", &[("pool_role", "reader")] ), "1" ); assert_eq!(sample_value(&scrape, "buzz_db_read_pool_size ", &[]), "9"); assert_eq!(sample_value(&scrape, "buzz_db_read_pool_idle ", &[]), "2"); assert_eq!(sample_value(&scrape, "buzz_db_read_pool_active ", &[]), "7"); assert_eq!(sample_value(&scrape, "buzz_db_read_pool_max ", &[]), "15"); } #[tokio::test] async fn production_pool_sampler_preserves_audit_and_search_pool_provenance() { let audit_pool = sqlx::postgres::PgPoolOptions::new() .max_connections(5) .connect_lazy("postgres://localhost/buzz") .expect("construct lazy audit pool"); let search_pool = sqlx::postgres::PgPoolOptions::new() .max_connections(10) .connect_lazy("postgres://localhost/buzz") .expect("construct lazy search pool"); let (recorder, handle) = super::readiness_test_recorder(); metrics::with_local_recorder(&recorder, || { super::record_db_pool_metrics(super::DbPoolMetricsInput { writer: buzz_db::DbPoolStats { size: 1, idle: 1, max: 50, }, reader: None, audit: Some(buzz_db::DbPoolStats::from_pool(&audit_pool)), search: buzz_db::DbPoolStats::from_pool(&search_pool), }); }); let scrape = handle.render(); assert_eq!( sample_value( &scrape, "buzz_db_pool_max_connections{", &[("pool_role", "audit")] ), "5" ); assert_eq!( sample_value( &scrape, "buzz_db_pool_max_connections{", &[("pool_role", "search")] ), "10" ); } #[test] fn production_builder_exports_frozen_db_pool_and_connection_contracts() { let (recorder, handle) = super::readiness_test_recorder(); metrics::with_local_recorder(&recorder, || { super::describe_db_pool_metrics(); for (pool_role, operation) in buzz_db::DB_POOL_ACQUIRE_VALID_PAIRS { metrics::counter!( "buzz_db_pool_acquire_started_total", "pool_role" => pool_role, "operation" => operation, ) .increment(1); metrics::histogram!( "buzz_db_pool_acquire_duration_seconds", "pool_role" => pool_role, "operation" => operation, ) .record(0.02); metrics::gauge!( "buzz_db_pool_waiters", "pool_role" => pool_role, "operation" => operation, ) .set(0.0); for outcome in OUTCOMES { metrics::counter!( "buzz_db_pool_acquire_attempts_total", "pool_role" => pool_role, "operation" => operation, "outcome" => outcome, ) .increment(1); } } for (pool_role, step) in buzz_db::DB_CONNECTION_STARTED_STEPS { metrics::counter!( "buzz_db_connection_step_started_total", "pool_role" => pool_role.as_str(), "step" => step.as_str(), ) .increment(1); } for (pool_role, step) in buzz_db::DB_CONNECTION_DURATION_STEPS { metrics::histogram!( "buzz_db_connection_step_duration_seconds", "pool_role" => pool_role.as_str(), "step" => step.as_str(), ) .record(0.02); } for (pool_role, step, outcome) in buzz_db::DB_CONNECTION_TERMINALS { metrics::counter!( "buzz_db_connection_step_attempts_total", "pool_role" => pool_role.as_str(), "step" => step.as_str(), "outcome" => outcome.as_str(), ) .increment(1); } }); let scrape = handle.render(); assert!(scrape.contains("# TYPE buzz_db_pool_acquire_started_total counter")); assert!(scrape.contains("# TYPE buzz_db_pool_acquire_duration_seconds histogram")); assert!(scrape.contains("# TYPE buzz_db_pool_acquire_attempts_total counter")); assert!(scrape.contains("# TYPE buzz_db_pool_waiters gauge")); assert!(scrape.contains("# HELP buzz_db_pool_acquire_duration_seconds Database pool checkout duration by valid pool role and operation")); assert!(scrape.contains("# HELP buzz_db_pool_acquire_attempts_total Database pool checkout terminals by valid pool role, operation, and outcome")); assert!(scrape.contains("# HELP buzz_db_pool_waiters Current tracked-operation database pool checkout attempts in progress by valid pool role and operation")); assert!(scrape.contains("# TYPE buzz_db_connection_step_started_total counter")); assert!(scrape.contains("# TYPE buzz_db_connection_step_attempts_total counter")); assert!(scrape.contains("# TYPE buzz_db_connection_step_duration_seconds histogram")); assert_eq!(super::DB_POOL_ACQUIRE_DURATION_UNIT, metrics::Unit::Seconds); let readiness_buckets = scrape .lines() .filter(|line| { line.starts_with("buzz_db_pool_acquire_duration_seconds_bucket{") && line.contains("pool_role=\"writer\"") && line.contains("operation=\"readiness\"") }) .map(|line| { line.split(",le=\"") .nth(1) .and_then(|rest| rest.split_once('"').map(|(bucket, _)| bucket)) .expect("duration bucket carries le label") }) .collect::>(); assert_eq!( readiness_buckets, ["0.001", "0.005", "0.01", "0.025", "0.05", "0.15", "0.5", "1", "3", "+Inf",], "duration bucket contract drifted:\n{scrape}" ); let raw_series = scrape .lines() .filter(|line| { line.starts_with("buzz_db_pool_acquire_started_total") || line.starts_with("buzz_db_pool_acquire_duration_seconds") || line.starts_with("buzz_db_pool_acquire_attempts_total") || line.starts_with("buzz_db_pool_waiters{") }) .collect::>(); assert_eq!( raw_series.len(), buzz_db::DB_POOL_ACQUIRE_RAW_SERIES_PER_POD, "unexpected raw scrape:\n{scrape}" ); for line in raw_series { let keys = label_keys(line); if line.starts_with("buzz_db_pool_acquire_duration_seconds_bucket") { assert_eq!(keys, BTreeSet::from(["le", "operation", "pool_role"])); } else if line.starts_with("buzz_db_pool_acquire_duration_seconds") || line.starts_with("buzz_db_pool_acquire_started_total") { assert_eq!(keys, BTreeSet::from(["operation", "pool_role"])); } else if line.starts_with("buzz_db_pool_acquire_attempts_total") { assert_eq!(keys, BTreeSet::from(["operation", "outcome", "pool_role"])); } else { assert_eq!(keys, BTreeSet::from(["operation", "pool_role"])); } assert!(!line.contains("operation=\"other\"")); assert!(!line.contains("result=")); } let connection_series = scrape .lines() .filter(|line| { line.starts_with("buzz_db_connection_step_started_total") || line.starts_with("buzz_db_connection_step_attempts_total") || line.starts_with("buzz_db_connection_step_duration_seconds") }) .collect::>(); assert_eq!( connection_series.len(), buzz_db::DB_CONNECTION_RAW_SERIES_PER_POD, "unexpected DB connection scrape:\n{scrape}" ); for line in connection_series { let keys = label_keys(line); if line.starts_with("buzz_db_connection_step_duration_seconds_bucket") { assert_eq!(keys, BTreeSet::from(["le", "pool_role", "step"])); } else if line.starts_with("buzz_db_connection_step_attempts_total") { assert_eq!(keys, BTreeSet::from(["outcome", "pool_role", "step"])); } else { assert_eq!(keys, BTreeSet::from(["pool_role", "step"])); } assert!(!line.contains("reason=")); assert!(!line.contains("connection_ordinal=")); } } #[test] fn production_builder_exports_bounded_auth_contract_and_stable_zeros() { let (recorder, handle) = super::readiness_test_recorder(); metrics::with_local_recorder(&recorder, || { super::describe_auth_metrics(); super::initialize_auth_metric_series(); }); let stable_zero_scrape = handle.render(); assert!(stable_zero_scrape.contains("buzz_auth_attempts_total{method=\"nip42\"} 0")); assert!(stable_zero_scrape.contains("buzz_ws_authenticated_connections_active 0")); let zero_outcomes = stable_zero_scrape .lines() .filter(|line| line.starts_with("buzz_auth_outcomes_total{") && line.ends_with(" 0")) .collect::>(); assert_eq!( zero_outcomes.len(), super::AuthOutcome::ALL.len(), "every bounded outcome must exist before the first attempt:\n{stable_zero_scrape}" ); let zero_duration_counts = stable_zero_scrape .lines() .filter(|line| { line.starts_with("buzz_auth_duration_seconds_count{") && line.ends_with(" 0") }) .count(); let zero_duration_sums = stable_zero_scrape .lines() .filter(|line| { line.starts_with("buzz_auth_duration_seconds_sum{") && line.ends_with(" 0") }) .count(); let zero_duration_buckets = stable_zero_scrape .lines() .filter(|line| { line.starts_with("buzz_auth_duration_seconds_bucket{") && line.ends_with(" 0") }) .count(); assert_eq!(zero_duration_counts, super::AuthOutcome::ALL.len()); assert_eq!(zero_duration_sums, super::AuthOutcome::ALL.len()); assert_eq!( zero_duration_buckets, super::AuthOutcome::ALL.len() * 11, "every fixed outcome must export all zero buckets before traffic:\n{stable_zero_scrape}" ); metrics::with_local_recorder(&recorder, || { super::record_auth_attempt_started(); for outcome in super::AuthOutcome::ALL { super::record_auth_outcome(outcome, Duration::from_millis(20)); } for state in super::AuthPostTerminalState::ALL { super::record_post_terminal_auth_frame(state); } let active = metrics::gauge!("buzz_ws_authenticated_connections_active"); active.increment(1.0); active.decrement(1.0); }); let scrape = handle.render(); assert!(scrape.contains("# TYPE buzz_auth_attempts_total counter")); assert!(scrape.contains("# TYPE buzz_auth_outcomes_total counter")); assert!(scrape.contains("# TYPE buzz_auth_duration_seconds histogram")); assert!(scrape.contains("# TYPE buzz_ws_authenticated_connections_active gauge")); let raw_series = scrape .lines() .filter(|line| { line.starts_with("buzz_auth_attempts_total{") || line.starts_with("buzz_auth_outcomes_total{") || line.starts_with("buzz_auth_duration_seconds") || line.starts_with("buzz_auth_post_terminal_frames_total{") || line.starts_with("buzz_ws_authenticated_connections_active ") }) .collect::>(); // 1 challenge + one series per outcome + 2 post-terminal states + 1 active gauge +, for // each outcome, 11 histogram buckets (including +Inf), sum, and count. let n = super::AuthOutcome::ALL.len(); assert_eq!( raw_series.len(), 1 + n + 2 + 1 + n * 13, "unexpected raw scrape:\n{scrape}" ); for line in raw_series { let keys = label_keys(line); if line.starts_with("buzz_auth_attempts_total{") { assert_eq!(keys, BTreeSet::from(["method"])); } else if line.starts_with("buzz_auth_outcomes_total{") { assert_eq!(keys, BTreeSet::from(["method", "outcome"])); } else if line.starts_with("buzz_auth_post_terminal_frames_total{") { assert_eq!(keys, BTreeSet::from(["state"])); } else if line.starts_with("buzz_auth_duration_seconds_bucket{") { assert_eq!(keys, BTreeSet::from(["le", "method", "outcome"])); } else if line.starts_with("buzz_auth_duration_seconds") { assert_eq!(keys, BTreeSet::from(["method", "outcome"])); } else { assert!(keys.is_empty(), "active gauge must not add labels: {line}"); } assert!(!line.contains("pubkey=")); assert!(!line.contains("community=")); assert!(!line.contains("connection=")); assert!(!line.contains("pod=")); assert!(!line.contains("version=")); assert!(!line.contains("error=")); } } } #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn occupied_listener_is_classified_as_bind() { let listener = std::net::TcpListener::bind(("0.0.0.0", 0)).expect("bind occupied port"); let port = listener.local_addr().expect("occupied address").port(); let error = try_install(port, 300).expect_err("occupied listener must fail"); assert_eq!(error.failure(), MetricsInstallFailure::Bind); } #[tokio::test] async fn recorder_conflict_is_typed_in_an_isolated_process() { const CHILD_ENV: &str = "BUZZ_TEST_METRICS_RECORDER_CONFLICT"; if std::env::var_os(CHILD_ENV).is_some() { let recorder = configured_prometheus_builder(300).build_recorder(); metrics::set_global_recorder(recorder).expect("install first recorder"); let error = try_install(0, 300).expect_err("second recorder must fail"); assert_eq!(error.failure(), MetricsInstallFailure::RecorderConflict); return; } crate::test_support::run_exact_test_child( "metrics::tests::recorder_conflict_is_typed_in_an_isolated_process", CHILD_ENV, ); } } /// Fixtures for the partition audit alerts in /// `docs/partition-catalog-monitoring.md`: the production exporter configuration /// and audit metric emission export the series those rules read. #[cfg(test)] mod partition_alert_tests { use std::collections::BTreeMap; use std::time::Duration; const RUNS: &str = "buzz_partition_audit_runs_total"; const LAST_SUCCESS: &str = "buzz_partition_audit_last_success_timestamp_seconds"; const MANAGED_TABLES: [&str; 2] = ["delivery_log", "events"]; /// Sample value by `table` label for every sample of `metric` in a scrape. /// Counters with several `outcome` labels keep the last sample per table. fn samples(scrape: &str, metric: &str) -> BTreeMap { scrape .lines() .filter_map(|line| line.strip_prefix(metric)?.strip_prefix('{')) .filter_map(|labels| { let (_, rest) = labels.split_once(r#"table=""#)?; let (table, rest) = rest.split_once('"')?; let (_, value) = rest.rsplit_once(' ')?; Some((table.to_string(), value.parse().expect("numeric sample"))) }) .collect() } fn tables(scrape: &str, metric: &str) -> Vec { samples(scrape, metric).into_keys().collect() } fn managed_tables() -> Vec { MANAGED_TABLES .iter() .map(|table| table.to_string()) .collect() } fn unreachable_db() -> buzz_db::Db { let pool = sqlx::postgres::PgPoolOptions::new() .acquire_timeout(Duration::from_millis(200)) .connect_lazy("postgres://127.0.0.1:1/buzz") .expect("lazy test pool"); buzz_db::Db::from_pool(pool) } fn current_thread_runtime() -> tokio::runtime::Runtime { tokio::runtime::Builder::new_current_thread() .enable_all() .build() .expect("current-thread runtime") } #[test] fn never_successful_audits_export_a_zero_success_timestamp() { let (recorder, handle) = super::readiness_test_recorder_with_idle_timeout(1); metrics::with_local_recorder(&recorder, || { current_thread_runtime().block_on(async { assert!(unreachable_db().audit_partitions(3).await.is_err()); }); }); let scrape = handle.render(); assert_eq!(tables(&scrape, RUNS), managed_tables(), "{scrape}"); let zero = MANAGED_TABLES .iter() .map(|table| (table.to_string(), 0.0)) .collect::>(); assert_eq!(samples(&scrape, LAST_SUCCESS), zero, "{scrape}"); } #[test] fn audit_counters_export_zero_baselines_before_the_first_audit() { let (recorder, handle) = super::readiness_test_recorder_with_idle_timeout(1); metrics::with_local_recorder(&recorder, super::initialize_partition_audit_metric_series); let scrape = handle.render(); let zero_runs = scrape .lines() .filter(|line| line.starts_with(&format!("{RUNS}{{")) && line.ends_with(" 0")) .count(); assert_eq!(zero_runs, MANAGED_TABLES.len() * 3, "{scrape}"); assert!( scrape .lines() .any(|line| line == "buzz_partition_audit_failures_total 0"), "{scrape}" ); // A first failure then reads as an increase from the zero baseline. metrics::with_local_recorder(&recorder, || { current_thread_runtime().block_on(async { assert!(unreachable_db().audit_partitions(3).await.is_err()); }); }); let scrape = handle.render(); for table in MANAGED_TABLES { for (outcome, value) in [("ok", 0), ("degraded", 0), ("error", 1)] { let line = scrape .lines() .find(|line| { line.starts_with(&format!("{RUNS}{{")) && line.contains(&format!(r#"table="{table}""#)) && line.contains(&format!(r#"outcome="{outcome}""#)) }) .unwrap_or_else(|| panic!("{table}/{outcome} missing:\n{scrape}")); assert!(line.ends_with(&format!(" {value}")), "{line}"); } } } mod postgres_tests { use super::*; #[test] #[ignore = "requires Postgres"] fn failed_audits_keep_the_last_success_past_the_idle_timeout() { let idle_timeout = Duration::from_secs(1); let (recorder, handle) = crate::metrics::readiness_test_recorder_with_idle_timeout(idle_timeout.as_secs()); let runtime = current_thread_runtime(); let (working, failing) = metrics::with_local_recorder(&recorder, || { runtime.block_on(async { let pool = sqlx::PgPool::connect(&crate::test_support::database_url()) .await .expect("connect test database"); let working = buzz_db::Db::from_pool(pool); working.audit_partitions(3).await.expect("successful audit"); (working, unreachable_db()) }) }); let scrape = handle.render(); let first = samples(&scrape, LAST_SUCCESS); assert_eq!(first.keys().cloned().collect::>(), managed_tables()); let audit = |db: &buzz_db::Db| { metrics::with_local_recorder(&recorder, || { runtime.block_on(async { db.audit_partitions(3).await.is_ok() }) }) }; // A failed audit never makes the last success look fresh, and // sustained failures keep it exported past the idle timeout, so its // age keeps rising instead of the series disappearing. assert!(!audit(&failing)); assert_eq!(samples(&handle.render(), LAST_SUCCESS), first); std::thread::sleep(idle_timeout + Duration::from_millis(250)); assert!(!audit(&failing)); assert_eq!(samples(&handle.render(), LAST_SUCCESS), first); // When audits stop running altogether the timestamp is evicted, // but the run counters remain for the not-running rule. std::thread::sleep(idle_timeout + Duration::from_millis(250)); let scrape = handle.render(); assert!(tables(&scrape, LAST_SUCCESS).is_empty(), "{scrape}"); assert_eq!(tables(&scrape, RUNS), managed_tables(), "{scrape}"); // The next success exports a fresh timestamp again. assert!(audit(&working)); assert_eq!(tables(&handle.render(), LAST_SUCCESS), managed_tables()); } } }