buzz/crates/buzz-relay/src/metrics.rs

//! Prometheus metrics: recorder setup, upkeep task, and HTTP middleware.
//!
//! ```text
//! ┌──────────────────────────────────────────────────────────┐
//! │  metrics-rs facade (metrics::counter!, histogram!, etc.) │
//! │         ↓                                                │
//! │  PrometheusBuilder → HTTP listener on :9102              │
//! │         ↓                                                │
//! │  GET /metrics → Prometheus text format                   │
//! └──────────────────────────────────────────────────────────┘
//! ```
//!
//! Framework metrics (`http_requests_total`, `http_request_latency_ms`) are
//! recorded by [`track_metrics`] middleware on the app router. Buzz-specific
//! metrics are recorded inline at their call sites.

use std::time::{Duration, Instant};

use axum::{
    extract::{MatchedPath, Request},
    middleware::Next,
    response::Response,
};
use metrics_exporter_prometheus::{BuildError, Matcher, PrometheusBuilder};
use metrics_util::MetricKindMask;

/// The fixed method label for the WebSocket NIP-42 challenge lifecycle.
pub(crate) const AUTH_METHOD_NIP42: &str = "nip42";

/// Bounded terminal results for one WebSocket authentication attempt.
///
/// Keep this enum free of user-controlled or error-string values. Rollout
/// analysis relies on these labels having a small, stable cardinality.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum AuthOutcome {
    Success,
    Invalid,
    Banned,
    BanCheckError,
    AllowlistCheckError,
    AllowlistDenied,
    RelayMembershipCheckError,
    NotRelayMember,
    /// NIP-FI key pairing mismatch: the NIP-42 key differs from the asserted key.
    PairingMismatch,
    Timeout,
    Disconnect,
    Shutdown,
}

impl AuthOutcome {
    pub(crate) const ALL: [Self; 12] = [
        Self::Success,
        Self::Invalid,
        Self::Banned,
        Self::BanCheckError,
        Self::AllowlistCheckError,
        Self::AllowlistDenied,
        Self::RelayMembershipCheckError,
        Self::NotRelayMember,
        Self::PairingMismatch,
        Self::Timeout,
        Self::Disconnect,
        Self::Shutdown,
    ];

    pub(crate) const fn as_str(self) -> &'static str {
        match self {
            Self::Success => "success",
            Self::Invalid => "invalid",
            Self::Banned => "banned",
            Self::BanCheckError => "ban_check_error",
            Self::AllowlistCheckError => "allowlist_check_error",
            Self::AllowlistDenied => "allowlist_denied",
            Self::RelayMembershipCheckError => "relay_membership_check_error",
            Self::NotRelayMember => "not_relay_member",
            Self::PairingMismatch => "pairing_mismatch",
            Self::Timeout => "timeout",
            Self::Disconnect => "disconnect",
            Self::Shutdown => "shutdown",
        }
    }
}

/// Bounded state of an AUTH frame received after the issued challenge had
/// already reached a terminal. These frames are protocol misuse, not recovery
/// lifecycles, and therefore stay outside rollout-gating attempts/outcomes.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum AuthPostTerminalState {
    Authenticated,
    Failed,
}

impl AuthPostTerminalState {
    pub(crate) const ALL: [Self; 2] = [Self::Authenticated, Self::Failed];

    pub(crate) const fn as_str(self) -> &'static str {
        match self {
            Self::Authenticated => "authenticated",
            Self::Failed => "failed",
        }
    }
}

/// HTTP latency buckets (milliseconds) — only for `http_request_latency_ms`.
const LATENCY_BUCKETS_MS: [f64; 11] = [
    5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, 5000.0, 10000.0,
];

/// Seconds-scale buckets for internal processing histograms (event, search, audit).
const DURATION_BUCKETS_S: [f64; 10] = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0];

/// Readiness buckets concentrate resolution near the two-second failure budget.
const READINESS_DURATION_BUCKETS_S: [f64; 15] = [
    0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5,
];

/// Pool checkout buckets: dense around normal sub-100ms waits, with explicit
/// coverage of the reader's 150ms and writer's default three-second budgets.
const DB_POOL_ACQUIRE_DURATION_BUCKETS_S: [f64; 9] =
    [0.001, 0.005, 0.01, 0.025, 0.05, 0.15, 0.5, 1.0, 3.0];
const DB_POOL_ACQUIRE_DURATION_UNIT: metrics::Unit = metrics::Unit::Seconds;

/// Writer pool/session buckets preserve sub-millisecond setup while retaining
/// seconds-scale failures in the final bucket.
const DB_CONNECTION_STEP_DURATION_BUCKETS_S: [f64; 10] = [
    0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.15, 0.5, 1.0, 3.0,
];

/// Seconds-scale buckets for Git hydration and pack streams.
const GIT_DURATION_BUCKETS_S: [f64; 13] = [
    0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
];

/// Mesh dial and reconcile-pass buckets. A pass dials peers serially, so a
/// large or stale peer table can stretch it to minutes; the default
/// five-second ceiling would hide exactly the stall these exist to show.
const MESH_DURATION_BUCKETS_S: [f64; 14] = [
    0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0,
];

/// Byte buckets for hydrated repositories and streamed clone/fetch responses.
const GIT_BYTES_BUCKETS: [f64; 9] = [
    0.0,
    64.0 * 1024.0,
    1024.0 * 1024.0,
    10.0 * 1024.0 * 1024.0,
    50.0 * 1024.0 * 1024.0,
    100.0 * 1024.0 * 1024.0,
    250.0 * 1024.0 * 1024.0,
    500.0 * 1024.0 * 1024.0,
    1024.0 * 1024.0 * 1024.0,
];

/// Pack-count buckets bounded by the manifest's maximum pack count.
const GIT_PACK_BUCKETS: [f64; 9] = [0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0];

/// Integer-count buckets for fan-out recipient histograms.
const FANOUT_BUCKETS: [f64; 9] = [0.0, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 500.0, 1000.0];

fn configured_prometheus_builder(gauge_idle_timeout_secs: u64) -> PrometheusBuilder {
    PrometheusBuilder::new()
        // Remove gauge series that the relay intentionally stops emitting.
        .idle_timeout(
            MetricKindMask::GAUGE,
            Some(Duration::from_secs(gauge_idle_timeout_secs)),
        )
        // Per-metric buckets: ms for HTTP latency, seconds for internal processing.
        .set_buckets_for_metric(
            Matcher::Full("http_request_latency_ms".to_owned()),
            &LATENCY_BUCKETS_MS,
        )
        .expect("valid ms bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full("buzz_git_hydrate_seconds".to_owned()),
            &GIT_DURATION_BUCKETS_S,
        )
        .expect("valid git hydration duration bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full("buzz_git_upload_pack_stream_seconds".to_owned()),
            &GIT_DURATION_BUCKETS_S,
        )
        .expect("valid git stream duration bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full("buzz_git_pack_cache_populate_seconds".to_owned()),
            &GIT_DURATION_BUCKETS_S,
        )
        .expect("valid git cache population duration bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full("buzz_git_pack_cache_population_wait_seconds".to_owned()),
            &GIT_DURATION_BUCKETS_S,
        )
        .expect("valid git cache population wait bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full("buzz_git_pack_compaction_seconds".to_owned()),
            &GIT_DURATION_BUCKETS_S,
        )
        .expect("valid git compaction duration bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full("buzz_readiness_check_duration_seconds".to_owned()),
            &READINESS_DURATION_BUCKETS_S,
        )
        .expect("valid readiness duration bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full("buzz_db_pool_acquire_duration_seconds".to_owned()),
            &DB_POOL_ACQUIRE_DURATION_BUCKETS_S,
        )
        .expect("valid DB pool acquisition duration bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full("buzz_db_connection_step_duration_seconds".to_owned()),
            &DB_CONNECTION_STEP_DURATION_BUCKETS_S,
        )
        .expect("valid DB connection-step duration bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full("buzz_git_hydrate_bytes".to_owned()),
            &GIT_BYTES_BUCKETS,
        )
        .expect("valid git hydration byte bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full("buzz_git_upload_pack_stream_bytes".to_owned()),
            &GIT_BYTES_BUCKETS,
        )
        .expect("valid git stream byte bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full("buzz_git_pack_compaction_bytes".to_owned()),
            &GIT_BYTES_BUCKETS,
        )
        .expect("valid git compaction byte bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full("buzz_git_hydrate_packs".to_owned()),
            &GIT_PACK_BUCKETS,
        )
        .expect("valid git pack-count bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full("buzz_git_pack_compaction_packs_before".to_owned()),
            &GIT_PACK_BUCKETS,
        )
        .expect("valid git compaction input pack-count bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full("buzz_git_pack_compaction_packs_after".to_owned()),
            &GIT_PACK_BUCKETS,
        )
        .expect("valid git compaction output pack-count bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full(buzz_relay_mesh::telemetry::DIAL_DURATION_SECONDS.to_owned()),
            &MESH_DURATION_BUCKETS_S,
        )
        .expect("valid mesh dial bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full(buzz_relay_mesh::telemetry::RECONCILE_PASS_SECONDS.to_owned()),
            &MESH_DURATION_BUCKETS_S,
        )
        .expect("valid mesh reconcile pass bucket boundaries")
        .set_buckets_for_metric(Matcher::Suffix("_seconds".to_owned()), &DURATION_BUCKETS_S)
        .expect("valid seconds bucket boundaries")
        .set_buckets_for_metric(
            Matcher::Full("buzz_fanout_recipients".to_owned()),
            &FANOUT_BUCKETS,
        )
        .expect("valid fanout bucket boundaries")
}

/// A bounded class of metrics installation failure.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum MetricsInstallFailure {
    /// The Prometheus listener could not bind.
    Bind,
    /// Another component already installed a global recorder.
    RecorderConflict,
    /// The exporter could not be built for another reason.
    ExporterBuild,
}

/// An error returned while installing Prometheus metrics.
#[derive(Debug, thiserror::Error)]
pub enum MetricsInstallError {
    /// Prometheus exporter construction failed.
    #[error("failed to build Prometheus exporter: {0}")]
    Build(#[source] BuildError),
    /// Another component already installed the process-global recorder.
    #[error("the global metrics recorder is already installed")]
    RecorderConflict,
}

impl MetricsInstallError {
    /// Return the secret-safe lifecycle classification.
    pub const fn failure(&self) -> MetricsInstallFailure {
        match self {
            Self::Build(BuildError::FailedToCreateHTTPListener(_)) => MetricsInstallFailure::Bind,
            Self::Build(_) => MetricsInstallFailure::ExporterBuild,
            Self::RecorderConflict => MetricsInstallFailure::RecorderConflict,
        }
    }
}

/// Try to install the global metrics recorder and spawn the Prometheus HTTP exporter.
///
/// `build()` returns the recorder + exporter future and internally spawns
/// the upkeep task, so no separate upkeep call is needed.
///
/// Must be called from within a Tokio runtime.
/// Listener and global-recorder failures are returned rather than panicking.
/// A later exporter exit remains detached from relay service; external scrape
/// coverage is authoritative for exporter availability.
pub fn try_install(port: u16, gauge_idle_timeout_secs: u64) -> Result<(), MetricsInstallError> {
    let (recorder, exporter) = configured_prometheus_builder(gauge_idle_timeout_secs)
        .with_http_listener(([0, 0, 0, 0], port))
        .build()
        .map_err(MetricsInstallError::Build)?;

    metrics::set_global_recorder(recorder)
        .map_err(|_error| MetricsInstallError::RecorderConflict)?;
    describe_readiness_metrics();
    describe_community_admission_metrics();
    describe_db_pool_metrics();
    describe_auth_metrics();
    crate::startup_steps::describe_metrics();
    buzz_relay_mesh::telemetry::describe_metrics();
    initialize_auth_metric_series();
    initialize_partition_audit_metric_series();
    tokio::spawn(exporter);
    Ok(())
}

/// Install the global metrics recorder and spawn the Prometheus HTTP exporter.
///
/// This compatibility entry point preserves the original panic-on-failure API.
/// New startup code should use [`try_install`] to report typed failures.
pub fn install(port: u16, gauge_idle_timeout_secs: u64) {
    try_install(port, gauge_idle_timeout_secs)
        .unwrap_or_else(|error| panic!("metrics exporter must install exactly once: {error}"));
}

/// Register the frozen readiness and dependency-diagnostic metric descriptions.
///
/// The two `buzz_readiness_*` probe families describe local process lifecycle.
/// The dependency families keep their names for dashboard continuity but are
/// published by the per-pod dependency sampler, not by the Kubernetes probe or
/// by an `/_status` request — a shared-dependency failure no longer deroutes
/// the pod, and nobody has to read the endpoint for the metrics to move.
pub(crate) fn describe_readiness_metrics() {
    metrics::describe_counter!(
        "buzz_readiness_checks_total",
        "Kubernetes health-listener readiness probes by lifecycle reason (ready, shutting_down)"
    );
    metrics::describe_counter!(
        "buzz_readiness_dependency_checks_total",
        "Completed dependency-sampler attempts by dependency and bounded outcome"
    );
    metrics::describe_histogram!(
        "buzz_readiness_check_duration_seconds",
        metrics::Unit::Seconds,
        "Completed dependency-sampler check duration without outcome label multiplication"
    );
    metrics::describe_gauge!(
        "buzz_readiness_state",
        "Latest private readiness-probe observation, where 1 is ready and 0 is shutting down"
    );
    metrics::describe_gauge!(
        "buzz_readiness_dependency_sample_completed_timestamp_seconds",
        "Unix time the cached /_status dependency report completed, absent until the first sample completes; sampler completion advances it and the publisher re-emits it"
    );
}

/// Register the bounded community-admission contract.
pub(crate) fn describe_community_admission_metrics() {
    metrics::describe_counter!(
        "buzz_community_admission_checks_total",
        "Durable community-active checks at socket admission by bounded outcome"
    );
}

/// Register the frozen operation-aware pool-acquisition contract.
pub(crate) fn describe_db_pool_metrics() {
    metrics::describe_counter!(
        "buzz_db_pool_acquire_started_total",
        "Database pool checkout starts by valid pool role and operation"
    );
    metrics::describe_histogram!(
        "buzz_db_pool_acquire_duration_seconds",
        DB_POOL_ACQUIRE_DURATION_UNIT,
        "Database pool checkout duration by valid pool role and operation"
    );
    metrics::describe_counter!(
        "buzz_db_pool_acquire_attempts_total",
        "Database pool checkout terminals by valid pool role, operation, and outcome"
    );
    metrics::describe_gauge!(
        "buzz_db_pool_waiters",
        "Current tracked-operation database pool checkout attempts in progress by valid pool role and operation"
    );
    metrics::describe_counter!(
        "buzz_db_connection_step_started_total",
        "Writer connection setup phase starts by fixed pool role and step"
    );
    metrics::describe_counter!(
        "buzz_db_connection_step_attempts_total",
        "Writer connection setup terminals by fixed pool role, step, and outcome"
    );
    metrics::describe_histogram!(
        "buzz_db_connection_step_duration_seconds",
        metrics::Unit::Seconds,
        "Writer connection setup phase duration by fixed pool role and step"
    );
    metrics::describe_gauge!(
        "buzz_db_pool_connections",
        "Current Postgres pool connections by physical pool role and bounded utilization state"
    );
    metrics::describe_gauge!(
        "buzz_db_pool_max_connections",
        "Maximum Postgres pool connections by physical pool role"
    );
    metrics::describe_gauge!(
        "buzz_db_pool_configured",
        "Whether a physical Postgres pool role is configured, where 1 is configured and 0 is not configured"
    );
}

/// Named utilization snapshots for every physical Postgres pool role.
pub struct DbPoolMetricsInput {
    /// Primary writer pool utilization.
    pub writer: buzz_db::DbPoolStats,
    /// Optional read-replica pool utilization.
    pub reader: Option<buzz_db::DbPoolStats>,
    /// Optional audit pool utilization.
    pub audit: Option<buzz_db::DbPoolStats>,
    /// Search pool utilization.
    pub search: buzz_db::DbPoolStats,
}

/// Record fixed-cardinality utilization for every physical Postgres pool role.
///
/// Optional roles emit zero-valued utilization when absent so the unified
/// scrape always contains four roles by two states, four maximum-capacity
/// series, and four configured series. The original writer and reader gauges
/// are also emitted unchanged for dashboard compatibility.
pub fn record_db_pool_metrics(input: DbPoolMetricsInput) {
    let DbPoolMetricsInput {
        writer,
        reader,
        audit,
        search,
    } = input;
    let pools = [Some(writer), reader, audit, Some(search)];
    for (role, stats) in buzz_db::DbPoolRole::ALL.into_iter().zip(pools) {
        let role = role.as_str();
        let configured = stats.is_some();
        let stats = stats.unwrap_or(buzz_db::DbPoolStats {
            size: 0,
            idle: 0,
            max: 0,
        });
        metrics::gauge!("buzz_db_pool_configured", "pool_role" => role).set(if configured {
            1.0
        } else {
            0.0
        });
        for (state, value) in [("idle", stats.idle), ("active", stats.active())] {
            metrics::gauge!(
                "buzz_db_pool_connections",
                "pool_role" => role,
                "state" => state,
            )
            .set(value as f64);
        }
        metrics::gauge!("buzz_db_pool_max_connections", "pool_role" => role).set(stats.max as f64);
    }

    metrics::gauge!("buzz_db_pool_size").set(writer.size as f64);
    metrics::gauge!("buzz_db_pool_idle").set(writer.idle as f64);
    metrics::gauge!("buzz_db_pool_active").set(writer.active() as f64);
    metrics::gauge!("buzz_db_pool_max").set(writer.max as f64);

    if let Some(reader) = reader {
        metrics::gauge!("buzz_db_read_pool_size").set(reader.size as f64);
        metrics::gauge!("buzz_db_read_pool_idle").set(reader.idle as f64);
        metrics::gauge!("buzz_db_read_pool_active").set(reader.active() as f64);
        metrics::gauge!("buzz_db_read_pool_max").set(reader.max as f64);
    }
}

/// Register the bounded WebSocket authentication metric contract.
pub(crate) fn describe_auth_metrics() {
    metrics::describe_counter!(
        "buzz_auth_attempts_total",
        "NIP-42 challenge lifecycles issued to WebSocket clients by bounded method"
    );
    metrics::describe_counter!(
        "buzz_auth_outcomes_total",
        "WebSocket authentication attempts by bounded method and terminal outcome"
    );
    metrics::describe_histogram!(
        "buzz_auth_duration_seconds",
        metrics::Unit::Seconds,
        "WebSocket authentication attempt duration by bounded method and terminal outcome"
    );
    metrics::describe_gauge!(
        "buzz_ws_authenticated_connections_active",
        "Current WebSocket connections that completed authentication on this pod"
    );
    metrics::describe_counter!(
        "buzz_auth_post_terminal_frames_total",
        "AUTH frames received after the issued challenge reached a terminal, by bounded state"
    );
}

/// Publish zero-valued authentication series at process start.
///
/// Counters are never idle-evicted by the configured recorder. The active
/// gauge is refreshed alongside the other lifecycle-relative gauges in
/// `main.rs`, so a quiet healthy pod remains distinguishable from a missing
/// scrape target without racing its increments and decrements.
pub(crate) fn initialize_auth_metric_series() {
    metrics::counter!(
        "buzz_auth_attempts_total",
        "method" => AUTH_METHOD_NIP42
    )
    .increment(0);
    for outcome in AuthOutcome::ALL {
        metrics::counter!(
            "buzz_auth_outcomes_total",
            "method" => AUTH_METHOD_NIP42,
            "outcome" => outcome.as_str()
        )
        .increment(0);
        // Register the fixed-label histogram without recording a fake sample.
        // Prometheus then exports zero buckets, `_sum`, and `_count` at boot.
        let _ = metrics::histogram!(
            "buzz_auth_duration_seconds",
            "method" => AUTH_METHOD_NIP42,
            "outcome" => outcome.as_str()
        );
    }
    for state in AuthPostTerminalState::ALL {
        metrics::counter!(
            "buzz_auth_post_terminal_frames_total",
            "state" => state.as_str()
        )
        .increment(0);
    }
    metrics::gauge!("buzz_ws_authenticated_connections_active").set(0.0);
}

/// Export the partition audit counters at zero before the first audit, so
/// `increase()`-based alerts count the first failure and every new outcome.
pub(crate) fn initialize_partition_audit_metric_series() {
    buzz_db::partition::initialize_audit_metric_series();
    metrics::counter!("buzz_partition_audit_failures_total").increment(0);
}

/// Record one issued NIP-42 challenge after it enters the connection writer.
///
/// Before this lifecycle contract, `buzz_auth_attempts_total` incremented only
/// when an AUTH frame reached the pending handler. It now uses one consistent,
/// recovery-oriented unit: challenges successfully queued to the connection.
pub(crate) fn record_auth_attempt_started() {
    metrics::counter!(
        "buzz_auth_attempts_total",
        "method" => AUTH_METHOD_NIP42
    )
    .increment(1);
}

/// Record exactly one bounded terminal for an authentication attempt.
pub(crate) fn record_auth_outcome(outcome: AuthOutcome, duration: Duration) {
    metrics::counter!(
        "buzz_auth_outcomes_total",
        "method" => AUTH_METHOD_NIP42,
        "outcome" => outcome.as_str()
    )
    .increment(1);
    metrics::histogram!(
        "buzz_auth_duration_seconds",
        "method" => AUTH_METHOD_NIP42,
        "outcome" => outcome.as_str()
    )
    .record(duration.as_secs_f64());
}

/// Record protocol noise after the issued challenge already terminalized.
pub(crate) fn record_post_terminal_auth_frame(state: AuthPostTerminalState) {
    metrics::counter!(
        "buzz_auth_post_terminal_frames_total",
        "state" => state.as_str()
    )
    .increment(1);
}

#[cfg(test)]
pub(crate) fn readiness_test_recorder() -> (
    metrics_exporter_prometheus::PrometheusRecorder,
    metrics_exporter_prometheus::PrometheusHandle,
) {
    readiness_test_recorder_with_idle_timeout(300)
}

#[cfg(test)]
pub(crate) fn readiness_test_recorder_with_idle_timeout(
    gauge_idle_timeout_secs: u64,
) -> (
    metrics_exporter_prometheus::PrometheusRecorder,
    metrics_exporter_prometheus::PrometheusHandle,
) {
    let recorder = configured_prometheus_builder(gauge_idle_timeout_secs).build_recorder();
    let handle = recorder.handle();
    (recorder, handle)
}

/// Axum middleware that records CAKE framework HTTP metrics.
///
/// Emits:
/// - `http_requests_total{code, caller, action}` — counter
/// - `http_request_latency_ms{code, caller, action}` — histogram
///
/// Skips health/metrics paths (`/_*`, `/health`) to avoid polluting dashboards.
///
/// Labels:
/// - `code`: exact HTTP status code (e.g. "200", "404")
/// - `caller`: upstream service from Istio `x-envoy-downstream-service-cluster` header
/// - `action`: matched route pattern (e.g. `/api/channels/{channel_id}`)
pub async fn track_metrics(req: Request, next: Next) -> Response {
    // Use the route pattern (e.g. "/api/channels/{channel_id}"), NOT the raw URI.
    // Falling back to raw URI on 404s would create unbounded cardinality from scanners.
    let path = req
        .extensions()
        .get::<MatchedPath>()
        .map(|p| p.as_str().to_owned());

    // Skip health probes, metrics endpoint, and unmatched paths (404 scanners).
    match path.as_deref() {
        Some(p) if p.starts_with("/_") || p == "/health" || p == "/metrics" => {
            return next.run(req).await;
        }
        None => {
            // No matched route — 404/scanner traffic. Skip to avoid cardinality bomb.
            return next.run(req).await;
        }
        _ => {}
    }
    let action = path.unwrap(); // safe: None case returned above

    // Caller from Istio header. In CAKE, this is set by the mesh (trusted).
    // On the public TCP listener it's client-controlled, so validate format:
    // only accept short alphanumeric-with-hyphens service names.
    let caller = req
        .headers()
        .get("x-envoy-downstream-service-cluster")
        .and_then(|v| v.to_str().ok())
        .filter(|s| {
            s.len() <= 64
                && s.bytes()
                    .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_')
        })
        .unwrap_or("unknown")
        .to_owned();

    let start = Instant::now();
    let response = next.run(req).await;
    let status = response.status().as_u16().to_string();
    let latency_ms = start.elapsed().as_secs_f64() * 1000.0;

    let labels = [("code", status), ("caller", caller), ("action", action)];
    metrics::counter!("http_requests_total", &labels).increment(1);
    metrics::histogram!("http_request_latency_ms", &labels).record(latency_ms);

    response
}
#[cfg(test)]
mod contract_tests {
    use std::collections::BTreeSet;
    use std::time::Duration;

    const OUTCOMES: [&str; 4] = ["success", "timeout", "error", "cancelled"];

    fn label_keys(line: &str) -> BTreeSet<&str> {
        line.split_once('{')
            .and_then(|(_, rest)| rest.split_once('}'))
            .map(|(labels, _)| {
                labels
                    .split(',')
                    .filter_map(|label| label.split_once('=').map(|(key, _)| key))
                    .collect()
            })
            .unwrap_or_default()
    }

    fn sample_value<'a>(scrape: &'a str, metric: &str, labels: &[(&str, &str)]) -> &'a str {
        scrape
            .lines()
            .find(|line| {
                line.starts_with(metric)
                    && labels
                        .iter()
                        .all(|(key, value)| line.contains(&format!(r#"{key}="{value}""#)))
            })
            .and_then(|line| line.rsplit_once(' ').map(|(_, value)| value))
            .unwrap_or_else(|| panic!("missing {metric} with {labels:?} in scrape:\n{scrape}"))
    }

    #[test]
    fn production_pool_sampler_has_fixed_roles_utilization_and_compatibility_metrics() {
        let (recorder, handle) = super::readiness_test_recorder();
        metrics::with_local_recorder(&recorder, || {
            super::describe_db_pool_metrics();
            super::record_db_pool_metrics(super::DbPoolMetricsInput {
                writer: buzz_db::DbPoolStats {
                    size: 12,
                    idle: 5,
                    max: 20,
                },
                reader: None,
                audit: None,
                search: buzz_db::DbPoolStats {
                    size: 4,
                    idle: 1,
                    max: 10,
                },
            });
        });

        let scrape = handle.render();
        let connections = scrape
            .lines()
            .filter(|line| line.starts_with("buzz_db_pool_connections{"))
            .collect::<Vec<_>>();
        let max_connections = scrape
            .lines()
            .filter(|line| line.starts_with("buzz_db_pool_max_connections{"))
            .collect::<Vec<_>>();
        let configured = scrape
            .lines()
            .filter(|line| line.starts_with("buzz_db_pool_configured{"))
            .collect::<Vec<_>>();
        assert_eq!(connections.len(), 8, "unexpected scrape:\n{scrape}");
        assert_eq!(max_connections.len(), 4, "unexpected scrape:\n{scrape}");
        assert_eq!(configured.len(), 4, "unexpected scrape:\n{scrape}");
        assert_eq!(
            connections.len() + max_connections.len() + configured.len(),
            16,
            "physical pool contract must stay fixed at 16 raw series:\n{scrape}"
        );

        let roles = connections
            .iter()
            .filter_map(|line| {
                ["writer", "reader", "audit", "search"]
                    .into_iter()
                    .find(|role| line.contains(&format!(r#"pool_role="{role}""#)))
            })
            .collect::<BTreeSet<_>>();
        assert_eq!(
            roles,
            BTreeSet::from(["writer", "reader", "audit", "search"])
        );
        assert!(connections
            .iter()
            .all(|line| label_keys(line) == BTreeSet::from(["pool_role", "state"])));
        assert!(max_connections
            .iter()
            .all(|line| label_keys(line) == BTreeSet::from(["pool_role"])));
        assert!(connections.iter().all(|line| {
            line.contains(r#"state="idle""#) || line.contains(r#"state="active""#)
        }));
        assert!(!connections
            .iter()
            .any(|line| line.contains(r#"state="size""#) || line.contains(r#"state="max""#)));

        assert_eq!(
            sample_value(
                &scrape,
                "buzz_db_pool_connections{",
                &[("pool_role", "writer"), ("state", "active")]
            ),
            "7"
        );
        for state in ["idle", "active"] {
            assert_eq!(
                sample_value(
                    &scrape,
                    "buzz_db_pool_connections{",
                    &[("pool_role", "reader"), ("state", state)]
                ),
                "0"
            );
        }
        assert_eq!(
            sample_value(
                &scrape,
                "buzz_db_pool_max_connections{",
                &[("pool_role", "writer")]
            ),
            "20"
        );
        assert_eq!(
            sample_value(
                &scrape,
                "buzz_db_pool_max_connections{",
                &[("pool_role", "reader")]
            ),
            "0"
        );
        assert_eq!(
            sample_value(
                &scrape,
                "buzz_db_pool_configured{",
                &[("pool_role", "reader")]
            ),
            "0"
        );
        assert_eq!(sample_value(&scrape, "buzz_db_pool_size ", &[]), "12");
        assert_eq!(sample_value(&scrape, "buzz_db_pool_idle ", &[]), "5");
        assert_eq!(sample_value(&scrape, "buzz_db_pool_active ", &[]), "7");
        assert_eq!(sample_value(&scrape, "buzz_db_pool_max ", &[]), "20");
        assert!(!scrape.contains("buzz_db_read_pool_size"));
    }

    #[test]
    fn production_pool_sampler_reports_configured_reader_and_legacy_reader_family() {
        let (recorder, handle) = super::readiness_test_recorder();
        metrics::with_local_recorder(&recorder, || {
            super::record_db_pool_metrics(super::DbPoolMetricsInput {
                writer: buzz_db::DbPoolStats {
                    size: 1,
                    idle: 1,
                    max: 50,
                },
                reader: Some(buzz_db::DbPoolStats {
                    size: 9,
                    idle: 2,
                    max: 15,
                }),
                audit: None,
                search: buzz_db::DbPoolStats {
                    size: 1,
                    idle: 1,
                    max: 10,
                },
            });
        });

        let scrape = handle.render();
        assert_eq!(
            sample_value(
                &scrape,
                "buzz_db_pool_configured{",
                &[("pool_role", "reader")]
            ),
            "1"
        );
        assert_eq!(sample_value(&scrape, "buzz_db_read_pool_size ", &[]), "9");
        assert_eq!(sample_value(&scrape, "buzz_db_read_pool_idle ", &[]), "2");
        assert_eq!(sample_value(&scrape, "buzz_db_read_pool_active ", &[]), "7");
        assert_eq!(sample_value(&scrape, "buzz_db_read_pool_max ", &[]), "15");
    }

    #[tokio::test]
    async fn production_pool_sampler_preserves_audit_and_search_pool_provenance() {
        let audit_pool = sqlx::postgres::PgPoolOptions::new()
            .max_connections(5)
            .connect_lazy("postgres://localhost/buzz")
            .expect("construct lazy audit pool");
        let search_pool = sqlx::postgres::PgPoolOptions::new()
            .max_connections(10)
            .connect_lazy("postgres://localhost/buzz")
            .expect("construct lazy search pool");
        let (recorder, handle) = super::readiness_test_recorder();

        metrics::with_local_recorder(&recorder, || {
            super::record_db_pool_metrics(super::DbPoolMetricsInput {
                writer: buzz_db::DbPoolStats {
                    size: 1,
                    idle: 1,
                    max: 50,
                },
                reader: None,
                audit: Some(buzz_db::DbPoolStats::from_pool(&audit_pool)),
                search: buzz_db::DbPoolStats::from_pool(&search_pool),
            });
        });

        let scrape = handle.render();
        assert_eq!(
            sample_value(
                &scrape,
                "buzz_db_pool_max_connections{",
                &[("pool_role", "audit")]
            ),
            "5"
        );
        assert_eq!(
            sample_value(
                &scrape,
                "buzz_db_pool_max_connections{",
                &[("pool_role", "search")]
            ),
            "10"
        );
    }

    #[test]
    fn production_builder_exports_frozen_db_pool_and_connection_contracts() {
        let (recorder, handle) = super::readiness_test_recorder();
        metrics::with_local_recorder(&recorder, || {
            super::describe_db_pool_metrics();
            for (pool_role, operation) in buzz_db::DB_POOL_ACQUIRE_VALID_PAIRS {
                metrics::counter!(
                    "buzz_db_pool_acquire_started_total",
                    "pool_role" => pool_role,
                    "operation" => operation,
                )
                .increment(1);
                metrics::histogram!(
                    "buzz_db_pool_acquire_duration_seconds",
                    "pool_role" => pool_role,
                    "operation" => operation,
                )
                .record(0.02);
                metrics::gauge!(
                    "buzz_db_pool_waiters",
                    "pool_role" => pool_role,
                    "operation" => operation,
                )
                .set(0.0);
                for outcome in OUTCOMES {
                    metrics::counter!(
                        "buzz_db_pool_acquire_attempts_total",
                        "pool_role" => pool_role,
                        "operation" => operation,
                        "outcome" => outcome,
                    )
                    .increment(1);
                }
            }
            for (pool_role, step) in buzz_db::DB_CONNECTION_STARTED_STEPS {
                metrics::counter!(
                    "buzz_db_connection_step_started_total",
                    "pool_role" => pool_role.as_str(),
                    "step" => step.as_str(),
                )
                .increment(1);
            }
            for (pool_role, step) in buzz_db::DB_CONNECTION_DURATION_STEPS {
                metrics::histogram!(
                    "buzz_db_connection_step_duration_seconds",
                    "pool_role" => pool_role.as_str(),
                    "step" => step.as_str(),
                )
                .record(0.02);
            }
            for (pool_role, step, outcome) in buzz_db::DB_CONNECTION_TERMINALS {
                metrics::counter!(
                    "buzz_db_connection_step_attempts_total",
                    "pool_role" => pool_role.as_str(),
                    "step" => step.as_str(),
                    "outcome" => outcome.as_str(),
                )
                .increment(1);
            }
        });

        let scrape = handle.render();
        assert!(scrape.contains("# TYPE buzz_db_pool_acquire_started_total counter"));
        assert!(scrape.contains("# TYPE buzz_db_pool_acquire_duration_seconds histogram"));
        assert!(scrape.contains("# TYPE buzz_db_pool_acquire_attempts_total counter"));
        assert!(scrape.contains("# TYPE buzz_db_pool_waiters gauge"));
        assert!(scrape.contains("# HELP buzz_db_pool_acquire_duration_seconds Database pool checkout duration by valid pool role and operation"));
        assert!(scrape.contains("# HELP buzz_db_pool_acquire_attempts_total Database pool checkout terminals by valid pool role, operation, and outcome"));
        assert!(scrape.contains("# HELP buzz_db_pool_waiters Current tracked-operation database pool checkout attempts in progress by valid pool role and operation"));
        assert!(scrape.contains("# TYPE buzz_db_connection_step_started_total counter"));
        assert!(scrape.contains("# TYPE buzz_db_connection_step_attempts_total counter"));
        assert!(scrape.contains("# TYPE buzz_db_connection_step_duration_seconds histogram"));
        assert_eq!(super::DB_POOL_ACQUIRE_DURATION_UNIT, metrics::Unit::Seconds);
        let readiness_buckets = scrape
            .lines()
            .filter(|line| {
                line.starts_with("buzz_db_pool_acquire_duration_seconds_bucket{")
                    && line.contains("pool_role=\"writer\"")
                    && line.contains("operation=\"readiness\"")
            })
            .map(|line| {
                line.split(",le=\"")
                    .nth(1)
                    .and_then(|rest| rest.split_once('"').map(|(bucket, _)| bucket))
                    .expect("duration bucket carries le label")
            })
            .collect::<Vec<_>>();
        assert_eq!(
            readiness_buckets,
            ["0.001", "0.005", "0.01", "0.025", "0.05", "0.15", "0.5", "1", "3", "+Inf",],
            "duration bucket contract drifted:\n{scrape}"
        );

        let raw_series = scrape
            .lines()
            .filter(|line| {
                line.starts_with("buzz_db_pool_acquire_started_total")
                    || line.starts_with("buzz_db_pool_acquire_duration_seconds")
                    || line.starts_with("buzz_db_pool_acquire_attempts_total")
                    || line.starts_with("buzz_db_pool_waiters{")
            })
            .collect::<Vec<_>>();
        assert_eq!(
            raw_series.len(),
            buzz_db::DB_POOL_ACQUIRE_RAW_SERIES_PER_POD,
            "unexpected raw scrape:\n{scrape}"
        );

        for line in raw_series {
            let keys = label_keys(line);
            if line.starts_with("buzz_db_pool_acquire_duration_seconds_bucket") {
                assert_eq!(keys, BTreeSet::from(["le", "operation", "pool_role"]));
            } else if line.starts_with("buzz_db_pool_acquire_duration_seconds")
                || line.starts_with("buzz_db_pool_acquire_started_total")
            {
                assert_eq!(keys, BTreeSet::from(["operation", "pool_role"]));
            } else if line.starts_with("buzz_db_pool_acquire_attempts_total") {
                assert_eq!(keys, BTreeSet::from(["operation", "outcome", "pool_role"]));
            } else {
                assert_eq!(keys, BTreeSet::from(["operation", "pool_role"]));
            }
            assert!(!line.contains("operation=\"other\""));
            assert!(!line.contains("result="));
        }

        let connection_series = scrape
            .lines()
            .filter(|line| {
                line.starts_with("buzz_db_connection_step_started_total")
                    || line.starts_with("buzz_db_connection_step_attempts_total")
                    || line.starts_with("buzz_db_connection_step_duration_seconds")
            })
            .collect::<Vec<_>>();
        assert_eq!(
            connection_series.len(),
            buzz_db::DB_CONNECTION_RAW_SERIES_PER_POD,
            "unexpected DB connection scrape:\n{scrape}"
        );
        for line in connection_series {
            let keys = label_keys(line);
            if line.starts_with("buzz_db_connection_step_duration_seconds_bucket") {
                assert_eq!(keys, BTreeSet::from(["le", "pool_role", "step"]));
            } else if line.starts_with("buzz_db_connection_step_attempts_total") {
                assert_eq!(keys, BTreeSet::from(["outcome", "pool_role", "step"]));
            } else {
                assert_eq!(keys, BTreeSet::from(["pool_role", "step"]));
            }
            assert!(!line.contains("reason="));
            assert!(!line.contains("connection_ordinal="));
        }
    }

    #[test]
    fn production_builder_exports_bounded_auth_contract_and_stable_zeros() {
        let (recorder, handle) = super::readiness_test_recorder();
        metrics::with_local_recorder(&recorder, || {
            super::describe_auth_metrics();
            super::initialize_auth_metric_series();
        });

        let stable_zero_scrape = handle.render();
        assert!(stable_zero_scrape.contains("buzz_auth_attempts_total{method=\"nip42\"} 0"));
        assert!(stable_zero_scrape.contains("buzz_ws_authenticated_connections_active 0"));
        let zero_outcomes = stable_zero_scrape
            .lines()
            .filter(|line| line.starts_with("buzz_auth_outcomes_total{") && line.ends_with(" 0"))
            .collect::<Vec<_>>();
        assert_eq!(
            zero_outcomes.len(),
            super::AuthOutcome::ALL.len(),
            "every bounded outcome must exist before the first attempt:\n{stable_zero_scrape}"
        );
        let zero_duration_counts = stable_zero_scrape
            .lines()
            .filter(|line| {
                line.starts_with("buzz_auth_duration_seconds_count{") && line.ends_with(" 0")
            })
            .count();
        let zero_duration_sums = stable_zero_scrape
            .lines()
            .filter(|line| {
                line.starts_with("buzz_auth_duration_seconds_sum{") && line.ends_with(" 0")
            })
            .count();
        let zero_duration_buckets = stable_zero_scrape
            .lines()
            .filter(|line| {
                line.starts_with("buzz_auth_duration_seconds_bucket{") && line.ends_with(" 0")
            })
            .count();
        assert_eq!(zero_duration_counts, super::AuthOutcome::ALL.len());
        assert_eq!(zero_duration_sums, super::AuthOutcome::ALL.len());
        assert_eq!(
            zero_duration_buckets,
            super::AuthOutcome::ALL.len() * 11,
            "every fixed outcome must export all zero buckets before traffic:\n{stable_zero_scrape}"
        );

        metrics::with_local_recorder(&recorder, || {
            super::record_auth_attempt_started();
            for outcome in super::AuthOutcome::ALL {
                super::record_auth_outcome(outcome, Duration::from_millis(20));
            }
            for state in super::AuthPostTerminalState::ALL {
                super::record_post_terminal_auth_frame(state);
            }
            let active = metrics::gauge!("buzz_ws_authenticated_connections_active");
            active.increment(1.0);
            active.decrement(1.0);
        });

        let scrape = handle.render();
        assert!(scrape.contains("# TYPE buzz_auth_attempts_total counter"));
        assert!(scrape.contains("# TYPE buzz_auth_outcomes_total counter"));
        assert!(scrape.contains("# TYPE buzz_auth_duration_seconds histogram"));
        assert!(scrape.contains("# TYPE buzz_ws_authenticated_connections_active gauge"));

        let raw_series = scrape
            .lines()
            .filter(|line| {
                line.starts_with("buzz_auth_attempts_total{")
                    || line.starts_with("buzz_auth_outcomes_total{")
                    || line.starts_with("buzz_auth_duration_seconds")
                    || line.starts_with("buzz_auth_post_terminal_frames_total{")
                    || line.starts_with("buzz_ws_authenticated_connections_active ")
            })
            .collect::<Vec<_>>();
        // 1 challenge + one series per outcome + 2 post-terminal states + 1 active gauge +, for
        // each outcome, 11 histogram buckets (including +Inf), sum, and count.
        let n = super::AuthOutcome::ALL.len();
        assert_eq!(
            raw_series.len(),
            1 + n + 2 + 1 + n * 13,
            "unexpected raw scrape:\n{scrape}"
        );

        for line in raw_series {
            let keys = label_keys(line);
            if line.starts_with("buzz_auth_attempts_total{") {
                assert_eq!(keys, BTreeSet::from(["method"]));
            } else if line.starts_with("buzz_auth_outcomes_total{") {
                assert_eq!(keys, BTreeSet::from(["method", "outcome"]));
            } else if line.starts_with("buzz_auth_post_terminal_frames_total{") {
                assert_eq!(keys, BTreeSet::from(["state"]));
            } else if line.starts_with("buzz_auth_duration_seconds_bucket{") {
                assert_eq!(keys, BTreeSet::from(["le", "method", "outcome"]));
            } else if line.starts_with("buzz_auth_duration_seconds") {
                assert_eq!(keys, BTreeSet::from(["method", "outcome"]));
            } else {
                assert!(keys.is_empty(), "active gauge must not add labels: {line}");
            }
            assert!(!line.contains("pubkey="));
            assert!(!line.contains("community="));
            assert!(!line.contains("connection="));
            assert!(!line.contains("pod="));
            assert!(!line.contains("version="));
            assert!(!line.contains("error="));
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn occupied_listener_is_classified_as_bind() {
        let listener = std::net::TcpListener::bind(("0.0.0.0", 0)).expect("bind occupied port");
        let port = listener.local_addr().expect("occupied address").port();
        let error = try_install(port, 300).expect_err("occupied listener must fail");
        assert_eq!(error.failure(), MetricsInstallFailure::Bind);
    }

    #[tokio::test]
    async fn recorder_conflict_is_typed_in_an_isolated_process() {
        const CHILD_ENV: &str = "BUZZ_TEST_METRICS_RECORDER_CONFLICT";
        if std::env::var_os(CHILD_ENV).is_some() {
            let recorder = configured_prometheus_builder(300).build_recorder();
            metrics::set_global_recorder(recorder).expect("install first recorder");
            let error = try_install(0, 300).expect_err("second recorder must fail");
            assert_eq!(error.failure(), MetricsInstallFailure::RecorderConflict);
            return;
        }

        crate::test_support::run_exact_test_child(
            "metrics::tests::recorder_conflict_is_typed_in_an_isolated_process",
            CHILD_ENV,
        );
    }
}

/// Fixtures for the partition audit alerts in
/// `docs/partition-catalog-monitoring.md`: the production exporter configuration
/// and audit metric emission export the series those rules read.
#[cfg(test)]
mod partition_alert_tests {
    use std::collections::BTreeMap;
    use std::time::Duration;

    const RUNS: &str = "buzz_partition_audit_runs_total";
    const LAST_SUCCESS: &str = "buzz_partition_audit_last_success_timestamp_seconds";
    const MANAGED_TABLES: [&str; 2] = ["delivery_log", "events"];

    /// Sample value by `table` label for every sample of `metric` in a scrape.
    /// Counters with several `outcome` labels keep the last sample per table.
    fn samples(scrape: &str, metric: &str) -> BTreeMap<String, f64> {
        scrape
            .lines()
            .filter_map(|line| line.strip_prefix(metric)?.strip_prefix('{'))
            .filter_map(|labels| {
                let (_, rest) = labels.split_once(r#"table=""#)?;
                let (table, rest) = rest.split_once('"')?;
                let (_, value) = rest.rsplit_once(' ')?;
                Some((table.to_string(), value.parse().expect("numeric sample")))
            })
            .collect()
    }

    fn tables(scrape: &str, metric: &str) -> Vec<String> {
        samples(scrape, metric).into_keys().collect()
    }

    fn managed_tables() -> Vec<String> {
        MANAGED_TABLES
            .iter()
            .map(|table| table.to_string())
            .collect()
    }

    fn unreachable_db() -> buzz_db::Db {
        let pool = sqlx::postgres::PgPoolOptions::new()
            .acquire_timeout(Duration::from_millis(200))
            .connect_lazy("postgres://127.0.0.1:1/buzz")
            .expect("lazy test pool");
        buzz_db::Db::from_pool(pool)
    }

    fn current_thread_runtime() -> tokio::runtime::Runtime {
        tokio::runtime::Builder::new_current_thread()
            .enable_all()
            .build()
            .expect("current-thread runtime")
    }

    #[test]
    fn never_successful_audits_export_a_zero_success_timestamp() {
        let (recorder, handle) = super::readiness_test_recorder_with_idle_timeout(1);
        metrics::with_local_recorder(&recorder, || {
            current_thread_runtime().block_on(async {
                assert!(unreachable_db().audit_partitions(3).await.is_err());
            });
        });

        let scrape = handle.render();
        assert_eq!(tables(&scrape, RUNS), managed_tables(), "{scrape}");
        let zero = MANAGED_TABLES
            .iter()
            .map(|table| (table.to_string(), 0.0))
            .collect::<BTreeMap<_, _>>();
        assert_eq!(samples(&scrape, LAST_SUCCESS), zero, "{scrape}");
    }

    #[test]
    fn audit_counters_export_zero_baselines_before_the_first_audit() {
        let (recorder, handle) = super::readiness_test_recorder_with_idle_timeout(1);
        metrics::with_local_recorder(&recorder, super::initialize_partition_audit_metric_series);

        let scrape = handle.render();
        let zero_runs = scrape
            .lines()
            .filter(|line| line.starts_with(&format!("{RUNS}{{")) && line.ends_with(" 0"))
            .count();
        assert_eq!(zero_runs, MANAGED_TABLES.len() * 3, "{scrape}");
        assert!(
            scrape
                .lines()
                .any(|line| line == "buzz_partition_audit_failures_total 0"),
            "{scrape}"
        );

        // A first failure then reads as an increase from the zero baseline.
        metrics::with_local_recorder(&recorder, || {
            current_thread_runtime().block_on(async {
                assert!(unreachable_db().audit_partitions(3).await.is_err());
            });
        });
        let scrape = handle.render();
        for table in MANAGED_TABLES {
            for (outcome, value) in [("ok", 0), ("degraded", 0), ("error", 1)] {
                let line = scrape
                    .lines()
                    .find(|line| {
                        line.starts_with(&format!("{RUNS}{{"))
                            && line.contains(&format!(r#"table="{table}""#))
                            && line.contains(&format!(r#"outcome="{outcome}""#))
                    })
                    .unwrap_or_else(|| panic!("{table}/{outcome} missing:\n{scrape}"));
                assert!(line.ends_with(&format!(" {value}")), "{line}");
            }
        }
    }

    mod postgres_tests {
        use super::*;

        #[test]
        #[ignore = "requires Postgres"]
        fn failed_audits_keep_the_last_success_past_the_idle_timeout() {
            let idle_timeout = Duration::from_secs(1);
            let (recorder, handle) =
                crate::metrics::readiness_test_recorder_with_idle_timeout(idle_timeout.as_secs());
            let runtime = current_thread_runtime();
            let (working, failing) = metrics::with_local_recorder(&recorder, || {
                runtime.block_on(async {
                    let pool = sqlx::PgPool::connect(&crate::test_support::database_url())
                        .await
                        .expect("connect test database");
                    let working = buzz_db::Db::from_pool(pool);
                    working.audit_partitions(3).await.expect("successful audit");
                    (working, unreachable_db())
                })
            });

            let scrape = handle.render();
            let first = samples(&scrape, LAST_SUCCESS);
            assert_eq!(first.keys().cloned().collect::<Vec<_>>(), managed_tables());

            let audit = |db: &buzz_db::Db| {
                metrics::with_local_recorder(&recorder, || {
                    runtime.block_on(async { db.audit_partitions(3).await.is_ok() })
                })
            };
            // A failed audit never makes the last success look fresh, and
            // sustained failures keep it exported past the idle timeout, so its
            // age keeps rising instead of the series disappearing.
            assert!(!audit(&failing));
            assert_eq!(samples(&handle.render(), LAST_SUCCESS), first);
            std::thread::sleep(idle_timeout + Duration::from_millis(250));
            assert!(!audit(&failing));
            assert_eq!(samples(&handle.render(), LAST_SUCCESS), first);

            // When audits stop running altogether the timestamp is evicted,
            // but the run counters remain for the not-running rule.
            std::thread::sleep(idle_timeout + Duration::from_millis(250));
            let scrape = handle.render();
            assert!(tables(&scrape, LAST_SUCCESS).is_empty(), "{scrape}");
            assert_eq!(tables(&scrape, RUNS), managed_tables(), "{scrape}");

            // The next success exports a fresh timestamp again.
            assert!(audit(&working));
            assert_eq!(tables(&handle.render(), LAST_SUCCESS), managed_tables());
        }
    }
}