mirror of
https://github.com/block/buzz.git
synced 2026-10-27 20:41:57 +00:00
## Summary Make Kubernetes readiness depend only on the relay process lifecycle, so a shared Postgres or Redis slowdown cannot withdraw every pod at once. Move dependency checks to one fixed-cadence loop per pod. The loop permits only one evaluation at a time, emits metrics after each sample, and caches a timestamped report. `/_status` reads that cache without calling Postgres, Redis, or the deletion catalog. It reports `not_yet_sampled`, `fresh`, or `stale`. Each completed sample also writes a Unix timestamp gauge, so Datadog can compute sample age at query time. Fail closed when the community lifecycle lookup errors. An inconclusive tenant-state lookup no longer admits a socket to AUTH or REQ handling. ### Related issue None found. This addresses the elevated relay HTTP 500 and reconnect incident investigated on 2026-09-03. ### Tradeoff The readiness gauge now reports the latest private-probe observation, not the shutdown transition itself. After shutdown starts, a scrape can still see `1` until the next readiness probe refreshes the gauge to `0`. Kubernetes routing is unaffected because it uses the readiness response from the authoritative process state. A terminal `0` was never guaranteed: even a transition-owned gauge could update in process and then exit before Prometheus scraped it. If no scrape occurs before exit, both designs miss the terminal sample. The new design adds at most the interval until the next readiness probe when a scrape occurs during shutdown. Dependency status is now sampled every 30 seconds rather than on demand. A slow evaluation delays the next sample instead of stacking more work. The status payload exposes the cached report age. The completion timestamp stays unchanged when sampling stops, so `time() - timestamp` crosses the stale threshold without a server aging loop. Outcome counters and duration histograms remain cumulative and cannot provide no-data detection. ### Testing Gimli exercised exact SHA `edff1ec4a5362e132e367bf7449d2b0a68f1e9a9` on an isolated Blox stack. Repeated `/_status` bursts caused no dependency calls, metrics advanced without status traffic, stale status remained visible, and lifecycle lookup failures admitted no AUTH or REQ handling. Follow-up verification at `fdbb394ed598c94cde85ed01cb11e9de0ee8f00d` confirmed sampler-owned outcome and duration emission without status traffic and bound the real startup seam. The final timestamp-gauge change is covered by targeted red/green tests and the same boot regression at `c0f36e22b527c2fd6a63a548d036d006ffa17171`. ### Complexity Readiness remains one process-local sample per probe. Dependency I/O now has one owner and one execution path; request volume cannot increase dependency-check load. One completion timestamp replaces the prior server-side age update path; Datadog computes age without another loop. Community admission also returns to one fail-closed rule for inactive and unknown tenant state. Generated with [Claude Code](https://claude.ai/code) --------- Signed-off-by: tornquist <tornquist@squareup.com> Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Amp <amp@ampcode.com>
252 lines
8.9 KiB
Bash
Executable File
252 lines
8.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# run-tests.sh — Run Buzz test suite
|
|
# =============================================================================
|
|
# Usage:
|
|
# ./scripts/run-tests.sh # run all tests (default)
|
|
# ./scripts/run-tests.sh unit # unit tests only (no infra needed)
|
|
# ./scripts/run-tests.sh integration # integration tests only
|
|
# ./scripts/run-tests.sh all # explicit all
|
|
# =============================================================================
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
|
MODE="${1:-all}"
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
NC='\033[0m'
|
|
|
|
log() { echo -e "${BLUE}[run-tests]${NC} $*"; }
|
|
success(){ echo -e "${GREEN}[run-tests]${NC} $*"; }
|
|
warn() { echo -e "${YELLOW}[run-tests]${NC} $*"; }
|
|
error() { echo -e "${RED}[run-tests]${NC} $*" >&2; }
|
|
section(){ echo -e "\n${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"; echo -e "${CYAN} $*${NC}"; echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"; }
|
|
|
|
cd "${REPO_ROOT}"
|
|
|
|
# ---- Load .env if present ---------------------------------------------------
|
|
|
|
if [[ -f ".env" ]]; then
|
|
log "Loading .env..."
|
|
set -o allexport
|
|
# shellcheck disable=SC1091
|
|
source .env
|
|
set +o allexport
|
|
else
|
|
# Use defaults matching docker-compose.yml
|
|
export DATABASE_URL="postgres://buzz:buzz_dev@localhost:5432/buzz" # sadscan:disable np.postgres.1
|
|
export PGHOST=localhost
|
|
export PGPORT=5432
|
|
export PGUSER=buzz
|
|
export PGPASSWORD=buzz_dev
|
|
export PGDATABASE=buzz
|
|
export REDIS_URL="redis://localhost:6379"
|
|
fi
|
|
|
|
# ---- Track results ----------------------------------------------------------
|
|
|
|
declare -a PASSED=()
|
|
declare -a FAILED=()
|
|
|
|
run_test_step() {
|
|
local name="$1"
|
|
shift
|
|
log "Running: ${name}"
|
|
if "$@"; then
|
|
success "${name} passed"
|
|
PASSED+=("${name}")
|
|
else
|
|
error "${name} FAILED"
|
|
FAILED+=("${name}")
|
|
fi
|
|
}
|
|
|
|
# ---- Check / start infra (for integration tests) ----------------------------
|
|
|
|
ensure_infra() {
|
|
"${REPO_ROOT}/bin/just" _ensure-migrations
|
|
}
|
|
|
|
# ---- Unit tests (no infra needed) -------------------------------------------
|
|
|
|
run_unit_tests() {
|
|
section "Unit Tests (no infra required)"
|
|
|
|
run_test_step "buzz-core tests" \
|
|
cargo test -p buzz-core --lib -- --nocapture
|
|
|
|
run_test_step "buzz-audit tests" \
|
|
cargo test -p buzz-audit --lib -- --nocapture
|
|
|
|
run_test_step "buzz-auth unit tests" \
|
|
cargo test -p buzz-auth --lib -- --nocapture
|
|
|
|
run_test_step "buzz-voice tests" \
|
|
cargo test -p buzz-voice --lib -- --nocapture
|
|
|
|
run_test_step "buzz-cli tests" \
|
|
cargo test -p buzz-cli -- --nocapture
|
|
|
|
# buzz-sdk builder/validation unit tests: pure event-builder and input
|
|
# validation, no infra. Mirrors the nextest path in `just test-unit` — the
|
|
# two lists must stay in step. `--lib` matches the nextest invocation and
|
|
# avoids the full-package rustdoc dependency-resolution flake.
|
|
run_test_step "buzz-sdk unit tests" \
|
|
cargo test -p buzz-sdk --lib -- --nocapture
|
|
|
|
# Keep the relay-to-agent trust-boundary regressions in the fallback path
|
|
# when cargo-nextest is unavailable.
|
|
run_test_step "buzz-acp tests" \
|
|
cargo test -p buzz-acp -- --nocapture
|
|
|
|
|
|
# buzz-db migrator/lint unit tests (no infra): guard the embedded-migrator
|
|
# invariant (exactly the consolidated 0001; cutover/backfill stays an operator
|
|
# script, not startup state) and the tenant-scoping lints. The Postgres-backed
|
|
# buzz-db tests are #[ignore]d; nothing here (or in integration mode below,
|
|
# which runs `cargo test -p buzz-db` without --ignored) runs them — they need a
|
|
# separate isolated-DB gate, so --lib keeps this step infra-free.
|
|
run_test_step "buzz-db unit tests" \
|
|
cargo test -p buzz-db --lib -- --nocapture
|
|
|
|
run_test_step "buzz-media storage snapshot serialization test" \
|
|
cargo test -p buzz-media --lib bucket_index::tests::bucket_snapshot_json_round_trip_preserves_community_keys -- --exact --nocapture
|
|
|
|
run_test_step "buzz-admin storage snapshot tests" \
|
|
cargo test -p buzz-admin storage_snapshot -- --nocapture
|
|
|
|
# Multi-tenant conformance gate: independent replay checker + golden
|
|
# fixtures (buzz-conformance). Pure in-process trace replay, no infra.
|
|
run_test_step "buzz-conformance tests" \
|
|
cargo test -p buzz-conformance -- --nocapture
|
|
|
|
run_test_step "buzz-push-gateway tests" \
|
|
cargo test -p buzz-push-gateway -- --nocapture
|
|
run_test_step "buzz-push-gateway personal development tests" \
|
|
cargo test -p buzz-push-gateway --features personal-dev-app-attest -- --nocapture
|
|
|
|
# Kubernetes backend provider: pure decision layers driven by a fake
|
|
# substrate, no cluster. Mirrors the nextest path in `just test-unit` —
|
|
# the two lists must stay in step or the fallback silently covers less.
|
|
run_test_step "buzz-backend-kubernetes tests" \
|
|
cargo test -p buzz-backend-kubernetes -- --nocapture
|
|
|
|
# Keep fallback parity with `just test-unit`: one LaunchDarkly-feature run
|
|
# exercises both default and feature-gated buzz-feature-flags tests.
|
|
run_test_step "buzz-feature-flags tests (launchdarkly)" \
|
|
cargo test -p buzz-feature-flags --features launchdarkly -- --nocapture
|
|
|
|
# buzz-agent model-capabilities corpus: the Rust half of the cross-language
|
|
# drift guard. model_capabilities.rs embeds scripts/model-capabilities.json +
|
|
# scripts/normative-corpus.json via include_str! and replays the full locked
|
|
# corpus as pure in-process tests (no infra). Mirrors the nextest path in
|
|
# `just test-unit` — the two lists must stay in step.
|
|
run_test_step "buzz-agent unit tests" \
|
|
cargo test -p buzz-agent --lib -- --nocapture
|
|
|
|
# ACP author-gate and queue tests are pure unit tests. Keep this fallback in
|
|
# step with `just test-unit`; ignored lifecycle tests run elsewhere.
|
|
run_test_step "buzz-acp unit tests" \
|
|
cargo test -p buzz-acp --lib -- --nocapture
|
|
|
|
# Mirror the relay filters from `just test-unit`: the three handler modules,
|
|
# storage-snapshot helpers, readiness and router unit suites, and the single
|
|
# scoped admission regression in state::tests.
|
|
run_test_step "buzz-relay channel authorization tests" \
|
|
cargo test -p buzz-relay --lib handlers::channel_authz:: -- --nocapture
|
|
|
|
run_test_step "buzz-relay moderation authorization tests" \
|
|
cargo test -p buzz-relay --lib handlers::moderation_authz:: -- --nocapture
|
|
|
|
run_test_step "buzz-relay side-effects helper tests" \
|
|
cargo test -p buzz-relay --lib handlers::side_effects::tests:: -- --nocapture
|
|
|
|
run_test_step "buzz-relay storage snapshot tests" \
|
|
cargo test -p buzz-relay --lib storage_sweep::tests:: -- --nocapture
|
|
|
|
run_test_step "buzz-relay readiness tests" \
|
|
cargo test -p buzz-relay --lib readiness::tests:: -- --nocapture
|
|
|
|
run_test_step "buzz-relay router tests" \
|
|
cargo test -p buzz-relay --lib router::tests:: -- --nocapture
|
|
|
|
run_test_step "buzz-relay admission regression test" \
|
|
cargo test -p buzz-relay --lib state::tests::neither_a_confirmed_inactive_community_nor_a_failed_lookup_admits_the_socket -- --nocapture
|
|
}
|
|
|
|
# ---- DB / integration tests (infra required) --------------------------------
|
|
|
|
run_integration_tests() {
|
|
section "Integration Tests (requires running services)"
|
|
|
|
ensure_infra
|
|
|
|
run_test_step "buzz-db tests" \
|
|
cargo test -p buzz-db -- --nocapture
|
|
|
|
if find crates/buzz-auth/tests -maxdepth 1 -name '*.rs' -print -quit 2>/dev/null | grep -q .; then
|
|
run_test_step "buzz-auth integration tests" \
|
|
cargo test -p buzz-auth --test '*' -- --nocapture
|
|
else
|
|
run_test_step "buzz-auth (no integration tests found)" true
|
|
fi
|
|
|
|
run_test_step "workspace integration tests" \
|
|
cargo test --test '*' -- --nocapture 2>/dev/null || \
|
|
run_test_step "workspace integration tests (none found)" true
|
|
}
|
|
|
|
# ---- Main -------------------------------------------------------------------
|
|
|
|
START_TIME=$(date +%s)
|
|
|
|
case "${MODE}" in
|
|
unit)
|
|
run_unit_tests
|
|
;;
|
|
integration)
|
|
run_integration_tests
|
|
;;
|
|
all|*)
|
|
run_unit_tests
|
|
run_integration_tests
|
|
;;
|
|
esac
|
|
|
|
END_TIME=$(date +%s)
|
|
ELAPSED=$((END_TIME - START_TIME))
|
|
|
|
# ---- Summary ----------------------------------------------------------------
|
|
|
|
section "Test Summary"
|
|
echo ""
|
|
echo -e " Duration: ${ELAPSED}s"
|
|
echo ""
|
|
|
|
if [[ ${#PASSED[@]} -gt 0 ]]; then
|
|
echo -e " ${GREEN}Passed (${#PASSED[@]}):${NC}"
|
|
for t in "${PASSED[@]}"; do
|
|
echo -e " ${GREEN}pass${NC} ${t}"
|
|
done
|
|
fi
|
|
|
|
if [[ ${#FAILED[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo -e " ${RED}Failed (${#FAILED[@]}):${NC}"
|
|
for t in "${FAILED[@]}"; do
|
|
echo -e " ${RED}fail${NC} ${t}"
|
|
done
|
|
echo ""
|
|
exit 1
|
|
fi
|
|
|
|
echo ""
|
|
success "All tests passed!"
|
|
exit 0
|