mirror of
https://github.com/block/buzz.git
synced 2026-11-10 11:21:05 +00:00
22d96d94ef
> **The pin is a temporary workaround for this week's public release.
The SQLx fix is still unmerged.**
>
> 1. We considered a maintained fork, but repository permissions blocked
it.
> 2. We will move to an official SQLx release once it includes the
required fixes and is verified with Buzz. Revisit the fork if needed.
## Bug
https://linear.app/squareup/issue/BUZZ-656/relay-writer-pool-connections-stranded-after-a-burst-bb-public-2026-11
A short database interruption can leave the relay unable to handle
database requests after traffic returns to normal. All connection slots
can remain occupied even though no request is using them and PostgreSQL
is idle. New requests then time out waiting for a slot.
## Cause
SQLx, our database library, returns connections to the pool in a
background task. If a socket stays open but stops answering, the final
health check has no timeout. That task keeps its slot, and
idle-connection cleanup cannot reclaim it
([#4349](https://github.com/transact-rs/sqlx/issues/4349)).
## Fix
Pin SQLx to
[`c3735abf`](c3735abf03)
from [PR #4407](https://github.com/transact-rs/sqlx/pull/4407). After
five seconds, it stops a stalled return operation, discards the
connection, and frees the slot so new requests can open a replacement.
Preserve the existing [transaction rollback
(#4394)](https://github.com/transact-rs/sqlx/pull/4394) and [SQLite
compatibility (#4402)](https://github.com/transact-rs/sqlx/pull/4402)
fixes. Give the reader's own check four seconds so it can record its
timeout before the outer deadline.
## Critical risk
**This fix depends on a SQLx patch that has not been merged.** Its
five-second limit affects every pool using this revision, so it can also
discard slow but healthy connections and cause more reconnections. The
local reproduction proves recovery from the injected failure; it does
not establish production behavior.
**Prevent accidental replacement:** require SQLx `=0.9.0`, disable
Renovate updates for all seven crates, and fail required CI if the
approved revision, lockfile source, or update restriction changes. These
guards prevent version drift; they do not remove the runtime risk of the
patch. An upgrade must explicitly update the policy after verifying all
required fixes.
## Before and after evidence
With real PostgreSQL 17.6 on macOS arm64, a proxy silenced 100 existing
connections while allowing new connections. All borrowers dropped their
connections; PostgreSQL showed 100 idle sessions.
| Observation | Before | After |
| --- | --- | --- |
| Pool slots still occupied | 100 after 10 seconds | 0 after 5.008
seconds |
| New work | Next checkout timed out | All 100 replacement connections
served `SELECT 1` |
<details>
<summary>Captured before and after output</summary>
**Before**
```text
REPRO burst: 100 dropped borrowers, all release pings held: size=100 idle=0 active=100
REPRO burst DB sessions: [("idle", "ClientRead", 100)]
REPRO burst after 10.000680583s: size=100 idle=0 active=100
REPRO burst: later checkout timed out despite PostgreSQL being healthy
```
**After**
```text
REPRO burst: 100 dropped borrowers, all release pings held: size=100 idle=0 active=100
REPRO burst DB sessions: [("idle", "ClientRead", 100)]
REPRO burst after 5.008456917s: size=0 idle=0 active=0
REPRO burst: all 100 replacement slots serve SELECT 1 while old sessions stay silent
```
Before evidence used Buzz `96af488d968a1ae9d961c2132bea27822735f8da`
with SQLx `f1e94ec3`. After evidence used Buzz
`4bfe60197e8c390a8701053228cd50c5391f37d3` with the SQLx pin committed
in `f253760085e24358728fa4f4fef059ccf892157f`. These are local
fault-injection results: they establish the stranded-slot mechanism and
recovery, not what originally interrupted production database traffic.
</details>
---------
Signed-off-by: Helen Tian <htian@squareup.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
222 lines
8.8 KiB
TOML
222 lines
8.8 KiB
TOML
[workspace]
|
|
members = [
|
|
"crates/buzz-relay",
|
|
"crates/buzz-core",
|
|
"crates/ifc-core",
|
|
"crates/buzz-conformance",
|
|
"crates/buzz-push-gateway",
|
|
"crates/buzz-db",
|
|
"crates/buzz-pubsub",
|
|
"crates/buzz-auth",
|
|
"crates/buzz-search",
|
|
"crates/buzz-audit",
|
|
"crates/buzz-acp",
|
|
"crates/buzz-agent",
|
|
"crates/sprig",
|
|
"crates/buzz-test-client",
|
|
"crates/buzz-ws-client",
|
|
"crates/buzz-admin",
|
|
"crates/buzz-deletion",
|
|
"crates/buzz-feature-flags",
|
|
"crates/buzz-workflow",
|
|
"crates/buzz-media",
|
|
"crates/buzz-cli",
|
|
"crates/buzz-pairing-cli",
|
|
"crates/buzz-sdk",
|
|
"crates/buzz-persona",
|
|
"crates/git-credential-nostr",
|
|
"crates/git-sign-nostr",
|
|
"crates/buzz-pair-relay",
|
|
"crates/buzz-relay-mesh",
|
|
"crates/buzz-mesh-smoke",
|
|
"crates/buzz-dev-mcp",
|
|
"crates/buzz-voice",
|
|
"crates/buzz-audio-client",
|
|
"crates/buzz-backend-kubernetes",
|
|
"crates/buzz-datastore-tracing",
|
|
"examples/countdown-bot",
|
|
]
|
|
exclude = ["desktop/src-tauri"]
|
|
resolver = "2"
|
|
|
|
[workspace.package]
|
|
version = "0.1.0"
|
|
edition = "2021"
|
|
rust-version = "1.88.0"
|
|
license = "Apache-2.0"
|
|
repository = "https://github.com/block/sprout"
|
|
|
|
[workspace.dependencies]
|
|
# Runtime
|
|
tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "time", "sync", "io-util", "signal", "process"] }
|
|
tokio-util = { version = "0.7", features = ["rt", "codec"] }
|
|
|
|
# HTTP + WebSocket
|
|
axum = { version = "0.8", features = ["ws", "macros"] }
|
|
tower = { version = "0.5", features = ["timeout", "util", "limit"] }
|
|
tower-http = { version = "0.6", features = ["trace", "cors", "compression-gzip", "limit", "timeout", "fs"] }
|
|
|
|
# Database
|
|
sqlx = { version = "=0.9.0", features = [
|
|
"runtime-tokio", "tls-rustls", "postgres", "uuid", "chrono", "json"
|
|
] }
|
|
|
|
# Redis
|
|
redis = { version = "1.0", features = ["tokio-comp", "tokio-rustls-comp"] }
|
|
|
|
# Kubernetes (buzz-backend-kubernetes provider). No `ring` feature here: the
|
|
# process-level CryptoProvider is installed explicitly at startup, matching
|
|
# buzz-cli/buzz-acp/buzz-admin/buzz-relay/buzz-dev-mcp — see the comment on the
|
|
# crate's own rustls dependency.
|
|
kube = { version = "2.0", default-features = false, features = ["client", "rustls-tls"] }
|
|
k8s-openapi = { version = "0.26", features = ["v1_31"] }
|
|
|
|
# Nostr
|
|
nostr = { version = "0.44", features = ["nip44", "nip98"] }
|
|
|
|
# Serialization
|
|
serde = { version = "1", features = ["derive"] }
|
|
postcard = { version = "1", default-features = false, features = ["use-std"] }
|
|
|
|
# Inter-relay mesh transport (buzz-relay-mesh)
|
|
iroh = { version = "1.0.0-rc.0", default-features = false, features = ["tls-ring"] }
|
|
serde_json = { version = "1", features = ["raw_value"] }
|
|
serde_yaml = "0.9"
|
|
evalexpr = "11"
|
|
cron = "0.16"
|
|
# Observability
|
|
tracing = "0.1"
|
|
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
|
|
tracing-opentelemetry = { version = "0.33" }
|
|
opentelemetry = { version = "0.32", features = ["trace"] }
|
|
opentelemetry_sdk = { version = "0.32", features = ["trace", "rt-tokio"] }
|
|
opentelemetry-otlp = { version = "0.32", default-features = false, features = ["trace", "grpc-tonic", "tls-ring"] }
|
|
metrics = "0.24"
|
|
metrics-exporter-prometheus = "0.18"
|
|
metrics-util = "0.20"
|
|
|
|
# Error handling
|
|
thiserror = "2"
|
|
anyhow = "1"
|
|
|
|
# Utilities
|
|
uuid = { version = "1", features = ["v4", "serde"] }
|
|
chrono = { version = "0.4", features = ["serde"] }
|
|
httpdate = "1"
|
|
|
|
# JWT / JWS verification (NIP-FI federated identity assertions)
|
|
jsonwebtoken = { version = "10.4.0", default-features = false, features = ["aws_lc_rs"] }
|
|
|
|
# HTTP client (webhook delivery)
|
|
reqwest = { version = "0.13", features = ["json", "rustls", "stream"], default-features = false }
|
|
|
|
# Cryptography
|
|
sha2 = "0.11"
|
|
hex = "0.4"
|
|
hmac = "0.13"
|
|
base64 = "0.22"
|
|
|
|
# Randomness
|
|
rand = "0.10"
|
|
subtle = "2.6"
|
|
zeroize = "1.8"
|
|
|
|
# Concurrent data structures
|
|
dashmap = "6"
|
|
moka = { version = "0.12", features = ["sync"] }
|
|
arc-swap = "1"
|
|
lru = "0.16"
|
|
|
|
# Async stream utilities
|
|
futures-util = "0.3"
|
|
|
|
# WebSocket client (test client)
|
|
tokio-tungstenite = { version = "0.29", features = ["rustls-tls-webpki-roots"] }
|
|
url = "2"
|
|
|
|
# Property-based testing (dev-only)
|
|
proptest = "1"
|
|
|
|
# MCP SDK (used by buzz-dev-mcp and buzz-agent)
|
|
rmcp = { version = "1.1.0", features = ["server", "transport-io", "macros"] }
|
|
schemars = { version = "1", default-features = false }
|
|
|
|
# Internal crates
|
|
buzz-core = { path = "crates/buzz-core" }
|
|
buzz-conformance = { path = "crates/buzz-conformance" }
|
|
buzz-db = { path = "crates/buzz-db" }
|
|
buzz-deletion = { path = "crates/buzz-deletion" }
|
|
buzz-feature-flags = { path = "crates/buzz-feature-flags" }
|
|
buzz-auth = { path = "crates/buzz-auth" }
|
|
buzz-pubsub = { path = "crates/buzz-pubsub" }
|
|
buzz-search = { path = "crates/buzz-search" }
|
|
buzz-audit = { path = "crates/buzz-audit" }
|
|
buzz-workflow = { path = "crates/buzz-workflow" }
|
|
buzz-media = { path = "crates/buzz-media" }
|
|
buzz-sdk = { path = "crates/buzz-sdk" }
|
|
buzz-ws-client = { path = "crates/buzz-ws-client" }
|
|
buzz-relay-mesh = { path = "crates/buzz-relay-mesh" }
|
|
buzz-datastore-tracing = { path = "crates/buzz-datastore-tracing" }
|
|
|
|
# CI profile — builds the relay for desktop e2e. Dependencies keep full
|
|
# release optimization (warm from main's cache; they carry the runtime hot
|
|
# path: tokio/sqlx/axum). Workspace crates build at opt-level 1 — enough for
|
|
# stable e2e timing (PR #307 flakiness was opt-0 + debug-assertions) at
|
|
# roughly half the codegen cost. Incremental stays disabled: rust-cache exports
|
|
# CARGO_INCREMENTAL=0, and the CI compiler cache requires non-incremental units.
|
|
[profile.ci]
|
|
inherits = "release"
|
|
lto = false
|
|
opt-level = 1
|
|
|
|
[profile.ci.package."*"]
|
|
opt-level = 3
|
|
|
|
# Sprig profile — optimized for deploy-anywhere Sprig release artifacts.
|
|
# Sprig is distributed over the network and installed on fresh hosts, so binary
|
|
# size matters more than compile speed here. Keep this separate from the normal
|
|
# `release` profile so desktop/dev release builds do not inherit the slower
|
|
# size-focused settings unless they opt in explicitly.
|
|
[profile.sprig]
|
|
inherits = "release"
|
|
opt-level = "z"
|
|
lto = "fat"
|
|
codegen-units = 1
|
|
panic = "abort"
|
|
strip = true
|
|
|
|
# Temporary fork pin: aws-creds 0.39.1 (via rust-s3) cannot read EKS Pod Identity
|
|
# credentials (AWS_CONTAINER_CREDENTIALS_FULL_URI + AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE),
|
|
# which the relay pod on bb-block requires for S3 media + git storage. This pins
|
|
# aws-creds to a fork that adopts the aws-creds portion of durch/rust-s3#449
|
|
# (FULL_URI + token-file + Authorization header, refresh-safe, with a loopback
|
|
# allowlist for the auth token). Revert to crates.io once #449 lands upstream.
|
|
[patch.crates-io]
|
|
aws-creds = { git = "https://github.com/tlongwell-block/rust-s3", rev = "c9fce3620dd434c1f810101d672cf384268dbb0f" }
|
|
# SQLx: pin every crate together; these crates share types. This unreleased
|
|
# revision includes the whole-return deadline from the open upstream PR #4407
|
|
# and retains the fixes below. Return to a released version once it includes
|
|
# all three fixes:
|
|
# - https://github.com/transact-rs/sqlx/pull/4407: bound the entire return to
|
|
# the pool, including the release hook and SQLx ping, to five seconds. Silent
|
|
# sockets must release their pool permits so new connections can replace them.
|
|
# - https://github.com/transact-rs/sqlx/pull/4394: a `BEGIN` whose future is
|
|
# dropped before Postgres answers no longer returns its pooled connection
|
|
# still inside the transaction. 0.9.0 queues no `ROLLBACK` for it, so the
|
|
# next borrower runs inside the leftover transaction (`25001`, then `25P02`
|
|
# once a statement in it fails).
|
|
# - https://github.com/transact-rs/sqlx/pull/4402: MeshLLM 0.76 uses
|
|
# libsqlite3-sys 0.38, which 0.9.0's optional SQLite driver rejects; Cargo
|
|
# resolves it even though Buzz only enables PostgreSQL.
|
|
# The exact version prevents lockfile updates from selecting a newer unpatched
|
|
# release. Renovate updates are disabled for these crates; CI checks this pin
|
|
# with scripts/check-sqlx-pin.py. Update that policy only after verifying all
|
|
# three fixes in the replacement release.
|
|
sqlx = { git = "https://github.com/launchbadge/sqlx", rev = "c3735abf034689bb1fab9d7bf9f040e492de28cf" }
|
|
sqlx-core = { git = "https://github.com/launchbadge/sqlx", rev = "c3735abf034689bb1fab9d7bf9f040e492de28cf" }
|
|
sqlx-macros = { git = "https://github.com/launchbadge/sqlx", rev = "c3735abf034689bb1fab9d7bf9f040e492de28cf" }
|
|
sqlx-macros-core = { git = "https://github.com/launchbadge/sqlx", rev = "c3735abf034689bb1fab9d7bf9f040e492de28cf" }
|
|
sqlx-mysql = { git = "https://github.com/launchbadge/sqlx", rev = "c3735abf034689bb1fab9d7bf9f040e492de28cf" }
|
|
sqlx-postgres = { git = "https://github.com/launchbadge/sqlx", rev = "c3735abf034689bb1fab9d7bf9f040e492de28cf" }
|
|
sqlx-sqlite = { git = "https://github.com/launchbadge/sqlx", rev = "c3735abf034689bb1fab9d7bf9f040e492de28cf" }
|