Files
buzz/crates/buzz-acp/src/lib.rs

8210 lines
328 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#![deny(unsafe_code)]
mod acp;
mod config;
mod engram_fetch;
mod filter;
mod observer;
mod pool;
mod pool_lifecycle;
mod queue;
mod relay;
mod setup_mode;
mod usage;
pub use usage::TurnUsage;
use std::collections::{HashMap, HashSet, VecDeque};
use std::sync::Arc;
use std::time::Duration;
use acp::{AcpClient, EnvVar, McpServer};
use anyhow::Result;
use buzz_core::kind::{
KIND_MEMBER_ADDED_NOTIFICATION, KIND_MEMBER_REMOVED_NOTIFICATION, KIND_STREAM_MESSAGE,
};
use buzz_core::observer::{
decrypt_observer_payload, encrypt_observer_payload, OBSERVER_FRAME_TELEMETRY,
OBSERVER_MAX_PLAINTEXT_LEN,
};
use clap::Parser;
use config::{
AuthAgentArgs, AuthMethodsArgs, AuthenticateArgs, Config, DedupMode, ModelsArgs,
MultipleEventHandling, RespondTo, SubscribeMode,
};
use filter::SubscriptionRule;
use futures_util::FutureExt;
use nostr::{PublicKey, ToBech32};
use pool::{
AgentPool, ControlSignal, IdleSwitchResult, OwnedAgent, PromptContext, PromptOutcome,
PromptResult, PromptSource, SessionState, TimeoutKind,
};
use pool_lifecycle::PoolLifecycle;
use queue::{CancelReason, EventQueue, FlushBatch, QueuedEvent, ThreadTags};
use relay::{HarnessRelay, RelayEventPublisher};
use tokio::sync::{mpsc, watch};
use tracing_subscriber::EnvFilter;
use uuid::Uuid;
/// Check if argv[1] matches a subcommand name, before any clap parsing.
///
/// This avoids clap rejecting harness flags (like `--private-key`) that aren't
/// declared on the subcommand's `Parser`. The `models` path has its own
/// dedicated parser; the default path uses the existing `CliArgs`.
///
/// **Constraint**: subcommand must be argv[1] — flags before the subcommand
/// name (e.g., `buzz-acp --verbose models`) are not supported.
fn is_subcommand(name: &str) -> bool {
std::env::args().nth(1).map(|a| a == name).unwrap_or(false)
}
/// Timeout for lightweight helper subcommands (spawn + initialize + model/method probes).
const MODELS_TIMEOUT: Duration = Duration::from_secs(10);
/// Timeout for `buzz-acp authenticate`. Browser-based vendor auth can require
/// human interaction, so it must not share the short probe timeout.
const AUTHENTICATE_TIMEOUT: Duration = Duration::from_secs(10 * 60);
/// Publish a kind:20001 presence update event via the WebSocket connection.
///
/// Ephemeral kinds (20000-29999) are rejected by the HTTP bridge, so presence
/// updates must be routed through the WS path.
///
/// Content is a bare status string (`"online"`, `"away"`, `"offline"`) matching
/// the desktop client's format. The relay stores this in Redis and synthesizes
/// it back on presence queries.
async fn publish_presence(
publisher: &relay::RelayEventPublisher,
keys: &nostr::Keys,
status: &str,
) -> Result<(), relay::RelayError> {
use buzz_core::kind::KIND_PRESENCE_UPDATE;
use nostr::{EventBuilder, Kind};
let event = EventBuilder::new(Kind::Custom(KIND_PRESENCE_UPDATE as u16), status)
.tags([])
.sign_with_keys(keys)
.map_err(|e| relay::RelayError::Http(format!("presence sign error: {e}")))?;
publisher.publish_event(event).await?;
Ok(())
}
fn emit_runtime_lifecycle(
observer: Option<&observer::ObserverHandle>,
start_nonce: &str,
pubkey: &str,
relay_url: &str,
lifecycle: &str,
error: Option<&str>,
) {
if let Some(observer) = observer {
observer.emit(
"managed_agent_runtime_lifecycle",
None,
&observer::ObserverContext::default(),
serde_json::json!({
"pubkey": pubkey,
"relayUrl": relay_url,
"startNonce": start_nonce,
"lifecycle": lifecycle,
"error": error,
}),
);
}
}
/// Resolve the agent's owner pubkey at startup.
///
/// Priority:
/// 1. `BUZZ_AUTH_TAG` env var — NIP-OA attestation signed by the owner.
/// Verified against the agent's own pubkey to extract the owner pubkey.
/// 2. `--agent-owner` CLI flag / `BUZZ_ACP_AGENT_OWNER` env var.
fn resolve_agent_owner(config: &Config) -> Option<String> {
// Try BUZZ_AUTH_TAG first (NIP-OA attestation).
if let Ok(auth_tag) = std::env::var("BUZZ_AUTH_TAG") {
if !auth_tag.is_empty() {
let agent_pk = config.keys.public_key();
match buzz_sdk::nip_oa::verify_auth_tag(&auth_tag, &agent_pk) {
Ok(owner_pk) => {
let owner_hex = owner_pk.to_hex().to_ascii_lowercase();
tracing::info!("owner resolved from BUZZ_AUTH_TAG: {owner_hex}");
return Some(owner_hex);
}
Err(e) => {
tracing::warn!("BUZZ_AUTH_TAG verification failed: {e} — falling back");
}
}
}
}
// Fall back to --agent-owner config.
config.agent_owner.clone()
}
/// Cache for the agent's owner pubkey.
///
/// Owner is now provided via `--agent-owner` config flag (no REST lookup).
/// Cache for the agent's owner pubkey + sibling lookups.
///
/// Siblings are other agents whose NIP-OA auth tag proves the same owner.
/// Lookup results are cached for the process lifetime (attestations are immutable).
struct OwnerCache {
pubkey: Option<String>,
/// author_hex → is_sibling (true = same owner, false = not)
siblings: std::sync::Mutex<HashMap<String, bool>>,
}
impl OwnerCache {
fn new(initial: Option<String>) -> Self {
Self {
pubkey: initial,
siblings: std::sync::Mutex::new(HashMap::new()),
}
}
/// Return the cached owner pubkey.
fn get(&self) -> Option<&str> {
self.pubkey.as_deref()
}
/// Check if author is a known sibling (cached result).
fn is_known_sibling(&self, author: &str) -> Option<bool> {
self.siblings.lock().ok()?.get(author).copied()
}
/// Cache a sibling lookup result.
fn cache_sibling(&self, author: String, is_sibling: bool) {
if let Ok(mut map) = self.siblings.lock() {
// Cap at 256 entries to prevent unbounded growth.
if map.len() >= 256 {
map.clear();
}
map.insert(author, is_sibling);
}
}
}
/// Check if `author` is the owner OR a sibling (same owner via NIP-OA).
///
/// For unknown authors, queries their kind:0 profile to extract the NIP-OA
/// auth tag and verify the owner matches. Result is cached.
async fn is_owner_or_sibling(
author: &str,
owner_cache: &OwnerCache,
rest_client: &relay::RestClient,
) -> bool {
let my_owner = match owner_cache.get() {
Some(o) => o,
None => return false, // no owner configured — fail closed
};
// Direct owner check.
if author == my_owner {
return true;
}
// Check sibling cache.
if let Some(cached) = owner_cache.is_known_sibling(author) {
return cached;
}
// Query the author's kind:0 profile to check for NIP-OA auth tag.
let is_sibling = check_sibling_via_profile(author, my_owner, rest_client).await;
owner_cache.cache_sibling(author.to_string(), is_sibling);
is_sibling
}
/// Inbound author gate decision: does this author's event fire a turn?
///
/// Coarse security policy applied before subscription rules. Both `OwnerOnly`
/// and `Allowlist` accept the owner and same-owner siblings; `Allowlist`
/// additionally accepts the explicit external pubkey list.
///
/// # DM hardening (`is_dm`)
///
/// Clients auto-p-tag every DM participant, so in a DM *any* participant's
/// message looks like a mention and would fire a turn. Combined with
/// agent-initiated DMs (the agent can be asked to DM a third party), that
/// turns `anyone`/`allowlist` modes into transitive access grants: whoever
/// lands in a DM with the agent can prompt it. To close that hole, when
/// `is_dm` is true only the owner and cryptographically verified same-owner
/// siblings may fire a turn — the explicit allowlist and `anyone` mode do
/// NOT apply inside DMs. `Nobody` still drops everything. Callers must
/// resolve `is_dm` fail-closed: unknown channel type ⇒ treat as DM.
async fn author_allowed(
respond_to: &RespondTo,
allowlist: &HashSet<String>,
author: &str,
is_dm: bool,
owner_cache: &OwnerCache,
rest_client: &relay::RestClient,
) -> bool {
if is_dm {
return match respond_to {
RespondTo::Nobody => false,
_ => is_owner_or_sibling(author, owner_cache, rest_client).await,
};
}
match respond_to {
RespondTo::Anyone => true,
RespondTo::Nobody => false,
RespondTo::OwnerOnly => is_owner_or_sibling(author, owner_cache, rest_client).await,
RespondTo::Allowlist => {
allowlist.contains(author)
|| is_owner_or_sibling(author, owner_cache, rest_client).await
}
}
}
/// Resolve whether `channel_id` is a DM, for the inbound author gate.
///
/// Resolution order:
/// 1. Startup discovery metadata (`startup_info`) — covers channels known at
/// process start.
/// 2. Per-loop resolution cache (`cache`) — covers channels resolved since.
/// 3. Lazy REST fetch of the channel's kind:39000 metadata — covers channels
/// the agent was added to *after* startup (the exploit path: an
/// agent-initiated DM is exactly such a channel).
///
/// Fail-closed: if the fetch fails or times out, the channel is treated as a
/// DM for this event and the result is NOT cached, so a later event retries
/// the fetch instead of pinning a mis-classification.
pub(crate) async fn is_dm_channel(
channel_id: Uuid,
channel_info: &pool::ChannelInfoResolver,
) -> bool {
match channel_info.resolve(channel_id).await {
Some(info) => info.channel_type == "dm",
None => {
tracing::warn!(
channel_id = %channel_id,
"channel type unresolved — treating as DM for author gate (fail closed)"
);
true
}
}
}
/// Query an author's kind:0 profile and check if their NIP-OA auth tag
/// proves the same owner as us.
async fn check_sibling_via_profile(
author: &str,
expected_owner: &str,
rest_client: &relay::RestClient,
) -> bool {
let filter = nostr::Filter::new()
.kind(nostr::Kind::Metadata)
.author(match nostr::PublicKey::from_hex(author) {
Ok(pk) => pk,
Err(_) => return false,
})
.limit(1);
let resp = match tokio::time::timeout(Duration::from_millis(2000), rest_client.query(&[filter]))
.await
{
Ok(Ok(v)) => v,
_ => return false, // timeout or error — fail closed
};
// Look for an "auth" tag in the profile event.
let events = match resp.as_array() {
Some(arr) => arr,
None => return false,
};
let event = match events.first() {
Some(e) => e,
None => return false,
};
let tags = match event.get("tags").and_then(|t| t.as_array()) {
Some(t) => t,
None => return false,
};
// Find ["auth", owner_pk, conditions, sig] and verify the Schnorr signature.
// Don't trust the relay — verify ourselves.
let agent_pk = match nostr::PublicKey::from_hex(author) {
Ok(pk) => pk,
Err(_) => return false,
};
for tag in tags {
let parts = match tag.as_array() {
Some(p) if p.len() >= 4 => p,
_ => continue,
};
if parts[0].as_str() != Some("auth") {
continue;
}
let tag_owner = match parts[1].as_str() {
Some(o) => o,
None => continue,
};
// Only verify if the owner field matches ours.
if !tag_owner.eq_ignore_ascii_case(expected_owner) {
continue;
}
// Cryptographically verify the NIP-OA attestation signature.
let tag_json = serde_json::to_string(tag).unwrap_or_default();
match buzz_sdk::nip_oa::verify_auth_tag(&tag_json, &agent_pk) {
Ok(_) => {
tracing::debug!(author, expected_owner, "sibling verified via NIP-OA");
return true;
}
Err(e) => {
tracing::debug!(author, "NIP-OA auth tag verification failed: {e}");
}
}
}
false
}
/// Observer frames are published at a global rate of AT MOST ONE relay frame
/// per tick — not one per channel, and not one per drain. Everything that
/// accumulates between ticks waits in [`ObserverPublishQueue`] as events and
/// is packed greedily into that single frame. One update per second is smooth
/// enough for a human watching the session viewer, and the global budget is
/// what makes the relay cost model flat: observer frames bill the agent's
/// `LimitType::Messages` quota (`agent_standard_messages_per_min` = 120,
/// enforced in relay `connection.rs::enforce_ws_admission`), shared with the
/// agent's real chat messages. At 1 frame/s telemetry spends at most 60/min —
/// half that budget — regardless of how many channels are active. A slower
/// tick (e.g. 2s → 30/min) would leave more quota headroom for chat at the
/// price of doubled viewer latency; this constant is the knob.
const OBSERVER_PUBLISH_TICK: Duration = Duration::from_secs(1);
/// Byte budget for EVERYTHING retained while awaiting a publish slot: the
/// event FIFO (serialized, post-`fit_observer_event_to_budget` bytes) PLUS
/// the chunk coalescer's pending buffer (serialized event skeletons + raw
/// accumulated text). Both stores count against this one cap — a
/// high-cardinality chunk flood (many distinct coalescer keys) is bounded
/// exactly like a plain event flood; neither buffer is a bypass around the
/// other. Lossless-ness is bounded by this budget: each publish slot packs
/// one ~64KB frame, gathered queue-wide for the front channel, so a single
/// channel drains at ~64KB/s and 4 MiB buys roughly **64 seconds** of
/// sustained over-production before the oldest items are dropped WITH
/// accounting (a warn carrying the dropped-event count). With C channels
/// producing concurrently the slots round-robin between them, so the
/// per-channel drain is ~64KB/Cs and the budget shortens accordingly —
/// still bytes-per-slot, never events-per-slot (see
/// [`ObserverPublishQueue::next_frame`]). Beyond-budget floods therefore
/// degrade to designed, visible loss — strictly better than the
/// pre-batching pacer's silent 90/min drop.
const OBSERVER_PENDING_QUEUE_MAX_BYTES: usize = 4 * 1024 * 1024;
/// Observer event kind for a batch envelope wrapping multiple events.
///
/// The payload is `{"events": [<ObserverEvent>, ...]}` with every inner event
/// carrying its own `seq`/`timestamp`, so consumers process inner events
/// exactly as they would unbatched ones. Single pending events are published
/// unwrapped, so the envelope only appears when there is something to batch.
const OBSERVER_BATCH_KIND: &str = "batch";
/// Collects observer events awaiting a publish slot.
///
/// Chunk-type events ride the [`ObserverChunkCoalescer`]; everything else is
/// appended in arrival order, force-flushing pending chunks first — the same
/// ordering rule the pre-batching publisher enforced, so merged chunk text can
/// never leapfrog a tool call that arrived mid-stream.
///
/// Events wait here as EVENTS, not pre-sealed frames: each publish slot packs
/// one frame at publish time ([`Self::next_frame`]), so a backlog keeps
/// compacting into full frames instead of freezing into a frame queue.
///
/// The queue is bounded by [`OBSERVER_PENDING_QUEUE_MAX_BYTES`]. When a
/// sustained flood outruns the one-frame-per-tick drain for longer than the
/// budget, the OLDEST events are dropped (the viewer wants recent state) with
/// accounting: a warning carrying the dropped-event count, and
/// `dropped_events` for tests.
#[derive(Default)]
struct ObserverPublishQueue {
coalescer: ObserverChunkCoalescer,
/// `(serialized_len, source_events, event)`, oldest first. Length is
/// captured at enqueue (post-fit) so byte accounting never re-serializes
/// on eviction; `source_events` is how many GENERATED observer events the
/// entry represents (a merged chunk carries every chunk it absorbed), so
/// eviction accounting stays in source units after flush.
events: VecDeque<(usize, u64, observer::ObserverEvent)>,
pending_bytes: usize,
/// SOURCE observer events lost to byte-budget eviction. Counted in
/// generated-event units, not retained entries: a coalesced entry that
/// merged N chunks accounts for N when evicted. A PUBLISHED merged entry
/// delivers all N sources' text in one event, so the invariant is
/// `ingested == dropped_events + Σ source_events over published events`.
dropped_events: u64,
}
impl ObserverPublishQueue {
fn ingest(&mut self, event: observer::ObserverEvent) {
// ObserverChunkCoalescer::ingest returns immediately-publishable events
// (force-flushed pending chunks + non-chunk passthrough, or a pending
// set displaced by the 60KB pre-flush); they join the queue in the
// order the coalescer emitted them, each carrying the count of source
// events it represents.
for (source_events, ready) in self.coalescer.ingest(event) {
self.enqueue(source_events, ready);
}
self.enforce_byte_budget();
}
fn enqueue(&mut self, source_events: u64, mut event: observer::ObserverEvent) {
// Pre-trim at enqueue so (a) byte accounting reflects what will ship
// and (b) one oversized leaf cannot force every frame it touches into
// whole-envelope elision downstream.
fit_observer_event_to_budget(&mut event);
let bytes = serialized_len(&event);
self.pending_bytes += bytes;
self.events.push_back((bytes, source_events, event));
}
/// Total bytes retained across BOTH stores — the event FIFO and the
/// coalescer's pending chunk buffer. The budget binds this sum; counting
/// only the FIFO would let a high-cardinality chunk flood (many distinct
/// coalescer keys, nothing ever flushing) grow unbounded outside the cap.
fn total_pending_bytes(&self) -> usize {
self.pending_bytes + self.coalescer.pending_bytes
}
/// Enforce [`OBSERVER_PENDING_QUEUE_MAX_BYTES`] over the total, dropping
/// OLDEST items first with accounting in SOURCE-event units. Global age
/// order across the two stores is structural: every enqueue path flushes
/// the coalescer first, so every pending coalescer entry is strictly newer
/// than every queued event — eviction is queue front, then coalescer
/// front. The `> 1` guard never drops the sole remaining item (any single
/// fitted event or pre-flush-capped chunk entry is far under the budget).
fn enforce_byte_budget(&mut self) {
let mut dropped = 0u64;
while self.total_pending_bytes() > OBSERVER_PENDING_QUEUE_MAX_BYTES
&& self.events.len() + self.coalescer.pending.len() > 1
{
if let Some((bytes, source_events, _)) = self.events.pop_front() {
self.pending_bytes -= bytes;
dropped += source_events;
} else {
dropped += self.coalescer.drop_oldest().expect("guard ensures an item");
}
}
if dropped > 0 {
self.dropped_events += dropped;
tracing::warn!(
dropped,
total_dropped = self.dropped_events,
pending_bytes = self.total_pending_bytes(),
"observer publish queue over byte budget; dropped oldest events"
);
}
}
/// True when nothing is waiting anywhere — the event queue AND the
/// coalescer's pending chunk buffer.
fn is_empty(&self) -> bool {
self.events.is_empty() && self.coalescer.pending.is_empty()
}
/// Pack and remove AT MOST ONE publishable frame: the front event's
/// channel, gathered queue-wide in FIFO order (packed greedily until
/// adding the next event would push the envelope over
/// `OBSERVER_MAX_PLAINTEXT_LEN`). Singletons ship unwrapped.
///
/// Two invariants bound the gather:
/// - A frame never mixes channels (the desktop archive indexes a frame
/// under its decrypted top-level `channelId`), and events keep their
/// FIFO order *within* each channel. Cross-channel frame order MAY
/// differ from arrival order — the desktop tolerates that everywhere:
/// the transcript store sorts + rebuilds on out-of-order arrival, the
/// archive is per-channel by construction, and the turn store's
/// watermark is keyed per (agent, channel).
/// - A NULL-channel event is a BARRIER nothing gathers across: null-scope
/// events (`agent_panic`-class) can causally couple to any channel, so
/// their relative order against every channel is preserved exactly.
/// Null-channel events themselves ship only as their contiguous front
/// run.
///
/// Gathering queue-wide (not just the front run) is what keeps the drain
/// rate in BYTES per slot rather than front-run-length events per slot:
/// with round-robin producers (channel A, B, A, B, ...) a front-run
/// packer degrades to ~1 event per slot regardless of size, silently
/// growing latency without ever tripping the byte budget.
///
/// Pending coalesced chunks are flushed into the queue first, so a
/// publish slot never leaves merged chunk text stranded behind the tick.
fn next_frame(&mut self) -> Option<observer::ObserverEvent> {
for (source_events, ready) in self.coalescer.flush() {
self.enqueue(source_events, ready);
}
let channel = self.events.front()?.2.channel_id.clone();
let mut picked: Vec<observer::ObserverEvent> = Vec::new();
let mut kept: VecDeque<(usize, u64, observer::ObserverEvent)> =
VecDeque::with_capacity(self.events.len());
let mut gathering = true;
while let Some((bytes, source_events, event)) = self.events.pop_front() {
if gathering && event.channel_id == channel {
picked.push(event);
if picked.len() > 1
&& serialized_len(&batch_envelope(&picked)) > OBSERVER_MAX_PLAINTEXT_LEN
{
// Frame full: the overflow event stays queued and leads
// its channel's next slot.
let event = picked.pop().expect("len > 1");
kept.push_back((bytes, source_events, event));
gathering = false;
} else {
self.pending_bytes -= bytes;
}
} else {
if gathering && (channel.is_none() || event.channel_id.is_none()) {
// Null-channel barrier (or, for a null-channel frame, the
// end of its contiguous front run): stop gathering.
gathering = false;
}
kept.push_back((bytes, source_events, event));
}
}
self.events = kept;
Some(seal_batch(picked))
}
}
/// A single event ships unwrapped; two or more get the batch envelope.
fn seal_batch(mut events: Vec<observer::ObserverEvent>) -> observer::ObserverEvent {
if events.len() == 1 {
return events.pop().expect("len == 1");
}
batch_envelope(&events)
}
/// Build the batch envelope for a set of same-channel events.
///
/// Envelope metadata mirrors the LAST inner event — the same convention the
/// chunk coalescer uses for merged chunks — so `(timestamp, seq)` ordering and
/// the desktop's latest-live-session tracking see the newest state.
fn batch_envelope(events: &[observer::ObserverEvent]) -> observer::ObserverEvent {
let last = events
.last()
.expect("batch envelope needs at least 1 event");
observer::ObserverEvent {
seq: last.seq,
timestamp: last.timestamp.clone(),
kind: OBSERVER_BATCH_KIND.to_string(),
agent_index: last.agent_index,
channel_id: last.channel_id.clone(),
session_id: last.session_id.clone(),
turn_id: last.turn_id.clone(),
started_at: last.started_at.clone(),
payload: serde_json::json!({
"events": serde_json::to_value(events).unwrap_or_default(),
}),
}
}
fn spawn_relay_observer_publisher(
observer: observer::ObserverHandle,
publisher: RelayEventPublisher,
keys: nostr::Keys,
agent_pubkey_hex: String,
owner_pubkey_hex: String,
owner_pubkey: PublicKey,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
// Subscribe BEFORE snapshotting so an event emitted between the two
// calls is never lost: it lands in the snapshot, the live receiver, or
// both. The overlap is deduped in the run loop via the snapshot's
// high-water `seq` (monotonic, assigned at emit).
let rx = observer.subscribe();
let snapshot = observer.snapshot();
run_relay_observer_publisher(
snapshot,
rx,
publisher,
keys,
agent_pubkey_hex,
owner_pubkey_hex,
owner_pubkey,
)
.await;
})
}
async fn run_relay_observer_publisher(
snapshot: Vec<observer::ObserverEvent>,
mut rx: tokio::sync::broadcast::Receiver<observer::ObserverEvent>,
publisher: RelayEventPublisher,
keys: nostr::Keys,
agent_pubkey_hex: String,
owner_pubkey_hex: String,
owner_pubkey: PublicKey,
) {
let mut queue = ObserverPublishQueue::default();
let max_snapshot_seq = snapshot.iter().map(|event| event.seq).max().unwrap_or(0);
for event in snapshot {
queue.ingest(event);
}
// Global pacer: AT MOST ONE relay frame per tick, no matter how many
// channels are active or how large the backlog is. `interval_at` starts
// the first tick a full period out, so a pre-loaded snapshot (up to the
// 1,000-event replay buffer on reconnect) cannot burst at t=0 — the old
// pacer's explicit "no initial burst" property, restored.
let mut publish_tick = tokio::time::interval_at(
tokio::time::Instant::now() + OBSERVER_PUBLISH_TICK,
OBSERVER_PUBLISH_TICK,
);
publish_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut closed = false;
loop {
tokio::select! {
result = rx.recv(), if !closed => {
match result {
Ok(event) => {
// Skip live events already delivered via the snapshot
// (the subscribe-before-snapshot overlap).
if event.seq <= max_snapshot_seq {
continue;
}
queue.ingest(event);
}
Err(tokio::sync::broadcast::error::RecvError::Lagged(count)) => {
tracing::warn!(dropped = count, "relay observer publisher lagged");
}
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
// Producer gone: stop selecting on the receiver and let
// the tick arm drain what remains — still one frame per
// tick. An unpaced final drain would be a burst bypass
// around everything the pacer exists to prevent.
closed = true;
}
}
}
_ = publish_tick.tick() => {
if let Some(frame) = queue.next_frame() {
publish_relay_observer_event(
&publisher, &keys, &agent_pubkey_hex,
&owner_pubkey_hex, &owner_pubkey, frame,
).await;
}
if closed && queue.is_empty() {
break;
}
}
}
}
}
#[derive(Default)]
struct ObserverChunkCoalescer {
pending: Vec<PendingObserverChunk>,
/// Approximate serialized bytes retained in `pending` (each entry's
/// serialized skeleton at creation plus appended chunk text). Counted
/// against [`OBSERVER_PENDING_QUEUE_MAX_BYTES`] by the owning
/// [`ObserverPublishQueue`] so this buffer can never grow outside the
/// queue's byte budget (a distinct-key chunk flood parks everything here
/// and nothing would otherwise bound it).
pending_bytes: usize,
}
struct PendingObserverChunk {
key: ObserverChunkKey,
event: observer::ObserverEvent,
text: String,
/// Bytes this entry contributes to `pending_bytes`.
bytes: usize,
/// GENERATED observer events merged into this entry (1 at creation, +1
/// per absorbed chunk). Evicting the entry loses this many source events,
/// so drop accounting must charge this count, not 1.
source_events: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct ObserverChunkKey {
update_type: String,
message_id: Option<String>,
channel_id: Option<String>,
session_id: Option<String>,
turn_id: Option<String>,
agent_index: Option<usize>,
}
/// Flush coalesced chunks before they exceed the NIP-44 plaintext limit (65,535 bytes).
/// Leave headroom for the JSON envelope wrapping the text. This is a SOFT pre-flush
/// of raw text below the hard cap; `fit_observer_event_to_budget` (the final ceiling,
/// keyed to `OBSERVER_MAX_PLAINTEXT_LEN` in buzz-core/observer.rs:25) is what actually
/// guarantees the serialized frame fits. Edit one of these two and review the other.
const OBSERVER_CHUNK_MAX_TEXT_BYTES: usize = 60_000;
impl ObserverChunkCoalescer {
/// Returns immediately-publishable events, each paired with the number of
/// SOURCE observer events it represents (merged chunks carry the count of
/// every chunk they absorbed; passthrough events are always 1).
fn ingest(&mut self, event: observer::ObserverEvent) -> Vec<(u64, observer::ObserverEvent)> {
let Some((key, text)) = observer_chunk_key_and_text(&event) else {
let mut events = self.flush();
events.push((1, event));
return events;
};
if let Some(pending) = self.pending.iter_mut().find(|pending| pending.key == key) {
// Flush before appending if this would exceed the plaintext size limit.
if pending.text.len() + text.len() >= OBSERVER_CHUNK_MAX_TEXT_BYTES {
let events = self.flush();
// Start a new pending entry with the current chunk.
self.push_pending(key, event, text);
return events;
}
pending.text.push_str(&text);
pending.bytes += text.len();
pending.source_events += 1;
self.pending_bytes += text.len();
pending.event.seq = event.seq;
pending.event.timestamp = event.timestamp;
return Vec::new();
}
self.push_pending(key, event, text);
Vec::new()
}
fn push_pending(
&mut self,
key: ObserverChunkKey,
event: observer::ObserverEvent,
text: String,
) {
// The entry RETAINS the first chunk's text twice until flush: once
// inside the serialized skeleton (`event.payload` still carries it)
// and once as the extracted `text` copy that appends grow. Both are
// real memory, so both count — charging only `serialized_len` lets a
// high-cardinality flood retain up to 2x the byte budget (each entry
// undercounts by exactly its first chunk's length).
let bytes = serialized_len(&event) + text.len();
self.pending_bytes += bytes;
self.pending.push(PendingObserverChunk {
key,
event,
text,
bytes,
source_events: 1,
});
}
/// Evict the OLDEST pending entry for byte-budget enforcement. Returns
/// the number of SOURCE events the entry represented (its merged chunk
/// count), or `None` when there is nothing to drop.
fn drop_oldest(&mut self) -> Option<u64> {
if self.pending.is_empty() {
return None;
}
let removed = self.pending.remove(0);
self.pending_bytes -= removed.bytes;
Some(removed.source_events)
}
fn flush(&mut self) -> Vec<(u64, observer::ObserverEvent)> {
self.pending_bytes = 0;
self.pending
.drain(..)
.map(|mut pending| {
set_observer_chunk_text(&mut pending.event.payload, pending.text);
(pending.source_events, pending.event)
})
.collect()
}
}
fn observer_chunk_key_and_text(
event: &observer::ObserverEvent,
) -> Option<(ObserverChunkKey, String)> {
let update = event.payload.get("params")?.get("update")?;
let update_type = update.get("sessionUpdate")?.as_str()?;
if !matches!(
update_type,
"agent_message_chunk" | "user_message_chunk" | "agent_thought_chunk"
) {
return None;
}
let text = update.get("content")?.get("text")?.as_str()?.to_string();
let message_id = update
.get("messageId")
.and_then(|value| value.as_str())
.map(ToOwned::to_owned);
Some((
ObserverChunkKey {
update_type: update_type.to_string(),
message_id,
channel_id: event.channel_id.clone(),
session_id: event.session_id.clone(),
turn_id: event.turn_id.clone(),
agent_index: event.agent_index,
},
text,
))
}
fn set_observer_chunk_text(payload: &mut serde_json::Value, text: String) {
let Some(content) = payload
.get_mut("params")
.and_then(|params| params.get_mut("update"))
.and_then(|update| update.get_mut("content"))
else {
return;
};
if let Some(content_object) = content.as_object_mut() {
content_object.insert("text".to_string(), serde_json::Value::String(text));
}
}
/// Bytes of head and tail to retain from an elided string leaf — the value
/// shown to the renderer at each end. The ONLY tuning knob here: large enough
/// that a clipped diff/tool-result still shows real content, small enough that
/// eliding actually shrinks the frame.
const OBSERVER_LEAF_RETAIN_BYTES: usize = 3_000;
/// Trim an oversized observer telemetry frame so its SERIALIZED form fits under
/// `OBSERVER_MAX_PLAINTEXT_LEN`, instead of dropping the whole frame (silent
/// telemetry loss). The common case — a frame already under budget — is left
/// byte-identical.
///
/// The cap is measured in SERIALIZED bytes (JSON escaping makes serialized
/// length differ from raw), so the stop condition is always a full reserialize
/// of the whole frame: that counts the envelope, the variable `Option<String>`
/// IDs, and any elision markers exactly. No separate margin constant is needed.
///
/// Termination is provable: each iteration elides the largest string leaf that
/// would STRICTLY shrink the serialized frame, then reserializes. Shrinkability
/// is re-evaluated against each leaf's CURRENT value, so a leaf already at its
/// retained floor can never be re-elided — the loop strictly decreases the
/// serialized length each pass and is bounded by the leaf count. When no leaf
/// can shrink the frame and it still overflows, the payload is replaced with a
/// tiny stub, which trivially fits. Monotone decrease, bounded below by the stub.
///
/// **Signature choice (`&mut`, double-serialize accepted):** on the common
/// under-budget path this serializes the frame once to decide it fits, then
/// `encrypt_observer_payload` serializes it again — one extra `to_string` of an
/// already-small frame. Reusing that string would mean changing buzz-core's
/// `encrypt_observer_payload` signature or adding a parallel encrypt path; both
/// are out of this change's scope (buzz-core stays untouched). The clean `&mut`
/// signature with one cheap redundant serialize is the deliberate tradeoff.
fn fit_observer_event_to_budget(event: &mut observer::ObserverEvent) {
if serialized_len(event) <= OBSERVER_MAX_PLAINTEXT_LEN {
return;
}
// Raw size of the payload we are about to trim, captured before mutation so
// the stub's `originalBytes` reports source bytes discarded, not serialized
// overflow — consistent with the per-leaf marker's raw byte count.
let original_payload_bytes = serde_json::to_string(&event.payload)
.map(|s| s.len())
.unwrap_or(0);
// Elide the largest shrinkable leaf, reserialize, repeat. Each successful
// elision strictly shrinks the serialized frame, and a floored leaf can
// never be re-elided, so the loop is bounded by the leaf count.
while let Some(leaf) = largest_shrinkable_leaf(&mut event.payload) {
elide_leaf(leaf);
if serialized_len(event) <= OBSERVER_MAX_PLAINTEXT_LEN {
return;
}
}
// No leaf can shrink the frame further and it still overflows: replace the
// whole payload with a stub that is trivially under-cap.
event.payload = serde_json::json!({
"elided": format!("{} payload too large", event.kind),
"originalBytes": original_payload_bytes,
});
}
fn serialized_len(event: &observer::ObserverEvent) -> usize {
serde_json::to_string(event).map(|s| s.len()).unwrap_or(0)
}
/// Find the longest string leaf that would STRICTLY shrink if elided, returning
/// a mutable handle to it. A leaf shrinks only if `head + marker + tail` is
/// shorter than its current value (the marker-pushback guard); a leaf already at
/// its retained floor fails this test and is skipped, which is what bounds the
/// loop. Returns `None` when no leaf can shrink.
fn largest_shrinkable_leaf(value: &mut serde_json::Value) -> Option<&mut serde_json::Value> {
// First pass: find the byte length of the best candidate without holding a
// borrow, then re-descend to return the matching mutable reference. Two
// immutable-style passes keep the borrow checker happy without unsafe.
let best_len = max_shrinkable_len(value)?;
find_leaf_with_len(value, best_len)
}
/// Largest current length among string leaves that can strictly shrink.
fn max_shrinkable_len(value: &serde_json::Value) -> Option<usize> {
match value {
serde_json::Value::String(s) if leaf_shrinks(s) => Some(s.len()),
serde_json::Value::String(_) => None,
serde_json::Value::Array(items) => items.iter().filter_map(max_shrinkable_len).max(),
serde_json::Value::Object(map) => map.values().filter_map(max_shrinkable_len).max(),
_ => None,
}
}
/// Return the first string leaf whose current length equals `target` and that
/// can strictly shrink. Used after `max_shrinkable_len` to re-acquire a mutable
/// borrow of the chosen leaf.
fn find_leaf_with_len(
value: &mut serde_json::Value,
target: usize,
) -> Option<&mut serde_json::Value> {
match value {
serde_json::Value::String(s) if s.len() == target && leaf_shrinks(s) => Some(value),
serde_json::Value::Array(items) => items
.iter_mut()
.find_map(|item| find_leaf_with_len(item, target)),
serde_json::Value::Object(map) => map
.values_mut()
.find_map(|item| find_leaf_with_len(item, target)),
_ => None,
}
}
/// True when eliding `s` to head + marker + tail yields a strictly shorter raw
/// string. The marker width grows with `N` (bytes removed), so a leaf only
/// marginally larger than the retained ends must NOT be touched.
fn leaf_shrinks(s: &str) -> bool {
let (head_end, tail_start) = elision_boundaries(s);
tail_start > head_end && {
let removed = tail_start - head_end;
let marker = elision_marker(removed);
head_end + marker.len() + (s.len() - tail_start) < s.len()
}
}
/// Replace the middle of a string leaf with `…[elided N bytes]…`, keeping a head
/// and tail slice on UTF-8 char boundaries. `N` is RAW bytes removed.
fn elide_leaf(leaf: &mut serde_json::Value) {
let serde_json::Value::String(s) = leaf else {
return;
};
let (head_end, tail_start) = elision_boundaries(s);
let removed = tail_start - head_end;
let mut elided = String::with_capacity(head_end + 32 + (s.len() - tail_start));
elided.push_str(&s[..head_end]);
elided.push_str(&elision_marker(removed));
elided.push_str(&s[tail_start..]);
*s = elided;
}
fn elision_marker(removed_bytes: usize) -> String {
format!("…[elided {removed_bytes} bytes]…")
}
/// Byte offsets bounding the elided middle, snapped to char boundaries so we
/// never split a multi-byte char. Returns `(head_end, tail_start)` with
/// `head_end <= tail_start`.
fn elision_boundaries(s: &str) -> (usize, usize) {
let head_end = floor_char_boundary(s, OBSERVER_LEAF_RETAIN_BYTES.min(s.len()));
let tail_start = ceil_char_boundary(s, s.len().saturating_sub(OBSERVER_LEAF_RETAIN_BYTES));
(head_end, tail_start.max(head_end))
}
fn floor_char_boundary(s: &str, mut i: usize) -> usize {
while i > 0 && !s.is_char_boundary(i) {
i -= 1;
}
i
}
fn ceil_char_boundary(s: &str, mut i: usize) -> usize {
while i < s.len() && !s.is_char_boundary(i) {
i += 1;
}
i
}
async fn publish_relay_observer_event(
publisher: &RelayEventPublisher,
keys: &nostr::Keys,
agent_pubkey_hex: &str,
owner_pubkey_hex: &str,
owner_pubkey: &PublicKey,
mut event: observer::ObserverEvent,
) {
// Trim oversized frames to fit the plaintext cap rather than letting
// encrypt_observer_payload reject and drop them whole (silent telemetry loss).
fit_observer_event_to_budget(&mut event);
let encrypted = match encrypt_observer_payload(keys, owner_pubkey, &event) {
Ok(encrypted) => encrypted,
Err(error) => {
tracing::warn!("failed to encrypt relay observer event: {error}");
return;
}
};
let builder = match buzz_sdk::build_agent_observer_frame(
owner_pubkey_hex,
agent_pubkey_hex,
OBSERVER_FRAME_TELEMETRY,
&encrypted,
) {
Ok(builder) => builder,
Err(error) => {
tracing::warn!("failed to build relay observer event: {error}");
return;
}
};
let signed = match builder.sign_with_keys(keys) {
Ok(event) => event,
Err(error) => {
tracing::warn!("failed to sign relay observer event: {error}");
return;
}
};
if let Err(error) = publisher.publish_event(signed).await {
tracing::warn!("relay observer event dropped: {error}");
}
}
/// Maximum age (seconds) for an observer control frame to be considered fresh.
const OBSERVER_CONTROL_FRESHNESS_SECS: i64 = 300;
fn handle_relay_observer_control_event(
keys: &nostr::Keys,
event: nostr::Event,
pool: &mut AgentPool,
observer: Option<&observer::ObserverHandle>,
owner_pubkey_hex: &str,
) {
// Defense-in-depth: verify signature even though the relay already checked.
if let Err(e) = buzz_core::verify_event(&event) {
tracing::warn!(error = %e, "observer control frame failed signature verification");
return;
}
// Defense-in-depth: verify the sender is the resolved owner.
if event.pubkey.to_hex() != owner_pubkey_hex {
tracing::warn!(
sender = %event.pubkey,
expected = %owner_pubkey_hex,
"observer control frame from non-owner — dropping"
);
return;
}
// Freshness: reject stale/replayed frames outside ±5 minute window.
let now = chrono::Utc::now().timestamp();
let event_ts = event.created_at.as_secs() as i64;
if (event_ts - now).unsigned_abs() > OBSERVER_CONTROL_FRESHNESS_SECS as u64 {
tracing::warn!(
event_ts,
now,
"observer control frame outside freshness window — dropping"
);
return;
}
let payload = match decrypt_observer_payload::<serde_json::Value>(keys, &event) {
Ok(payload) => payload,
Err(error) => {
tracing::warn!("failed to decrypt observer control frame: {error}");
return;
}
};
let command_type = payload.get("type").and_then(|value| value.as_str());
match command_type {
Some("cancel_turn") => {
handle_cancel_turn_control(&payload, pool, observer);
}
Some("switch_model") => {
handle_switch_model_control(&payload, pool, observer);
}
_ => {
tracing::debug!(payload = %payload, "ignoring unknown observer control frame");
}
}
}
/// Handle a `cancel_turn` control frame: signal the in-flight task to cancel.
fn handle_cancel_turn_control(
payload: &serde_json::Value,
pool: &mut AgentPool,
observer: Option<&observer::ObserverHandle>,
) {
let Some(channel_id) = payload
.get("channelId")
.and_then(|value| value.as_str())
.and_then(|value| value.parse::<Uuid>().ok())
else {
tracing::warn!("observer cancel_turn control frame missing valid channelId");
return;
};
let fired = signal_in_flight_task(pool, channel_id, ControlSignal::Cancel);
let status = if fired { "sent" } else { "no_active_turn" };
if let Some(observer) = observer {
observer.emit(
"control_result",
None,
&observer::ObserverContext {
channel_id: Some(channel_id.to_string()),
session_id: None,
turn_id: None,
started_at: None,
},
serde_json::json!({
"type": "cancel_turn",
"status": status,
}),
);
}
}
/// Handle a `switch_model` control frame (Phase 3a, Option ii).
///
/// Busy path: deliver `SwitchModel` over the in-flight task's oneshot — the
/// task cancels the turn, sets `desired_model`, and requeues the batch so it
/// re-runs on a fresh session under the new model. A catalog miss surfaces
/// post-cancel via `create_session_and_apply_model` (the turn restarts on the
/// unchanged model + an `unsupported_model` result).
///
/// Idle path: validate against the cached catalog *before* invalidating
/// (pre-cancel guard), then set `desired_model` + invalidate. The override
/// takes visible effect on the agent's next turn.
fn handle_switch_model_control(
payload: &serde_json::Value,
pool: &mut AgentPool,
observer: Option<&observer::ObserverHandle>,
) {
let Some(channel_id) = payload
.get("channelId")
.and_then(|value| value.as_str())
.and_then(|value| value.parse::<Uuid>().ok())
else {
tracing::warn!("observer switch_model control frame missing valid channelId");
return;
};
let Some(model_id) = payload.get("modelId").and_then(|value| value.as_str()) else {
tracing::warn!("observer switch_model control frame missing modelId");
return;
};
// A turn is in flight for this channel iff a task_map entry exists. The
// agent is moved out of the pool during a turn, so the control oneshot is
// the only reachable lever; an idle channel has no such entry.
let turn_in_flight = pool
.task_map()
.values()
.any(|m| m.channel_id == Some(channel_id));
let status = if turn_in_flight {
// Busy path: deliver over the oneshot. `false` means the oneshot was
// already consumed this turn (a prior cancel/interrupt) — the turn is
// already ending, so the switch cannot land on it.
if signal_in_flight_task(
pool,
channel_id,
ControlSignal::SwitchModel(model_id.to_string()),
) {
"sent"
} else {
"turn_ending"
}
} else {
// Idle path: validate against the cached catalog before invalidating.
match pool.switch_idle_agent_model(channel_id, model_id) {
IdleSwitchResult::Switched => "switched",
IdleSwitchResult::UnsupportedModel => "unsupported_model",
IdleSwitchResult::NoIdleAgent => "no_active_turn",
}
};
if let Some(observer) = observer {
observer.emit(
"control_result",
None,
&observer::ObserverContext {
channel_id: Some(channel_id.to_string()),
session_id: None,
turn_id: None,
started_at: None,
},
serde_json::json!({
"type": "switch_model",
"status": status,
"modelId": model_id,
}),
);
}
}
/// Maximum crashes in a 60-second window before a slot's circuit opens.
const CIRCUIT_BREAKER_THRESHOLD: usize = 3;
/// Window for circuit-breaker crash counting.
const CIRCUIT_BREAKER_WINDOW: Duration = Duration::from_secs(60);
/// Cooldown before a tripped circuit breaker allows a probe respawn.
const CIRCUIT_BREAKER_COOLDOWN: Duration = Duration::from_secs(300); // 5 minutes
/// Base backoff delay for respawn (doubles per recent crash, capped at 30s).
const RESPAWN_BASE_DELAY: Duration = Duration::from_secs(1);
/// Maximum respawn backoff delay.
const RESPAWN_MAX_DELAY: Duration = Duration::from_secs(30);
/// Per-slot circuit breaker state.
///
/// `crash_times` holds timestamps of recent crashes within `CIRCUIT_BREAKER_WINDOW`.
/// `open_until` is set when the threshold is hit; the circuit stays open until that
/// instant, then allows one probe respawn (half-open). If the probe crashes, the
/// circuit re-opens for another `CIRCUIT_BREAKER_COOLDOWN` period.
///
/// All state transitions go through methods on this struct — callers never
/// manipulate `crash_times` or `open_until` directly.
struct SlotCircuit {
crash_times: Vec<std::time::Instant>,
open_until: Option<std::time::Instant>,
/// True while a background respawn/refill task is in flight for this slot.
/// Prevents duplicate spawns from maintenance ticks that fire before the
/// previous spawn_and_init completes.
respawn_in_flight: bool,
}
/// Result of [`SlotCircuit::record_crash`].
enum CrashVerdict {
/// Respawn is allowed after sleeping for this duration (jittered backoff).
Respawn(Duration),
/// Circuit is open — do not respawn.
CircuitOpen,
/// Circuit was open but cooldown has elapsed — one probe respawn is allowed
/// (no backoff sleep). If the probe crashes, the next `record_crash` will
/// immediately re-open the circuit.
HalfOpenProbe,
}
impl SlotCircuit {
/// Record a crash and decide whether to respawn.
///
/// This is the **single canonical path** for all crash → respawn decisions.
/// Called by `respawn_agent_into`, `recover_panicked_agent`, and slot refill.
fn record_crash(&mut self) -> CrashVerdict {
let now = std::time::Instant::now();
// Half-open: cooldown elapsed → allow one probe.
if let Some(open_until) = self.open_until {
if now >= open_until {
// Pre-seed crash_times to threshold-1 so that if the probe
// itself crashes on the *next* call, the threshold is hit
// immediately and the circuit re-opens. This implements a
// "prove stability for one full window" policy.
self.crash_times.clear();
for _ in 0..(CIRCUIT_BREAKER_THRESHOLD - 1) {
self.crash_times.push(now);
}
self.open_until = None;
return CrashVerdict::HalfOpenProbe;
} else {
return CrashVerdict::CircuitOpen;
}
}
// Record this crash and prune old entries.
self.crash_times.push(now);
self.crash_times
.retain(|&t| now.duration_since(t) < CIRCUIT_BREAKER_WINDOW);
let recent = self.crash_times.len();
if recent >= CIRCUIT_BREAKER_THRESHOLD {
self.open_until = Some(now + CIRCUIT_BREAKER_COOLDOWN);
return CrashVerdict::CircuitOpen;
}
// Exponential backoff: 1s * 2^(recent-1), capped at 30s, with ±20% jitter.
let base = RESPAWN_BASE_DELAY.saturating_mul(1u32 << (recent - 1).min(5));
let capped = base.min(RESPAWN_MAX_DELAY);
let jitter = (std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.subsec_nanos() as f64)
/ 1_000_000_000.0; // 0.0..1.0
let factor = 0.8 + jitter * 0.4; // 0.8..1.2
CrashVerdict::Respawn(capped.mul_f64(factor))
}
/// Mark a spawn failure — opens the circuit so the slot isn't retried
/// on every heartbeat tick. Uses fresh `Instant::now()` so spawn latency
/// doesn't shorten the effective cooldown.
fn mark_spawn_failed(&mut self) {
self.open_until = Some(std::time::Instant::now() + CIRCUIT_BREAKER_COOLDOWN);
}
/// Check if an empty slot can be refilled. Unlike `record_crash`, this
/// does NOT record a new crash — it only checks whether the circuit
/// allows a respawn attempt.
///
/// Returns `true` if respawn is allowed. For half-open probes, pre-seeds
/// crash_times so the next crash re-opens immediately. For normal refills
/// (no circuit was ever opened), crash history is preserved so the breaker
/// can still trip if the refilled agent crashes quickly.
fn can_refill(&mut self) -> bool {
let now = std::time::Instant::now();
match self.open_until {
Some(open_until) => {
if now >= open_until {
// Half-open probe: pre-seed crash_times.
self.crash_times.clear();
for _ in 0..(CIRCUIT_BREAKER_THRESHOLD - 1) {
self.crash_times.push(now);
}
self.open_until = None;
true
} else {
false // cooldown not elapsed
}
}
None => true, // no circuit open — normal refill, preserve crash history
}
}
}
/// True if any slot has a respawn task in flight. Used to prevent premature
/// "all agents dead" exits — a respawning agent may succeed in seconds.
fn any_respawn_in_flight(crash_history: &[SlotCircuit]) -> bool {
crash_history.iter().any(|s| s.respawn_in_flight)
}
/// Result of a background respawn task.
struct RespawnResult {
index: usize,
/// Tuple: (initialized client, protocol version, agent name).
result: Result<(AcpClient, u32, String)>,
}
/// Outcome of a non-cancelling steer attempt, forwarded from a per-attempt
/// watcher task (which awaits the `SteerRequest.ack_tx` oneshot) back to
/// the main loop's `select!`. The main loop drives queue side-effects from
/// this — it cannot await the oneshot itself without blocking the relay
/// stream.
///
/// Carries enough identity to operate on the right withheld event in
/// `EventQueue::withheld_native_steer`: `channel_id` is the routing key,
/// `event_id` is the hex id of the single event the steer carried.
struct NativeSteerPrepared {
channel_id: Uuid,
event: nostr::Event,
prompt_blocks: Vec<String>,
}
/// Async edit enrichment may finish after channel membership was revoked.
/// Such prepared work is stale: removal drains its queue reservation, and it
/// must never be forwarded to an otherwise still-live ACP turn.
fn native_steer_preparation_is_stale(removed_channels: &HashSet<Uuid>, channel_id: Uuid) -> bool {
removed_channels.contains(&channel_id)
}
struct SteerAckEvent {
channel_id: Uuid,
event_id: String,
/// `Ok` if the read loop sent any of the locked `SteerAck` variants.
/// `Err` if the oneshot was dropped without a send — should not happen
/// under the current read-loop drains, but if it ever does the main
/// loop treats it as `PromptCompletedNeutral` (release withheld, no
/// fallback signal) to avoid leaking the withheld event.
ack: std::result::Result<pool::SteerAck, tokio::sync::oneshot::error::RecvError>,
}
/// RAII guard that ensures a `RespawnResult` is sent even if the task panics.
/// Without this, a panicked respawn task would leave `respawn_in_flight = true`
/// permanently, silently losing the slot forever.
struct RespawnGuard {
index: usize,
tx: mpsc::Sender<RespawnResult>,
sent: bool,
}
impl RespawnGuard {
fn new(index: usize, tx: mpsc::Sender<RespawnResult>) -> Self {
Self {
index,
tx,
sent: false,
}
}
/// Send the result and disarm the guard. Uses `try_send` (sync) so there
/// is no await boundary between marking `sent` and actually enqueueing —
/// cancellation cannot slip between the two.
fn send(mut self, result: Result<(AcpClient, u32, String)>) {
// Invariant: try_send succeeds because the channel capacity equals the
// slot count, and respawn_in_flight guarantees at most one outstanding
// result per slot. If this ever fails, the channel sizing or the
// respawn_in_flight guard has drifted — that's a bug, not a transient.
match self.tx.try_send(RespawnResult {
index: self.index,
result,
}) {
Ok(()) => self.sent = true,
Err(e) => {
tracing::error!(
agent = self.index,
"respawn result channel full or closed: {e}"
);
// Drop will fire and send a failure result as fallback.
}
}
}
}
impl Drop for RespawnGuard {
fn drop(&mut self) {
if !self.sent {
tracing::error!(
agent = self.index,
"respawn task exited without sending result — sending failure"
);
// Best-effort: try_send in Drop (can't await).
let _ = self.tx.try_send(RespawnResult {
index: self.index,
result: Err(anyhow::anyhow!("respawn task panicked or was cancelled")),
});
}
}
}
//
// Sync env-var propagation must run before the tokio runtime starts so that
// any child processes inherit the correct environment. This must happen in the
// sync entry point — `std::env::set_var` is only safe before tokio spawns
// worker threads (Rust 2024 edition safety requirement).
fn inactivity_expired(
last_activity: tokio::time::Instant,
now: tokio::time::Instant,
bound: Duration,
turn_in_flight: bool,
) -> bool {
!bound.is_zero() && !turn_in_flight && now.duration_since(last_activity) >= bound
}
#[cfg(test)]
mod inactivity_tests {
use super::*;
#[test]
fn zero_disables_expiry_and_in_flight_turns_defer_it() {
let started = tokio::time::Instant::now();
let after_bound = started + Duration::from_secs(61);
assert!(!inactivity_expired(
started,
after_bound,
Duration::ZERO,
false
));
assert!(!inactivity_expired(
started,
after_bound,
Duration::from_secs(60),
true
));
assert!(inactivity_expired(
started,
after_bound,
Duration::from_secs(60),
false
));
}
#[test]
fn dispatched_activity_restarts_the_inactivity_bound() {
let started = tokio::time::Instant::now();
let dispatched = started + Duration::from_secs(50);
let checked = started + Duration::from_secs(61);
assert!(!inactivity_expired(
dispatched,
checked,
Duration::from_secs(60),
false
));
}
}
pub fn run() -> Result<()> {
config::propagate_legacy_env_vars();
tokio_main()
}
#[tokio::main]
async fn tokio_main() -> Result<()> {
// Install the ring crypto provider for rustls (required for wss:// connections).
rustls::crypto::ring::default_provider()
.install_default()
.expect("failed to install rustls crypto provider");
if is_subcommand("models") {
// Strip the subcommand token so clap doesn't reject it as a positional.
// Keeps argv[0] (binary name) and passes everything after the subcommand.
let filtered: Vec<String> = std::env::args()
.enumerate()
.filter(|(i, _)| *i != 1)
.map(|(_, a)| a)
.collect();
let args = ModelsArgs::parse_from(&filtered);
return run_models(args).await;
}
if is_subcommand("auth-methods") {
let filtered: Vec<String> = std::env::args()
.enumerate()
.filter(|(i, _)| *i != 1)
.map(|(_, a)| a)
.collect();
let args = AuthMethodsArgs::parse_from(&filtered);
return run_auth_methods(args).await;
}
if is_subcommand("authenticate") {
let filtered: Vec<String> = std::env::args()
.enumerate()
.filter(|(i, _)| *i != 1)
.map(|(_, a)| a)
.collect();
let args = AuthenticateArgs::parse_from(&filtered);
return run_authenticate(args).await;
}
tracing_subscriber::fmt()
.with_env_filter(
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("buzz_acp=info")),
)
.compact()
.init();
let mut config = Config::from_cli().map_err(|e| anyhow::anyhow!("configuration error: {e}"))?;
// ── Setup-mode early branch ───────────────────────────────────────────────
//
// When the desktop determines an agent is not ready (missing credentials,
// model, or provider), it spawns buzz-acp with BUZZ_ACP_SETUP_PAYLOAD set.
// We enter the minimal setup-listener path and never start the agent pool.
if let Some(payload) = setup_mode::SetupPayload::from_env()
.map_err(|e| anyhow::anyhow!("setup payload error: {e}"))?
{
tracing::info!("buzz-acp: setup payload present, entering setup-listener mode");
return setup_mode::run_setup_listener(config, payload).await;
}
tracing::info!("buzz-acp starting: {}", config.summary());
let observer = config
.relay_observer
.then(observer::ObserverHandle::in_process);
if let Some(handle) = &observer {
handle.emit(
"harness_started",
None,
&observer::ObserverContext::default(),
serde_json::json!({
"relayUrl": config.relay_url,
"agentCommand": config.agent_command,
"agentArgs": config.agent_args,
"parallelism": config.agents,
"relayObserver": config.relay_observer,
}),
);
}
let mut pool = if config.lazy_pool {
AgentPool::from_slots((0..config.agents).map(|_| None).collect())
} else {
initialize_agent_pool(&PoolStartup::from_config(&config, observer.clone()), None).await?
};
let mut pool_ready = !config.lazy_pool;
let mut pool_lifecycle: PoolLifecycle<AgentPool> = PoolLifecycle::listening();
// Capture a startup watermark BEFORE connecting to the relay. This timestamp
// is used for membership notification replay (via startup_watermark) and as
// the initial subscribe_since for channels discovered at startup. The Subscribe
// handler falls back to subscribe_since when last_seen is None, closing the
// blind spot between "agents ready" and "first REQ sent".
let startup_watermark: u64 = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let pubkey_hex = config.keys.public_key().to_hex();
// Parse BUZZ_AUTH_TAG into a nostr::Tag for NIP-OA relay membership delegation.
let relay_auth_tag: Option<nostr::Tag> = std::env::var("BUZZ_AUTH_TAG")
.ok()
.filter(|s| !s.is_empty())
.and_then(|s| buzz_sdk::nip_oa::parse_auth_tag(&s).ok());
let mut relay =
HarnessRelay::connect(&config.relay_url, &config.keys, &pubkey_hex, relay_auth_tag)
.await
.map_err(|e| anyhow::anyhow!("relay connect error: {e}"))?;
// Tell the relay background task the watermark so it can use
// `since = watermark - 5s` on the first REQ instead of `since=now`.
// Best-effort: a failure here is non-fatal (we just lose the startup window
// protection, which is the same as the pre-fix behaviour).
if let Err(e) = relay.set_startup_watermark(startup_watermark).await {
tracing::warn!("failed to set startup watermark: {e}");
}
tracing::info!("connected to relay at {}", config.relay_url);
relay
.subscribe_membership_notifications()
.await
.map_err(|e| anyhow::anyhow!("membership notification subscribe error: {e}"))?;
tracing::info!("subscribed to membership notifications");
let presence_publisher = relay.event_publisher();
let presence_keys = config.keys.clone();
// Priority: BUZZ_AUTH_TAG (NIP-OA attestation) → --agent-owner flag.
let startup_owner: Option<String> = resolve_agent_owner(&config);
if let Some(ref owner) = startup_owner {
tracing::info!("agent owner: {owner}");
} else {
tracing::info!("no agent owner configured");
}
// Warn if owner-dependent mode but no owner resolved yet.
if startup_owner.is_none() {
match &config.respond_to {
RespondTo::OwnerOnly => {
tracing::warn!(
"respond-to=owner-only but no owner is set — all events will be \
dropped. Set BUZZ_AUTH_TAG or --agent-owner, or use --respond-to=anyone."
);
}
RespondTo::Allowlist => {
tracing::warn!(
"respond-to=allowlist but no owner is set — allowlisted pubkeys \
will still be accepted, but owner-based matching is unavailable \
until owner is resolved."
);
}
_ => {} // anyone/nobody don't depend on owner
}
}
let owner_cache = OwnerCache::new(startup_owner.clone());
let mut relay_observer_control_rx = None;
let mut relay_observer_publisher_task = None;
let mut relay_observer_publisher = None;
if config.relay_observer {
if let (Some(observer), Some(owner_pubkey_hex)) =
(observer.clone(), owner_cache.pubkey.clone())
{
match PublicKey::from_hex(&owner_pubkey_hex) {
Ok(owner_pubkey) => {
relay_observer_publisher = Some((
observer,
relay.event_publisher(),
config.keys.clone(),
pubkey_hex.clone(),
owner_pubkey_hex,
owner_pubkey,
));
relay
.subscribe_observer_controls()
.await
.map_err(|e| anyhow::anyhow!("observer control subscribe error: {e}"))?;
relay_observer_control_rx = relay.take_observer_control_rx();
tracing::info!("relay observer enabled");
}
Err(error) => {
tracing::warn!("relay observer disabled: invalid owner pubkey: {error}");
}
}
} else {
tracing::warn!(
"relay observer requested but no agent owner was resolved at startup; \
observer frames will not be published"
);
}
}
let channel_info_map = relay
.discover_channels()
.await
.map_err(|e| anyhow::anyhow!("channel discovery error: {e}"))?;
tracing::info!("discovered {} channel(s)", channel_info_map.len());
let channel_ids: Vec<Uuid> = channel_info_map.keys().copied().collect();
let rules: Vec<SubscriptionRule> = match config.subscribe_mode {
SubscribeMode::Mentions => {
vec![SubscriptionRule {
name: "mentions".into(),
channels: filter::ChannelScope::All("all".into()),
kinds: config
.kinds_override
.clone()
.unwrap_or_else(config::default_mention_kinds),
require_mention: !config.no_mention_filter,
filter: None,
compiled_filter: None,
consecutive_timeouts: std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0)),
prompt_tag: Some("@mention".into()),
}]
}
SubscribeMode::All => {
vec![SubscriptionRule {
name: "all".into(),
channels: filter::ChannelScope::All("all".into()),
kinds: config.kinds_override.clone().unwrap_or_default(),
require_mention: false,
filter: None,
compiled_filter: None,
consecutive_timeouts: std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0)),
prompt_tag: Some("all".into()),
}]
}
SubscribeMode::Config => {
// load_rules() already warns if the config file has zero rules.
config::load_rules(&config.config_path)?
}
};
let channel_filters = config::resolve_channel_filters(&config, &channel_ids, &rules);
if channel_filters.is_empty() {
tracing::warn!("no channel subscriptions resolved — agent will sit idle");
}
let mut subscribed_channel_ids = HashSet::with_capacity(channel_filters.len());
for (channel_id, filter) in &channel_filters {
if let Err(e) = relay.subscribe_channel(*channel_id, filter.clone()).await {
tracing::warn!("failed to subscribe to channel {channel_id}: {e}");
} else {
subscribed_channel_ids.insert(*channel_id);
tracing::info!("subscribed to channel {channel_id}");
}
}
if let Some((observer, publisher, keys, agent_pubkey, owner_pubkey, owner)) =
relay_observer_publisher.take()
{
relay_observer_publisher_task = Some(spawn_relay_observer_publisher(
observer,
publisher,
keys,
agent_pubkey,
owner_pubkey,
owner,
));
}
let runtime_start_nonce = std::env::var("BUZZ_MANAGED_AGENT_START_NONCE").unwrap_or_default();
let dedup_mode = config.dedup_mode;
let mut queue =
EventQueue::new(dedup_mode).with_in_flight_deadline(config.max_turn_duration_secs);
// Online means the harness can receive work, not merely that its socket is
// connected. Publishing after channel subscriptions gives desktop callers
// a durable readiness boundary before they send a startup mention.
if config.presence_enabled {
match publish_presence(&presence_publisher, &presence_keys, "online").await {
Ok(_) => tracing::info!("presence set to online"),
Err(e) => tracing::warn!("failed to set initial presence: {e}"),
}
}
if config.lazy_pool {
emit_runtime_lifecycle(
observer.as_ref(),
&runtime_start_nonce,
&pubkey_hex,
&config.relay_url,
"listening",
None,
);
}
let base_prompt_content = config.base_prompt_content.take();
let ctx = Arc::new(PromptContext {
mcp_servers: build_mcp_servers(&config),
initial_message: config.initial_message.clone(),
idle_timeout: Duration::from_secs(config.idle_timeout_secs),
max_turn_duration: Duration::from_secs(config.max_turn_duration_secs),
turn_liveness_interval: Duration::from_secs(config.turn_liveness_secs),
dedup_mode: config.dedup_mode,
system_prompt: config.system_prompt.clone(),
session_title: config.session_title.clone(),
team_instructions: config.team_instructions.clone(),
base_prompt: if config.no_base_prompt {
None
} else if let Some(content) = base_prompt_content {
Some(Box::leak(content.into_boxed_str()))
} else {
Some(include_str!("base_prompt.md"))
},
heartbeat_prompt: config.heartbeat_prompt.clone(),
cwd: std::env::current_dir()
.unwrap_or_else(|_| std::path::PathBuf::from("/"))
.to_string_lossy()
.to_string(),
rest_client: relay.rest_client(),
channel_info: pool::ChannelInfoResolver::new(channel_info_map, relay.rest_client()),
context_message_limit: config.context_message_limit,
max_turns_per_session: config.max_turns_per_session,
permission_mode: config.permission_mode,
agent_keys: config.keys.clone(),
agent_owner_pubkey: startup_owner
.as_deref()
.and_then(|hex| nostr::PublicKey::from_hex(hex).ok()),
memory_enabled: config.memory_enabled,
harness_name: crate::config::normalize_agent_command_identity(&config.agent_command),
relay_url: config.relay_url.clone(),
});
if !config.memory_enabled {
tracing::info!(
target: "engram::core",
"NIP-AE core memory injection disabled (re-enable by removing --no-memory / BUZZ_ACP_NO_MEMORY)"
);
}
let mut heartbeat = if config.heartbeat_interval_secs > 0 {
let interval = Duration::from_secs(config.heartbeat_interval_secs);
Some(tokio::time::interval_at(
tokio::time::Instant::now() + interval,
interval,
))
} else {
None
};
let mut heartbeat_in_flight = false;
let mut presence_heartbeat = if config.presence_enabled {
let interval = Duration::from_secs(60);
Some(tokio::time::interval_at(
tokio::time::Instant::now() + interval,
interval,
))
} else {
None
};
let mut typing_refresh = if config.typing_enabled {
let interval = Duration::from_secs(3);
Some(tokio::time::interval_at(
tokio::time::Instant::now() + interval,
interval,
))
} else {
None
};
let mut typing_channels: HashMap<Uuid, ThreadTags> = HashMap::new();
let mut presence_task: Option<tokio::task::JoinHandle<()>> = None;
// Independent of pool readiness: a never-mentioned lazy agent must still
// self-terminate. The watch interval is capped so small configured bounds
// remain reasonably precise without waking long-lived agents frequently.
let inactivity_bound = Duration::from_secs(config.exit_after_inactivity_secs);
let mut last_activity = tokio::time::Instant::now();
let mut inactivity_reaper = if inactivity_bound.is_zero() {
None
} else {
let interval = inactivity_bound.min(Duration::from_secs(30));
Some(tokio::time::interval_at(
tokio::time::Instant::now() + interval,
interval,
))
};
// Runs at the TOP of every loop iteration via Instant check — cannot be
// starved by the biased select. Slot refill spawns background tasks so
// spawn_and_init never blocks the main loop.
let maintenance_interval = Duration::from_secs(30);
let mut last_maintenance = std::time::Instant::now();
// Channel for background respawn tasks to return completed agents.
// Bounded to agent count — at most one respawn per slot in flight.
let (respawn_tx, mut respawn_rx) = mpsc::channel::<RespawnResult>(config.agents as usize);
// JoinSet for respawn tasks so shutdown can abort them.
let mut respawn_tasks: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
let (wake_tx, mut wake_rx) = mpsc::channel::<(u32, Result<AgentPool, String>)>(1);
let mut wake_tasks: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
// Channel for non-cancelling steer ack watchers to forward outcomes back
// to the main loop. Each `pool.send_steer(...) == Ok(())` spawns a
// short-lived task that awaits the `SteerRequest.ack_tx` oneshot and
// forwards a `SteerAckEvent`. Unbounded because:
// 1. The producer count is bounded by in-flight goose turns
// (`agents` slots, capacity-1 `steer_tx` each), so the channel
// cannot legitimately back up under steady state.
// 2. We must never drop a steer outcome — losing an ack would leak a
// withheld event in `EventQueue::withheld_native_steer` until
// `IN_FLIGHT_DEADLINE_SECS` expires.
let (steer_ack_tx, mut steer_ack_rx) = mpsc::unbounded_channel::<SteerAckEvent>();
// Edit-aware native steer enrichment performs bounded REST lookups. Keep it
// off the relay select arm, then return the immutable prompt payload here
// for queue/pool mutation on the main loop.
let (native_steer_tx, mut native_steer_rx) = mpsc::unbounded_channel::<NativeSteerPrepared>();
// ── Step 7: Shutdown signal ───────────────────────────────────────────────
let (shutdown_tx, mut shutdown_rx) = watch::channel(());
let tx = shutdown_tx.clone();
tokio::spawn(async move {
tokio::signal::ctrl_c().await.ok();
let _ = tx.send(());
});
#[cfg(unix)]
{
let tx = shutdown_tx.clone();
tokio::spawn(async move {
use tokio::signal::unix::{signal, SignalKind};
let mut sigterm = signal(SignalKind::terminate()).expect("SIGTERM handler");
sigterm.recv().await;
let _ = tx.send(());
});
}
// Track the newest membership notification timestamp per channel.
// On reconnect the relay replays events newest-first, so the first event
// per channel is authoritative. Any later event with ts < newest is stale.
// Exact duplicates (same event ID) are caught by seen_membership_ids.
//
// Uses strict `<` (not `<=`) so that legitimate live events at the same
// second are both processed. The seen_membership_ids set handles exact
// replays that share the same timestamp.
let mut membership_newest_ts: HashMap<Uuid, u64> = HashMap::new();
// Two-generation dedup for membership event replays (bounded, no amnesia).
// Rotates at 1000 entries instead of clearing the entire set at 2000.
let mut seen_membership_current: HashSet<String> = HashSet::new();
let mut seen_membership_previous: HashSet<String> = HashSet::new();
// Channels the agent has been removed from. When a checked-out agent is
// returned to the pool, its sessions for these channels are stripped, and
// failed/panicked batches for these channels are dropped instead of requeued.
//
// Cleared on re-add (KIND_MEMBER_ADDED_NOTIFICATION) so re-joined channels
// regain session affinity.
//
// Known limitation: if a batch is in-flight when the channel is removed AND
// re-added before the batch returns, the stale batch may be requeued. This
// is acceptable because: (a) the agent is a member again and has access,
// (b) the events are from the agent's authorized history, (c) the window
// is extremely narrow (membership changes are rare, prompt turns are seconds),
// and (d) fixing this would require per-channel epoch tracking on TaskMeta
// and PromptResult — significant complexity for a benign edge case. If strict
// causal invalidation is needed, add a monotonic epoch counter per channel
// and capture it in TaskMeta at dispatch time.
let mut removed_channels: HashSet<Uuid> = HashSet::new();
//
// One SlotCircuit per agent slot. crash_times entries are pruned to the last
// CIRCUIT_BREAKER_WINDOW on each respawn attempt. The Vec is indexed by
// agent slot index, so it must be sized to the configured pool capacity
// (not the live count, which may be smaller after partial startup).
let mut crash_history: Vec<SlotCircuit> = (0..config.agents as usize)
.map(|_| SlotCircuit {
crash_times: Vec::new(),
open_until: None,
respawn_in_flight: false,
})
.collect();
//
// Branches 1 & 2 both need to borrow `pool`, but they access different
// fields (result_rx vs join_set). We use `rx_and_join_set()` to split the
// borrow, yielding a typed enum so the outer code can dispatch cleanly.
enum PoolEvent {
Result(Box<PromptResult>),
Panic(tokio::task::JoinError),
SteerAck(SteerAckEvent),
NativeSteerPrepared(NativeSteerPrepared),
Wake(u32, Result<AgentPool, String>),
}
loop {
// Whether buffered work is waiting on a lazy pool. Also gates the
// retry-deadline sleep arm below: a `Failed` lifecycle keeps its
// (possibly past) `retry_at` until the next wake, so sleeping on it
// unconditionally would complete instantly on every iteration — a
// busy spin — whenever the queued work drained after a failed wake.
let mut lazy_wake_work_pending = false;
if config.lazy_pool && !pool_ready {
lazy_wake_work_pending = queue.has_flushable_work();
if let Some(attempt) = pool_lifecycle
.start_wake_if_due(lazy_wake_work_pending, tokio::time::Instant::now())
{
emit_runtime_lifecycle(
observer.as_ref(),
&runtime_start_nonce,
&pubkey_hex,
&config.relay_url,
"waking",
None,
);
let startup = PoolStartup::from_config(&config, observer.clone());
let wake_tx = wake_tx.clone();
let wake_shutdown = shutdown_rx.clone();
wake_tasks.spawn(async move {
let result = initialize_agent_pool(&startup, Some(wake_shutdown))
.await
.map_err(|error| error.to_string());
if let Err(error) = wake_tx.send((attempt, result)).await {
let (_attempt, result) = error.0;
if let Ok(mut abandoned_pool) = result {
shutdown_agent_pool(&mut abandoned_pool).await;
}
}
});
}
}
if pool_ready && last_maintenance.elapsed() >= maintenance_interval {
last_maintenance = std::time::Instant::now();
queue.compact_expired_state();
// Slot refill: spawn background tasks for empty slots whose
// circuit breaker allows it. spawn_and_init runs off the main
// loop so it never blocks event processing.
for (idx, slot) in crash_history.iter_mut().enumerate() {
if pool.slot_alive(idx) || slot.respawn_in_flight {
continue;
}
if !slot.can_refill() {
continue;
}
slot.respawn_in_flight = true;
tracing::info!(agent = idx, "slot refill: spawning background respawn");
let cmd = config.agent_command.clone();
let args = config.agent_args.clone();
let env = config.persona_env_vars.clone();
let has_codex = config.has_generated_codex_config;
let observer = observer.clone();
let guard = RespawnGuard::new(idx, respawn_tx.clone());
respawn_tasks.spawn(async move {
let result = spawn_and_init(&cmd, &args, &env, has_codex, idx, observer).await;
guard.send(result);
});
}
// Flush requeued batches whose retry_after has expired. Without
// this, a batch requeued during crash recovery can sit idle
// indefinitely on quiet channels — dispatch_pending is only
// called on relay events or pool results, neither of which
// arrive when the channel is silent.
if queue.has_flushable_work() {
for (channel_id, thread_tags) in
dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity)
{
typing_channels.insert(channel_id, thread_tags);
}
}
}
let mut respawn_collected = false;
while let Ok(rr) = respawn_rx.try_recv() {
crash_history[rr.index].respawn_in_flight = false;
match rr.result {
Ok((acp, protocol_version, agent_name)) => {
let agent = OwnedAgent {
index: rr.index,
acp,
state: SessionState::default(),
model_capabilities: None,
desired_model: config.model.clone(),
model_overridden: false,
agent_name,
goose_system_prompt_supported: None,
protocol_version,
};
pool.return_agent(agent);
tracing::info!(agent = rr.index, "respawn complete");
respawn_collected = true;
}
Err(e) => {
crash_history[rr.index].mark_spawn_failed();
tracing::warn!(agent = rr.index, "respawn failed: {e} — circuit re-opened");
}
}
}
// Flush requeued events that were waiting for a live agent. Without
// this, batches requeued during crash recovery sit idle until the
// next relay event arrives — which can be minutes on quiet channels.
if respawn_collected {
for (channel_id, thread_tags) in
dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity)
{
typing_channels.insert(channel_id, thread_tags);
}
}
// Borrow result_rx and join_set simultaneously via split-borrow helper.
let pool_event: Option<PoolEvent> = {
let (result_rx, join_set) = pool.rx_and_join_set();
tokio::select! {
biased;
// recv() returning None means all senders dropped (pool was torn down).
// Break cleanly instead of panicking.
r = result_rx.recv(), if pool_ready => match r {
Some(result) => Some(PoolEvent::Result(Box::new(result))),
None => {
tracing::info!("result channel closed — exiting main loop");
break;
}
},
// Guard: join_next() returns None immediately when JoinSet is
// empty, which would cause a tight spin. Only poll when there
// are in-flight tasks.
Some(Err(e)) = join_set.join_next(), if !join_set.is_empty() => {
Some(PoolEvent::Panic(e))
}
// Goose-native steer ack from a watcher task. Outcomes drive
// queue side-effects (drop / release withheld event) and
// optionally the cancel+merge fallback signal. See the
// `Some(PoolEvent::SteerAck(...))` match arm below for the
// locked semantics (Eva + Max + Perci).
Some(ack_event) = steer_ack_rx.recv() => {
Some(PoolEvent::SteerAck(ack_event))
}
Some(prepared) = native_steer_rx.recv() => {
Some(PoolEvent::NativeSteerPrepared(prepared))
}
Some((attempt, result)) = wake_rx.recv(), if config.lazy_pool && !pool_ready => {
Some(PoolEvent::Wake(attempt, result))
}
// Gated on pending work: with an empty queue there is nothing
// for the retry to dispatch, and a past `retry_at` would
// otherwise complete instantly on every iteration (busy spin).
// The next accepted event re-enables the arm.
_ = async {
match pool_lifecycle.retry_at() {
Some(retry_at) if lazy_wake_work_pending => {
tokio::time::sleep_until(retry_at).await
}
_ => std::future::pending().await,
}
} => None,
Some(Err(error)) = wake_tasks.join_next(), if !wake_tasks.is_empty() => {
if let Some(attempt) = pool_lifecycle.waking_attempt() {
let message = format!("pool wake task failed: {error}");
if pool_lifecycle.cancel_wake(
attempt,
message.clone(),
tokio::time::Instant::now(),
) {
emit_runtime_lifecycle(
observer.as_ref(),
&runtime_start_nonce,
&pubkey_hex,
&config.relay_url,
"failed",
Some(&message),
);
}
}
None
}
control_event = async {
match relay_observer_control_rx.as_mut() {
Some(rx) => rx.recv().await,
None => std::future::pending().await,
}
} => {
let _ = result_rx;
match control_event {
Some(event) => {
if let Some(ref owner_hex) = owner_cache.pubkey {
handle_relay_observer_control_event(&config.keys, event, &mut pool, observer.as_ref(), owner_hex);
} else {
tracing::warn!("observer control frame received but no owner resolved — dropping");
}
}
None => {
relay_observer_control_rx = None;
tracing::warn!("relay observer control channel closed");
}
}
None
}
// Remaining branches don't touch pool — evaluated when pool is idle.
buzz_event = relay.next_event() => {
let _ = result_rx; // end split borrow before relay handling
match buzz_event {
Some(buzz_event) => {
let kind_u32 = buzz_event.event.kind.as_u16() as u32;
if kind_u32 == KIND_MEMBER_ADDED_NOTIFICATION
|| kind_u32 == KIND_MEMBER_REMOVED_NOTIFICATION
{
let ch = buzz_event.channel_id;
let ts = buzz_event.event.created_at.as_secs();
let eid = buzz_event.event.id.to_hex();
// Two-layer membership dedup:
//
// 1. Exact duplicate rejection (seen_membership_ids):
// Catches the same event replayed on reconnect.
//
// 2. Timestamp watermark (membership_newest_ts):
// Uses strict `<` so that older events from reconnect
// replay are dropped, but legitimate live events at the
// same second are both processed. This is safe because
// exact duplicates are already caught by layer 1.
//
// Why not `<=`? That would suppress legitimate live
// add→remove (or remove→add) sequences in the same second,
// leaving the harness in the wrong membership state.
// Two-generation dedup: check both sets before inserting.
if seen_membership_current.contains(&eid)
|| seen_membership_previous.contains(&eid)
{
tracing::debug!(
channel_id = %ch,
kind = kind_u32,
"skipping duplicate membership notification (same event_id)"
);
continue;
}
seen_membership_current.insert(eid);
// Rotate at 1000: current → previous, no amnesia window.
if seen_membership_current.len() >= 1000 {
seen_membership_previous =
std::mem::take(&mut seen_membership_current);
}
if let Some(&newest) = membership_newest_ts.get(&ch) {
if ts < newest {
tracing::debug!(
channel_id = %ch,
kind = kind_u32,
ts,
newest,
"skipping stale membership notification (older than newest)"
);
continue;
}
}
membership_newest_ts.insert(ch, ts);
if kind_u32 == KIND_MEMBER_ADDED_NOTIFICATION {
// Clear removal tracking so sessions are not
// stripped for a legitimately re-added channel.
removed_channels.remove(&ch);
if subscribed_channel_ids.contains(&ch) {
tracing::debug!(channel_id = %ch, "membership notification: channel already subscribed");
} else if let Some(filter) = config::resolve_dynamic_channel_filter(&config, ch, &rules) {
tracing::info!(channel_id = %ch, "membership notification: subscribing to new channel");
if let Err(e) = relay.subscribe_channel_from(ch, filter, Some(ts)).await {
tracing::warn!("failed to subscribe to new channel {ch}: {e}");
} else {
subscribed_channel_ids.insert(ch);
}
} else {
tracing::debug!(channel_id = %ch, "membership notification: no matching rules — skipping");
}
} else {
subscribed_channel_ids.remove(&ch);
tracing::info!(channel_id = %ch, "membership notification: unsubscribing from channel");
if let Err(e) = relay.unsubscribe_channel(ch).await {
tracing::warn!("failed to unsubscribe from channel {ch}: {e}");
}
// Drain queued events and invalidate sessions for the
// removed channel. Events already in-flight will
// complete normally (the relay may reject actions if
// the agent lost access).
let drained_ids = queue.drain_channel(ch);
let invalidated = if pool_ready {
pool.invalidate_channel_sessions(ch)
} else {
0
};
// Track removed channels so checked-out agents get
// their sessions stripped when they return to the pool.
removed_channels.insert(ch);
typing_channels.remove(&ch);
// Best-effort: clean up 👀 on drained events.
// Note: the relay revokes membership before
// emitting the notification, so this DELETE may
// 403 on non-open channels. Stale 👀 in that
// case is a known limitation — fix belongs in
// the relay (clean up bot reactions on removal).
if !drained_ids.is_empty() {
let rc = ctx.rest_client.clone();
let ids = drained_ids.clone();
tokio::spawn(async move {
for eid in &ids {
pool::reaction_remove(&rc, eid, "👀").await;
}
});
}
if !drained_ids.is_empty() || invalidated > 0 {
tracing::info!(
channel_id = %ch,
drained = drained_ids.len(),
invalidated,
"cleaned up after membership removal"
);
}
}
continue;
}
if config.ignore_self && buzz_event.event.pubkey.to_hex() == pubkey_hex {
tracing::debug!(channel_id = %buzz_event.channel_id, "dropping self-authored event");
continue;
}
// Check: kind:9, content "!shutdown", from owner, mentions THIS agent.
let is_shutdown = is_owner_control_command(
&buzz_event.event,
kind_u32,
"!shutdown",
&pubkey_hex,
);
if is_shutdown {
let owner = owner_cache.get();
if let Some(owner) = owner {
if buzz_event.event.pubkey.to_hex() == *owner {
tracing::info!(
channel_id = %buzz_event.channel_id,
sender = %buzz_event.event.pubkey.to_hex(),
"shutdown command from owner — exiting gracefully"
);
let _ = shutdown_tx.send(());
continue;
}
}
// Not from owner — fall through to normal prompt handling.
// Don't drop it — it's a regular message that happens to
// contain "!shutdown" from a non-owner.
}
// Mirrors !shutdown: kind:9, content "!cancel", from
// owner, mentions THIS agent. Must be BEFORE
// queue.push() — the event content is moved by push.
//
// Mode-independent: !cancel fires regardless of
// --multiple-event-handling. It is explicit user
// intent, not an automatic policy decision.
let is_cancel = is_owner_control_command(
&buzz_event.event,
kind_u32,
"!cancel",
&pubkey_hex,
);
if is_cancel {
if let Some(owner) = owner_cache.get() {
if buzz_event.event.pubkey.to_hex() == *owner {
let fired = signal_in_flight_task(
&mut pool,
buzz_event.channel_id,
ControlSignal::Cancel,
);
if !fired {
tracing::warn!(
channel_id = %buzz_event.channel_id,
"!cancel received but no in-flight task — no-op"
);
}
continue; // consume event — do NOT push to queue
}
}
// Not from owner — fall through to normal prompt handling.
}
// Mirrors !shutdown / !cancel: kind:9, content
// "!rotate", from owner, mentions THIS agent.
//
// Rotation is explicit owner intent to start the
// next turn in this channel with a fresh ACP
// session. It is consumed by the harness and never
// forwarded to the agent. If a turn is in-flight,
// cancel it, drop its triggering batch, and
// invalidate the channel session when the task
// returns. If idle, invalidate the cached channel
// session immediately. Queued future events remain
// queued and will create a fresh session on dispatch.
let is_rotate = is_owner_control_command(
&buzz_event.event,
kind_u32,
"!rotate",
&pubkey_hex,
);
if is_rotate {
if let Some(owner) = owner_cache.get() {
if buzz_event.event.pubkey.to_hex() == *owner {
let fired = signal_in_flight_task(
&mut pool,
buzz_event.channel_id,
ControlSignal::Rotate,
);
if fired {
tracing::info!(
channel_id = %buzz_event.channel_id,
"!rotate received — cancelling in-flight turn and rotating session"
);
} else {
let invalidated = pool.invalidate_channel_sessions(buzz_event.channel_id);
tracing::info!(
channel_id = %buzz_event.channel_id,
invalidated,
"!rotate received — invalidated idle channel session(s)"
);
}
continue; // consume event — do NOT push to queue
}
}
// Not from owner — fall through to normal prompt handling.
}
// Coarse security policy: drop events from disallowed
// authors before they reach subscription rules or the
// agent. Must be AFTER !shutdown (owner can always
// shut down regardless of gate mode).
//
// Both OwnerOnly and Allowlist accept events from
// "siblings" — pubkeys whose agent_owner_pubkey
// matches this agent's owner (e.g. other bots
// launched by the same human). Allowlist adds the
// explicit pubkey list on top, for external people;
// it never revokes same-owner team bots.
{
let author = buzz_event.event.pubkey.to_hex();
// DM hardening: resolve channel type (fail-closed
// to DM) so allowlist/anyone modes cannot be
// exercised by non-owner authors inside DMs.
let is_dm =
is_dm_channel(buzz_event.channel_id, &ctx.channel_info).await;
let allowed = author_allowed(
&config.respond_to,
&config.respond_to_allowlist,
&author,
is_dm,
&owner_cache,
&ctx.rest_client,
)
.await;
if !allowed {
tracing::debug!(
channel_id = %buzz_event.channel_id,
author = %buzz_event.event.pubkey.to_hex(),
mode = %config.respond_to,
is_dm,
"inbound author gate — dropping event"
);
continue;
}
}
let matched = filter::match_event(&buzz_event.event, buzz_event.channel_id, &rules, &pubkey_hex).await;
let prompt_tag = match matched {
Some(m) => m.prompt_tag,
None => {
tracing::debug!(channel_id = %buzz_event.channel_id, kind = buzz_event.event.kind.as_u16(), "event matched no rule — dropping");
continue;
}
};
// Capture author pubkey before queue.push() moves
// buzz_event.event (needed for mode gate below).
let author_hex = buzz_event.event.pubkey.to_hex();
let event_id_hex = buzz_event.event.id.to_hex();
// Clone for the non-cancelling steer fork, which
// needs the event to render the steer body. The
// clone is unconditional because we don't know
// yet whether the mode gate will demand a steer
// — checking `multiple_event_handling` here
// would couple the queueing path to the mode
// and break the existing invariant that every
// accepted event goes through `queue.push`
// first. `nostr::Event::clone` is cheap (Arc-
// backed payload) so the cost is negligible.
let event_for_steer = buzz_event.event.clone();
let prompt_tag_for_steer = prompt_tag.clone();
let accepted = queue.push(QueuedEvent {
channel_id: buzz_event.channel_id,
event: buzz_event.event,
received_at: std::time::Instant::now(),
prompt_tag,
});
// 👀 — immediate "seen" reaction, only if the event
// was actually queued (not dropped by DedupMode::Drop).
// Fire-and-forget: on rare fast-failure paths the
// guard's cleanup may race with this add, leaving a
// cosmetic stale 👀. Acceptable — see ReactionGuard docs.
if accepted {
let rc = ctx.rest_client.clone();
let eid = event_id_hex.clone();
tokio::spawn(async move {
pool::reaction_add(&rc, &eid, "👀").await;
});
}
// Event is already queued. If mode requires it AND
// the channel has an in-flight task, fire cancel —
// OR take the non-cancelling (ACP steer) fork for Steer signals.
if accepted && queue.is_channel_in_flight(buzz_event.channel_id) {
// Author eligibility (owner ∪ allowlist ∪ siblings)
// is already enforced by the inbound author gate
// above, so the mid-turn signal fires for every
// event that reaches here.
let signal = mode_gate_signal(
config.multiple_event_handling,
&author_hex,
owner_cache.get(),
);
if let Some(signal) = signal {
// Non-cancelling fork: when the mode
// wants a Steer, attempt the
// non-cancelling path first. On accept,
// withhold the queued event and spawn an
// ack watcher; the main loop's
// `PoolEvent::SteerAck` arm decides
// success/release/fallback. On reject
// (including agents that advertise no
// steer transport at all), fall through
// to the universal cancel+merge `Steer`
// signal so the event still reaches the
// agent.
let native_attempted = if matches!(signal, ControlSignal::Steer)
{
if queue::edit_target_id(&event_for_steer).is_some() {
let event_id = event_for_steer.id.to_hex();
let reserved = queue.mark_native_steer_pending(
buzz_event.channel_id,
&event_id,
);
debug_assert!(reserved, "accepted edit must still be queued");
let tx = native_steer_tx.clone();
let ctx = Arc::clone(&ctx);
let channel_id = buzz_event.channel_id;
tokio::spawn(async move {
let prompt_blocks = pool::format_native_steer_prompt(
channel_id,
event_for_steer.clone(),
prompt_tag_for_steer,
&ctx,
)
.await;
let _ = tx.send(NativeSteerPrepared {
channel_id,
event: event_for_steer,
prompt_blocks,
});
});
true
} else {
let prompt_blocks =
pool::format_native_steer_prompt_sync(
buzz_event.channel_id,
&event_for_steer,
&prompt_tag_for_steer,
);
try_native_steer(
&mut pool,
&mut queue,
buzz_event.channel_id,
event_for_steer,
prompt_blocks,
&steer_ack_tx,
)
}
} else {
false
};
if !native_attempted {
signal_in_flight_task(
&mut pool,
buzz_event.channel_id,
signal,
);
}
}
}
if pool_ready {
for (channel_id, thread_tags) in
dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity)
{
typing_channels.insert(channel_id, thread_tags);
}
}
}
None => {
tracing::warn!("relay event stream ended — requesting reconnect");
if let Err(e) = relay.reconnect().await {
tracing::error!("relay background task is gone: {e} — exiting");
tokio::time::sleep(Duration::from_secs(1)).await;
break;
}
}
}
None
}
_ = async {
match inactivity_reaper.as_mut() {
Some(timer) => timer.tick().await,
None => std::future::pending().await,
}
} => {
let _ = result_rx;
if inactivity_expired(
last_activity,
tokio::time::Instant::now(),
inactivity_bound,
queue.has_in_flight() || heartbeat_in_flight,
) {
tracing::info!(
inactivity_seconds = config.exit_after_inactivity_secs,
"inactivity bound reached — exiting gracefully"
);
let _ = shutdown_tx.send(());
}
None
}
_ = async {
match heartbeat.as_mut() {
Some(hb) => hb.tick().await,
None => std::future::pending().await,
}
} => {
let _ = result_rx;
if !pool_ready {
tracing::debug!("heartbeat_skipped_pool_not_ready");
} else if queue.has_flushable_work() {
tracing::debug!("heartbeat_skipped_events");
for (channel_id, thread_tags) in
dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity)
{
typing_channels.insert(channel_id, thread_tags);
}
} else if pool.any_idle() {
dispatch_heartbeat(&mut pool, &ctx, &mut heartbeat_in_flight);
} else {
tracing::debug!("heartbeat_skipped_busy");
}
None
}
_ = async {
match presence_heartbeat.as_mut() {
Some(t) => t.tick().await,
None => std::future::pending().await,
}
} => {
let _ = result_rx;
// Abort previous heartbeat if still in flight (prevents race on shutdown).
if let Some(h) = presence_task.take() {
h.abort();
}
let pp = presence_publisher.clone();
let pk = presence_keys.clone();
presence_task = Some(tokio::spawn(async move {
if let Err(e) = publish_presence(&pp, &pk, "online").await {
tracing::warn!("presence heartbeat failed: {e}");
}
}));
None
}
_ = async {
match typing_refresh.as_mut() {
Some(t) => t.tick().await,
None => std::future::pending().await,
}
} => {
let _ = result_rx;
// Use try_publish (non-blocking) for typing indicators —
// they're ephemeral and must not block the main loop during
// relay reconnection (#35).
for (&ch, thread_tags) in &typing_channels {
if let Ok(event) = relay.build_typing_event(
ch,
thread_tags.root_event_id.as_deref(),
thread_tags.parent_event_id.as_deref(),
) {
if let Err(e) = relay.try_publish_event(event) {
tracing::debug!("typing indicator dropped for {ch}: {e}");
}
}
}
None
}
_ = shutdown_rx.changed() => {
tracing::info!("shutting down");
break;
}
}
};
match pool_event {
Some(PoolEvent::Result(result)) => {
// Stop typing indicator for the completed channel.
if let PromptSource::Channel(ch) = &result.source {
typing_channels.remove(ch);
}
if handle_prompt_result(
&mut pool,
&mut queue,
&config,
*result,
&mut heartbeat_in_flight,
&removed_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
observer.clone(),
Some(&ctx.rest_client),
) == LoopAction::Exit
{
break;
}
if drain_ready_join_results(
&mut pool,
&mut queue,
&config,
&mut heartbeat_in_flight,
&removed_channels,
&mut typing_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
observer.clone(),
) == LoopAction::Exit
{
break;
}
for (channel_id, thread_tags) in
dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity)
{
typing_channels.insert(channel_id, thread_tags);
}
}
Some(PoolEvent::Panic(join_error)) => {
tracing::error!("agent task panicked: {join_error}");
recover_panicked_agent(
&mut pool,
&mut queue,
&config,
join_error,
&mut heartbeat_in_flight,
&removed_channels,
&mut typing_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
observer.clone(),
);
if pool.live_count() == 0 && !any_respawn_in_flight(&crash_history) {
tracing::error!("all agents dead — exiting");
break;
}
for (channel_id, thread_tags) in
dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity)
{
typing_channels.insert(channel_id, thread_tags);
}
}
Some(PoolEvent::NativeSteerPrepared(NativeSteerPrepared {
channel_id,
event,
prompt_blocks,
})) => {
if native_steer_preparation_is_stale(&removed_channels, channel_id) {
tracing::debug!(
%channel_id,
event_id = %event.id.to_hex(),
"discarding native steer prepared after channel removal"
);
continue;
}
if !try_native_steer(
&mut pool,
&mut queue,
channel_id,
event.clone(),
prompt_blocks,
&steer_ack_tx,
) {
queue.release_native_steer(channel_id, &event.id.to_hex());
signal_in_flight_task(&mut pool, channel_id, ControlSignal::Steer);
}
}
Some(PoolEvent::SteerAck(SteerAckEvent {
channel_id,
event_id,
ack,
})) => {
// Mid-turn steer attempt resolved (either transport:
// `_goose/unstable/session/steer` or `_session/steering`).
// Locked semantics (Eva + Max + Perci, unanimous on Option X):
//
// Success
// The agent received the steer via the non-cancelling
// path. Drop the withheld event so normal dispatch
// never redelivers it.
//
// Also covers `_session/steering`'s `startedNewTurn`
// outcome: the message was delivered, but into a fresh
// turn because the one being steered had already
// finished. Delivery is what this arm keys on, so the
// event is still dropped. The read loop deliberately
// does NOT renew its hard deadline in that case (the
// awaited turn is settled), while
// `extend_in_flight_deadline` below still applies —
// the agent really is running more work, so the
// channel's in-flight budget should reflect it.
//
// Err(_) where the write never landed (Transport /
// ExpectedRunIdMissing):
// Delivery state of the underlying message is "never
// attempted on the wire". Release withheld back to the
// queue front AND issue the cancel+merge fallback so
// the message still reaches the agent.
//
// Err(OutcomeRejected { .. })
// A `_session/steering` request returned a JSON-RPC
// success whose `outcome` was not `injected` or
// `startedNewTurn` (codex's `failed`, an unknown value,
// or a bare `{}` with no `outcome` at all). The steer
// did not land, so this is treated exactly like a write
// that never happened: release withheld AND fire the
// cancel+merge fallback. Handled by the catch-all
// `Err(_)` arm below.
//
// Err(AgentError { code: -32601, .. })
// The agent returned method_not_found — it does not
// implement the steer extension. Release withheld AND
// fire the cancel+merge fallback so the message still
// reaches the agent via the universal path.
//
// Err(AgentError { code: other, .. })
// The write landed and the agent returned a JSON-RPC
// error at the application level (e.g. wrong run id).
// The agent's turn is still running (or just completed).
// Release withheld for normal dispatch; do NOT fire the
// fallback signal — the agent already saw the steer
// attempt. If the turn is still running, normal dispatch
// re-delivers when it completes. If the turn already
// ended, there is nothing to cancel.
//
// PromptCompletedNeutral
// The read loop wrote the steer (or was preparing to)
// but the prompt completed before the response landed.
// Delivery state is unknown — but the prompt completing
// means there is no in-flight turn to signal anymore.
// Release withheld for normal dispatch; do NOT fire
// the fallback signal (it would target a turn that
// just ended; normal dispatch already handles
// redelivery via the released queue entry).
//
// Err(PromptCompleted)
// `SteerError::PromptCompleted` is returned synchronously
// by `pool::send_steer` when no task is in flight (handled
// in `try_native_steer`'s Err branch, which falls through
// to cancel+merge). It is never routed through the ack
// channel, so this variant never appears in `SteerAckEvent`.
//
// Watcher Err (oneshot dropped)
// Should not happen — the read loop drains
// pending_steer on every return path. If it does,
// treat as PromptCompletedNeutral to avoid leaking
// the withheld event in `withheld_native_steer`.
let (release_withheld, drop_withheld, signal_fallback) = match &ack {
Ok(pool::SteerAck::Success { .. }) => (false, true, false),
// -32601 = method_not_found: agent does not implement the
// steer extension. Fire cancel+merge so the message still
// reaches the agent.
Ok(pool::SteerAck::Err(pool::SteerError::AgentError { code, .. }))
if *code == -32601 =>
{
(true, false, true)
}
// AgentError: write landed, agent rejected it at the
// application level (e.g. wrong run id). Release for
// normal dispatch; no fallback signal (the turn is still
// running or just ended — either way there is nothing to
// cancel).
Ok(pool::SteerAck::Err(pool::SteerError::AgentError { .. })) => {
(true, false, false)
}
// Transport / ExpectedRunIdMissing / OutcomeRejected: the
// steer did not land. Release and fire the cancel+merge
// fallback so the message still reaches the agent.
Ok(pool::SteerAck::Err(_)) => (true, false, true),
Ok(pool::SteerAck::PromptCompletedNeutral) => (true, false, false),
Err(_recv_err) => (true, false, false),
};
tracing::info!(
channel = %channel_id,
event_id = %event_id,
?ack,
release_withheld,
drop_withheld,
signal_fallback,
"non-cancelling steer ack received"
);
if let Ok(pool::SteerAck::Success { session_id }) = &ack {
queue.extend_in_flight_deadline(channel_id, config.max_turn_duration_secs);
if !pool.record_successful_steer(
channel_id,
event_id.clone(),
session_id.clone(),
) {
tracing::warn!(
channel = %channel_id,
event_id = %event_id,
"successful steer lost its in-flight delivery ledger"
);
}
}
if drop_withheld {
queue.remove_event(channel_id, &event_id);
}
if release_withheld {
queue.release_native_steer(channel_id, &event_id);
}
if signal_fallback {
// Universal cancel+merge fallback. Note: the
// queued event has already been released to the
// front of `queues[channel_id]`, so the cancel
// will pick it up as part of the merged batch and
// re-prompt the agent.
signal_in_flight_task(&mut pool, channel_id, ControlSignal::Steer);
}
// After releasing a withheld event, give dispatch a chance
// to re-flush. If the prompt is still in flight, the
// channel stays `in_flight_channels` and `flush_next`
// skips it — but a Steer fallback signal sent above will
// tear down the in-flight task; on its completion the
// queue drains. We still try here in case the in-flight
// task has already returned.
for (channel_id, thread_tags) in
dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity)
{
typing_channels.insert(channel_id, thread_tags);
}
}
Some(PoolEvent::Wake(attempt, result)) => {
let completion = result.as_ref().map(|_| ()).map_err(|error| error.clone());
if let Err(error) =
pool_lifecycle.complete_wake(attempt, result, tokio::time::Instant::now())
{
tracing::warn!(attempt, error, "discarding stale pool wake result");
continue;
}
match completion {
Ok(()) => {
pool = pool_lifecycle
.take_ready()
.expect("successful wake stores a ready pool");
pool_ready = true;
emit_runtime_lifecycle(
observer.as_ref(),
&runtime_start_nonce,
&pubkey_hex,
&config.relay_url,
"ready",
None,
);
for (channel_id, thread_tags) in
dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity)
{
typing_channels.insert(channel_id, thread_tags);
}
}
Err(error) => {
debug_assert_eq!(pool_lifecycle.failed_error(), Some(error.as_str()));
emit_runtime_lifecycle(
observer.as_ref(),
&runtime_start_nonce,
&pubkey_hex,
&config.relay_url,
"failed",
Some(&error),
);
}
}
}
None => {} // relay/heartbeat/shutdown branches handled inline above
}
}
// Drain wake tasks gracefully rather than aborting: an in-flight
// initialize_agent_pool observes the shutdown watch at its biased per-slot
// select and reaps its partially-spawned agents itself. `shutdown()` here
// would abort the task mid-init and drop those AcpClients via best-effort
// Drop — the exact zombie class the eager path's spawn-outside-the-timeout
// comment exists to prevent. Fire the watch first so exits that bypass the
// signal handlers (result channel closed, LoopAction::Exit) cancel the wake
// just as promptly. Timeout is a backstop for a slot stuck outside the
// select (e.g. in spawn); only then do we fall back to aborting.
let _ = shutdown_tx.send(());
let wake_drain = tokio::time::timeout(Duration::from_secs(30), async {
while wake_tasks.join_next().await.is_some() {}
})
.await;
if wake_drain.is_err() {
tracing::warn!("wake task did not drain within grace period — aborting");
wake_tasks.shutdown().await;
}
while let Ok((_attempt, result)) = wake_rx.try_recv() {
if let Ok(mut awakened_pool) = result {
shutdown_agent_pool(&mut awakened_pool).await;
}
}
tracing::info!("shutdown: waiting for in-flight prompts");
// 30 s is generous for in-flight prompts to be cancelled; using
// max_turn_duration here would cause Ctrl+C to hang for up to an hour.
let grace = Duration::from_secs(30);
// Best-effort drain of both join_set and result_rx during the grace period.
// Tasks that finish normally send their OwnedAgent through result_rx — we
// explicitly shut them down here to reap child processes. If the grace
// period expires, remaining tasks are aborted and fall back to
// AcpClient::Drop (start_kill + try_wait — best-effort, not guaranteed).
let (rx_ref, js_ref) = pool.rx_and_join_set();
let shutdown_result = tokio::time::timeout(grace, async {
loop {
tokio::select! {
result = js_ref.join_next() => {
match result {
Some(Err(e)) => tracing::warn!("task error during shutdown: {e}"),
Some(Ok(())) => {}
None => break, // join_set empty
}
}
maybe_result = rx_ref.recv() => {
if let Some(mut pr) = maybe_result {
let idx = pr.agent.index;
pr.agent.acp.shutdown().await;
tracing::debug!(agent = idx, "reaped checked-out agent on shutdown");
}
// If None, channel closed — tasks are done.
}
}
}
})
.await;
if shutdown_result.is_err() {
tracing::warn!("grace period expired, aborting remaining tasks");
pool.join_set.shutdown().await;
}
// Drain any remaining results that arrived after join_set drained but
// before tasks were aborted.
while let Ok(mut pr) = pool.result_rx_try_recv() {
let idx = pr.agent.index;
pr.agent.acp.shutdown().await;
tracing::debug!(agent = idx, "reaped late-arriving agent on shutdown");
}
// Explicitly shut down idle agents still sitting in their slots.
for slot in pool.agents_mut().iter_mut() {
if let Some(agent) = slot.take() {
let idx = agent.index;
let mut acp = agent.acp;
acp.shutdown().await;
tracing::debug!(agent = idx, "reaped idle agent on shutdown");
}
}
drop(pool);
// Abort any in-flight respawn tasks. They may be sleeping in backoff or
// running spawn_and_init — either way, we don't want them spawning new
// children after the main loop has exited. RespawnGuard::Drop sends a
// failure result for aborted tasks, so respawn_in_flight is cleared.
respawn_tasks.shutdown().await;
// Drain any respawn results that completed before the abort. Explicitly
// shut down returned agents instead of relying on AcpClient::Drop.
while let Ok(rr) = respawn_rx.try_recv() {
if let Ok((mut acp, _, _)) = rr.result {
acp.shutdown().await;
tracing::debug!(agent = rr.index, "reaped respawned agent on shutdown");
}
}
// Cancel any in-flight presence heartbeat before sending offline.
if let Some(h) = presence_task.take() {
h.abort();
}
// Best-effort: set presence to offline before exiting.
if config.presence_enabled {
match tokio::time::timeout(
Duration::from_secs(2),
publish_presence(&presence_publisher, &presence_keys, "offline"),
)
.await
{
Ok(Ok(_)) => tracing::info!("presence set to offline"),
Ok(Err(e)) => tracing::warn!("failed to set offline presence: {e}"),
Err(_) => tracing::warn!("offline presence timed out"),
}
}
if let Some(handle) = relay_observer_publisher_task.take() {
handle.abort();
}
// Graceful relay shutdown — sends WebSocket close frame and waits up to 5s
// for the background task to finish, rather than aborting immediately (#40).
relay.shutdown().await;
tracing::info!("buzz-acp stopped");
Ok(())
}
#[derive(PartialEq)]
enum LoopAction {
Continue,
Exit,
}
fn event_mentions_agent(event: &nostr::Event, agent_pubkey_hex: &str) -> bool {
event.tags.iter().any(|t| {
t.as_slice().first().map(|s| s.as_str()) == Some("p")
&& t.as_slice().get(1).map(|s| s.as_str()) == Some(agent_pubkey_hex)
})
}
fn is_owner_control_command(
event: &nostr::Event,
kind_u32: u32,
command: &str,
agent_pubkey_hex: &str,
) -> bool {
kind_u32 == KIND_STREAM_MESSAGE
&& event.content.trim() == command
&& event_mentions_agent(event, agent_pubkey_hex)
}
// ── signal_in_flight_task ─────────────────────────────────────────────────────
/// Decide which [`ControlSignal`] (if any) to send to an in-flight turn when a
/// new, already-author-gated event arrives for that channel.
///
/// Returns `None` to leave the in-flight turn untouched (the event waits in the
/// queue and is delivered when the turn completes). Author eligibility — owner
/// ∪ allowlist ∪ siblings — is enforced upstream by the inbound author gate, so
/// `Steer`/`Interrupt` apply to every event that reaches this point; only
/// `OwnerInterrupt` re-checks authorship (owner-only) here.
///
/// `owner` is the resolved owner pubkey hex, if known.
fn mode_gate_signal(
handling: MultipleEventHandling,
author_hex: &str,
owner: Option<&str>,
) -> Option<ControlSignal> {
match handling {
MultipleEventHandling::Queue => None,
MultipleEventHandling::Steer => Some(ControlSignal::Steer),
MultipleEventHandling::Interrupt => Some(ControlSignal::Interrupt),
MultipleEventHandling::OwnerInterrupt => match owner {
Some(o) if author_hex == o => Some(ControlSignal::Interrupt),
_ => None,
},
}
}
/// Send a control signal to the in-flight task for `channel_id`.
/// Returns `true` if a signal was sent, `false` if no in-flight task was found.
fn signal_in_flight_task(
pool: &mut AgentPool,
channel_id: uuid::Uuid,
mode: ControlSignal,
) -> bool {
let entry = pool
.task_map_mut()
.values_mut()
.find(|m| m.channel_id == Some(channel_id));
if let Some(meta) = entry {
if let Some(tx) = meta.control_tx.take() {
tracing::info!(channel = %channel_id, ?mode, "control signal sent to in-flight task");
let _ = tx.send(mode);
return true;
}
}
false
}
/// Attempt the non-cancelling (ACP) steer for a freshly-queued event.
///
/// Caller invariants:
/// - `event` has already been pushed into `EventQueue::queues[channel_id]`
/// via [`EventQueue::push`] — its `event.id` must still be locatable
/// there so [`EventQueue::mark_native_steer_pending`] can move it to the
/// side table.
/// - `multiple_event_handling` resolved to `ControlSignal::Steer`; this
/// function is the non-cancelling fork of that signal.
///
/// Returns `true` if the native attempt was accepted by the read loop
/// (capacity-1 mpsc `try_send` succeeded, event withheld synchronously,
/// ack watcher spawned). On `true` the caller MUST NOT issue the
/// universal cancel+merge `ControlSignal::Steer` fallback — the watcher
/// will issue it from the ack arm if the native attempt fails.
///
/// Returns `false` if `pool.send_steer` failed (no in-flight task,
/// `steer_tx` already full from a prior in-flight steer, or read loop
/// torn down). The caller MUST fall through to
/// `signal_in_flight_task(channel_id, ControlSignal::Steer)` so the
/// event still reaches the agent via the universal path.
///
/// The withheld event is NOT released here on `false` because no withhold
/// was established: `mark_native_steer_pending` only runs on `Ok(())`.
fn try_native_steer(
pool: &mut AgentPool,
queue: &mut EventQueue,
channel_id: uuid::Uuid,
event: nostr::Event,
prompt_blocks: Vec<String>,
steer_ack_tx: &mpsc::UnboundedSender<SteerAckEvent>,
) -> bool {
let event_id_hex = event.id.to_hex();
let (ack_tx, ack_rx) = tokio::sync::oneshot::channel::<pool::SteerAck>();
let request = pool::SteerRequest {
prompt_blocks,
ack_tx,
};
match pool.send_steer(channel_id, request) {
Ok(()) => {
// Ordinary events are withheld after send. Edit preparation reserves
// its event before leaving the main loop, so this is idempotent.
let withheld = queue.mark_native_steer_pending(channel_id, &event_id_hex);
if !withheld {
tracing::debug!(
channel = %channel_id,
event_id = %event_id_hex,
"native steer event was already reserved during async preparation"
);
}
let ack_tx_clone = steer_ack_tx.clone();
let event_id_for_watcher = event_id_hex.clone();
tokio::spawn(async move {
let ack = ack_rx.await;
let _ = ack_tx_clone.send(SteerAckEvent {
channel_id,
event_id: event_id_for_watcher,
ack,
});
});
true
}
Err(e) => {
tracing::info!(
channel = %channel_id,
error = ?e,
"non-cancelling steer not accepted — falling back to cancel+merge"
);
false
}
}
}
// ── dispatch_pending ──────────────────────────────────────────────────────────
/// Flush queued work to available agents.
fn dispatch_pending(
pool: &mut AgentPool,
queue: &mut EventQueue,
ctx: &Arc<PromptContext>,
last_activity: &mut tokio::time::Instant,
) -> Vec<(Uuid, ThreadTags)> {
let mut dispatched_channels = Vec::new();
loop {
let batch = match queue.flush_next() {
Some(b) => b,
None => break,
};
let channel_id = batch.channel_id;
let typing_scope = batch
.events
.last()
.map(|event| queue::parse_thread_tags(&event.event))
.unwrap_or_default();
let affinity_hit = pool.has_session_for(channel_id);
let mut agent = match pool.try_claim(Some(channel_id)) {
Some(a) => a,
None => {
let pending = queue.pending_channels();
tracing::debug!(pending_channels = pending, "pool_exhausted");
queue.requeue_preserve_timestamps(batch);
queue.mark_complete(channel_id);
break;
}
};
tracing::debug!(agent = agent.index, channel = %channel_id, affinity_hit, "agent_claimed");
let recoverable_batch = match ctx.dedup_mode {
DedupMode::Queue => Some(batch.clone()),
DedupMode::Drop => None,
};
let result_tx = pool.result_tx();
let ctx_clone = Arc::clone(ctx);
let agent_index = agent.index;
// Mid-turn non-cancelling steer seam: install the per-turn steer
// receiver on the read loop so the main loop's mode-gate fork
// (see the `if accepted && queue.is_channel_in_flight(...)` block
// in the relay event branch of the main `select!` loop) can drive
// it via the matching sender stored in `TaskMeta.steer_tx`.
// Installed for every prompt task: the read loop picks the steer
// transport at write time from `active_run_id` and the agent's
// advertised `_session/steering` capability, and acks
// `ExpectedRunIdMissing` (→ cancel+merge) when it has neither.
let (tx, rx) = tokio::sync::mpsc::channel::<pool::SteerRequest>(1);
agent.acp.install_steer_rx(rx);
let steer_tx = Some(tx);
// Prompt text is now built inside run_prompt_task (needs async for
// context fetching). Pass None for prompt_text; batch carries the data.
let (control_tx, control_rx) = tokio::sync::oneshot::channel::<ControlSignal>();
let turn_id = Uuid::new_v4().to_string();
let task_turn_id = turn_id.clone();
let abort_handle = pool.join_set.spawn(async move {
pool::run_prompt_task(
agent,
Some(batch),
None,
ctx_clone,
result_tx,
Some(control_rx),
task_turn_id,
)
.await;
});
pool.task_map_mut().insert(
abort_handle.id(),
pool::TaskMeta {
agent_index,
channel_id: Some(channel_id),
turn_id,
recoverable_batch,
control_tx: Some(control_tx),
steer_tx,
successful_steer_deliveries: HashSet::new(),
},
);
dispatched_channels.push((channel_id, typing_scope));
*last_activity = tokio::time::Instant::now();
}
tracing::debug!(
dispatched = dispatched_channels.len(),
queue_depth = queue.pending_channels(),
"dispatch_pending"
);
dispatched_channels
}
/// Returns `true` when `error` is a non-retryable authentication failure.
///
/// Retrying auth errors is harmful: the token won't self-repair between
/// attempts, so each retry wastes an attempt slot, delays the visible failure,
/// and burns the user's context window. Dead-letter immediately and surface a
/// re-authentication hint instead.
///
/// # Classification rationale
///
/// Auth failures arrive as [`acp::AcpError::AgentError`] with a message
/// surfaced from the upstream CLI. Two narrow patterns reliably identify
/// non-transient auth failures observed in the field:
///
/// - `"Re-authenticate"` — emitted by the Claude CLI when an OAuth token has
/// expired ("OAuth access token has expired. Re-authenticate to continue.").
/// Specific to the auth-expiry flow; does not appear in unrelated errors.
/// - `"API Error: 401"` — present in Claude/Codex HTTP-401 responses; 401 is
/// the standard auth-failure status and does not arise from network blips.
///
/// False positives (misclassifying a transient error as non-retryable) silently
/// drop a user message, which is worse than a false negative (extra retries on
/// an auth error). Both patterns are therefore chosen for high precision.
fn is_auth_error(error: &acp::AcpError) -> bool {
let acp::AcpError::AgentError { message, .. } = error else {
return false;
};
message.contains("Re-authenticate") || message.contains("API Error: 401")
}
/// Spawn a task that posts a user-visible failure notice to the relay.
///
/// Shared by the hard-cap immediate dead-letter path and the retries-exhausted
/// dead-letter path so neither duplicates the tokio::spawn block.
fn spawn_failure_notice(
rest_client: Option<&relay::RestClient>,
batch: &FlushBatch,
content: String,
) {
if let Some(rest) = rest_client {
let thread_tags = batch
.events
.last()
.map(|be| queue::parse_thread_tags(&be.event))
.unwrap_or_default();
let rest = rest.clone();
let channel_id = batch.channel_id;
tokio::spawn(async move {
pool::post_failure_notice(&rest, channel_id, &thread_tags, &content).await;
});
}
}
#[allow(clippy::too_many_arguments)]
fn handle_prompt_result(
pool: &mut AgentPool,
queue: &mut EventQueue,
config: &Config,
mut result: PromptResult,
heartbeat_in_flight: &mut bool,
removed_channels: &HashSet<Uuid>,
crash_history: &mut [SlotCircuit],
respawn_tx: &mpsc::Sender<RespawnResult>,
respawn_tasks: &mut tokio::task::JoinSet<()>,
observer: Option<observer::ObserverHandle>,
rest_client: Option<&relay::RestClient>,
) -> LoopAction {
let before = pool.task_map().len();
let agent_index = result.agent.index;
let successful_steer_deliveries = pool
.task_map()
.values()
.find(|meta| meta.agent_index == agent_index)
.map(|meta| meta.successful_steer_deliveries.clone())
.unwrap_or_default();
pool.task_map_mut()
.retain(|_, meta| meta.agent_index != agent_index);
debug_assert_eq!(before, pool.task_map().len() + 1);
if let PromptSource::Channel(channel_id) = &result.source {
// The task may have invalidated this session before returning. Never
// resurrect delivery state for a dead session; its replacement must
// receive fresh standing context and history.
if let Some(live_session_id) = result.agent.state.sessions.get(channel_id).cloned() {
let event_ids = successful_steer_deliveries
.into_iter()
.filter(|delivery| delivery.session_id == live_session_id)
.map(|delivery| delivery.event_id);
result
.agent
.state
.mark_channel_delivery_success(*channel_id, false, event_ids);
}
}
// The hard-timeout death_message (below) must describe the batch's
// *actual* fate, not just the `recently_active` eligibility flag — a
// recently-active batch that exhausts the retry budget in queue.requeue()
// is dead-lettered same as an immediate one, and both differ from a
// channel-removed drop or a heartbeat call with no batch at all. Each
// branch below records what actually happened; only the hard-timeout
// match arm in the death_message construction reads it.
let mut hard_timeout_fate_suffix: Option<&'static str> = None;
// Requeue BEFORE mark_complete: requeue() sets retry_after with a future
// deadline, and mark_complete() checks for it to decide whether to preserve
// retry_counts. If mark_complete runs first, retry_counts is cleared and
// every retry starts at attempt 1 — defeating exponential backoff and
// dead-letter protection.
if let Some(batch) = result.batch.take() {
// Don't requeue batches for channels the agent was removed from —
// those events are stale and should be silently dropped.
if !removed_channels.contains(&batch.channel_id) {
if matches!(
result.outcome,
PromptOutcome::Cancelled | PromptOutcome::CancelDrainTimeout(_)
) {
// Cancel re-prompt: store as cancelled events so flush_next()
// merges them into the next FlushBatch.cancelled_events,
// enabling the annotated merged-prompt format. The batch's
// cancel_reason (set by the pool task per the control signal)
// selects steer vs interrupt framing. It is always set on this
// path; if somehow unset, fall back to the gentler Steer framing
// — consistent with MergeFraming::for_reason(None) and the
// system default — rather than telling the agent to supersede.
//
// CancelDrainTimeout shares this path with Cancelled: a failed
// 5s drain after a control-signal cancel is a cleanup-deadline
// problem, not the deterministic hard-cap death below — the
// original batch must survive with no retry/dead-letter
// accounting, same as a clean cancel.
let reason = batch.cancel_reason.unwrap_or(CancelReason::Steer);
queue.requeue_as_cancelled(batch, reason);
} else if matches!(
result.outcome,
PromptOutcome::Timeout(TimeoutKind::Hard {
recently_active: false
})
) {
tracing::error!(
channel_id = %batch.channel_id,
events = batch.events.len(),
"dead-lettering batch after hard-cap timeout (no recent activity) — discarding {} events",
batch.events.len(),
);
let content = format!(
"⚠️ I couldn't process the last request (the turn exceeded the maximum duration ({}s)). Please re-send if it's still needed.",
config.max_turn_duration_secs
);
spawn_failure_notice(rest_client, &batch, content);
hard_timeout_fate_suffix = Some(" — dead-lettered (no recent activity)");
} else if matches!(
result.outcome,
PromptOutcome::Timeout(TimeoutKind::Hard {
recently_active: true
})
) {
tracing::warn!(
channel_id = %batch.channel_id,
events = batch.events.len(),
"hard-cap timeout with recent activity — requeueing for retry"
);
if let Some(dead) = queue.requeue(batch) {
let content = format!(
"⚠️ I couldn't process the last request after multiple retries (the turn exceeded the maximum duration ({}s)). Please re-send if it's still needed.",
config.max_turn_duration_secs
);
spawn_failure_notice(rest_client, &dead, content);
hard_timeout_fate_suffix = Some(" — dead-lettered (retry budget exhausted)");
} else {
hard_timeout_fate_suffix = Some(" — requeued for retry (recently active)");
}
} else if matches!(&result.outcome, PromptOutcome::Error(e) if is_auth_error(e)) {
// Auth errors are non-retryable: the token won't self-repair
// between retries, so requeueing only wastes attempt slots and
// delays the visible failure. Dead-letter immediately and tell
// the user to re-authenticate the CLI.
tracing::warn!(
channel_id = %batch.channel_id,
events = batch.events.len(),
"dead-lettering batch immediately — non-retryable auth error"
);
let content = "⚠️ I couldn't process the last request: authentication failed. \
Please re-authenticate the CLI (e.g. run `claude /login` or `codex login`) \
and then re-send."
.to_string();
spawn_failure_notice(rest_client, &batch, content);
} else if let Some(dead) = queue.requeue(batch) {
let reason = match &result.outcome {
PromptOutcome::Timeout(TimeoutKind::Idle) => "the turn timed out".to_string(),
PromptOutcome::Timeout(TimeoutKind::Hard { .. }) => {
"the turn exceeded the maximum duration".to_string()
}
PromptOutcome::AgentExited => "the agent process exited".to_string(),
PromptOutcome::Error(e) => format!("{e}"),
_ => "repeated failures".to_string(),
};
let content = format!(
"⚠️ I couldn't process the last request after multiple retries ({reason}). Please re-send if it's still needed."
);
spawn_failure_notice(rest_client, &dead, content);
}
} else {
tracing::debug!(
channel_id = %batch.channel_id,
events = batch.events.len(),
"dropping failed batch for removed channel"
);
hard_timeout_fate_suffix = Some(" — batch dropped (channel removed)");
}
}
match &result.source {
PromptSource::Channel(ch) => queue.mark_complete(*ch),
PromptSource::Heartbeat => *heartbeat_in_flight = false,
}
// Strip sessions for channels the agent was removed from while this
// agent was checked out. This covers the gap where invalidate_channel_sessions
// only touches idle agents.
for ch in removed_channels {
result.agent.state.invalidate_channel(ch);
}
let outcome_label = match &result.outcome {
PromptOutcome::Ok(_) => "ok",
PromptOutcome::Error(_) => "error",
PromptOutcome::Timeout(TimeoutKind::Idle) => "idle_timeout",
PromptOutcome::Timeout(TimeoutKind::Hard { .. }) => "hard_timeout",
PromptOutcome::AgentExited => "exited",
PromptOutcome::Cancelled => "cancelled",
PromptOutcome::CancelDrainTimeout(_) => "cancel_drain_timeout",
};
let agent_index = result.agent.index;
// Capture the spawn-time configured model and our PID before the agent is
// moved into match arms below. `desired_model` reflects the config/persona
// model at spawn time — it does NOT reflect `session/set_model` overrides,
// which live in buzz-agent's session state and are what `llm: (model) …`
// errors carry. The two can legitimately differ; `configured_model=` is
// still valuable for identifying a stale orphan running an old model.
let harness_configured_model = result
.agent
.desired_model
.as_deref()
.unwrap_or("<none>")
.to_string();
let harness_pid = std::process::id();
let channel_id = match &result.source {
PromptSource::Channel(ch) => Some(*ch),
PromptSource::Heartbeat => None,
};
let turn_id = result.turn_id.clone();
let emit_turn_error = |error_msg: &str, error_code: Option<i64>| {
if let Some(ref observer) = observer {
let mut payload = serde_json::json!({
"outcome": outcome_label,
"error": error_msg,
});
if let Some(code) = error_code {
payload["code"] = serde_json::json!(code);
}
observer.emit(
"turn_error",
Some(agent_index),
&observer::context_for(channel_id, None, Some(turn_id.clone())),
payload,
);
}
};
match result.outcome {
// Successful prompt — return agent to pool.
PromptOutcome::Ok(_) => {
tracing::debug!(
agent = agent_index,
outcome = outcome_label,
"agent_returned"
);
pool.return_agent(result.agent);
}
// Fatal outcomes: the agent subprocess is dead or poisoned — respawn it.
PromptOutcome::AgentExited | PromptOutcome::Timeout(_) => {
tracing::warn!(
agent = agent_index,
outcome = outcome_label,
configured_model = %harness_configured_model,
pid = harness_pid,
"agent_returned — respawning"
);
let death_message: String = match outcome_label {
"exited" => "Agent process exited unexpectedly".to_string(),
"hard_timeout" => {
// Neutral wording when no fate was recorded above: a
// heartbeat hard timeout carries no batch at all, so
// nothing was requeued or dead-lettered.
let suffix = hard_timeout_fate_suffix.unwrap_or(" (no batch to retry)");
format!(
"Agent turn exceeded the maximum duration ({}s){}",
config.max_turn_duration_secs, suffix
)
}
_ => "Agent session timed out due to inactivity".to_string(),
};
emit_turn_error(&death_message, None);
let index = result.agent.index;
let slot_history = &mut crash_history[index];
if !spawn_respawn_task(
result.agent,
config,
slot_history,
respawn_tx,
respawn_tasks,
observer.clone(),
) {
// Circuit open — slot stays empty until maintenance refill.
if pool.live_count() == 0 && !any_respawn_in_flight(crash_history) {
tracing::error!("all agents dead — exiting");
return LoopAction::Exit;
}
}
}
// Cancel-drain expiry: a control-signal cancel (steer fallback,
// interrupt, or explicit stop) did not drain within its bounded
// grace window. The process is poisoned/uncertain like a hard
// timeout — respawn it — but this is NOT the configured max-turn
// cap, so the message must name the actual grace, not
// `max_turn_duration_secs`. The triggering batch's fate (preserved
// for Steer/Interrupt, dropped for explicit Cancel/Rotate or a
// removed channel) is decided above — the message stays fate-neutral
// since it must be true in every case.
PromptOutcome::CancelDrainTimeout(grace) => {
tracing::warn!(
agent = agent_index,
outcome = outcome_label,
configured_model = %harness_configured_model,
pid = harness_pid,
grace = ?grace,
"agent_returned — respawning (cancel-drain timeout)"
);
let death_message = format!(
"Agent did not stop within {grace:?} after cancellation; the agent process is being replaced."
);
emit_turn_error(&death_message, None);
let index = result.agent.index;
let slot_history = &mut crash_history[index];
if !spawn_respawn_task(
result.agent,
config,
slot_history,
respawn_tx,
respawn_tasks,
observer.clone(),
) {
// Circuit open — slot stays empty until maintenance refill.
if pool.live_count() == 0 && !any_respawn_in_flight(crash_history) {
tracing::error!("all agents dead — exiting");
return LoopAction::Exit;
}
}
}
// Errors fall into two categories:
//
// 1. Transport-class (Io, WriteTimeout, Timeout, Protocol): the stdio
// pipe may be corrupted or the agent desynchronized. These are fatal
// to the agent regardless of whether they occurred during session
// creation or an active prompt — respawn unconditionally.
//
// 2. Application-class (IdleTimeout, HardTimeout, Json): the pipe is
// intact but the prompt failed. Return the agent to the pool so it
// can be reused for the next event.
// Intentional cancel — agent is healthy, return it to the pool.
// No respawn, no retry penalty. The cancelled batch was already stored
// via requeue_as_cancelled() above and will be merged into the next
// FlushBatch by flush_next().
PromptOutcome::Cancelled => {
tracing::debug!(
agent = agent_index,
outcome = outcome_label,
configured_model = %harness_configured_model,
pid = harness_pid,
"agent_returned (cancelled)"
);
pool.return_agent(result.agent);
}
PromptOutcome::Error(ref e) => {
let is_transport_error = matches!(
e,
acp::AcpError::Io(_)
| acp::AcpError::WriteTimeout(_)
| acp::AcpError::Timeout(_)
| acp::AcpError::Protocol(_)
);
let error_code = match &e {
acp::AcpError::AgentError { code, .. } => Some(*code),
_ => None,
};
if is_transport_error {
tracing::warn!(
agent = agent_index,
outcome = outcome_label,
configured_model = %harness_configured_model,
pid = harness_pid,
error = %e,
"transport/protocol error — respawning agent"
);
emit_turn_error(&e.to_string(), error_code);
let index = result.agent.index;
let slot_history = &mut crash_history[index];
if !spawn_respawn_task(
result.agent,
config,
slot_history,
respawn_tx,
respawn_tasks,
observer,
) && pool.live_count() == 0
&& !any_respawn_in_flight(crash_history)
{
tracing::error!("all agents dead — exiting");
return LoopAction::Exit;
}
} else {
tracing::warn!(
agent = agent_index,
outcome = outcome_label,
configured_model = %harness_configured_model,
pid = harness_pid,
error = %e,
"agent_returned (application error — pipe intact)"
);
emit_turn_error(&e.to_string(), error_code);
pool.return_agent(result.agent);
}
}
}
LoopAction::Continue
}
#[allow(clippy::too_many_arguments)]
fn recover_panicked_agent(
pool: &mut AgentPool,
queue: &mut EventQueue,
config: &Config,
join_error: tokio::task::JoinError,
heartbeat_in_flight: &mut bool,
removed_channels: &HashSet<Uuid>,
typing_channels: &mut HashMap<Uuid, ThreadTags>,
crash_history: &mut [SlotCircuit],
respawn_tx: &mpsc::Sender<RespawnResult>,
respawn_tasks: &mut tokio::task::JoinSet<()>,
observer: Option<observer::ObserverHandle>,
) {
let task_id = join_error.id();
let Some(meta) = pool.task_map_mut().remove(&task_id) else {
tracing::error!("panic for unknown task {task_id:?} — bug");
return;
};
let i = meta.agent_index;
// Requeue BEFORE mark_complete (same rationale as handle_prompt_result).
if let Some(batch) = meta.recoverable_batch {
if let Some(ch) = meta.channel_id {
if !removed_channels.contains(&ch) {
// Dead-letter on exhaustion is logged inside requeue(); a
// panic path has no outcome to report, so no notice here.
let _ = queue.requeue(batch);
tracing::warn!("requeued batch for panicked agent {i}");
} else {
tracing::debug!(
channel_id = %ch,
"dropping panicked batch for removed channel"
);
}
}
}
if let Some(ch) = meta.channel_id {
queue.mark_complete(ch);
typing_channels.remove(&ch);
tracing::warn!("cleared wedged in-flight channel {ch} from panicked agent {i}");
} else {
*heartbeat_in_flight = false;
tracing::warn!("cleared wedged heartbeat_in_flight from panicked agent {i}");
}
if let Some(ref observer) = observer {
observer.emit(
"agent_panic",
Some(i),
&observer::context_for(meta.channel_id, None, Some(meta.turn_id)),
serde_json::json!({
"outcome": "panic",
"error": format!("Agent task panicked: {join_error}"),
}),
);
}
// Panics count as crashes for the circuit breaker.
// The panicked task already dropped the AcpClient, so we just need to
// check the circuit and spawn a fresh agent in the background.
let slot = &mut crash_history[i];
let delay = match slot.record_crash() {
CrashVerdict::CircuitOpen => {
tracing::error!(agent = i, "circuit open after panic — not respawning");
return;
}
CrashVerdict::HalfOpenProbe => {
tracing::info!(agent = i, "circuit half-open — probe respawn after panic");
Duration::ZERO
}
CrashVerdict::Respawn(d) => {
tracing::info!(
agent = i,
delay_ms = d.as_millis(),
"respawn backoff after panic"
);
d
}
};
// Spawn respawn work off the main loop.
slot.respawn_in_flight = true;
let cmd = config.agent_command.clone();
let args = config.agent_args.clone();
let env = config.persona_env_vars.clone();
let has_codex = config.has_generated_codex_config;
let guard = RespawnGuard::new(i, respawn_tx.clone());
respawn_tasks.spawn(async move {
if !delay.is_zero() {
tokio::time::sleep(delay).await;
}
let result = spawn_and_init(&cmd, &args, &env, has_codex, i, observer).await;
guard.send(result);
});
}
#[allow(clippy::too_many_arguments)]
fn drain_ready_join_results(
pool: &mut AgentPool,
queue: &mut EventQueue,
config: &Config,
heartbeat_in_flight: &mut bool,
removed_channels: &HashSet<Uuid>,
typing_channels: &mut HashMap<Uuid, ThreadTags>,
crash_history: &mut [SlotCircuit],
respawn_tx: &mpsc::Sender<RespawnResult>,
respawn_tasks: &mut tokio::task::JoinSet<()>,
observer: Option<observer::ObserverHandle>,
) -> LoopAction {
while let Some(Some(join_result)) = pool.join_set.join_next().now_or_never() {
if let Err(join_error) = join_result {
tracing::error!("agent task panicked: {join_error}");
recover_panicked_agent(
pool,
queue,
config,
join_error,
heartbeat_in_flight,
removed_channels,
typing_channels,
crash_history,
respawn_tx,
respawn_tasks,
observer.clone(),
);
if pool.live_count() == 0 && !any_respawn_in_flight(crash_history) {
return LoopAction::Exit;
}
}
}
LoopAction::Continue
}
fn dispatch_heartbeat(
pool: &mut AgentPool,
ctx: &Arc<PromptContext>,
heartbeat_in_flight: &mut bool,
) {
if *heartbeat_in_flight {
return;
}
let agent = match pool.try_claim(None) {
Some(a) => a,
None => return,
};
let prompt_text = ctx
.heartbeat_prompt
.clone()
.unwrap_or_else(default_heartbeat_prompt);
let result_tx = pool.result_tx();
let ctx_clone = Arc::clone(ctx);
let agent_index = agent.index;
let turn_id = Uuid::new_v4().to_string();
let task_turn_id = turn_id.clone();
let abort_handle = pool.join_set.spawn(async move {
pool::run_prompt_task(
agent,
None,
Some(prompt_text),
ctx_clone,
result_tx,
None,
task_turn_id,
)
.await;
});
pool.task_map_mut().insert(
abort_handle.id(),
pool::TaskMeta {
agent_index,
channel_id: None,
turn_id,
recoverable_batch: None,
control_tx: None,
steer_tx: None,
successful_steer_deliveries: HashSet::new(),
},
);
*heartbeat_in_flight = true;
tracing::info!(agent = agent_index, "heartbeat_fired");
}
#[cfg(test)]
mod agent_draft_prompt_tests {
#[test]
fn shared_base_prompt_teaches_portable_agent_drafts() {
let prompt = include_str!("base_prompt.md");
assert!(prompt.contains("buzz agents draft-create"));
assert!(prompt.contains("ask for at most two things"));
assert!(prompt.contains("what it should do day-to-day"));
assert!(prompt.contains("owner saves it"));
assert!(prompt.contains("Do not ask about runtime, provider, model, credentials"));
}
#[test]
fn shared_base_prompt_teaches_real_newlines_for_multiline_messages() {
let prompt = include_str!("base_prompt.md");
assert!(prompt.contains("pass real newline bytes through stdin"));
assert!(prompt.contains("single-quoted shell strings preserve `\\n` literally"));
assert!(prompt.contains("buzz messages send ... --content -"));
}
#[test]
fn shared_base_prompt_teaches_single_command_mentions_and_preflight() {
let prompt = include_str!("base_prompt.md");
assert!(prompt.contains("--mention <hex-or-npub>"));
assert!(prompt.contains("every presentation-only name that should notify"));
assert!(
prompt.contains("permits unresolved or ambiguous `@Name` text as presentation-only")
);
assert!(prompt.contains("success JSON's `mention_pubkeys`"));
assert!(prompt.contains("no follow-up verification command is needed"));
assert!(prompt.contains("stops before sending"));
assert!(prompt
.contains("add them explicitly with `buzz channels add-member` only when authorized"));
assert!(prompt.contains("never changes membership automatically"));
}
}
fn default_heartbeat_prompt() -> String {
let now = chrono::Utc::now().to_rfc3339();
format!(
"[System: Heartbeat]\nTime: {now}\n\n\
You have been awakened for a routine heartbeat. You have NO incoming messages or\n\
active channel context for this turn.\n\n\
Your tasks:\n\
1. Run `buzz feed get --types needs_action` to check for pending workflow approvals or\n\
high-priority requests addressed to you.\n\
2. Run `buzz feed get --types mentions` to check for unanswered @mentions.\n\
3. If you find actionable items, address them using the appropriate CLI commands\n\
(e.g., `buzz workflows approve --token <UUID>`, `buzz messages send`,\n\
`buzz messages send --reply-to <event-id>`).\n\
4. If there are no pending actions or mentions, end your turn immediately.\n\n\
Do not run `buzz channels list` or `buzz messages search` unless you have a specific reason.\n\
Do not invent work — only act on items surfaced by the feed commands."
)
}
/// Spawn a background respawn task for a crashed agent slot.
///
/// Does the circuit breaker check synchronously (non-blocking), then spawns
/// the actual shutdown + backoff + spawn_and_init work into a background task.
/// The result comes back through `respawn_tx` so the main loop stays responsive.
///
/// Returns `true` if a respawn task was spawned, `false` if the circuit is open.
fn spawn_respawn_task(
old_agent: OwnedAgent,
config: &Config,
slot: &mut SlotCircuit,
respawn_tx: &mpsc::Sender<RespawnResult>,
respawn_tasks: &mut tokio::task::JoinSet<()>,
observer: Option<observer::ObserverHandle>,
) -> bool {
let index = old_agent.index;
// Circuit breaker: record crash, decide whether to respawn.
let delay = match slot.record_crash() {
CrashVerdict::CircuitOpen => {
tracing::error!(agent = index, "circuit open — not respawning");
return false;
}
CrashVerdict::HalfOpenProbe => {
tracing::info!(agent = index, "circuit half-open — probe respawn");
Duration::ZERO
}
CrashVerdict::Respawn(d) => {
tracing::info!(agent = index, delay_ms = d.as_millis(), "respawn backoff");
d
}
};
slot.respawn_in_flight = true;
// Spawn the actual work (shutdown + sleep + spawn + init) off the main loop.
let cmd = config.agent_command.clone();
let args = config.agent_args.clone();
let env = config.persona_env_vars.clone();
let has_codex = config.has_generated_codex_config;
let guard = RespawnGuard::new(index, respawn_tx.clone());
respawn_tasks.spawn(async move {
// Shutdown old agent (reap child, prevent zombie).
let mut agent = old_agent;
agent.acp.shutdown().await;
drop(agent);
if !delay.is_zero() {
tokio::time::sleep(delay).await;
}
let result = spawn_and_init(&cmd, &args, &env, has_codex, index, observer).await;
guard.send(result);
});
true
}
fn normalized_agent_name(init_result: &serde_json::Value) -> String {
init_result
.get("agentInfo")
.or_else(|| init_result.get("serverInfo"))
.and_then(|info| info.get("name"))
.and_then(|value| value.as_str())
.unwrap_or("unknown")
.trim()
.to_ascii_lowercase()
}
async fn shutdown_agent_slots(slots: &mut [Option<OwnedAgent>]) {
for slot in slots {
if let Some(mut agent) = slot.take() {
agent.acp.shutdown().await;
}
}
}
async fn shutdown_agent_pool(pool: &mut AgentPool) {
pool.join_set.shutdown().await;
while let Ok(mut result) = pool.result_rx_try_recv() {
result.agent.acp.shutdown().await;
}
for slot in pool.agents_mut() {
if let Some(mut agent) = slot.take() {
agent.acp.shutdown().await;
}
}
}
struct PoolStartup {
agents: u32,
command: String,
args: Vec<String>,
extra_env: Vec<(String, String)>,
has_generated_codex_config: bool,
model: Option<String>,
observer: Option<observer::ObserverHandle>,
}
impl PoolStartup {
fn from_config(config: &Config, observer: Option<observer::ObserverHandle>) -> Self {
Self {
agents: config.agents,
command: config.agent_command.clone(),
args: config.agent_args.clone(),
extra_env: config.persona_env_vars.clone(),
has_generated_codex_config: config.has_generated_codex_config,
model: config.model.clone(),
observer,
}
}
}
async fn initialize_agent_pool(
startup: &PoolStartup,
mut shutdown: Option<watch::Receiver<()>>,
) -> Result<AgentPool> {
// One agent failing to start must not kill the whole pool.
// Attempt each spawn under a 60-second timeout; a partial pool is valid.
let mut agent_slots: Vec<Option<OwnedAgent>> = Vec::with_capacity(startup.agents as usize);
for i in 0..startup.agents as usize {
let spawn_result = AcpClient::spawn(
&startup.command,
&startup.args,
&startup.extra_env,
startup.has_generated_codex_config,
)
.await;
match spawn_result {
Ok(mut acp) => {
acp.set_observer(startup.observer.clone(), i);
let initialize = tokio::time::timeout(Duration::from_secs(60), acp.initialize());
let initialize_result = match shutdown.as_mut() {
Some(shutdown) => tokio::select! {
biased;
_ = shutdown.changed() => {
acp.shutdown().await;
shutdown_agent_slots(&mut agent_slots).await;
return Err(anyhow::anyhow!("pool initialization cancelled by shutdown"));
}
result = initialize => result,
},
None => initialize.await,
};
match initialize_result {
Ok(Ok(init_result)) => {
tracing::info!(agent = i, "agent initialized: {init_result}");
let protocol_version =
init_result["protocolVersion"].as_u64().unwrap_or(1) as u32;
tracing::info!(
agent = i,
name = init_result
.get("agentInfo")
.or_else(|| init_result.get("serverInfo"))
.and_then(|info| info.get("name"))
.and_then(|v| v.as_str())
.unwrap_or("unknown"),
steering_supported = acp.steering_supported(),
"agent initialized"
);
acp.observe(
"agent_initialized",
serde_json::json!({
"agentIndex": i,
"initializeResult": init_result,
}),
);
let agent_name = normalized_agent_name(&init_result);
agent_slots.push(Some(OwnedAgent {
index: i,
acp,
state: SessionState::default(),
model_capabilities: None,
desired_model: startup.model.clone(),
model_overridden: false,
agent_name,
goose_system_prompt_supported: None,
protocol_version,
}));
}
Ok(Err(e)) => {
tracing::error!(agent = i, "agent initialize failed: {e}");
acp.shutdown().await;
agent_slots.push(None);
}
Err(_) => {
tracing::error!(agent = i, "agent timed out during init (60s)");
acp.shutdown().await;
agent_slots.push(None);
}
}
}
Err(e) => {
tracing::error!(agent = i, "agent failed to spawn: {e}");
agent_slots.push(None);
}
}
}
let live_count = agent_slots.iter().filter(|slot| slot.is_some()).count();
if live_count == 0 {
return Err(anyhow::anyhow!(
"all {} agents failed to start — cannot continue",
startup.agents
));
}
if live_count < startup.agents as usize {
tracing::warn!(
"started {}/{} agents — continuing with reduced pool",
live_count,
startup.agents
);
}
tracing::info!("agent_pool_ready agents={}", live_count);
Ok(AgentPool::from_slots(agent_slots))
}
// ── spawn_and_init ────────────────────────────────────────────────────────────
/// Spawn an agent subprocess and run the MCP `initialize` handshake.
///
/// Takes owned args so it can run in a background `tokio::spawn` task without
/// borrowing `Config`. All respawn/refill paths use this.
async fn spawn_and_init(
command: &str,
args: &[String],
extra_env: &[(String, String)],
has_generated_codex_config: bool,
agent_index: usize,
observer: Option<observer::ObserverHandle>,
) -> Result<(AcpClient, u32, String)> {
let mut acp = AcpClient::spawn(command, args, extra_env, has_generated_codex_config)
.await
.map_err(|e| anyhow::anyhow!("failed to spawn agent: {e}"))?;
acp.set_observer(observer, agent_index);
match acp.initialize().await {
Ok(init_result) => {
tracing::info!("agent initialized: {init_result}");
let protocol_version = init_result["protocolVersion"].as_u64().unwrap_or(1) as u32;
acp.observe(
"agent_initialized",
serde_json::json!({
"agentIndex": agent_index,
"initializeResult": init_result,
}),
);
let agent_name = normalized_agent_name(&init_result);
Ok((acp, protocol_version, agent_name))
}
Err(e) => {
// Explicitly shut down the spawned child to prevent zombie/leak.
// Drop only does start_kill + try_wait (best-effort); shutdown()
// does start_kill + bounded wait (guaranteed reap).
acp.shutdown().await;
Err(anyhow::anyhow!("agent initialize failed: {e}"))
}
}
}
async fn spawn_auth_client(agent: &AuthAgentArgs) -> Result<AcpClient, acp::AcpError> {
let agent_args = config::normalize_agent_args(&agent.agent_command, agent.agent_args.clone());
AcpClient::spawn(&agent.agent_command, &agent_args, &[], false).await
}
fn extract_auth_methods(init_result: &serde_json::Value) -> Vec<serde_json::Value> {
init_result
.get("authMethods")
.and_then(|methods| methods.as_array())
.cloned()
.unwrap_or_default()
}
/// `buzz-acp auth-methods` — spawn an adapter, initialize it, print authMethods.
async fn run_auth_methods(args: AuthMethodsArgs) -> Result<()> {
let mut client = match spawn_auth_client(&args.agent).await {
Ok(c) => c,
Err(e) => {
eprintln!("error: failed to spawn agent: {e}");
std::process::exit(1);
}
};
let init_result = match tokio::time::timeout(MODELS_TIMEOUT, client.initialize()).await {
Ok(Ok(result)) => result,
Ok(Err(e)) => {
client.shutdown().await;
eprintln!("error: agent initialize failed: {e}");
std::process::exit(1);
}
Err(_) => {
client.shutdown().await;
eprintln!("error: agent timed out ({MODELS_TIMEOUT:?})");
std::process::exit(1);
}
};
let methods = extract_auth_methods(&init_result);
client.shutdown().await;
if args.json {
let output = serde_json::json!({ "methods": methods });
println!("{}", serde_json::to_string_pretty(&output)?);
} else if methods.is_empty() {
println!("No auth methods advertised.");
} else {
for method in methods {
let id = method
.get("id")
.and_then(|value| value.as_str())
.unwrap_or("unknown");
let name = method
.get("name")
.and_then(|value| value.as_str())
.unwrap_or(id);
println!("{id}\t{name}");
}
}
Ok(())
}
/// `buzz-acp authenticate` — invoke one adapter-owned auth method.
async fn run_authenticate(args: AuthenticateArgs) -> Result<()> {
let mut client = match spawn_auth_client(&args.agent).await {
Ok(c) => c,
Err(e) => {
eprintln!("error: failed to spawn agent: {e}");
std::process::exit(1);
}
};
let init_result = match tokio::time::timeout(MODELS_TIMEOUT, client.initialize()).await {
Ok(Ok(result)) => result,
Ok(Err(e)) => {
client.shutdown().await;
eprintln!("error: agent initialize failed: {e}");
std::process::exit(1);
}
Err(_) => {
client.shutdown().await;
eprintln!("error: agent initialize timed out ({MODELS_TIMEOUT:?})");
std::process::exit(1);
}
};
let supports_method = extract_auth_methods(&init_result)
.iter()
.any(|method| method.get("id").and_then(|id| id.as_str()) == Some(args.method_id.as_str()));
if !supports_method {
client.shutdown().await;
eprintln!(
"error: auth method '{}' is not advertised by this adapter",
args.method_id
);
std::process::exit(1);
}
let result =
tokio::time::timeout(AUTHENTICATE_TIMEOUT, client.authenticate(&args.method_id)).await;
match result {
Ok(Ok(_)) => {
client.shutdown().await;
Ok(())
}
Ok(Err(e)) => {
client.shutdown().await;
eprintln!("error: authenticate failed: {e}");
std::process::exit(1);
}
Err(_) => {
client.shutdown().await;
eprintln!("error: authenticate timed out ({AUTHENTICATE_TIMEOUT:?})");
std::process::exit(1);
}
}
}
/// Flow: spawn → initialize → session/new → print models → shutdown.
/// No relay connection, no MCP servers, no subscriptions. ~2-5s total.
async fn run_models(args: ModelsArgs) -> Result<()> {
use acp::{extract_model_config_options, extract_model_state};
let agent_args = config::normalize_agent_args(&args.agent.agent_command, args.agent.agent_args);
let cwd = std::env::current_dir()
.unwrap_or_else(|_| std::path::PathBuf::from("/"))
.to_string_lossy()
.to_string();
// Spawn outside the timeout so we always own the child for cleanup.
// `models` subcommand doesn't use persona packs — no extra env, no codex config.
let mut client =
match AcpClient::spawn(&args.agent.agent_command, &agent_args, &[], false).await {
Ok(c) => c,
Err(e) => {
eprintln!("error: failed to spawn agent: {e}");
std::process::exit(1);
}
};
// Initialize + session/new under a timeout. Client is owned above,
// so shutdown() runs on all paths (success, error, timeout).
let protocol_result = tokio::time::timeout(MODELS_TIMEOUT, async {
let init = client.initialize().await?;
let session = client.session_new_full(&cwd, vec![], None, None).await?;
Ok::<_, acp::AcpError>((init, session))
})
.await;
let (init_result, session_resp) = match protocol_result {
Ok(Ok(tuple)) => tuple,
Ok(Err(e)) => {
client.shutdown().await;
eprintln!("error: agent communication failed: {e}");
std::process::exit(1);
}
Err(_) => {
client.shutdown().await;
eprintln!("error: agent timed out ({MODELS_TIMEOUT:?})");
std::process::exit(1);
}
};
// Extract agent info from initialize response.
// ACP spec uses "serverInfo" (MCP heritage); some agents may use "agentInfo".
let info_obj = init_result
.get("serverInfo")
.or_else(|| init_result.get("agentInfo"));
let agent_name = info_obj
.and_then(|ai| ai.get("name"))
.and_then(|v| v.as_str())
.unwrap_or("unknown");
let agent_version = info_obj
.and_then(|ai| ai.get("version"))
.and_then(|v| v.as_str())
.unwrap_or("unknown");
// Extract model info from session/new response.
let config_options = extract_model_config_options(&session_resp.raw);
let model_state = extract_model_state(&session_resp.raw);
if args.json {
// Structured JSON output — consumed by Phase 3 `get_agent_models`.
let output = serde_json::json!({
"agent": {
"name": agent_name,
"version": agent_version,
},
"stable": {
"configOptions": config_options,
},
"unstable": model_state.as_ref().map(|ms| serde_json::json!({
"currentModelId": ms.get("currentModelId"),
"availableModels": ms.get("availableModels"),
})),
});
println!("{}", serde_json::to_string_pretty(&output)?);
} else {
// Human-readable output.
println!("Agent: {} v{}", agent_name, agent_version);
println!();
let mut has_models = false;
if !config_options.is_empty() {
println!("Models (stable configOptions):");
for opt in &config_options {
let config_id = opt.get("configId").and_then(|v| v.as_str()).unwrap_or("?");
let display = opt
.get("displayName")
.and_then(|v| v.as_str())
.unwrap_or(config_id);
println!(" {display} (configId: {config_id})");
if let Some(options) = opt.get("options").and_then(|v| v.as_array()) {
for o in options {
let val = o.get("value").and_then(|v| v.as_str()).unwrap_or("?");
let name = o.get("displayName").and_then(|v| v.as_str()).unwrap_or(val);
println!(" - {name} (value: {val})");
}
}
}
has_models = true;
}
if let Some(ref ms) = model_state {
let current = ms
.get("currentModelId")
.and_then(|v| v.as_str())
.unwrap_or("(none)");
println!("Models (unstable SessionModelState):");
println!(" Current: {current}");
if let Some(available) = ms.get("availableModels").and_then(|v| v.as_array()) {
println!(" Available:");
for m in available {
let id = m.get("modelId").and_then(|v| v.as_str()).unwrap_or("?");
let name = m.get("name").and_then(|v| v.as_str()).unwrap_or(id);
let desc = m.get("description").and_then(|v| v.as_str()).unwrap_or("");
if desc.is_empty() {
println!(" - {name} (id: {id})");
} else {
println!(" - {name} (id: {id}) — {desc}");
}
}
}
has_models = true;
}
if !has_models {
println!("No model information available from this agent.");
}
}
client.shutdown().await;
Ok(())
}
fn build_mcp_servers(config: &Config) -> Vec<McpServer> {
if config.mcp_command.is_empty() {
return vec![];
}
vec![McpServer {
name: std::path::Path::new(&config.mcp_command)
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("mcp")
.to_string(),
command: config.mcp_command.clone(),
args: vec![],
env: {
let mut env = vec![
EnvVar {
name: "BUZZ_RELAY_URL".into(),
value: config.relay_url.clone(),
},
EnvVar {
name: "BUZZ_PRIVATE_KEY".into(),
// bech32 encoding of a valid secret key is infallible.
// Panic here is correct: injecting a bogus secret would cause
// delayed, hard-to-diagnose agent failures downstream.
value: config
.keys
.secret_key()
.to_bech32()
.expect("secret key bech32 encoding should never fail"),
},
];
// Forward BUZZ_AUTH_TAG (NIP-OA owner attestation credential)
// so the MCP server can attach it to every signed event.
if let Ok(auth_tag) = std::env::var("BUZZ_AUTH_TAG") {
if !auth_tag.is_empty() {
env.push(EnvVar {
name: "BUZZ_AUTH_TAG".into(),
value: auth_tag,
});
}
}
// Forward the agent's display name so dev-mcp can use it as the git
// author name instead of the raw npub. Read from the process env
// rather than Config: this is a pass-through of a contract owned
// upstream, and absent simply means dev-mcp falls back to the npub.
if let Ok(display_name) = std::env::var("BUZZ_ACP_DISPLAY_NAME") {
if !display_name.is_empty() {
env.push(EnvVar {
name: "BUZZ_ACP_DISPLAY_NAME".into(),
value: display_name,
});
}
}
env
},
}]
}
#[cfg(test)]
mod heartbeat_base_prompt_tests {
use super::*;
// Pins the heartbeat dispatch path (dispatch_heartbeat, ~line 2359): a
// legacy agent WITH a base_prompt must get [Base] prepended to the
// heartbeat user message, composed as `[Base]\n{bp}\n\n{prompt}`. This is
// the second half of the round-2 regression (the first being initial_message).
fn heartbeat_standing() -> queue::StandingContext<'static> {
queue::StandingContext {
base_prompt: Some("you are a helpful agent"),
..Default::default()
}
}
#[test]
fn test_heartbeat_legacy_agent_gets_base_prepended() {
// protocol_version 1 + Some(base_prompt): heartbeat prompt is prefixed
// with the [Base] section exactly as the legacy session/new path would.
let prompt = "[System: Heartbeat]\nrun feed get";
let composed = pool::prepend_standing_for_legacy(1, &heartbeat_standing(), prompt);
assert_eq!(
composed,
"[Base]\nyou are a helpful agent\n\n[System: Heartbeat]\nrun feed get"
);
}
#[test]
fn test_heartbeat_modern_agent_omits_base() {
// protocol_version 2 gets base_prompt via session/new; the heartbeat
// prompt is sent verbatim.
let prompt = "[System: Heartbeat]\nrun feed get";
let composed = pool::prepend_standing_for_legacy(2, &heartbeat_standing(), prompt);
assert_eq!(composed, prompt);
}
}
#[cfg(test)]
mod owner_control_command_tests {
use super::*;
use nostr::{EventBuilder, Keys, Kind, Tag};
fn make_event(kind: u32, content: &str, p_hex: Option<&str>) -> nostr::Event {
let keys = Keys::generate();
let tags = match p_hex {
Some(hex) => vec![Tag::parse(["p", hex]).expect("p tag")],
None => vec![],
};
EventBuilder::new(Kind::Custom(kind as u16), content)
.tags(tags)
.sign_with_keys(&keys)
.unwrap()
}
#[test]
fn owner_control_command_requires_kind_content_and_agent_mention() {
let agent = "ab".repeat(32);
let event = make_event(KIND_STREAM_MESSAGE, " !rotate ", Some(&agent));
assert!(is_owner_control_command(
&event,
KIND_STREAM_MESSAGE,
"!rotate",
&agent
));
let wrong_kind = make_event(1, "!rotate", Some(&agent));
assert!(!is_owner_control_command(&wrong_kind, 1, "!rotate", &agent));
let wrong_content = make_event(KIND_STREAM_MESSAGE, "!cancel", Some(&agent));
assert!(!is_owner_control_command(
&wrong_content,
KIND_STREAM_MESSAGE,
"!rotate",
&agent
));
let no_mention = make_event(KIND_STREAM_MESSAGE, "!rotate", None);
assert!(!is_owner_control_command(
&no_mention,
KIND_STREAM_MESSAGE,
"!rotate",
&agent
));
}
#[test]
fn mode_gate_signal_maps_handling_to_control_signal() {
let owner = "a".repeat(64);
let other = "b".repeat(64);
// Queue: never signals — events wait for the turn to finish.
assert!(mode_gate_signal(MultipleEventHandling::Queue, &owner, Some(&owner)).is_none());
// Steer: always steers (eligibility already enforced upstream).
assert!(matches!(
mode_gate_signal(MultipleEventHandling::Steer, &other, Some(&owner)),
Some(ControlSignal::Steer)
));
// Steer even when owner is unknown — gate doesn't re-check authorship.
assert!(matches!(
mode_gate_signal(MultipleEventHandling::Steer, &other, None),
Some(ControlSignal::Steer)
));
// Interrupt: always interrupts for any eligible author.
assert!(matches!(
mode_gate_signal(MultipleEventHandling::Interrupt, &other, Some(&owner)),
Some(ControlSignal::Interrupt)
));
// OwnerInterrupt: interrupts only for the owner.
assert!(matches!(
mode_gate_signal(MultipleEventHandling::OwnerInterrupt, &owner, Some(&owner)),
Some(ControlSignal::Interrupt)
));
assert!(
mode_gate_signal(MultipleEventHandling::OwnerInterrupt, &other, Some(&owner)).is_none(),
"owner-interrupt must not fire for a non-owner author"
);
assert!(
mode_gate_signal(MultipleEventHandling::OwnerInterrupt, &owner, None).is_none(),
"owner-interrupt must not fire when the owner is unknown"
);
}
#[tokio::test]
async fn signal_in_flight_task_sends_rotate_once() {
let mut pool = AgentPool::from_slots(vec![]);
let channel_id = Uuid::new_v4();
let other_channel_id = Uuid::new_v4();
let (control_tx, control_rx) = tokio::sync::oneshot::channel();
let abort_handle = pool.join_set.spawn(async {});
pool.task_map_mut().insert(
abort_handle.id(),
pool::TaskMeta {
agent_index: 0,
channel_id: Some(channel_id),
turn_id: "test-turn-id".to_string(),
recoverable_batch: None,
control_tx: Some(control_tx),
steer_tx: None,
successful_steer_deliveries: HashSet::new(),
},
);
assert!(!signal_in_flight_task(
&mut pool,
other_channel_id,
ControlSignal::Rotate
));
assert!(signal_in_flight_task(
&mut pool,
channel_id,
ControlSignal::Rotate
));
assert_eq!(control_rx.await.unwrap(), ControlSignal::Rotate);
assert!(!signal_in_flight_task(
&mut pool,
channel_id,
ControlSignal::Rotate
));
}
}
#[cfg(test)]
mod owner_cache_tests {
use super::*;
#[test]
fn new_with_some_caches_immediately() {
let cache = OwnerCache::new(Some("abcd".into()));
assert_eq!(cache.get(), Some("abcd"));
}
#[test]
fn new_with_none_returns_none() {
let cache = OwnerCache::new(None);
assert!(cache.get().is_none());
}
#[test]
fn get_returns_cached_value() {
let cache = OwnerCache::new(Some("ab".repeat(32)));
assert_eq!(cache.get(), Some("ab".repeat(32)).as_deref());
}
}
#[cfg(test)]
mod author_gate_tests {
use super::*;
/// A `RestClient` for tests. The author-gate decisions exercised here all
/// resolve from the owner pubkey or sibling cache before any HTTP call, so
/// this client is never actually used to make a request.
fn dummy_rest_client() -> relay::RestClient {
relay::RestClient {
http: reqwest::Client::new(),
base_url: "http://localhost:0".into(),
keys: nostr::Keys::generate(),
auth_tag_json: None,
}
}
const OWNER: &str = "00";
const SIBLING: &str = "11";
const EXTERNAL: &str = "22";
const STRANGER: &str = "33";
/// Owner + a known sibling, none of them on the explicit allowlist.
fn cache_with_sibling() -> OwnerCache {
let cache = OwnerCache::new(Some(OWNER.into()));
cache.cache_sibling(SIBLING.into(), true);
cache.cache_sibling(STRANGER.into(), false);
cache.cache_sibling(EXTERNAL.into(), false);
cache
}
#[tokio::test]
async fn test_allowlist_accepts_sibling_not_in_allowlist() {
let cache = cache_with_sibling();
let allowlist = HashSet::from([EXTERNAL.to_string()]);
assert!(
author_allowed(
&RespondTo::Allowlist,
&allowlist,
SIBLING,
false,
&cache,
&dummy_rest_client()
)
.await,
"a same-owner sibling must fire a turn under Allowlist even when not listed"
);
}
#[tokio::test]
async fn test_allowlist_accepts_explicit_external_pubkey() {
let cache = cache_with_sibling();
let allowlist = HashSet::from([EXTERNAL.to_string()]);
assert!(
author_allowed(
&RespondTo::Allowlist,
&allowlist,
EXTERNAL,
false,
&cache,
&dummy_rest_client()
)
.await,
"an explicitly allowlisted external pubkey must still be accepted"
);
}
#[tokio::test]
async fn test_allowlist_rejects_non_sibling_not_in_allowlist() {
let cache = cache_with_sibling();
let allowlist = HashSet::from([EXTERNAL.to_string()]);
assert!(
!author_allowed(
&RespondTo::Allowlist,
&allowlist,
STRANGER,
false,
&cache,
&dummy_rest_client()
)
.await,
"a non-sibling absent from the allowlist must be dropped"
);
}
#[tokio::test]
async fn test_allowlist_accepts_owner() {
let cache = cache_with_sibling();
let allowlist = HashSet::new();
assert!(
author_allowed(
&RespondTo::Allowlist,
&allowlist,
OWNER,
false,
&cache,
&dummy_rest_client()
)
.await,
"the owner must always be accepted under Allowlist"
);
}
// The default `respond-to` is OwnerOnly. Under steering, "an ineligible
// author must NOT steer" is enforced *here* — author_allowed drops the
// event before it reaches the mode gate — not in the gate itself. These
// pin that invariant against the default mode.
#[tokio::test]
async fn test_owner_only_rejects_stranger_so_no_steer() {
let cache = cache_with_sibling();
assert!(
!author_allowed(
&RespondTo::OwnerOnly,
&HashSet::new(),
STRANGER,
false,
&cache,
&dummy_rest_client()
)
.await,
"under the default OwnerOnly, a stranger must be dropped — so it can never reach the mode gate to steer"
);
}
#[tokio::test]
async fn test_owner_only_admits_owner_and_sibling_to_steer() {
let cache = cache_with_sibling();
for (who, label) in [(OWNER, "owner"), (SIBLING, "sibling")] {
assert!(
author_allowed(
&RespondTo::OwnerOnly,
&HashSet::new(),
who,
false,
&cache,
&dummy_rest_client()
)
.await,
"under default OwnerOnly, the {label} must be admitted so steering can fire"
);
}
}
// ── DM hardening ──────────────────────────────────────────────────────
//
// In a DM, clients auto-p-tag every participant, and an agent can be
// asked to open a DM with a third party. The gate must therefore ignore
// the allowlist and `anyone` mode inside DMs: only owner + verified
// siblings fire turns.
#[tokio::test]
async fn test_dm_rejects_allowlisted_external_pubkey() {
let cache = cache_with_sibling();
let allowlist = HashSet::from([EXTERNAL.to_string()]);
assert!(
!author_allowed(
&RespondTo::Allowlist,
&allowlist,
EXTERNAL,
true,
&cache,
&dummy_rest_client()
)
.await,
"an allowlisted external pubkey must NOT fire a turn inside a DM"
);
}
#[tokio::test]
async fn test_dm_rejects_stranger_under_anyone() {
let cache = cache_with_sibling();
assert!(
!author_allowed(
&RespondTo::Anyone,
&HashSet::new(),
STRANGER,
true,
&cache,
&dummy_rest_client()
)
.await,
"respond_to=anyone must still drop non-owner authors inside a DM"
);
}
#[tokio::test]
async fn test_dm_admits_owner_and_sibling_in_every_responding_mode() {
let cache = cache_with_sibling();
for mode in [
RespondTo::OwnerOnly,
RespondTo::Allowlist,
RespondTo::Anyone,
] {
for (who, label) in [(OWNER, "owner"), (SIBLING, "sibling")] {
assert!(
author_allowed(
&mode,
&HashSet::new(),
who,
true,
&cache,
&dummy_rest_client()
)
.await,
"in a DM under {mode}, the {label} must still be admitted"
);
}
}
}
#[tokio::test]
async fn test_dm_nobody_rejects_even_owner() {
let cache = cache_with_sibling();
assert!(
!author_allowed(
&RespondTo::Nobody,
&HashSet::new(),
OWNER,
true,
&cache,
&dummy_rest_client()
)
.await,
"respond_to=nobody must drop everything, DMs included"
);
}
// ── is_dm_channel resolution ──────────────────────────────────────────
fn resolver(startup: HashMap<Uuid, relay::ChannelInfo>) -> pool::ChannelInfoResolver {
pool::ChannelInfoResolver::new(startup, dummy_rest_client())
}
#[tokio::test]
async fn test_is_dm_channel_uses_definitive_startup_metadata() {
let dm_id = Uuid::new_v4();
let stream_id = Uuid::new_v4();
let startup = HashMap::from([
(
dm_id,
relay::ChannelInfo {
name: "dm".into(),
channel_type: "dm".into(),
},
),
(
stream_id,
relay::ChannelInfo {
name: "stream".into(),
channel_type: "stream".into(),
},
),
]);
let resolver = resolver(startup);
assert!(is_dm_channel(dm_id, &resolver).await);
assert!(!is_dm_channel(stream_id, &resolver).await);
}
#[tokio::test]
async fn test_is_dm_channel_fails_closed_for_unknown_startup_metadata() {
let id = Uuid::new_v4();
let startup = HashMap::from([(
id,
relay::ChannelInfo {
name: "unknown".into(),
channel_type: "unknown".into(),
},
)]);
assert!(
is_dm_channel(id, &resolver(startup)).await,
"missing startup metadata must not be trusted as a stream"
);
}
async fn lazy_resolver_with_response(
response: serde_json::Value,
) -> (
pool::ChannelInfoResolver,
std::sync::Arc<std::sync::atomic::AtomicUsize>,
tokio::task::JoinHandle<()>,
) {
use std::sync::atomic::{AtomicUsize, Ordering};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind test HTTP server");
let base_url = format!("http://{}", listener.local_addr().unwrap());
let requests = std::sync::Arc::new(AtomicUsize::new(0));
let server_requests = requests.clone();
let body = response.to_string();
let server = tokio::spawn(async move {
loop {
let Ok((mut socket, _)) = listener.accept().await else {
break;
};
let mut request = vec![0; 8192];
let _ = socket.read(&mut request).await;
server_requests.fetch_add(1, Ordering::SeqCst);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
let _ = socket.write_all(response.as_bytes()).await;
}
});
let rest = relay::RestClient {
http: reqwest::Client::new(),
base_url,
keys: nostr::Keys::generate(),
auth_tag_json: None,
};
(
pool::ChannelInfoResolver::new(HashMap::new(), rest),
requests,
server,
)
}
#[tokio::test]
async fn test_is_dm_channel_lazy_resolves_declared_dm_and_caches_it() {
use std::sync::atomic::Ordering;
let id = Uuid::new_v4();
let response = serde_json::json!([{
"tags": [["d", id.to_string()], ["name", "DM"], ["t", "dm"]]
}]);
let (resolver, requests, server) = lazy_resolver_with_response(response).await;
assert!(is_dm_channel(id, &resolver).await);
assert!(is_dm_channel(id, &resolver).await);
assert_eq!(
requests.load(Ordering::SeqCst),
1,
"second resolution uses cache"
);
server.abort();
}
#[tokio::test]
async fn test_discovery_without_metadata_stays_fail_closed_at_author_gate() {
let id = Uuid::new_v4();
let discovered = relay::merge_discovered_channels(vec![id], &serde_json::json!([]));
let channel_info = resolver(discovered);
let owner_cache = cache_with_sibling();
let allowlist = HashSet::from([EXTERNAL.to_string()]);
let is_dm = is_dm_channel(id, &channel_info).await;
assert!(is_dm, "unknown startup metadata must fail closed as DM");
assert!(
!author_allowed(
&RespondTo::Allowlist,
&allowlist,
EXTERNAL,
is_dm,
&owner_cache,
&dummy_rest_client(),
)
.await,
"an external author must not pass when startup discovery omitted metadata"
);
}
#[tokio::test]
async fn test_is_dm_channel_fails_closed_when_lazy_resolution_fails() {
assert!(
is_dm_channel(Uuid::new_v4(), &resolver(HashMap::new())).await,
"an unresolvable channel type must be treated as a DM"
);
}
}
#[cfg(test)]
mod observer_snapshot_race_tests {
use super::*;
use nostr::Keys;
fn emit_marker(observer: &observer::ObserverHandle, marker: &str) {
observer.emit(
"test_event",
None,
&observer::context_for(None, None, None),
serde_json::json!({ "marker": marker }),
);
}
/// An event emitted between `subscribe()` and `snapshot()` lands in BOTH
/// the snapshot and the live receiver; the seq high-water dedupe must
/// deliver it exactly once — and never lose events on either side of it.
#[tokio::test(start_paused = true)]
async fn overlap_between_subscribe_and_snapshot_publishes_exactly_once() {
let observer = observer::ObserverHandle::in_process();
let agent_keys = Keys::generate();
let owner_keys = Keys::generate();
let (publisher, mut published_rx) = RelayEventPublisher::test_pair();
// Before the publisher starts: replay-buffer only.
emit_marker(&observer, "before");
// The race window: emitted after subscribe() but before snapshot(),
// so it is present in the snapshot AND queued on the receiver.
let rx = observer.subscribe();
emit_marker(&observer, "overlap");
let snapshot = observer.snapshot();
assert_eq!(snapshot.len(), 2, "overlap event must be in the snapshot");
// After the snapshot: live receiver only.
emit_marker(&observer, "after");
// Close the broadcast channel so the run loop drains and exits.
drop(observer);
run_relay_observer_publisher(
snapshot,
rx,
publisher,
agent_keys.clone(),
agent_keys.public_key().to_hex(),
owner_keys.public_key().to_hex(),
owner_keys.public_key(),
)
.await;
// The run loop has exited, dropping the publisher; drain the forwarded
// events until the channel closes (deterministic — no try_recv race
// with the test_pair forwarding task). With per-tick batching the three
// events arrive inside batch envelopes (or unwrapped when a drain held
// exactly one event); unwrap both shapes.
let mut markers = Vec::new();
while let Some(event) = published_rx.recv().await {
let payload: serde_json::Value =
decrypt_observer_payload(&owner_keys, &event).expect("decrypt published frame");
match payload["payload"]["events"].as_array() {
Some(inner) => markers.extend(
inner
.iter()
.map(|e| e["payload"]["marker"].as_str().unwrap().to_string()),
),
None => markers.push(payload["payload"]["marker"].as_str().unwrap().to_string()),
}
}
assert_eq!(
markers,
["before", "overlap", "after"],
"each event must be published exactly once, in order"
);
}
}
#[cfg(test)]
mod observer_publish_queue_tests {
use super::*;
fn event(seq: u64, kind: &str, channel: Option<&str>) -> observer::ObserverEvent {
observer::ObserverEvent {
seq,
timestamp: format!("2026-04-29T04:00:{:02}Z", seq.min(59)),
kind: kind.to_string(),
agent_index: Some(0),
channel_id: channel.map(ToOwned::to_owned),
session_id: Some("session-1".to_string()),
turn_id: Some("turn-1".to_string()),
started_at: None,
payload: serde_json::json!({ "seq": seq }),
}
}
fn queue_of(events: Vec<observer::ObserverEvent>) -> ObserverPublishQueue {
let mut queue = ObserverPublishQueue::default();
for event in events {
queue.ingest(event);
}
queue
}
/// Collect every frame the queue will produce, one publish slot at a time.
fn drain_frames(queue: &mut ObserverPublishQueue) -> Vec<observer::ObserverEvent> {
let mut frames = Vec::new();
while !queue.is_empty() {
frames.push(queue.next_frame().expect("queue not empty"));
}
frames
}
/// Inner seqs of a frame, whether it is an envelope or an unwrapped
/// singleton.
fn frame_seqs(frame: &observer::ObserverEvent) -> Vec<u64> {
match frame.payload.get("events").and_then(|v| v.as_array()) {
Some(inner) => inner.iter().map(|e| e["seq"].as_u64().unwrap()).collect(),
None => vec![frame.seq],
}
}
/// Retained bytes computed by WALKING the entries, independently of the
/// queue's own accumulator. Cap regressions must assert on this, not on
/// `total_pending_bytes()` — asserting the counter against itself passed
/// while the process retained ~2x the budget (Sami/Max round 3: each
/// pending coalescer entry holds the first chunk's text twice, in the
/// serialized skeleton AND the extracted `text` copy).
fn walked_retained_bytes(queue: &ObserverPublishQueue) -> usize {
let fifo: usize = queue
.events
.iter()
.map(|(_, _, event)| serialized_len(event))
.sum();
let coalescer: usize = queue
.coalescer
.pending
.iter()
.map(|pending| serialized_len(&pending.event) + pending.text.len())
.sum();
fifo + coalescer
}
/// The walker above is itself an instrument, and every cap test asks it
/// only for `<= CAP` — a blinded walker (missing an arm, or returning 0)
/// would satisfy all of them while hiding exactly the 2x overshoot it was
/// added to catch (Sami round 5, M17-M20). Pin it two-sided: it must SEE
/// the double retention, and it must agree with the accumulator EXACTLY
/// while both stores are non-empty — neither may drift.
#[test]
fn walked_retained_bytes_agrees_with_the_accumulator_exactly() {
fn chunk(seq: u64, message_id: &str, text: &str) -> observer::ObserverEvent {
let mut e = event(seq, "acp_read", Some("chan-a"));
e.payload = serde_json::json!({
"jsonrpc": "2.0",
"method": "session/update",
"params": {
"sessionId": "session-1",
"update": {
"sessionUpdate": "agent_message_chunk",
"messageId": message_id,
"content": { "type": "text", "text": text },
},
},
});
e
}
let text = "w".repeat(7_000);
let mut queue = ObserverPublishQueue::default();
// One pending chunk: its text lives in the serialized skeleton AND
// the extracted copy, so a walker blind to either arm reads short.
queue.ingest(chunk(1, "message-a", &text));
assert!(
walked_retained_bytes(&queue) >= 2 * text.len(),
"the walker must SEE the first chunk's text twice \
(skeleton + extracted copy), got {}",
walked_retained_bytes(&queue)
);
// Populate BOTH stores: the non-chunk event flushes message-a into
// the FIFO and queues itself; fresh pending keys (plus a same-key
// append) rebuild the coalescer side.
queue.ingest(event(2, "tool_call", Some("chan-a")));
queue.ingest(chunk(3, "message-b", &text));
queue.ingest(chunk(4, "message-b", &text));
queue.ingest(chunk(5, "message-c", &text));
assert!(
!queue.events.is_empty() && !queue.coalescer.pending.is_empty(),
"both arms must be non-empty for the agreement check to bind"
);
assert_eq!(
queue.total_pending_bytes(),
walked_retained_bytes(&queue),
"accumulator and entry-walk must agree exactly: neither may drift"
);
}
/// Two or more pending events for one channel ship as a single batch
/// envelope whose payload carries every inner event in arrival order.
#[test]
fn multiple_events_ship_as_one_envelope_in_order() {
let mut queue = queue_of(vec![
event(1, "turn_started", Some("chan-a")),
event(2, "acp_read", Some("chan-a")),
event(3, "acp_write", Some("chan-a")),
]);
let frame = queue.next_frame().expect("one frame");
assert!(queue.is_empty(), "one channel, one publish slot");
assert_eq!(frame.kind, OBSERVER_BATCH_KIND);
assert_eq!(frame.seq, 3, "envelope mirrors the last inner event");
assert_eq!(frame_seqs(&frame), [1, 2, 3], "arrival order preserved");
let inner = frame.payload["events"].as_array().expect("events array");
assert_eq!(inner[1]["kind"], "acp_read", "inner events keep their kind");
}
/// A single pending event is published unwrapped — no envelope, so
/// consumers that predate batching still understand quiet periods.
#[test]
fn a_single_event_stays_unwrapped() {
let mut queue = queue_of(vec![event(7, "turn_started", Some("chan-a"))]);
let frame = queue.next_frame().expect("one frame");
assert!(queue.is_empty());
assert_eq!(frame.kind, "turn_started");
assert_eq!(frame.seq, 7);
}
/// An empty queue yields no frame — a tick with nothing pending must not
/// publish anything.
#[test]
fn empty_queue_yields_no_frame() {
let mut queue = ObserverPublishQueue::default();
assert!(queue.next_frame().is_none());
assert!(queue.is_empty());
}
/// Frames never mix channels, and each channel's events keep their FIFO
/// order. Gathering is QUEUE-WIDE: the front event's channel collects its
/// events from anywhere in the queue (that is what keeps the drain rate
/// in bytes per slot under interleaving), so cross-channel frame order
/// MAY differ from arrival order — but a null-channel event is a barrier
/// nothing gathers across.
#[test]
fn frames_never_mix_channels_and_gather_queue_wide() {
let mut queue = queue_of(vec![
event(1, "acp_read", Some("chan-a")),
event(2, "acp_write", Some("chan-a")),
event(3, "acp_read", Some("chan-b")),
event(4, "acp_read", Some("chan-a")),
event(5, "acp_read", None),
]);
let frames = drain_frames(&mut queue);
assert_eq!(
frames.len(),
3,
"gathered: [1,2,4]@a, [3]@b, [5]@None — one frame each"
);
for frame in &frames {
let channels: HashSet<Option<String>> = match frame.payload.get("events") {
Some(serde_json::Value::Array(inner)) => inner
.iter()
.map(|e| e["channelId"].as_str().map(ToOwned::to_owned))
.collect(),
_ => std::iter::once(frame.channel_id.clone()).collect(),
};
assert_eq!(channels.len(), 1, "a frame never mixes channels");
}
assert_eq!(
frame_seqs(&frames[0]),
[1, 2, 4],
"chan-a gathers queue-wide, FIFO within the channel"
);
assert_eq!(frames[0].channel_id.as_deref(), Some("chan-a"));
assert_eq!(frames[1].kind, "acp_read", "singleton stays unwrapped");
assert_eq!(frames[1].channel_id.as_deref(), Some("chan-b"));
assert_eq!(frames[2].channel_id, None);
}
/// A NULL-channel event is a barrier: channel events queued BEHIND it
/// must not gather into a frame ahead of it, so causally-global events
/// (`agent_panic`-class) keep their exact order against every channel.
/// The null event itself ships only its contiguous front run.
#[test]
fn null_channel_events_are_gather_barriers() {
let mut queue = queue_of(vec![
event(1, "acp_read", Some("chan-a")),
event(2, "acp_read", Some("chan-b")),
event(3, "agent_panic", None),
event(4, "acp_write", Some("chan-a")),
]);
let frames = drain_frames(&mut queue);
let published: Vec<Vec<u64>> = frames.iter().map(frame_seqs).collect();
assert_eq!(
published,
[vec![1], vec![2], vec![3], vec![4]],
"seq 4 must not gather past the null barrier into frame 1"
);
}
/// The drain-rate regression Sami measured: with two channels strictly
/// alternating, a front-run packer degrades to ONE event per slot
/// (~275 B/s regardless of the 64KB frame budget). Queue-wide gathering
/// must drain an interleaved backlog in ~ceil(events / per-frame-fit)
/// slots per channel, not one slot per event.
#[test]
fn interleaved_channels_drain_at_bytes_per_slot_not_events_per_slot() {
let mut events = Vec::new();
for i in 0..100u64 {
events.push(event(2 * i + 1, "acp_read", Some("chan-a")));
events.push(event(2 * i + 2, "acp_read", Some("chan-b")));
}
let mut queue = queue_of(events);
let frames = drain_frames(&mut queue);
assert!(
frames.len() <= 4,
"200 tiny alternating events must gather into a few full frames, \
got {} (front-run packing would need 200 slots)",
frames.len()
);
for frame in &frames {
assert!(serialized_len(frame) <= OBSERVER_MAX_PLAINTEXT_LEN);
}
// Within each channel, FIFO order survives the gather.
let mut seqs_a = Vec::new();
let mut seqs_b = Vec::new();
for frame in &frames {
match frame.channel_id.as_deref() {
Some("chan-a") => seqs_a.extend(frame_seqs(frame)),
Some("chan-b") => seqs_b.extend(frame_seqs(frame)),
other => panic!("unexpected channel {other:?}"),
}
}
assert!(seqs_a.windows(2).all(|w| w[0] < w[1]), "chan-a FIFO");
assert!(seqs_b.windows(2).all(|w| w[0] < w[1]), "chan-b FIFO");
assert_eq!(seqs_a.len() + seqs_b.len(), 200, "nothing lost");
}
/// A same-channel backlog that cannot fit one 64KB frame splits across
/// SUCCESSIVE publish slots — never multiple frames from one slot — with
/// every frame under the cap and no event lost or reordered.
#[test]
fn oversized_backlogs_split_across_publish_slots_under_the_cap() {
let big_text = "x".repeat(30_000);
let mut queue = queue_of(
(1..=6)
.map(|seq| {
let mut e = event(seq, "acp_read", Some("chan-a"));
e.payload = serde_json::json!({ "seq": seq, "text": big_text });
e
})
.collect(),
);
let frames = drain_frames(&mut queue);
assert!(
frames.len() > 1,
"six 30KB events cannot fit one 64KB frame"
);
let mut seen = Vec::new();
for frame in &frames {
assert!(
serialized_len(frame) <= OBSERVER_MAX_PLAINTEXT_LEN,
"every emitted frame must fit the plaintext cap"
);
seen.extend(frame_seqs(frame));
}
assert_eq!(
seen,
[1, 2, 3, 4, 5, 6],
"no event lost or reordered by splitting"
);
}
/// The queue preserves the coalescer's ordering rule: a non-chunk event
/// force-flushes pending chunk text ahead of itself, so merged chunks can
/// never leapfrog a tool call that arrived after them.
#[test]
fn non_chunk_events_flush_pending_chunks_ahead_of_themselves() {
fn chunk(seq: u64, text: &str) -> observer::ObserverEvent {
let mut e = event(seq, "acp_read", Some("chan-a"));
e.payload = serde_json::json!({
"params": { "update": {
"sessionUpdate": "agent_message_chunk",
"messageId": "m1",
"content": { "text": text },
}}
});
e
}
let mut queue = ObserverPublishQueue::default();
queue.ingest(chunk(1, "hello "));
queue.ingest(chunk(2, "world"));
queue.ingest(event(3, "tool_call", Some("chan-a")));
let frame = queue.next_frame().expect("one frame");
assert!(queue.is_empty());
let inner = frame.payload["events"].as_array().expect("batch of 2");
assert_eq!(inner.len(), 2, "two chunks coalesce into one event");
assert_eq!(
inner[0]["payload"]["params"]["update"]["content"]["text"], "hello world",
"chunk text merged before the tool call"
);
assert_eq!(inner[1]["kind"], "tool_call");
assert!(inner[0]["seq"].as_u64() < inner[1]["seq"].as_u64());
}
/// Chunks still pending inside the coalescer (no non-chunk flushed them)
/// are picked up by the publish slot itself, not stranded.
#[test]
fn a_publish_slot_flushes_pending_coalesced_chunks() {
let mut e = event(1, "acp_read", Some("chan-a"));
e.payload = serde_json::json!({
"params": { "update": {
"sessionUpdate": "agent_message_chunk",
"messageId": "m1",
"content": { "text": "buffered" },
}}
});
let mut queue = ObserverPublishQueue::default();
queue.ingest(e);
assert!(!queue.is_empty(), "pending chunk counts as queued work");
let frame = queue.next_frame().expect("chunk must ship");
assert!(queue.is_empty());
assert_eq!(
frame.payload["params"]["update"]["content"]["text"],
"buffered"
);
}
/// Sami's ceiling assertion: when sustained input outruns the one-frame
/// drain budget for longer than the queue's byte budget, the OLDEST events
/// drop with accounting — never silently — and everything that survives
/// publishes in order with nothing else lost.
#[test]
fn over_budget_floods_drop_oldest_with_accounting() {
let big_text = "y".repeat(10_000);
let total = 500usize; // ~5MB of ~10KB events > 4MiB budget
let mut queue = ObserverPublishQueue::default();
for seq in 1..=total as u64 {
let mut e = event(seq, "acp_read", Some("chan-a"));
e.payload = serde_json::json!({ "seq": seq, "text": big_text });
queue.ingest(e);
}
assert!(
queue.dropped_events > 0,
"a 5MB backlog must overflow the 4MiB budget"
);
assert!(
walked_retained_bytes(&queue) <= OBSERVER_PENDING_QUEUE_MAX_BYTES,
"eviction must restore the byte budget (entry-walked), got {}",
walked_retained_bytes(&queue)
);
let frames = drain_frames(&mut queue);
let published: Vec<u64> = frames.iter().flat_map(frame_seqs).collect();
let expected: Vec<u64> = (queue.dropped_events + 1..=total as u64).collect();
assert_eq!(
published, expected,
"exactly the oldest `dropped_events` events are missing; the rest \
publish in order"
);
assert_eq!(
published.len() as u64 + queue.dropped_events,
total as u64,
"accounting: published + dropped == ingested"
);
}
/// Max's coalescer-bypass regression: a flood of chunks with DISTINCT
/// messageIds never flushes on its own, so every chunk sits in the
/// coalescer's pending buffer. TRUE retained bytes — walked from the
/// entries, never the queue's own accumulator — MUST respect the byte
/// budget with event-level drop accounting. Pre-fix this retained ~25MB
/// against the 4 MiB cap with `pending_bytes == 0` and zero drops; the
/// round-3 refinement (Sami/Max) caught the accumulator itself reading
/// under cap while true retention was 1.99x over.
#[test]
fn distinct_key_chunk_floods_are_bounded_by_the_byte_budget() {
let big_text = "z".repeat(50_000);
let total = 500u64; // ~25MB pending chunk text vs a 4MiB budget
let mut queue = ObserverPublishQueue::default();
for seq in 1..=total {
let mut e = event(seq, "acp_read", Some("chan-a"));
e.payload = serde_json::json!({
"jsonrpc": "2.0",
"method": "session/update",
"params": {
"sessionId": "session-1",
"update": {
"sessionUpdate": "agent_message_chunk",
"messageId": format!("message-{seq}"),
"content": { "type": "text", "text": big_text },
},
},
});
queue.ingest(e);
}
let walked = walked_retained_bytes(&queue);
assert!(
walked <= OBSERVER_PENDING_QUEUE_MAX_BYTES,
"TRUE retained bytes (walked from entries) must respect the cap, \
got {walked}"
);
assert!(
queue.total_pending_bytes() >= walked,
"the accumulator must never under-count true retention \
(accumulator {} < walked {walked})",
queue.total_pending_bytes()
);
assert!(
queue.dropped_events > 0,
"a ~25MB distinct-key chunk flood must record drops"
);
// Event-level accounting: everything that survives publishes, and
// survivors + dropped == ingested.
let frames = drain_frames(&mut queue);
let survived: u64 = frames.iter().map(|f| frame_seqs(f).len() as u64).sum();
assert_eq!(
survived + queue.dropped_events,
total,
"accounting: published + dropped == ingested"
);
// The survivors are the NEWEST events (drop-oldest).
let last_frame_seqs = frame_seqs(frames.last().expect("frames"));
assert_eq!(*last_frame_seqs.last().expect("seqs"), total);
}
/// Max's merged-chunk accounting regression: one coalescer entry can
/// represent MANY generated observer events (same-messageId chunks merge
/// in place), so evicting it must charge every merged source event to
/// `dropped_events`, not 1 per retained entry. Pre-fix, evicting an entry
/// that merged 50 chunks recorded `dropped_events == 1` and 49 generated
/// events vanished from the accounting.
#[test]
fn evicting_a_merged_chunk_entry_accounts_every_source_event() {
fn chunk(seq: u64, message_id: &str, text: &str) -> observer::ObserverEvent {
let mut e = event(seq, "acp_read", Some("chan-a"));
e.payload = serde_json::json!({
"jsonrpc": "2.0",
"method": "session/update",
"params": {
"sessionId": "session-1",
"update": {
"sessionUpdate": "agent_message_chunk",
"messageId": message_id,
"content": { "type": "text", "text": text },
},
},
});
e
}
let mut queue = ObserverPublishQueue::default();
// 50 × 1KB chunks under ONE messageId merge into a single pending
// coalescer entry — the oldest item anywhere in the queue.
let merged_text = "m".repeat(1_000);
let merged_sources = 50u64;
for seq in 1..=merged_sources {
queue.ingest(chunk(seq, "message-merged", &merged_text));
}
// Flood with distinct-key 50KB chunks until the byte budget evicts
// the oldest entries — the merged entry goes first.
let flood_text = "f".repeat(50_000);
let flood = 100u64;
for seq in 1..=flood {
queue.ingest(chunk(
merged_sources + seq,
&format!("message-{seq}"),
&flood_text,
));
}
assert!(
walked_retained_bytes(&queue) <= OBSERVER_PENDING_QUEUE_MAX_BYTES,
"eviction must restore the byte budget (entry-walked), got {}",
walked_retained_bytes(&queue)
);
let frames = drain_frames(&mut queue);
assert!(
!frames
.iter()
.flat_map(frame_seqs)
.any(|seq| seq <= merged_sources),
"the merged entry (globally oldest) must have been evicted"
);
// Every survivor is an unmerged distinct-key chunk (1 source each),
// so source-event accounting must close exactly: the merged entry's
// eviction charges all 50 sources.
let survived: u64 = frames.iter().map(|f| frame_seqs(f).len() as u64).sum();
assert_eq!(
survived + queue.dropped_events,
merged_sources + flood,
"accounting: published sources + dropped sources == ingested"
);
}
/// Sami's M13 / Max's forced-flush probe: the OTHER eviction arm. A
/// merged entry FLUSHED into the publish FIFO (by a non-chunk event) must
/// still charge every absorbed source on eviction — the FIFO stores the
/// per-entry count precisely so the ledger survives flush. The
/// coalescer-side regression above never exercises this arm; mutating the
/// FIFO eviction to `dropped += 1` survived all 687 tests until this one.
#[test]
fn evicting_a_flushed_merged_entry_from_the_fifo_accounts_every_source_event() {
fn chunk(seq: u64, message_id: &str, text: &str) -> observer::ObserverEvent {
let mut e = event(seq, "acp_read", Some("chan-a"));
e.payload = serde_json::json!({
"jsonrpc": "2.0",
"method": "session/update",
"params": {
"sessionId": "session-1",
"update": {
"sessionUpdate": "agent_message_chunk",
"messageId": message_id,
"content": { "type": "text", "text": text },
},
},
});
e
}
let mut queue = ObserverPublishQueue::default();
// 50 × 1KB chunks merge under one messageId in the coalescer…
let merged_text = "m".repeat(1_000);
let merged_sources = 50u64;
for seq in 1..=merged_sources {
queue.ingest(chunk(seq, "message-merged", &merged_text));
}
// …then a non-chunk event force-flushes the merged entry into the
// publish FIFO. From here eviction happens on the FIFO arm.
queue.ingest(event(merged_sources + 1, "tool_call", Some("chan-a")));
assert!(
queue.coalescer.pending.is_empty(),
"the non-chunk event must have flushed the merged entry"
);
assert_eq!(
queue.events.front().expect("flushed entry queued").1,
merged_sources,
"the FIFO front must carry the merged source count"
);
// Distinct-key flood forces byte-budget eviction of the FIFO front.
let flood_text = "f".repeat(50_000);
let flood = 100u64;
for seq in 1..=flood {
queue.ingest(chunk(
merged_sources + 1 + seq,
&format!("message-{seq}"),
&flood_text,
));
}
assert!(
walked_retained_bytes(&queue) <= OBSERVER_PENDING_QUEUE_MAX_BYTES,
"eviction must restore the byte budget (entry-walked), got {}",
walked_retained_bytes(&queue)
);
let frames = drain_frames(&mut queue);
assert!(
!frames
.iter()
.flat_map(frame_seqs)
.any(|seq| seq <= merged_sources),
"the flushed merged entry (globally oldest) must have been evicted"
);
// Ledger in source units: survivors are unmerged (1 source each), the
// evicted merged FIFO entry must charge all 50 sources.
let survived: u64 = frames.iter().map(|f| frame_seqs(f).len() as u64).sum();
let ingested = merged_sources + 1 + flood;
assert_eq!(
survived + queue.dropped_events,
ingested,
"accounting: published sources + dropped sources == ingested"
);
}
/// Under the byte budget the queue is lossless: every ingested event
/// publishes exactly once.
#[test]
fn under_budget_backlogs_are_lossless() {
let mut queue = queue_of(
(1..=200)
.map(|seq| event(seq, "acp_read", Some("chan-a")))
.collect(),
);
let frames = drain_frames(&mut queue);
let published: Vec<u64> = frames.iter().flat_map(frame_seqs).collect();
assert_eq!(published, (1..=200).collect::<Vec<u64>>());
assert_eq!(queue.dropped_events, 0);
}
}
#[cfg(test)]
mod observer_publish_cadence_tests {
use super::*;
use nostr::Keys;
/// Let every spawned task (publisher loop, test_pair forwarder) run to
/// quiescence WITHOUT advancing paused time. `yield_now` keeps this task
/// runnable, so tokio's auto-advance never fires here — time only moves
/// when the test says so.
async fn settle() {
for _ in 0..64 {
tokio::task::yield_now().await;
}
}
fn recv_all(rx: &mut tokio::sync::mpsc::Receiver<nostr::Event>) -> Vec<nostr::Event> {
let mut out = Vec::new();
while let Ok(event) = rx.try_recv() {
out.push(event);
}
out
}
fn count_inner(owner: &Keys, event: &nostr::Event) -> usize {
let payload: serde_json::Value =
decrypt_observer_payload(owner, event).expect("decrypt frame");
match payload["payload"]["events"].as_array() {
Some(inner) => inner.len(),
None => 1,
}
}
fn emit_on(observer: &observer::ObserverHandle, channel: Option<uuid::Uuid>, marker: &str) {
observer.emit(
"test_event",
None,
&observer::context_for(channel, None, None),
serde_json::json!({ "marker": marker }),
);
}
/// THE regression Max demanded: with a backlog needing multiple frames
/// (two channels — a frame never mixes channels, so the backlog takes two
/// publish slots), no frame publishes before its tick. Startup publishes
/// NOTHING at t=0 (Sami's Finding 1: a full replay buffer must not burst
/// on reconnect), frame 1 arrives at +1s, frame 2 no earlier than +2s.
#[tokio::test(start_paused = true)]
async fn one_frame_per_second_and_no_startup_burst() {
let observer = observer::ObserverHandle::in_process();
let agent_keys = Keys::generate();
let owner_keys = Keys::generate();
let (publisher, mut published_rx) = RelayEventPublisher::test_pair();
// Interleave channels so the backlog cannot fit one frame: each run
// boundary forces a new publish slot.
let chan_a = uuid::Uuid::new_v4();
let chan_b = uuid::Uuid::new_v4();
emit_on(&observer, Some(chan_a), "a1");
emit_on(&observer, Some(chan_b), "b1");
emit_on(&observer, Some(chan_a), "a2");
let rx = observer.subscribe();
let snapshot = observer.snapshot();
assert_eq!(snapshot.len(), 3, "all three preloaded in the snapshot");
let task = tokio::spawn(run_relay_observer_publisher(
snapshot,
rx,
publisher,
agent_keys.clone(),
agent_keys.public_key().to_hex(),
owner_keys.public_key().to_hex(),
owner_keys.public_key(),
));
// t=0: nothing may publish, no matter how full the snapshot was.
settle().await;
assert_eq!(
recv_all(&mut published_rx).len(),
0,
"startup must not burst at t=0"
);
// t=0.999s: still nothing.
tokio::time::advance(Duration::from_millis(999)).await;
settle().await;
assert_eq!(
recv_all(&mut published_rx).len(),
0,
"no frame may publish before the first tick"
);
// t=1s: exactly ONE frame — chan-a gathered queue-wide, so a1 AND a2
// ride the first slot together.
tokio::time::advance(Duration::from_millis(1)).await;
settle().await;
let frames = recv_all(&mut published_rx);
assert_eq!(frames.len(), 1, "tick 1 publishes exactly one frame");
assert_eq!(count_inner(&owner_keys, &frames[0]), 2, "a1 + a2 gathered");
// t=1.5s: between ticks, nothing.
tokio::time::advance(Duration::from_millis(500)).await;
settle().await;
assert_eq!(
recv_all(&mut published_rx).len(),
0,
"frame 2 must wait for tick 2"
);
// t=2s: the chan-b frame drains on its own tick.
tokio::time::advance(Duration::from_millis(500)).await;
settle().await;
assert_eq!(recv_all(&mut published_rx).len(), 1, "tick 2: one frame");
// Backlog drained; a quiet tick publishes nothing.
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
assert_eq!(recv_all(&mut published_rx).len(), 0, "quiet tick is quiet");
task.abort();
}
/// Shutdown is NOT a burst bypass: when the producer closes with a
/// backlog, the remaining frames still publish one per tick, and the loop
/// exits only after the queue is empty — paced, lossless, in order.
#[tokio::test(start_paused = true)]
async fn shutdown_drain_is_paced_and_lossless() {
let observer = observer::ObserverHandle::in_process();
let agent_keys = Keys::generate();
let owner_keys = Keys::generate();
let (publisher, mut published_rx) = RelayEventPublisher::test_pair();
let chan_a = uuid::Uuid::new_v4();
let chan_b = uuid::Uuid::new_v4();
emit_on(&observer, Some(chan_a), "a1");
emit_on(&observer, Some(chan_b), "b1");
emit_on(&observer, Some(chan_a), "a2");
let rx = observer.subscribe();
let snapshot = observer.snapshot();
// Close the broadcast channel immediately: the entire drain happens
// in "shutdown" mode.
drop(observer);
let task = tokio::spawn(run_relay_observer_publisher(
snapshot,
rx,
publisher,
agent_keys.clone(),
agent_keys.public_key().to_hex(),
owner_keys.public_key().to_hex(),
owner_keys.public_key(),
));
settle().await;
assert_eq!(
recv_all(&mut published_rx).len(),
0,
"shutdown drain must not burst at t=0"
);
let mut markers = Vec::new();
for tick in 1..=2 {
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
let frames = recv_all(&mut published_rx);
assert_eq!(frames.len(), 1, "shutdown tick {tick}: exactly one frame");
let payload: serde_json::Value =
decrypt_observer_payload(&owner_keys, &frames[0]).expect("decrypt");
match payload["payload"]["events"].as_array() {
Some(inner) => markers.extend(
inner
.iter()
.map(|e| e["payload"]["marker"].as_str().unwrap().to_string()),
),
None => markers.push(payload["payload"]["marker"].as_str().unwrap().to_string()),
}
}
// Gather-packing: chan-a (a1+a2) ships tick 1, chan-b tick 2.
assert_eq!(markers, ["a1", "a2", "b1"], "paced drain loses nothing");
// Queue empty + closed: the loop must have exited on its own.
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
assert!(task.is_finished(), "publisher exits after paced drain");
}
/// Pins `MissedTickBehavior::Skip` (Sami's M6 mutant): when the publisher
/// misses ticks — relay backpressure can stall the tick arm past several
/// deadlines, since `publish_event` awaits a bounded mpsc — the interval
/// must fire ONE catch-up tick and realign, not fire once per missed
/// deadline. With `Burst`, a 10s stall against a multi-frame backlog
/// would replay all 10 missed ticks back-to-back: an unpaced burst that
/// bypasses exactly what the pacer exists to prevent.
#[tokio::test(start_paused = true)]
async fn missed_ticks_skip_instead_of_bursting() {
let observer = observer::ObserverHandle::in_process();
let agent_keys = Keys::generate();
let owner_keys = Keys::generate();
let (publisher, mut published_rx) = RelayEventPublisher::test_pair();
// Three channels => three frames pending (a frame never mixes
// channels), so a bursting interval would have work for every
// spurious catch-up tick.
for chan in 0..3 {
emit_on(&observer, Some(uuid::Uuid::new_v4()), &format!("c{chan}"));
}
let rx = observer.subscribe();
let snapshot = observer.snapshot();
let task = tokio::spawn(run_relay_observer_publisher(
snapshot,
rx,
publisher,
agent_keys.clone(),
agent_keys.public_key().to_hex(),
owner_keys.public_key().to_hex(),
owner_keys.public_key(),
));
settle().await;
// Jump 10 seconds in ONE advance — the loop was never polled in
// between, exactly like a stall across 10 deadlines.
tokio::time::advance(Duration::from_secs(10)).await;
settle().await;
assert_eq!(
recv_all(&mut published_rx).len(),
1,
"Skip: one catch-up frame after a stall — Burst would publish \
one per missed deadline"
);
// The interval realigned: the remaining backlog stays paced.
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
assert_eq!(recv_all(&mut published_rx).len(), 1, "paced after realign");
task.abort();
}
}
#[cfg(test)]
mod observer_chunk_coalescer_tests {
use super::*;
fn chunk_event(
seq: u64,
update_type: &str,
message_id: &str,
text: &str,
) -> observer::ObserverEvent {
observer::ObserverEvent {
seq,
timestamp: format!("2026-04-29T04:00:0{seq}Z"),
kind: "acp_read".to_string(),
agent_index: Some(0),
channel_id: Some("channel-1".to_string()),
session_id: Some("session-1".to_string()),
turn_id: Some("turn-1".to_string()),
started_at: None,
payload: serde_json::json!({
"jsonrpc": "2.0",
"method": "session/update",
"params": {
"sessionId": "session-1",
"update": {
"sessionUpdate": update_type,
"messageId": message_id,
"content": {
"type": "text",
"text": text,
},
},
},
}),
}
}
fn non_chunk_event(seq: u64) -> observer::ObserverEvent {
observer::ObserverEvent {
seq,
timestamp: format!("2026-04-29T04:00:0{seq}Z"),
kind: "turn_started".to_string(),
agent_index: Some(0),
channel_id: Some("channel-1".to_string()),
session_id: Some("session-1".to_string()),
turn_id: Some("turn-1".to_string()),
started_at: None,
payload: serde_json::json!({ "type": "turn_started" }),
}
}
fn chunk_text(event: &observer::ObserverEvent) -> &str {
event.payload["params"]["update"]["content"]["text"]
.as_str()
.expect("chunk text")
}
#[test]
fn coalesces_chunks_until_non_chunk_event() {
let mut coalescer = ObserverChunkCoalescer::default();
assert!(coalescer
.ingest(chunk_event(1, "agent_message_chunk", "message-1", "hello "))
.is_empty());
assert!(coalescer
.ingest(chunk_event(2, "agent_message_chunk", "message-1", "world"))
.is_empty());
let events = coalescer.ingest(non_chunk_event(3));
assert_eq!(events.len(), 2);
assert_eq!(events[0].1.seq, 2);
assert_eq!(chunk_text(&events[0].1), "hello world");
assert_eq!(
events[0].0, 2,
"a merged entry reports every source chunk it absorbed"
);
assert_eq!(events[1].1.kind, "turn_started");
assert_eq!(events[1].0, 1);
}
#[test]
fn keeps_independent_chunk_streams_separate() {
let mut coalescer = ObserverChunkCoalescer::default();
assert!(coalescer
.ingest(chunk_event(1, "agent_message_chunk", "message-1", "answer"))
.is_empty());
assert!(coalescer
.ingest(chunk_event(
2,
"agent_thought_chunk",
"thought-1",
"thinking"
))
.is_empty());
let events = coalescer.flush();
assert_eq!(events.len(), 2);
assert_eq!(chunk_text(&events[0].1), "answer");
assert_eq!(chunk_text(&events[1].1), "thinking");
}
}
#[cfg(test)]
mod build_mcp_servers_tests {
use super::*;
use std::sync::Mutex;
/// Env-var-touching tests must run serially — env vars are process-global.
static ENV_LOCK: Mutex<()> = Mutex::new(());
fn test_config() -> Config {
Config {
keys: nostr::Keys::generate(),
relay_url: "ws://localhost:3000".into(),
agent_command: "goose".into(),
agent_args: vec!["acp".into()],
mcp_command: "test-mcp-server".into(),
idle_timeout_secs: config::DEFAULT_IDLE_TIMEOUT_SECS,
max_turn_duration_secs: config::DEFAULT_MAX_TURN_DURATION_SECS,
agents: 1,
heartbeat_interval_secs: 0,
turn_liveness_secs: 10,
heartbeat_prompt: None,
system_prompt: None,
team_instructions: None,
initial_message: None,
subscribe_mode: config::SubscribeMode::All,
dedup_mode: config::DedupMode::Queue,
multiple_event_handling: config::MultipleEventHandling::Queue,
ignore_self: true,
kinds_override: None,
channels_override: None,
no_mention_filter: false,
config_path: std::path::PathBuf::from("./buzz-acp.toml"),
context_message_limit: 12,
max_turns_per_session: 0,
presence_enabled: true,
typing_enabled: true,
memory_enabled: false,
model: None,
session_title: None,
permission_mode: config::PermissionMode::BypassPermissions,
respond_to: config::RespondTo::Anyone,
respond_to_allowlist: std::collections::HashSet::new(),
allowed_respond_to: vec![],
persona_env_vars: vec![],
has_generated_codex_config: false,
relay_observer: false,
exit_after_inactivity_secs: 0,
lazy_pool: false,
agent_owner: None,
no_base_prompt: false,
base_prompt_content: None,
}
}
#[test]
fn session_new_mcp_server_has_required_fields() {
let config = test_config();
let servers = build_mcp_servers(&config);
assert_eq!(servers.len(), 1);
let server = &servers[0];
assert_eq!(server.name, "test-mcp-server");
let names: Vec<&str> = server.env.iter().map(|e| e.name.as_str()).collect();
assert!(
names.contains(&"BUZZ_RELAY_URL"),
"missing BUZZ_RELAY_URL; got {names:?}"
);
assert!(
names.contains(&"BUZZ_PRIVATE_KEY"),
"missing BUZZ_PRIVATE_KEY; got {names:?}"
);
}
#[test]
fn session_new_mcp_server_forwards_buzz_auth_tag() {
let _guard = ENV_LOCK.lock().unwrap();
std::env::set_var("BUZZ_AUTH_TAG", "test-attestation-tag");
let config = test_config();
let servers = build_mcp_servers(&config);
std::env::remove_var("BUZZ_AUTH_TAG");
let server = &servers[0];
let auth_tag_env = server.env.iter().find(|e| e.name == "BUZZ_AUTH_TAG");
assert!(
auth_tag_env.is_some(),
"BUZZ_AUTH_TAG should be forwarded when set"
);
assert_eq!(auth_tag_env.unwrap().value, "test-attestation-tag");
}
#[test]
fn session_new_mcp_server_skips_empty_buzz_auth_tag() {
let _guard = ENV_LOCK.lock().unwrap();
std::env::set_var("BUZZ_AUTH_TAG", "");
let config = test_config();
let servers = build_mcp_servers(&config);
std::env::remove_var("BUZZ_AUTH_TAG");
let server = &servers[0];
let has_auth_tag = server.env.iter().any(|e| e.name == "BUZZ_AUTH_TAG");
assert!(!has_auth_tag, "empty BUZZ_AUTH_TAG should not be forwarded");
}
#[test]
fn test_display_name_set_is_forwarded_to_mcp_server() {
let _guard = ENV_LOCK.lock().unwrap();
std::env::set_var("BUZZ_ACP_DISPLAY_NAME", "Duncan");
let config = test_config();
let servers = build_mcp_servers(&config);
std::env::remove_var("BUZZ_ACP_DISPLAY_NAME");
let entry = servers[0]
.env
.iter()
.find(|e| e.name == "BUZZ_ACP_DISPLAY_NAME");
assert_eq!(
entry.map(|e| e.value.as_str()),
Some("Duncan"),
"a set display name should reach the MCP server verbatim"
);
}
#[test]
fn test_display_name_unset_omits_the_key_entirely() {
let _guard = ENV_LOCK.lock().unwrap();
std::env::remove_var("BUZZ_ACP_DISPLAY_NAME");
let config = test_config();
let servers = build_mcp_servers(&config);
// Absent, not empty-valued: dev-mcp distinguishes the two and only
// falls back to the npub when the key is missing or blank.
assert!(
!servers[0]
.env
.iter()
.any(|e| e.name == "BUZZ_ACP_DISPLAY_NAME"),
"unset display name should not add the key"
);
}
#[test]
fn test_display_name_empty_omits_the_key_entirely() {
let _guard = ENV_LOCK.lock().unwrap();
std::env::set_var("BUZZ_ACP_DISPLAY_NAME", "");
let config = test_config();
let servers = build_mcp_servers(&config);
std::env::remove_var("BUZZ_ACP_DISPLAY_NAME");
assert!(
!servers[0]
.env
.iter()
.any(|e| e.name == "BUZZ_ACP_DISPLAY_NAME"),
"empty display name should not be forwarded"
);
}
#[test]
fn empty_mcp_command_returns_no_servers() {
let mut config = test_config();
config.mcp_command = "".into();
let servers = build_mcp_servers(&config);
assert!(
servers.is_empty(),
"empty mcp_command should produce no MCP servers"
);
}
#[test]
fn absolute_path_mcp_command_uses_file_stem_as_name() {
let mut config = test_config();
config.mcp_command = "/opt/bin/my-mcp-server".into();
let servers = build_mcp_servers(&config);
assert_eq!(servers.len(), 1);
assert_eq!(servers[0].name, "my-mcp-server");
}
#[test]
fn mcp_command_with_no_stem_falls_back_to_mcp() {
// Path::new("").file_stem() returns None — exercises the unwrap_or("mcp") path.
let mut config = test_config();
config.mcp_command = "".into();
// Empty command returns no servers; test the stem logic directly.
assert_eq!(
std::path::Path::new("")
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("mcp"),
"mcp"
);
// Confirm a non-empty command with no stem (e.g. just a dot) also falls back.
config.mcp_command = ".".into();
let servers = build_mcp_servers(&config);
assert_eq!(servers.len(), 1);
assert_eq!(
servers[0].name, "mcp",
"Path::new(\".\").file_stem() is None — should fall back to \"mcp\""
);
}
}
#[cfg(test)]
mod error_outcome_emission_tests {
//! Pins the policy that error-class outcomes surface to the activity feed
//! and never to the channel:
//!
//! - Channel silence is enforced *structurally* — `handle_prompt_result`
//! takes no relay handle, so it has no way to post a channel message. A
//! future re-introduction of channel notices would have to add the relay
//! parameter back, which these tests' construction would then refuse to
//! compile against.
//! - Feed coverage is the regression-prone half and is asserted at runtime:
//! each error outcome must emit exactly one `turn_error` observer event.
//! If any branch drops its `emit_turn_error` call, the matching test goes
//! red.
use super::*;
use crate::acp::{AcpClient, AcpError};
use crate::observer::ObserverHandle;
use crate::pool::{
AgentPool, OwnedAgent, PromptOutcome, PromptResult, PromptSource, TimeoutKind,
};
use crate::queue::{BatchEvent, FlushBatch};
use nostr::{EventBuilder, Keys, Kind};
use std::collections::HashSet;
fn test_config() -> Config {
Config {
keys: nostr::Keys::generate(),
relay_url: "ws://localhost:3000".into(),
// `true` exits cleanly, so the async respawn fails fast and
// harmlessly off the JoinSet — irrelevant to the synchronous
// feed emission under test.
agent_command: "true".into(),
agent_args: vec![],
mcp_command: "test-mcp-server".into(),
idle_timeout_secs: config::DEFAULT_IDLE_TIMEOUT_SECS,
max_turn_duration_secs: config::DEFAULT_MAX_TURN_DURATION_SECS,
agents: 1,
heartbeat_interval_secs: 0,
turn_liveness_secs: 10,
heartbeat_prompt: None,
system_prompt: None,
team_instructions: None,
initial_message: None,
subscribe_mode: config::SubscribeMode::All,
dedup_mode: config::DedupMode::Queue,
multiple_event_handling: config::MultipleEventHandling::Queue,
ignore_self: true,
kinds_override: None,
channels_override: None,
no_mention_filter: false,
config_path: std::path::PathBuf::from("./buzz-acp.toml"),
context_message_limit: 12,
max_turns_per_session: 0,
presence_enabled: true,
typing_enabled: true,
memory_enabled: false,
model: None,
session_title: None,
permission_mode: config::PermissionMode::BypassPermissions,
respond_to: config::RespondTo::Anyone,
respond_to_allowlist: HashSet::new(),
allowed_respond_to: vec![],
persona_env_vars: vec![],
has_generated_codex_config: false,
relay_observer: false,
exit_after_inactivity_secs: 0,
lazy_pool: false,
agent_owner: None,
no_base_prompt: false,
base_prompt_content: None,
}
}
#[test]
fn normalizes_agent_name_from_initialize_result() {
assert_eq!(
normalized_agent_name(&serde_json::json!({
"agentInfo": { "name": " Goose ", "version": "1.43.0" }
})),
"goose"
);
assert_eq!(
normalized_agent_name(&serde_json::json!({
"serverInfo": { "name": "buzz-agent" }
})),
"buzz-agent"
);
}
/// Spawn a real but inert agent subprocess (`cat`) so the error paths have
/// an `OwnedAgent` to move into respawn or return to the pool. The error
/// branches never talk to the subprocess.
async fn dummy_agent(index: usize) -> OwnedAgent {
OwnedAgent {
index,
acp: AcpClient::spawn("cat", &[], &[], false)
.await
.expect("spawn cat as inert agent"),
state: Default::default(),
model_capabilities: None,
desired_model: None,
model_overridden: false,
agent_name: "unknown".into(),
goose_system_prompt_supported: None,
// Error branches under test never read this; 1 is the legacy
// non-systemPrompt path, the simplest valid value.
protocol_version: 1,
}
}
#[tokio::test]
async fn successful_native_steer_is_transferred_to_live_session_delivery_state() {
let channel_id = Uuid::new_v4();
let steer_event_id = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
let mut agent = dummy_agent(0).await;
agent
.state
.sessions
.insert(channel_id, "live-session".into());
agent
.state
.deliveries
.insert(channel_id, Default::default());
let mut pool = AgentPool::from_slots(vec![None]);
let task_id = pool.join_set.spawn(async {}).id();
pool.task_map_mut().insert(
task_id,
crate::pool::TaskMeta {
agent_index: 0,
channel_id: Some(channel_id),
turn_id: "test-turn-id".into(),
recoverable_batch: None,
control_tx: None,
steer_tx: None,
successful_steer_deliveries: HashSet::from([
crate::pool::SuccessfulSteerDelivery {
event_id: steer_event_id.into(),
session_id: "live-session".into(),
},
]),
},
);
let mut queue = EventQueue::new(config::DedupMode::Queue);
let config = test_config();
let mut heartbeat_in_flight = false;
let removed_channels = HashSet::new();
let mut crash_history = vec![SlotCircuit {
crash_times: Vec::new(),
open_until: None,
respawn_in_flight: false,
}];
let (respawn_tx, _respawn_rx) = mpsc::channel(8);
let mut respawn_tasks = tokio::task::JoinSet::new();
let result = PromptResult {
agent,
source: PromptSource::Channel(channel_id),
turn_id: "test-turn-id".into(),
outcome: PromptOutcome::Ok(crate::acp::StopReason::EndTurn),
batch: None,
};
handle_prompt_result(
&mut pool,
&mut queue,
&config,
result,
&mut heartbeat_in_flight,
&removed_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
None,
None,
);
let returned = pool.agents_mut()[0].as_ref().expect("returned agent");
assert!(returned.state.deliveries[&channel_id]
.delivered_event_ids
.contains(steer_event_id));
}
#[tokio::test]
async fn in_flight_stale_native_steer_ack_cannot_update_replacement_session() {
let channel_id = Uuid::new_v4();
let mut agent = dummy_agent(0).await;
agent
.state
.sessions
.insert(channel_id, "replacement-session".into());
agent
.state
.deliveries
.insert(channel_id, Default::default());
let mut pool = AgentPool::from_slots(vec![None]);
let task_id = pool.join_set.spawn(async {}).id();
pool.task_map_mut().insert(
task_id,
crate::pool::TaskMeta {
agent_index: 0,
channel_id: Some(channel_id),
turn_id: "test-turn-id".into(),
recoverable_batch: None,
control_tx: None,
steer_tx: None,
successful_steer_deliveries: HashSet::from([
crate::pool::SuccessfulSteerDelivery {
event_id: "stale-event".into(),
session_id: "old-session".into(),
},
]),
},
);
let mut queue = EventQueue::new(config::DedupMode::Queue);
let config = test_config();
let mut heartbeat_in_flight = false;
let removed_channels = HashSet::new();
let mut crash_history = vec![SlotCircuit {
crash_times: Vec::new(),
open_until: None,
respawn_in_flight: false,
}];
let (respawn_tx, _respawn_rx) = mpsc::channel(8);
let mut respawn_tasks = tokio::task::JoinSet::new();
let result = PromptResult {
agent,
source: PromptSource::Channel(channel_id),
turn_id: "test-turn-id".into(),
outcome: PromptOutcome::Ok(crate::acp::StopReason::EndTurn),
batch: None,
};
handle_prompt_result(
&mut pool,
&mut queue,
&config,
result,
&mut heartbeat_in_flight,
&removed_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
None,
None,
);
let returned = pool.agents_mut()[0].as_ref().expect("returned agent");
assert!(returned.state.deliveries[&channel_id]
.delivered_event_ids
.is_empty());
}
#[tokio::test]
async fn successful_native_steer_ack_after_task_return_updates_matching_live_session() {
let channel_id = Uuid::new_v4();
let steer_event_id = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
let mut agent = dummy_agent(0).await;
agent
.state
.sessions
.insert(channel_id, "live-session".into());
agent
.state
.deliveries
.insert(channel_id, Default::default());
let mut pool = AgentPool::from_slots(vec![Some(agent)]);
assert!(pool.record_successful_steer(
channel_id,
steer_event_id.into(),
"live-session".into(),
));
let returned = pool.agents_mut()[0].as_ref().expect("idle returned agent");
assert!(returned.state.deliveries[&channel_id]
.delivered_event_ids
.contains(steer_event_id));
}
#[tokio::test]
async fn late_native_steer_ack_cannot_update_replacement_session() {
let channel_id = Uuid::new_v4();
let mut agent = dummy_agent(0).await;
agent
.state
.sessions
.insert(channel_id, "replacement-session".into());
agent
.state
.deliveries
.insert(channel_id, Default::default());
let mut pool = AgentPool::from_slots(vec![Some(agent)]);
assert!(!pool.record_successful_steer(
channel_id,
"stale-event".into(),
"old-session".into(),
));
let returned = pool.agents_mut()[0].as_ref().expect("replacement agent");
assert!(returned.state.deliveries[&channel_id]
.delivered_event_ids
.is_empty());
}
#[tokio::test]
async fn invalidated_session_does_not_resurrect_successful_steer_delivery_state() {
let channel_id = Uuid::new_v4();
let agent = dummy_agent(0).await;
// No live session: simulates the prompt task invalidating before return.
let mut pool = AgentPool::from_slots(vec![None]);
let task_id = pool.join_set.spawn(async {}).id();
pool.task_map_mut().insert(
task_id,
crate::pool::TaskMeta {
agent_index: 0,
channel_id: Some(channel_id),
turn_id: "test-turn-id".into(),
recoverable_batch: None,
control_tx: None,
steer_tx: None,
successful_steer_deliveries: HashSet::from([
crate::pool::SuccessfulSteerDelivery {
event_id: "stale-event".into(),
session_id: "invalidated-session".into(),
},
]),
},
);
let mut queue = EventQueue::new(config::DedupMode::Queue);
let config = test_config();
let mut heartbeat_in_flight = false;
let removed_channels = HashSet::new();
let mut crash_history = vec![SlotCircuit {
crash_times: Vec::new(),
open_until: None,
respawn_in_flight: false,
}];
let (respawn_tx, _respawn_rx) = mpsc::channel(8);
let mut respawn_tasks = tokio::task::JoinSet::new();
let result = PromptResult {
agent,
source: PromptSource::Channel(channel_id),
turn_id: "test-turn-id".into(),
outcome: PromptOutcome::Ok(crate::acp::StopReason::EndTurn),
batch: None,
};
handle_prompt_result(
&mut pool,
&mut queue,
&config,
result,
&mut heartbeat_in_flight,
&removed_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
None,
None,
);
let returned = pool.agents_mut()[0].as_ref().expect("returned agent");
assert!(!returned.state.deliveries.contains_key(&channel_id));
}
/// Drive one error outcome through `handle_prompt_result` and return how
/// many `turn_error` events it emitted to the observer feed.
async fn turn_errors_emitted_for(outcome: PromptOutcome) -> usize {
let agent = dummy_agent(0).await;
let mut pool = AgentPool::from_slots(vec![None]);
// `handle_prompt_result` asserts it removes exactly one in-flight task
// for the completing agent (the slot was checked out, not idle). Mirror
// the real dispatch path by registering a TaskMeta keyed on a genuine
// `task::Id` — only obtainable from inside a spawned task.
let task_id = pool.join_set.spawn(async {}).id();
pool.task_map_mut().insert(
task_id,
crate::pool::TaskMeta {
agent_index: 0,
channel_id: None,
turn_id: "test-turn-id".to_string(),
recoverable_batch: None,
control_tx: None,
steer_tx: None,
successful_steer_deliveries: HashSet::new(),
},
);
let mut queue = EventQueue::new(config::DedupMode::Queue);
let config = test_config();
let mut heartbeat_in_flight = false;
let removed_channels = HashSet::new();
let mut crash_history = vec![SlotCircuit {
crash_times: Vec::new(),
open_until: None,
respawn_in_flight: false,
}];
let (respawn_tx, _respawn_rx) = mpsc::channel(8);
let mut respawn_tasks = tokio::task::JoinSet::new();
let observer = ObserverHandle::in_process();
let result = PromptResult {
agent,
source: PromptSource::Channel(Uuid::new_v4()),
turn_id: "test-turn-id".to_string(),
outcome,
batch: None,
};
handle_prompt_result(
&mut pool,
&mut queue,
&config,
result,
&mut heartbeat_in_flight,
&removed_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
Some(observer.clone()),
None,
);
let turn_errors: Vec<_> = observer
.snapshot()
.into_iter()
.filter(|e| e.kind == "turn_error")
.collect();
assert!(
turn_errors
.iter()
.all(|event| event.turn_id.as_deref() == Some("test-turn-id")),
"turn_error must retain the completed turn id"
);
turn_errors.len()
}
#[tokio::test]
async fn agent_exited_emits_exactly_one_feed_event() {
assert_eq!(turn_errors_emitted_for(PromptOutcome::AgentExited).await, 1);
}
#[tokio::test]
async fn panic_event_retains_task_turn_id() {
let mut pool = AgentPool::from_slots(vec![]);
let channel_id = Uuid::new_v4();
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let abort_handle = pool.join_set.spawn(async move {
let _ = started_tx.send(());
std::future::pending::<()>().await;
});
let task_id = abort_handle.id();
pool.task_map_mut().insert(
task_id,
crate::pool::TaskMeta {
agent_index: 0,
channel_id: Some(channel_id),
turn_id: "panic-turn-id".to_string(),
recoverable_batch: None,
control_tx: None,
steer_tx: None,
successful_steer_deliveries: HashSet::new(),
},
);
started_rx.await.unwrap();
abort_handle.abort();
let join_error = pool.join_set.join_next().await.unwrap().unwrap_err();
let mut queue = EventQueue::new(config::DedupMode::Queue);
let config = test_config();
let mut heartbeat_in_flight = false;
let removed_channels = HashSet::new();
let mut typing_channels = HashMap::new();
let mut crash_history = vec![SlotCircuit {
crash_times: Vec::new(),
open_until: None,
respawn_in_flight: false,
}];
let (respawn_tx, _respawn_rx) = mpsc::channel(8);
let mut respawn_tasks = tokio::task::JoinSet::new();
let observer = ObserverHandle::in_process();
recover_panicked_agent(
&mut pool,
&mut queue,
&config,
join_error,
&mut heartbeat_in_flight,
&removed_channels,
&mut typing_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
Some(observer.clone()),
);
let panic = observer
.snapshot()
.into_iter()
.find(|event| event.kind == "agent_panic")
.expect("panic recovery emits an observer event");
assert_eq!(
panic.channel_id.as_deref(),
Some(channel_id.to_string().as_str())
);
assert_eq!(panic.turn_id.as_deref(), Some("panic-turn-id"));
}
#[tokio::test]
async fn idle_timeout_emits_exactly_one_feed_event() {
assert_eq!(
turn_errors_emitted_for(PromptOutcome::Timeout(TimeoutKind::Idle)).await,
1
);
}
#[tokio::test]
async fn hard_timeout_emits_exactly_one_feed_event() {
assert_eq!(
turn_errors_emitted_for(PromptOutcome::Timeout(TimeoutKind::Hard {
recently_active: false
}))
.await,
1
);
}
#[tokio::test]
async fn cancel_drain_timeout_emits_exactly_one_feed_event() {
assert_eq!(
turn_errors_emitted_for(PromptOutcome::CancelDrainTimeout(
std::time::Duration::from_secs(5)
))
.await,
1
);
}
/// idle_timeout outcome_label is "idle_timeout"; hard_timeout is "hard_timeout".
#[tokio::test]
async fn timeout_outcome_labels_differ() {
let check_label = |outcome: PromptOutcome, expected_label: &'static str| async move {
let agent = dummy_agent(0).await;
let mut pool = AgentPool::from_slots(vec![None]);
let task_id = pool.join_set.spawn(async {}).id();
pool.task_map_mut().insert(
task_id,
crate::pool::TaskMeta {
agent_index: 0,
channel_id: None,
turn_id: "test-turn-id".to_string(),
recoverable_batch: None,
control_tx: None,
steer_tx: None,
successful_steer_deliveries: HashSet::new(),
},
);
let mut queue = EventQueue::new(config::DedupMode::Queue);
let config = test_config();
let mut heartbeat_in_flight = false;
let removed_channels = HashSet::new();
let mut crash_history = vec![SlotCircuit {
crash_times: Vec::new(),
open_until: None,
respawn_in_flight: false,
}];
let (respawn_tx, _respawn_rx) = mpsc::channel(8);
let mut respawn_tasks = tokio::task::JoinSet::new();
let observer = ObserverHandle::in_process();
let result = PromptResult {
agent,
source: PromptSource::Channel(Uuid::new_v4()),
turn_id: "test-turn-id".to_string(),
outcome,
batch: None,
};
handle_prompt_result(
&mut pool,
&mut queue,
&config,
result,
&mut heartbeat_in_flight,
&removed_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
Some(observer.clone()),
None,
);
let events = observer.snapshot();
let turn_error = events.iter().find(|e| e.kind == "turn_error").unwrap();
assert_eq!(
turn_error.payload["outcome"].as_str().unwrap(),
expected_label
);
};
check_label(PromptOutcome::Timeout(TimeoutKind::Idle), "idle_timeout").await;
check_label(
PromptOutcome::Timeout(TimeoutKind::Hard {
recently_active: false,
}),
"hard_timeout",
)
.await;
check_label(
PromptOutcome::CancelDrainTimeout(std::time::Duration::from_secs(5)),
"cancel_drain_timeout",
)
.await;
}
/// hard-cap timeout dead-letters immediately (no requeue); idle timeout is requeued.
#[tokio::test]
async fn hard_timeout_not_requeued_idle_timeout_is_requeued() {
let make_batch = || {
let keys = Keys::generate();
let event = EventBuilder::new(Kind::Custom(9), "test")
.sign_with_keys(&keys)
.unwrap();
FlushBatch {
channel_id: Uuid::new_v4(),
events: vec![BatchEvent {
event,
prompt_tag: "test".into(),
received_at: std::time::Instant::now(),
}],
cancelled_events: vec![],
cancel_reason: None,
}
};
// Returns (pending_channels, queued_event_count_for_channel).
let run = |outcome: PromptOutcome, batch: FlushBatch| async move {
let channel_id = batch.channel_id;
let agent = dummy_agent(0).await;
let mut pool = AgentPool::from_slots(vec![None]);
let task_id = pool.join_set.spawn(async {}).id();
pool.task_map_mut().insert(
task_id,
crate::pool::TaskMeta {
agent_index: 0,
channel_id: None,
turn_id: "test-turn-id".to_string(),
recoverable_batch: None,
control_tx: None,
steer_tx: None,
successful_steer_deliveries: HashSet::new(),
},
);
let mut queue = EventQueue::new(config::DedupMode::Queue);
let config = test_config();
let mut heartbeat_in_flight = false;
let removed_channels = HashSet::new();
let mut crash_history = vec![SlotCircuit {
crash_times: Vec::new(),
open_until: None,
respawn_in_flight: false,
}];
let (respawn_tx, _respawn_rx) = mpsc::channel(8);
let mut respawn_tasks = tokio::task::JoinSet::new();
let result = PromptResult {
agent,
source: PromptSource::Channel(channel_id),
turn_id: "test-turn-id".to_string(),
outcome,
batch: Some(batch),
};
handle_prompt_result(
&mut pool,
&mut queue,
&config,
result,
&mut heartbeat_in_flight,
&removed_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
None,
None,
);
(
queue.pending_channels(),
queue.queued_event_count(&channel_id),
)
};
// Hard timeout (not recently active): dead-lettered immediately.
let hard_batch = make_batch();
let (hard_channels, hard_events) = run(
PromptOutcome::Timeout(TimeoutKind::Hard {
recently_active: false,
}),
hard_batch,
)
.await;
assert_eq!(
hard_channels, 0,
"hard-cap timeout (not recently active) must not requeue the batch"
);
assert_eq!(
hard_events, 0,
"hard-cap timeout (not recently active) must drop all events"
);
// Idle timeout: batch IS requeued (first attempt, not yet dead-lettered).
let idle_batch = make_batch();
let (idle_channels, idle_events) =
run(PromptOutcome::Timeout(TimeoutKind::Idle), idle_batch).await;
assert_eq!(
idle_channels, 1,
"idle timeout must requeue the batch for retry"
);
assert_eq!(
idle_events, 1,
"idle timeout must preserve the event for retry"
);
}
#[tokio::test]
async fn hard_timeout_recently_active_requeues_batch() {
let channel_id = Uuid::new_v4();
let make_batch = || {
let keys = Keys::generate();
let event = EventBuilder::new(Kind::Custom(9), "test")
.sign_with_keys(&keys)
.unwrap();
FlushBatch {
channel_id,
events: vec![BatchEvent {
event,
prompt_tag: "test".into(),
received_at: std::time::Instant::now(),
}],
cancelled_events: vec![],
cancel_reason: None,
}
};
let run = |outcome: PromptOutcome, batch: FlushBatch| async move {
let channel_id = batch.channel_id;
let agent = dummy_agent(0).await;
let mut pool = AgentPool::from_slots(vec![None]);
let task_id = pool.join_set.spawn(async {}).id();
pool.task_map_mut().insert(
task_id,
crate::pool::TaskMeta {
agent_index: 0,
channel_id: None,
turn_id: "test-turn-id".to_string(),
recoverable_batch: None,
control_tx: None,
steer_tx: None,
successful_steer_deliveries: HashSet::new(),
},
);
let mut queue = EventQueue::new(config::DedupMode::Queue);
let config = test_config();
let mut heartbeat_in_flight = false;
let removed_channels = HashSet::new();
let mut crash_history = vec![SlotCircuit {
crash_times: Vec::new(),
open_until: None,
respawn_in_flight: false,
}];
let (respawn_tx, _respawn_rx) = mpsc::channel(8);
let mut respawn_tasks = tokio::task::JoinSet::new();
let result = PromptResult {
agent,
source: PromptSource::Channel(channel_id),
turn_id: "test-turn-id".to_string(),
outcome,
batch: Some(batch),
};
handle_prompt_result(
&mut pool,
&mut queue,
&config,
result,
&mut heartbeat_in_flight,
&removed_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
None,
None,
);
(
queue.pending_channels(),
queue.queued_event_count(&channel_id),
)
};
let batch = make_batch();
let (channels, events) = run(
PromptOutcome::Timeout(TimeoutKind::Hard {
recently_active: true,
}),
batch,
)
.await;
assert_eq!(
channels, 1,
"hard-cap timeout with recent activity must requeue the batch"
);
assert_eq!(
events, 1,
"hard-cap timeout with recent activity must preserve the event"
);
}
/// The hard-timeout `death_message` must report what actually happened to
/// the batch, not just the `recently_active` eligibility flag: a
/// recently-active batch within its retry budget is requeued, so the
/// observer payload must say so.
#[tokio::test]
async fn hard_timeout_recently_active_requeue_success_reports_requeued_for_retry() {
let channel_id = Uuid::new_v4();
let agent = dummy_agent(0).await;
let mut pool = AgentPool::from_slots(vec![None]);
let task_id = pool.join_set.spawn(async {}).id();
pool.task_map_mut().insert(
task_id,
crate::pool::TaskMeta {
agent_index: 0,
channel_id: None,
turn_id: "test-turn-id".to_string(),
recoverable_batch: None,
control_tx: None,
steer_tx: None,
successful_steer_deliveries: HashSet::new(),
},
);
let mut queue = EventQueue::new(config::DedupMode::Queue);
let config = test_config();
let mut heartbeat_in_flight = false;
let removed_channels = HashSet::new();
let mut crash_history = vec![SlotCircuit {
crash_times: Vec::new(),
open_until: None,
respawn_in_flight: false,
}];
let (respawn_tx, _respawn_rx) = mpsc::channel(8);
let mut respawn_tasks = tokio::task::JoinSet::new();
let observer = ObserverHandle::in_process();
let batch = FlushBatch {
channel_id,
events: vec![BatchEvent {
event: EventBuilder::new(Kind::Custom(9), "test")
.sign_with_keys(&Keys::generate())
.unwrap(),
prompt_tag: "test".into(),
received_at: std::time::Instant::now(),
}],
cancelled_events: vec![],
cancel_reason: None,
};
let result = PromptResult {
agent,
source: PromptSource::Channel(channel_id),
turn_id: "test-turn-id".to_string(),
outcome: PromptOutcome::Timeout(TimeoutKind::Hard {
recently_active: true,
}),
batch: Some(batch),
};
handle_prompt_result(
&mut pool,
&mut queue,
&config,
result,
&mut heartbeat_in_flight,
&removed_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
Some(observer.clone()),
None,
);
let events = observer.snapshot();
let turn_error = events
.iter()
.find(|e| e.kind == "turn_error")
.expect("exactly one turn_error event must be emitted");
assert_eq!(
turn_error.payload["error"].as_str().unwrap(),
format!(
"Agent turn exceeded the maximum duration ({}s) — requeued for retry (recently active)",
config.max_turn_duration_secs
),
);
assert_eq!(
queue.pending_channels(),
1,
"batch must be requeued, not dead-lettered, while within the retry budget"
);
}
/// Same recently-active hard timeout, but the channel has already
/// exhausted its retry budget ([`crate::queue::MAX_RETRIES`] prior
/// attempts) — `queue.requeue()` dead-letters instead of requeueing, and
/// the observer payload must report that fate, not the requeue wording
/// above.
#[tokio::test]
async fn hard_timeout_recently_active_budget_exhausted_reports_dead_lettered() {
let channel_id = Uuid::new_v4();
let mut queue = EventQueue::new(config::DedupMode::Queue);
// Simulate MAX_RETRIES prior failed attempts on this channel so the
// upcoming requeue() call in handle_prompt_result crosses the
// dead-letter threshold.
queue.set_retry_count_for_test(channel_id, crate::queue::MAX_RETRIES);
let agent = dummy_agent(0).await;
let mut pool = AgentPool::from_slots(vec![None]);
let task_id = pool.join_set.spawn(async {}).id();
pool.task_map_mut().insert(
task_id,
crate::pool::TaskMeta {
agent_index: 0,
channel_id: None,
turn_id: "test-turn-id".to_string(),
recoverable_batch: None,
control_tx: None,
steer_tx: None,
successful_steer_deliveries: HashSet::new(),
},
);
let config = test_config();
let mut heartbeat_in_flight = false;
let removed_channels = HashSet::new();
let mut crash_history = vec![SlotCircuit {
crash_times: Vec::new(),
open_until: None,
respawn_in_flight: false,
}];
let (respawn_tx, _respawn_rx) = mpsc::channel(8);
let mut respawn_tasks = tokio::task::JoinSet::new();
let observer = ObserverHandle::in_process();
let batch = FlushBatch {
channel_id,
events: vec![BatchEvent {
event: EventBuilder::new(Kind::Custom(9), "final-attempt")
.sign_with_keys(&Keys::generate())
.unwrap(),
prompt_tag: "test".into(),
received_at: std::time::Instant::now(),
}],
cancelled_events: vec![],
cancel_reason: None,
};
let result = PromptResult {
agent,
source: PromptSource::Channel(channel_id),
turn_id: "test-turn-id".to_string(),
outcome: PromptOutcome::Timeout(TimeoutKind::Hard {
recently_active: true,
}),
batch: Some(batch),
};
handle_prompt_result(
&mut pool,
&mut queue,
&config,
result,
&mut heartbeat_in_flight,
&removed_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
Some(observer.clone()),
None,
);
let events = observer.snapshot();
let turn_error = events
.iter()
.find(|e| e.kind == "turn_error")
.expect("exactly one turn_error event must be emitted");
assert_eq!(
turn_error.payload["error"].as_str().unwrap(),
format!(
"Agent turn exceeded the maximum duration ({}s) — dead-lettered (retry budget exhausted)",
config.max_turn_duration_secs
),
);
assert_eq!(
queue.queued_event_count(&channel_id),
0,
"batch with an exhausted retry budget must be dead-lettered, not requeued"
);
}
/// Cancel-drain-timeout batches are requeued as cancelled (merge into the
/// next flush, `CancelReason` preserved) — never dead-lettered like a real
/// hard-cap. The agent itself is NOT returned to the idle pool: it is
/// handed to `spawn_respawn_task` instead, mirroring a fatal `Timeout`.
///
/// This reproduces the full steer-fallback incident, not just the
/// original batch in isolation: the steer ack handler already released
/// the new triggering event back to `queue` (`lib.rs`'s
/// `ExpectedRunIdMissing` path) before the cancel-drain expiry fires. The
/// next `flush_next()` must merge the surviving original event (via
/// `cancelled_events`) with that already-queued new event (via `events`)
/// exactly once each — proving no loss and no duplication.
#[tokio::test]
async fn cancel_drain_timeout_requeues_batch_and_does_not_return_agent() {
let keys = Keys::generate();
let original_event = EventBuilder::new(Kind::Custom(9), "original")
.sign_with_keys(&keys)
.unwrap();
let new_event = EventBuilder::new(Kind::Custom(9), "new")
.sign_with_keys(&keys)
.unwrap();
assert_ne!(
original_event.id, new_event.id,
"test fixture must use two distinct events"
);
let channel_id = Uuid::new_v4();
let batch = FlushBatch {
channel_id,
events: vec![BatchEvent {
event: original_event.clone(),
prompt_tag: "test".into(),
received_at: std::time::Instant::now(),
}],
cancelled_events: vec![],
cancel_reason: Some(CancelReason::Steer),
};
let agent = dummy_agent(0).await;
let mut pool = AgentPool::from_slots(vec![None]);
let task_id = pool.join_set.spawn(async {}).id();
pool.task_map_mut().insert(
task_id,
crate::pool::TaskMeta {
agent_index: 0,
channel_id: None,
turn_id: "test-turn-id".to_string(),
recoverable_batch: None,
control_tx: None,
steer_tx: None,
successful_steer_deliveries: HashSet::new(),
},
);
let mut queue = EventQueue::new(config::DedupMode::Queue);
// The steer ack handler releases the new event to the queue BEFORE
// signaling the fallback ControlSignal::Steer that ultimately times
// out on drain — so it is already queued by the time
// handle_prompt_result runs.
queue.push(QueuedEvent {
channel_id,
event: new_event.clone(),
received_at: std::time::Instant::now(),
prompt_tag: "test".into(),
});
let config = test_config();
let mut heartbeat_in_flight = false;
let removed_channels = HashSet::new();
let mut crash_history = vec![SlotCircuit {
crash_times: Vec::new(),
open_until: None,
respawn_in_flight: false,
}];
let (respawn_tx, _respawn_rx) = mpsc::channel(8);
let mut respawn_tasks = tokio::task::JoinSet::new();
let observer = ObserverHandle::in_process();
let grace = std::time::Duration::from_secs(5);
let result = PromptResult {
agent,
source: PromptSource::Channel(channel_id),
turn_id: "test-turn-id".to_string(),
outcome: PromptOutcome::CancelDrainTimeout(grace),
batch: Some(batch),
};
handle_prompt_result(
&mut pool,
&mut queue,
&config,
result,
&mut heartbeat_in_flight,
&removed_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
Some(observer.clone()),
None,
);
// Batch preserved as a cancelled merge, not dead-lettered — same
// treatment as a normal `Cancelled` outcome. `handle_prompt_result`
// already called `mark_complete` internally, releasing the channel.
// `flush_next()` must merge the already-queued new event with the
// preserved original: each exactly once, in the correct bucket.
let requeued = queue.flush_next().expect("batch must be requeued");
assert_eq!(
requeued.events.len(),
1,
"exactly one new event must be in the regular events bucket"
);
assert_eq!(
requeued.events[0].event.id, new_event.id,
"the regular events bucket must hold the new (already-queued) event"
);
assert_eq!(
requeued.cancelled_events.len(),
1,
"exactly one original event must be in the cancelled_events bucket"
);
assert_eq!(
requeued.cancelled_events[0].event.id, original_event.id,
"the cancelled_events bucket must hold the original (interrupted) event"
);
assert_ne!(
requeued.events[0].event.id, requeued.cancelled_events[0].event.id,
"the new and original events must not be the same event"
);
assert_eq!(
requeued.cancel_reason,
Some(CancelReason::Steer),
"CancelReason must ride through to the requeued batch"
);
// Agent must NOT be back in the idle pool — it was handed to respawn.
assert_eq!(
pool.live_count(),
0,
"agent must not be returned to the pool after a cancel-drain timeout"
);
assert_eq!(
respawn_tasks.len(),
1,
"a respawn task must be spawned for the poisoned agent"
);
// The observer payload must be fate-neutral: it names the grace and
// the process replacement, and must NOT claim the batch was
// preserved — that claim is false for explicit Stop/removed-channel
// drops (see the sibling dropped-Stop test below), so the same
// wording is used regardless of fate.
let events = observer.snapshot();
let turn_error = events
.iter()
.find(|e| e.kind == "turn_error")
.expect("exactly one turn_error event must be emitted");
assert_eq!(
turn_error.payload["outcome"].as_str().unwrap(),
"cancel_drain_timeout"
);
assert_eq!(
turn_error.payload["error"].as_str().unwrap(),
format!("Agent did not stop within {grace:?} after cancellation; the agent process is being replaced."),
"observer message must name the actual grace and must not claim preservation"
);
assert_eq!(
events.iter().filter(|e| e.kind == "turn_error").count(),
1,
"exactly one turn_error event must be emitted"
);
}
/// Explicit Stop (`ControlSignal::Cancel`) on cancel-drain expiry drops
/// the triggering batch — `requeue_cancelled_batch` returns `None` for
/// `Cancel`/`Rotate`. The observer payload must be the SAME fate-neutral
/// text as the preserved-Steer case above: it must never claim work was
/// preserved when it was intentionally discarded. The poisoned agent is
/// still respawned exactly as in the preserved case.
#[tokio::test]
async fn cancel_drain_timeout_dropped_stop_batch_none_same_neutral_payload() {
let agent = dummy_agent(0).await;
let mut pool = AgentPool::from_slots(vec![None]);
let task_id = pool.join_set.spawn(async {}).id();
pool.task_map_mut().insert(
task_id,
crate::pool::TaskMeta {
agent_index: 0,
channel_id: None,
turn_id: "test-turn-id".to_string(),
recoverable_batch: None,
control_tx: None,
steer_tx: None,
successful_steer_deliveries: HashSet::new(),
},
);
let mut queue = EventQueue::new(config::DedupMode::Queue);
let config = test_config();
let mut heartbeat_in_flight = false;
let removed_channels = HashSet::new();
let mut crash_history = vec![SlotCircuit {
crash_times: Vec::new(),
open_until: None,
respawn_in_flight: false,
}];
let (respawn_tx, _respawn_rx) = mpsc::channel(8);
let mut respawn_tasks = tokio::task::JoinSet::new();
let observer = ObserverHandle::in_process();
let grace = std::time::Duration::from_secs(5);
let result = PromptResult {
agent,
source: PromptSource::Channel(Uuid::new_v4()),
turn_id: "test-turn-id".to_string(),
outcome: PromptOutcome::CancelDrainTimeout(grace),
// Explicit Stop already dropped the batch upstream in
// `classify_control_cancel_failure` — `handle_prompt_result`
// never sees one to requeue.
batch: None,
};
handle_prompt_result(
&mut pool,
&mut queue,
&config,
result,
&mut heartbeat_in_flight,
&removed_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
Some(observer.clone()),
None,
);
// No batch to merge — the queue has nothing pending for any channel.
assert_eq!(
queue.pending_channels(),
0,
"a dropped Stop batch must not leave anything queued"
);
// Same respawn treatment as the preserved case: never returned idle.
assert_eq!(
pool.live_count(),
0,
"agent must not be returned to the pool after a cancel-drain timeout"
);
assert_eq!(
respawn_tasks.len(),
1,
"a respawn task must be spawned for the poisoned agent"
);
// The observer payload is byte-identical to the preserved-Steer case:
// fate-neutral, naming the grace, with no preservation claim.
let events = observer.snapshot();
let turn_error = events
.iter()
.find(|e| e.kind == "turn_error")
.expect("exactly one turn_error event must be emitted");
assert_eq!(
turn_error.payload["outcome"].as_str().unwrap(),
"cancel_drain_timeout"
);
assert_eq!(
turn_error.payload["error"].as_str().unwrap(),
format!("Agent did not stop within {grace:?} after cancellation; the agent process is being replaced."),
"observer message must be fate-neutral even though the batch was dropped"
);
assert_eq!(
events.iter().filter(|e| e.kind == "turn_error").count(),
1,
"exactly one turn_error event must be emitted"
);
}
#[tokio::test]
async fn transport_error_emits_exactly_one_feed_event() {
let io = AcpError::Io(std::io::Error::other("pipe broke"));
assert_eq!(turn_errors_emitted_for(PromptOutcome::Error(io)).await, 1);
}
#[tokio::test]
async fn application_error_emits_exactly_one_feed_event() {
let app = AcpError::IdleTimeout(std::time::Duration::from_secs(1));
assert_eq!(turn_errors_emitted_for(PromptOutcome::Error(app)).await, 1);
}
// ── is_auth_error classification ───────────────────────────────────────
#[test]
fn is_auth_error_matches_reauthenticate_message() {
let e = acp::AcpError::AgentError {
code: -32000,
message: "API Error: OAuth access token has expired. Re-authenticate to continue."
.to_string(),
};
assert!(
is_auth_error(&e),
"Re-authenticate variant must be classified as auth error"
);
}
#[test]
fn is_auth_error_matches_401_message() {
let e = acp::AcpError::AgentError {
code: -32000,
message: "Internal error: API Error: 401 OAuth access token has expired.".to_string(),
};
assert!(
is_auth_error(&e),
"API Error: 401 variant must be classified as auth error"
);
}
#[test]
fn is_auth_error_rejects_other_agent_error_message() {
let e = acp::AcpError::AgentError {
code: -32601,
message: "Usage credits required for 1M context — turn on usage credits".to_string(),
};
assert!(
!is_auth_error(&e),
"usage-credit error must NOT be classified as auth error"
);
}
#[test]
fn is_auth_error_rejects_transport_errors() {
let io = acp::AcpError::Io(std::io::Error::other("pipe broke"));
assert!(
!is_auth_error(&io),
"I/O error must not be classified as auth error"
);
let timeout = acp::AcpError::WriteTimeout(std::time::Duration::from_secs(5));
assert!(
!is_auth_error(&timeout),
"WriteTimeout must not be classified as auth error"
);
}
// ── auth error dead-letter behavior ────────────────────────────────────
/// An auth-class `PromptOutcome::Error` must dead-letter immediately
/// (the batch is never requeued) so the user sees a re-auth hint at once
/// rather than after 10 futile retries.
#[tokio::test]
async fn auth_error_dead_letters_immediately_without_requeueing() {
let keys = nostr::Keys::generate();
let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "test")
.sign_with_keys(&keys)
.unwrap();
let channel_id = uuid::Uuid::new_v4();
let batch = FlushBatch {
channel_id,
events: vec![BatchEvent {
event,
prompt_tag: "test".into(),
received_at: std::time::Instant::now(),
}],
cancelled_events: vec![],
cancel_reason: None,
};
let auth_error = acp::AcpError::AgentError {
code: -32000,
message: "API Error: 401 OAuth access token has expired. Re-authenticate to continue."
.to_string(),
};
let agent = dummy_agent(0).await;
let mut pool = AgentPool::from_slots(vec![None]);
let task_id = pool.join_set.spawn(async {}).id();
pool.task_map_mut().insert(
task_id,
crate::pool::TaskMeta {
agent_index: 0,
channel_id: None,
turn_id: "test-turn-id".to_string(),
recoverable_batch: None,
control_tx: None,
steer_tx: None,
successful_steer_deliveries: HashSet::new(),
},
);
let mut queue = EventQueue::new(config::DedupMode::Queue);
let config = test_config();
let mut heartbeat_in_flight = false;
let removed_channels = std::collections::HashSet::new();
let mut crash_history = vec![SlotCircuit {
crash_times: Vec::new(),
open_until: None,
respawn_in_flight: false,
}];
let (respawn_tx, _respawn_rx) = mpsc::channel(8);
let mut respawn_tasks = tokio::task::JoinSet::new();
let result = PromptResult {
agent,
source: PromptSource::Channel(channel_id),
turn_id: "test-turn-id".to_string(),
outcome: PromptOutcome::Error(auth_error),
batch: Some(batch),
};
handle_prompt_result(
&mut pool,
&mut queue,
&config,
result,
&mut heartbeat_in_flight,
&removed_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
None,
None,
);
// The batch must not be requeued: pending_channels returns 0.
assert_eq!(
queue.pending_channels(),
0,
"auth error must dead-letter immediately — batch must not be requeued"
);
assert_eq!(
queue.queued_event_count(&channel_id),
0,
"auth error must dead-letter immediately — no events should be pending"
);
}
/// A non-auth application error (e.g. usage credits) must still follow the
/// standard requeue path so today's behavior is unchanged.
#[tokio::test]
async fn non_auth_application_error_is_requeued() {
let keys = nostr::Keys::generate();
let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "test")
.sign_with_keys(&keys)
.unwrap();
let channel_id = uuid::Uuid::new_v4();
let batch = FlushBatch {
channel_id,
events: vec![BatchEvent {
event,
prompt_tag: "test".into(),
received_at: std::time::Instant::now(),
}],
cancelled_events: vec![],
cancel_reason: None,
};
// Usage-credits error — AgentError but NOT an auth error.
let usage_error = acp::AcpError::AgentError {
code: -32000,
message: "Usage credits required for 1M context".to_string(),
};
let agent = dummy_agent(0).await;
let mut pool = AgentPool::from_slots(vec![None]);
let task_id = pool.join_set.spawn(async {}).id();
pool.task_map_mut().insert(
task_id,
crate::pool::TaskMeta {
agent_index: 0,
channel_id: None,
turn_id: "test-turn-id".to_string(),
recoverable_batch: None,
control_tx: None,
steer_tx: None,
successful_steer_deliveries: HashSet::new(),
},
);
let mut queue = EventQueue::new(config::DedupMode::Queue);
let config = test_config();
let mut heartbeat_in_flight = false;
let removed_channels = std::collections::HashSet::new();
let mut crash_history = vec![SlotCircuit {
crash_times: Vec::new(),
open_until: None,
respawn_in_flight: false,
}];
let (respawn_tx, _respawn_rx) = mpsc::channel(8);
let mut respawn_tasks = tokio::task::JoinSet::new();
let result = PromptResult {
agent,
source: PromptSource::Channel(channel_id),
turn_id: "test-turn-id".to_string(),
outcome: PromptOutcome::Error(usage_error),
batch: Some(batch),
};
handle_prompt_result(
&mut pool,
&mut queue,
&config,
result,
&mut heartbeat_in_flight,
&removed_channels,
&mut crash_history,
&respawn_tx,
&mut respawn_tasks,
None,
None,
);
// Non-auth application error: batch IS requeued (first attempt, retry budget > 0).
assert_eq!(
queue.pending_channels(),
1,
"non-auth application error must requeue the batch for retry"
);
assert_eq!(
queue.queued_event_count(&channel_id),
1,
"non-auth application error must preserve the event for retry"
);
}
}
#[cfg(test)]
mod observer_payload_trim_tests {
use super::*;
fn event_with_payload(kind: &str, payload: serde_json::Value) -> observer::ObserverEvent {
observer::ObserverEvent {
seq: 1,
timestamp: "2026-06-16T00:00:00Z".to_string(),
kind: kind.to_string(),
agent_index: Some(0),
channel_id: Some("11111111-1111-1111-1111-111111111111".to_string()),
session_id: Some("sess-1".to_string()),
turn_id: Some("turn-1".to_string()),
started_at: None,
payload,
}
}
fn serialized(event: &observer::ObserverEvent) -> String {
serde_json::to_string(event).unwrap()
}
#[test]
fn test_under_budget_frame_passes_through_byte_identical() {
let mut event = event_with_payload("acp_read", serde_json::json!({ "body": "small" }));
let before = serialized(&event);
fit_observer_event_to_budget(&mut event);
assert_eq!(
serialized(&event),
before,
"under-budget frame must not be mutated"
);
}
#[test]
fn test_single_giant_leaf_is_elided_to_fit_with_envelope_intact() {
let big = "x".repeat(100_000);
let mut event = event_with_payload("acp_read", serde_json::json!({ "body": big }));
fit_observer_event_to_budget(&mut event);
assert!(
serialized(&event).len() <= OBSERVER_MAX_PLAINTEXT_LEN,
"frame must fit after trimming"
);
// Envelope intact.
assert_eq!(event.kind, "acp_read");
assert_eq!(event.turn_id.as_deref(), Some("turn-1"));
assert_eq!(
event.channel_id.as_deref(),
Some("11111111-1111-1111-1111-111111111111")
);
assert_eq!(event.seq, 1);
let leaf = event.payload["body"].as_str().unwrap();
assert!(
leaf.starts_with(&"x".repeat(OBSERVER_LEAF_RETAIN_BYTES)),
"head retained"
);
assert!(
leaf.ends_with(&"x".repeat(OBSERVER_LEAF_RETAIN_BYTES)),
"tail retained"
);
// N in the marker is RAW bytes removed: original len minus retained len.
let removed = 100_000 - leaf.chars().filter(|c| *c == 'x').count();
assert!(
leaf.contains(&format!("…[elided {removed} bytes]…")),
"marker reports raw bytes removed"
);
}
#[test]
fn test_multi_block_prompt_retains_every_section_header_after_elision() {
// The real session/prompt fix: format_prompt now emits one block per
// section, so the observer payload is params.prompt = [{text: "[Base]…"},
// {text: "[Agent Memory — core]…"}, … {text: "[Buzz event: …]…<huge>"}].
// An oversized section is its own leaf, so eliding its body keeps the
// leaf's head-3000 (which begins with the section's [Header] line) — every
// header survives, so the desktop "Prompt context" panel counts them all.
// This is the regression the single-fat-leaf shape caused (the trailing
// [Buzz event] header fell into the elided middle and the count collapsed
// to 1).
let sections = [
"[Base]\nyou are a helpful agent".to_string(),
"[System]\npersona text".to_string(),
"[Agent Memory — core]\nremember this".to_string(),
"[Context]\nScope: thread".to_string(),
// The triggering event body, oversized on its own.
format!("[Buzz event: @mention]\nContent: {}", "E".repeat(90_000)),
];
let block_refs: Vec<&str> = sections.iter().map(String::as_str).collect();
// Mirror the wire shape build_prompt_params produces: each block is its
// own {type:"text", text} leaf under params.prompt.
let prompt_blocks: Vec<serde_json::Value> = block_refs
.iter()
.map(|text| serde_json::json!({ "type": "text", "text": text }))
.collect();
let mut event = event_with_payload(
"acp_write",
serde_json::json!({
"method": "session/prompt",
"params": { "sessionId": "sess-1", "prompt": prompt_blocks },
}),
);
assert!(
serialized(&event).len() > OBSERVER_MAX_PLAINTEXT_LEN,
"precondition: oversized event body pushes the frame over the cap"
);
fit_observer_event_to_budget(&mut event);
assert!(
serialized(&event).len() <= OBSERVER_MAX_PLAINTEXT_LEN,
"frame must fit after trimming"
);
let blocks = event.payload["params"]["prompt"]
.as_array()
.expect("prompt array survives");
let texts: Vec<&str> = blocks.iter().map(|b| b["text"].as_str().unwrap()).collect();
for header in [
"[Base]",
"[System]",
"[Agent Memory — core]",
"[Context]",
"[Buzz event: @mention]",
] {
assert!(
texts.iter().any(|t| t.starts_with(header)),
"section header {header} must survive at the head of its own block"
);
}
// The oversized event body was elided in place (header kept, middle cut).
let event_block = texts
.iter()
.find(|t| t.starts_with("[Buzz event: @mention]"))
.unwrap();
assert!(
event_block.contains("…[elided"),
"the oversized event body is elided, not dropped"
);
}
#[test]
fn test_multi_leaf_elides_largest_shrinkable_first_and_stops_when_it_fits() {
// One leaf alone over the cap; a second smaller-but-still-large leaf.
// Eliding the biggest should suffice, leaving the smaller intact.
let mut event = event_with_payload(
"acp_write",
serde_json::json!({
"huge": "a".repeat(90_000),
"medium": "b".repeat(20_000),
}),
);
fit_observer_event_to_budget(&mut event);
assert!(serialized(&event).len() <= OBSERVER_MAX_PLAINTEXT_LEN);
assert!(
event.payload["huge"].as_str().unwrap().contains("…[elided"),
"the largest leaf is elided"
);
assert_eq!(
event.payload["medium"].as_str().unwrap().len(),
20_000,
"the smaller leaf is left untouched once the frame fits"
);
}
#[test]
fn test_coalesced_chunk_nested_leaf_is_reached_by_recursive_walk() {
// The coalesced-chunk big leaf lives at params.update.content.text,
// not a top-level field — the walk must recurse to reach it.
let big = "z".repeat(80_000);
let mut event = event_with_payload(
"session_update",
serde_json::json!({
"params": {
"update": {
"sessionUpdate": "agent_message_chunk",
"content": { "text": big }
}
}
}),
);
fit_observer_event_to_budget(&mut event);
assert!(serialized(&event).len() <= OBSERVER_MAX_PLAINTEXT_LEN);
let text = event.payload["params"]["update"]["content"]["text"]
.as_str()
.unwrap();
assert!(text.contains("…[elided"), "nested leaf was elided");
}
#[test]
fn test_many_medium_leaves_terminate_via_stub() {
// Many leaves each too small to shrink on their own (below 2x retain),
// collectively over the cap. No leaf can strictly shrink, so the trimmer
// must terminate via the stub rather than loop forever.
let leaf = "m".repeat(OBSERVER_LEAF_RETAIN_BYTES); // shorter than head+tail → cannot shrink
let items: Vec<serde_json::Value> = (0..40)
.map(|_| serde_json::Value::String(leaf.clone()))
.collect();
let mut event = event_with_payload("acp_read", serde_json::json!({ "items": items }));
assert!(
serialized(&event).len() > OBSERVER_MAX_PLAINTEXT_LEN,
"precondition: frame is over the cap"
);
fit_observer_event_to_budget(&mut event);
assert!(serialized(&event).len() <= OBSERVER_MAX_PLAINTEXT_LEN);
assert_eq!(
event.payload["elided"].as_str().unwrap(),
"acp_read payload too large",
"fell back to the stub"
);
assert!(event.payload.get("originalBytes").is_some());
}
#[test]
fn test_leaf_too_small_to_shrink_is_not_mutated() {
// A frame already under budget whose only leaf is below the shrink floor:
// nothing should change. (Under-budget short-circuits, and even if forced,
// leaf_shrinks would reject it.)
let short = "s".repeat(OBSERVER_LEAF_RETAIN_BYTES); // == head; cannot strictly shrink
assert!(
!leaf_shrinks(&short),
"a leaf at the retain floor must not shrink"
);
let longer = "L".repeat(OBSERVER_LEAF_RETAIN_BYTES * 2 + 100);
assert!(leaf_shrinks(&longer), "a clearly larger leaf must shrink");
}
#[test]
fn test_utf8_multibyte_leaf_elides_on_char_boundary() {
// A leaf of 3-byte chars (… = U+2026) — eliding must land on char
// boundaries and never panic or produce invalid UTF-8.
let big: String = "…".repeat(40_000); // 120_000 bytes
let mut event = event_with_payload("acp_read", serde_json::json!({ "body": big }));
fit_observer_event_to_budget(&mut event);
assert!(serialized(&event).len() <= OBSERVER_MAX_PLAINTEXT_LEN);
let leaf = event.payload["body"].as_str().unwrap();
// Valid UTF-8 by construction (it's a &str); confirm head/tail are whole
// multi-byte chars and the marker is present.
assert!(leaf.starts_with('…'));
assert!(leaf.ends_with('…'));
assert!(leaf.contains("[elided"));
}
}