#![deny(unsafe_code)] mod acp; mod config; mod engram_fetch; mod filter; mod observer; mod pool; mod queue; mod relay; mod setup_mode; mod usage; pub use usage::TurnUsage; use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::Duration; use acp::{AcpClient, EnvVar, McpServer}; use anyhow::Result; use buzz_core::kind::{ KIND_MEMBER_ADDED_NOTIFICATION, KIND_MEMBER_REMOVED_NOTIFICATION, KIND_STREAM_MESSAGE, KIND_STREAM_REMINDER, KIND_WORKFLOW_APPROVAL_REQUESTED, }; use buzz_core::observer::{ decrypt_observer_payload, encrypt_observer_payload, OBSERVER_FRAME_TELEMETRY, OBSERVER_MAX_PLAINTEXT_LEN, }; use clap::Parser; use config::{ AuthAgentArgs, AuthMethodsArgs, AuthenticateArgs, Config, DedupMode, ModelsArgs, MultipleEventHandling, RespondTo, SubscribeMode, }; use filter::SubscriptionRule; use futures_util::FutureExt; use nostr::{PublicKey, ToBech32}; use pool::{ AgentPool, ControlSignal, IdleSwitchResult, OwnedAgent, PromptContext, PromptOutcome, PromptResult, PromptSource, SessionState, TimeoutKind, }; use queue::{CancelReason, EventQueue, FlushBatch, QueuedEvent, ThreadTags}; use relay::{HarnessRelay, RelayEventPublisher}; use tokio::sync::{mpsc, watch}; use tracing_subscriber::EnvFilter; use uuid::Uuid; /// Check if argv[1] matches a subcommand name, before any clap parsing. /// /// This avoids clap rejecting harness flags (like `--private-key`) that aren't /// declared on the subcommand's `Parser`. The `models` path has its own /// dedicated parser; the default path uses the existing `CliArgs`. /// /// **Constraint**: subcommand must be argv[1] — flags before the subcommand /// name (e.g., `buzz-acp --verbose models`) are not supported. fn is_subcommand(name: &str) -> bool { std::env::args().nth(1).map(|a| a == name).unwrap_or(false) } /// Timeout for lightweight helper subcommands (spawn + initialize + model/method probes). const MODELS_TIMEOUT: Duration = Duration::from_secs(10); /// Timeout for `buzz-acp authenticate`. Browser-based vendor auth can require /// human interaction, so it must not share the short probe timeout. const AUTHENTICATE_TIMEOUT: Duration = Duration::from_secs(10 * 60); /// Publish a kind:20001 presence update event via the WebSocket connection. /// /// Ephemeral kinds (20000-29999) are rejected by the HTTP bridge, so presence /// updates must be routed through the WS path. /// /// Content is a bare status string (`"online"`, `"away"`, `"offline"`) matching /// the desktop client's format. The relay stores this in Redis and synthesizes /// it back on presence queries. async fn publish_presence( publisher: &relay::RelayEventPublisher, keys: &nostr::Keys, status: &str, ) -> Result<(), relay::RelayError> { use buzz_core::kind::KIND_PRESENCE_UPDATE; use nostr::{EventBuilder, Kind}; let event = EventBuilder::new(Kind::Custom(KIND_PRESENCE_UPDATE as u16), status) .tags([]) .sign_with_keys(keys) .map_err(|e| relay::RelayError::Http(format!("presence sign error: {e}")))?; publisher.publish_event(event).await?; Ok(()) } /// Resolve the agent's owner pubkey at startup. /// /// Priority: /// 1. `BUZZ_AUTH_TAG` env var — NIP-OA attestation signed by the owner. /// Verified against the agent's own pubkey to extract the owner pubkey. /// 2. `--agent-owner` CLI flag / `BUZZ_ACP_AGENT_OWNER` env var. fn resolve_agent_owner(config: &Config) -> Option { // Try BUZZ_AUTH_TAG first (NIP-OA attestation). if let Ok(auth_tag) = std::env::var("BUZZ_AUTH_TAG") { if !auth_tag.is_empty() { let agent_pk = config.keys.public_key(); match buzz_sdk::nip_oa::verify_auth_tag(&auth_tag, &agent_pk) { Ok(owner_pk) => { let owner_hex = owner_pk.to_hex().to_ascii_lowercase(); tracing::info!("owner resolved from BUZZ_AUTH_TAG: {owner_hex}"); return Some(owner_hex); } Err(e) => { tracing::warn!("BUZZ_AUTH_TAG verification failed: {e} — falling back"); } } } } // Fall back to --agent-owner config. config.agent_owner.clone() } /// Cache for the agent's owner pubkey. /// /// Owner is now provided via `--agent-owner` config flag (no REST lookup). /// Cache for the agent's owner pubkey + sibling lookups. /// /// Siblings are other agents whose NIP-OA auth tag proves the same owner. /// Lookup results are cached for the process lifetime (attestations are immutable). struct OwnerCache { pubkey: Option, /// author_hex → is_sibling (true = same owner, false = not) siblings: std::sync::Mutex>, } impl OwnerCache { fn new(initial: Option) -> Self { Self { pubkey: initial, siblings: std::sync::Mutex::new(HashMap::new()), } } /// Return the cached owner pubkey. fn get(&self) -> Option<&str> { self.pubkey.as_deref() } /// Check if author is a known sibling (cached result). fn is_known_sibling(&self, author: &str) -> Option { self.siblings.lock().ok()?.get(author).copied() } /// Cache a sibling lookup result. fn cache_sibling(&self, author: String, is_sibling: bool) { if let Ok(mut map) = self.siblings.lock() { // Cap at 256 entries to prevent unbounded growth. if map.len() >= 256 { map.clear(); } map.insert(author, is_sibling); } } } /// Check if `author` is the owner OR a sibling (same owner via NIP-OA). /// /// For unknown authors, queries their kind:0 profile to extract the NIP-OA /// auth tag and verify the owner matches. Result is cached. async fn is_owner_or_sibling( author: &str, owner_cache: &OwnerCache, rest_client: &relay::RestClient, ) -> bool { let my_owner = match owner_cache.get() { Some(o) => o, None => return false, // no owner configured — fail closed }; // Direct owner check. if author == my_owner { return true; } // Check sibling cache. if let Some(cached) = owner_cache.is_known_sibling(author) { return cached; } // Query the author's kind:0 profile to check for NIP-OA auth tag. let is_sibling = check_sibling_via_profile(author, my_owner, rest_client).await; owner_cache.cache_sibling(author.to_string(), is_sibling); is_sibling } /// Inbound author gate decision: does this author's event fire a turn? /// /// Coarse security policy applied before subscription rules. Both `OwnerOnly` /// and `Allowlist` accept the owner and same-owner siblings; `Allowlist` /// additionally accepts the explicit external pubkey list. async fn author_allowed( respond_to: &RespondTo, allowlist: &HashSet, author: &str, owner_cache: &OwnerCache, rest_client: &relay::RestClient, ) -> bool { match respond_to { RespondTo::Anyone => true, RespondTo::Nobody => false, RespondTo::OwnerOnly => is_owner_or_sibling(author, owner_cache, rest_client).await, RespondTo::Allowlist => { allowlist.contains(author) || is_owner_or_sibling(author, owner_cache, rest_client).await } } } /// Query an author's kind:0 profile and check if their NIP-OA auth tag /// proves the same owner as us. async fn check_sibling_via_profile( author: &str, expected_owner: &str, rest_client: &relay::RestClient, ) -> bool { let filter = nostr::Filter::new() .kind(nostr::Kind::Metadata) .author(match nostr::PublicKey::from_hex(author) { Ok(pk) => pk, Err(_) => return false, }) .limit(1); let resp = match tokio::time::timeout(Duration::from_millis(2000), rest_client.query(&[filter])) .await { Ok(Ok(v)) => v, _ => return false, // timeout or error — fail closed }; // Look for an "auth" tag in the profile event. let events = match resp.as_array() { Some(arr) => arr, None => return false, }; let event = match events.first() { Some(e) => e, None => return false, }; let tags = match event.get("tags").and_then(|t| t.as_array()) { Some(t) => t, None => return false, }; // Find ["auth", owner_pk, conditions, sig] and verify the Schnorr signature. // Don't trust the relay — verify ourselves. let agent_pk = match nostr::PublicKey::from_hex(author) { Ok(pk) => pk, Err(_) => return false, }; for tag in tags { let parts = match tag.as_array() { Some(p) if p.len() >= 4 => p, _ => continue, }; if parts[0].as_str() != Some("auth") { continue; } let tag_owner = match parts[1].as_str() { Some(o) => o, None => continue, }; // Only verify if the owner field matches ours. if !tag_owner.eq_ignore_ascii_case(expected_owner) { continue; } // Cryptographically verify the NIP-OA attestation signature. let tag_json = serde_json::to_string(tag).unwrap_or_default(); match buzz_sdk::nip_oa::verify_auth_tag(&tag_json, &agent_pk) { Ok(_) => { tracing::debug!(author, expected_owner, "sibling verified via NIP-OA"); return true; } Err(e) => { tracing::debug!(author, "NIP-OA auth tag verification failed: {e}"); } } } false } fn spawn_relay_observer_publisher( observer: observer::ObserverHandle, publisher: RelayEventPublisher, keys: nostr::Keys, agent_pubkey_hex: String, owner_pubkey_hex: String, owner_pubkey: PublicKey, ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { let mut coalescer = ObserverChunkCoalescer::default(); for event in observer.snapshot() { for event in coalescer.ingest(event) { publish_relay_observer_event( &publisher, &keys, &agent_pubkey_hex, &owner_pubkey_hex, &owner_pubkey, event, ) .await; } } let mut rx = observer.subscribe(); let mut flush_interval = tokio::time::interval(std::time::Duration::from_millis(500)); flush_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { tokio::select! { result = rx.recv() => { match result { Ok(event) => { for event in coalescer.ingest(event) { publish_relay_observer_event( &publisher, &keys, &agent_pubkey_hex, &owner_pubkey_hex, &owner_pubkey, event, ).await; } } Err(tokio::sync::broadcast::error::RecvError::Lagged(count)) => { for event in coalescer.flush() { publish_relay_observer_event( &publisher, &keys, &agent_pubkey_hex, &owner_pubkey_hex, &owner_pubkey, event, ).await; } tracing::warn!(dropped = count, "relay observer publisher lagged"); } Err(tokio::sync::broadcast::error::RecvError::Closed) => { for event in coalescer.flush() { publish_relay_observer_event( &publisher, &keys, &agent_pubkey_hex, &owner_pubkey_hex, &owner_pubkey, event, ).await; } break; } } } _ = flush_interval.tick() => { // Periodic flush ensures live streaming even during continuous chunk delivery. for event in coalescer.flush() { publish_relay_observer_event( &publisher, &keys, &agent_pubkey_hex, &owner_pubkey_hex, &owner_pubkey, event, ).await; } } } } }) } #[derive(Default)] struct ObserverChunkCoalescer { pending: Vec, } struct PendingObserverChunk { key: ObserverChunkKey, event: observer::ObserverEvent, text: String, } #[derive(Debug, Clone, PartialEq, Eq)] struct ObserverChunkKey { update_type: String, message_id: Option, channel_id: Option, session_id: Option, turn_id: Option, agent_index: Option, } /// Flush coalesced chunks before they exceed the NIP-44 plaintext limit (65,535 bytes). /// Leave headroom for the JSON envelope wrapping the text. This is a SOFT pre-flush /// of raw text below the hard cap; `fit_observer_event_to_budget` (the final ceiling, /// keyed to `OBSERVER_MAX_PLAINTEXT_LEN` in buzz-core/observer.rs:25) is what actually /// guarantees the serialized frame fits. Edit one of these two and review the other. const OBSERVER_CHUNK_MAX_TEXT_BYTES: usize = 60_000; impl ObserverChunkCoalescer { fn ingest(&mut self, event: observer::ObserverEvent) -> Vec { let Some((key, text)) = observer_chunk_key_and_text(&event) else { let mut events = self.flush(); events.push(event); return events; }; if let Some(pending) = self.pending.iter_mut().find(|pending| pending.key == key) { // Flush before appending if this would exceed the plaintext size limit. if pending.text.len() + text.len() >= OBSERVER_CHUNK_MAX_TEXT_BYTES { let events = self.flush(); // Start a new pending entry with the current chunk. self.pending.push(PendingObserverChunk { key, event, text }); return events; } pending.text.push_str(&text); pending.event.seq = event.seq; pending.event.timestamp = event.timestamp; return Vec::new(); } self.pending.push(PendingObserverChunk { key, event, text }); Vec::new() } fn flush(&mut self) -> Vec { self.pending .drain(..) .map(|mut pending| { set_observer_chunk_text(&mut pending.event.payload, pending.text); pending.event }) .collect() } } fn observer_chunk_key_and_text( event: &observer::ObserverEvent, ) -> Option<(ObserverChunkKey, String)> { let update = event.payload.get("params")?.get("update")?; let update_type = update.get("sessionUpdate")?.as_str()?; if !matches!( update_type, "agent_message_chunk" | "user_message_chunk" | "agent_thought_chunk" ) { return None; } let text = update.get("content")?.get("text")?.as_str()?.to_string(); let message_id = update .get("messageId") .and_then(|value| value.as_str()) .map(ToOwned::to_owned); Some(( ObserverChunkKey { update_type: update_type.to_string(), message_id, channel_id: event.channel_id.clone(), session_id: event.session_id.clone(), turn_id: event.turn_id.clone(), agent_index: event.agent_index, }, text, )) } fn set_observer_chunk_text(payload: &mut serde_json::Value, text: String) { let Some(content) = payload .get_mut("params") .and_then(|params| params.get_mut("update")) .and_then(|update| update.get_mut("content")) else { return; }; if let Some(content_object) = content.as_object_mut() { content_object.insert("text".to_string(), serde_json::Value::String(text)); } } /// Bytes of head and tail to retain from an elided string leaf — the value /// shown to the renderer at each end. The ONLY tuning knob here: large enough /// that a clipped diff/tool-result still shows real content, small enough that /// eliding actually shrinks the frame. const OBSERVER_LEAF_RETAIN_BYTES: usize = 3_000; /// Trim an oversized observer telemetry frame so its SERIALIZED form fits under /// `OBSERVER_MAX_PLAINTEXT_LEN`, instead of dropping the whole frame (silent /// telemetry loss). The common case — a frame already under budget — is left /// byte-identical. /// /// The cap is measured in SERIALIZED bytes (JSON escaping makes serialized /// length differ from raw), so the stop condition is always a full reserialize /// of the whole frame: that counts the envelope, the variable `Option` /// IDs, and any elision markers exactly. No separate margin constant is needed. /// /// Termination is provable: each iteration elides the largest string leaf that /// would STRICTLY shrink the serialized frame, then reserializes. Shrinkability /// is re-evaluated against each leaf's CURRENT value, so a leaf already at its /// retained floor can never be re-elided — the loop strictly decreases the /// serialized length each pass and is bounded by the leaf count. When no leaf /// can shrink the frame and it still overflows, the payload is replaced with a /// tiny stub, which trivially fits. Monotone decrease, bounded below by the stub. /// /// **Signature choice (`&mut`, double-serialize accepted):** on the common /// under-budget path this serializes the frame once to decide it fits, then /// `encrypt_observer_payload` serializes it again — one extra `to_string` of an /// already-small frame. Reusing that string would mean changing buzz-core's /// `encrypt_observer_payload` signature or adding a parallel encrypt path; both /// are out of this change's scope (buzz-core stays untouched). The clean `&mut` /// signature with one cheap redundant serialize is the deliberate tradeoff. fn fit_observer_event_to_budget(event: &mut observer::ObserverEvent) { if serialized_len(event) <= OBSERVER_MAX_PLAINTEXT_LEN { return; } // Raw size of the payload we are about to trim, captured before mutation so // the stub's `originalBytes` reports source bytes discarded, not serialized // overflow — consistent with the per-leaf marker's raw byte count. let original_payload_bytes = serde_json::to_string(&event.payload) .map(|s| s.len()) .unwrap_or(0); // Elide the largest shrinkable leaf, reserialize, repeat. Each successful // elision strictly shrinks the serialized frame, and a floored leaf can // never be re-elided, so the loop is bounded by the leaf count. while let Some(leaf) = largest_shrinkable_leaf(&mut event.payload) { elide_leaf(leaf); if serialized_len(event) <= OBSERVER_MAX_PLAINTEXT_LEN { return; } } // No leaf can shrink the frame further and it still overflows: replace the // whole payload with a stub that is trivially under-cap. event.payload = serde_json::json!({ "elided": format!("{} payload too large", event.kind), "originalBytes": original_payload_bytes, }); } fn serialized_len(event: &observer::ObserverEvent) -> usize { serde_json::to_string(event).map(|s| s.len()).unwrap_or(0) } /// Find the longest string leaf that would STRICTLY shrink if elided, returning /// a mutable handle to it. A leaf shrinks only if `head + marker + tail` is /// shorter than its current value (the marker-pushback guard); a leaf already at /// its retained floor fails this test and is skipped, which is what bounds the /// loop. Returns `None` when no leaf can shrink. fn largest_shrinkable_leaf(value: &mut serde_json::Value) -> Option<&mut serde_json::Value> { // First pass: find the byte length of the best candidate without holding a // borrow, then re-descend to return the matching mutable reference. Two // immutable-style passes keep the borrow checker happy without unsafe. let best_len = max_shrinkable_len(value)?; find_leaf_with_len(value, best_len) } /// Largest current length among string leaves that can strictly shrink. fn max_shrinkable_len(value: &serde_json::Value) -> Option { match value { serde_json::Value::String(s) if leaf_shrinks(s) => Some(s.len()), serde_json::Value::String(_) => None, serde_json::Value::Array(items) => items.iter().filter_map(max_shrinkable_len).max(), serde_json::Value::Object(map) => map.values().filter_map(max_shrinkable_len).max(), _ => None, } } /// Return the first string leaf whose current length equals `target` and that /// can strictly shrink. Used after `max_shrinkable_len` to re-acquire a mutable /// borrow of the chosen leaf. fn find_leaf_with_len( value: &mut serde_json::Value, target: usize, ) -> Option<&mut serde_json::Value> { match value { serde_json::Value::String(s) if s.len() == target && leaf_shrinks(s) => Some(value), serde_json::Value::Array(items) => items .iter_mut() .find_map(|item| find_leaf_with_len(item, target)), serde_json::Value::Object(map) => map .values_mut() .find_map(|item| find_leaf_with_len(item, target)), _ => None, } } /// True when eliding `s` to head + marker + tail yields a strictly shorter raw /// string. The marker width grows with `N` (bytes removed), so a leaf only /// marginally larger than the retained ends must NOT be touched. fn leaf_shrinks(s: &str) -> bool { let (head_end, tail_start) = elision_boundaries(s); tail_start > head_end && { let removed = tail_start - head_end; let marker = elision_marker(removed); head_end + marker.len() + (s.len() - tail_start) < s.len() } } /// Replace the middle of a string leaf with `…[elided N bytes]…`, keeping a head /// and tail slice on UTF-8 char boundaries. `N` is RAW bytes removed. fn elide_leaf(leaf: &mut serde_json::Value) { let serde_json::Value::String(s) = leaf else { return; }; let (head_end, tail_start) = elision_boundaries(s); let removed = tail_start - head_end; let mut elided = String::with_capacity(head_end + 32 + (s.len() - tail_start)); elided.push_str(&s[..head_end]); elided.push_str(&elision_marker(removed)); elided.push_str(&s[tail_start..]); *s = elided; } fn elision_marker(removed_bytes: usize) -> String { format!("…[elided {removed_bytes} bytes]…") } /// Byte offsets bounding the elided middle, snapped to char boundaries so we /// never split a multi-byte char. Returns `(head_end, tail_start)` with /// `head_end <= tail_start`. fn elision_boundaries(s: &str) -> (usize, usize) { let head_end = floor_char_boundary(s, OBSERVER_LEAF_RETAIN_BYTES.min(s.len())); let tail_start = ceil_char_boundary(s, s.len().saturating_sub(OBSERVER_LEAF_RETAIN_BYTES)); (head_end, tail_start.max(head_end)) } fn floor_char_boundary(s: &str, mut i: usize) -> usize { while i > 0 && !s.is_char_boundary(i) { i -= 1; } i } fn ceil_char_boundary(s: &str, mut i: usize) -> usize { while i < s.len() && !s.is_char_boundary(i) { i += 1; } i } async fn publish_relay_observer_event( publisher: &RelayEventPublisher, keys: &nostr::Keys, agent_pubkey_hex: &str, owner_pubkey_hex: &str, owner_pubkey: &PublicKey, mut event: observer::ObserverEvent, ) { // Trim oversized frames to fit the plaintext cap rather than letting // encrypt_observer_payload reject and drop them whole (silent telemetry loss). fit_observer_event_to_budget(&mut event); let encrypted = match encrypt_observer_payload(keys, owner_pubkey, &event) { Ok(encrypted) => encrypted, Err(error) => { tracing::warn!("failed to encrypt relay observer event: {error}"); return; } }; let builder = match buzz_sdk::build_agent_observer_frame( owner_pubkey_hex, agent_pubkey_hex, OBSERVER_FRAME_TELEMETRY, &encrypted, ) { Ok(builder) => builder, Err(error) => { tracing::warn!("failed to build relay observer event: {error}"); return; } }; let signed = match builder.sign_with_keys(keys) { Ok(event) => event, Err(error) => { tracing::warn!("failed to sign relay observer event: {error}"); return; } }; if let Err(error) = publisher.publish_event(signed).await { tracing::warn!("relay observer event dropped: {error}"); } } /// Maximum age (seconds) for an observer control frame to be considered fresh. const OBSERVER_CONTROL_FRESHNESS_SECS: i64 = 300; fn handle_relay_observer_control_event( keys: &nostr::Keys, event: nostr::Event, pool: &mut AgentPool, observer: Option<&observer::ObserverHandle>, owner_pubkey_hex: &str, ) { // Defense-in-depth: verify signature even though the relay already checked. if let Err(e) = buzz_core::verify_event(&event) { tracing::warn!(error = %e, "observer control frame failed signature verification"); return; } // Defense-in-depth: verify the sender is the resolved owner. if event.pubkey.to_hex() != owner_pubkey_hex { tracing::warn!( sender = %event.pubkey, expected = %owner_pubkey_hex, "observer control frame from non-owner — dropping" ); return; } // Freshness: reject stale/replayed frames outside ±5 minute window. let now = chrono::Utc::now().timestamp(); let event_ts = event.created_at.as_secs() as i64; if (event_ts - now).unsigned_abs() > OBSERVER_CONTROL_FRESHNESS_SECS as u64 { tracing::warn!( event_ts, now, "observer control frame outside freshness window — dropping" ); return; } let payload = match decrypt_observer_payload::(keys, &event) { Ok(payload) => payload, Err(error) => { tracing::warn!("failed to decrypt observer control frame: {error}"); return; } }; let command_type = payload.get("type").and_then(|value| value.as_str()); match command_type { Some("cancel_turn") => { handle_cancel_turn_control(&payload, pool, observer); } Some("switch_model") => { handle_switch_model_control(&payload, pool, observer); } _ => { tracing::debug!(payload = %payload, "ignoring unknown observer control frame"); } } } /// Handle a `cancel_turn` control frame: signal the in-flight task to cancel. fn handle_cancel_turn_control( payload: &serde_json::Value, pool: &mut AgentPool, observer: Option<&observer::ObserverHandle>, ) { let Some(channel_id) = payload .get("channelId") .and_then(|value| value.as_str()) .and_then(|value| value.parse::().ok()) else { tracing::warn!("observer cancel_turn control frame missing valid channelId"); return; }; let fired = signal_in_flight_task(pool, channel_id, ControlSignal::Cancel); let status = if fired { "sent" } else { "no_active_turn" }; if let Some(observer) = observer { observer.emit( "control_result", None, &observer::ObserverContext { channel_id: Some(channel_id.to_string()), session_id: None, turn_id: None, }, serde_json::json!({ "type": "cancel_turn", "status": status, }), ); } } /// Handle a `switch_model` control frame (Phase 3a, Option ii). /// /// Busy path: deliver `SwitchModel` over the in-flight task's oneshot — the /// task cancels the turn, sets `desired_model`, and requeues the batch so it /// re-runs on a fresh session under the new model. A catalog miss surfaces /// post-cancel via `create_session_and_apply_model` (the turn restarts on the /// unchanged model + an `unsupported_model` result). /// /// Idle path: validate against the cached catalog *before* invalidating /// (pre-cancel guard), then set `desired_model` + invalidate. The override /// takes visible effect on the agent's next turn. fn handle_switch_model_control( payload: &serde_json::Value, pool: &mut AgentPool, observer: Option<&observer::ObserverHandle>, ) { let Some(channel_id) = payload .get("channelId") .and_then(|value| value.as_str()) .and_then(|value| value.parse::().ok()) else { tracing::warn!("observer switch_model control frame missing valid channelId"); return; }; let Some(model_id) = payload.get("modelId").and_then(|value| value.as_str()) else { tracing::warn!("observer switch_model control frame missing modelId"); return; }; // A turn is in flight for this channel iff a task_map entry exists. The // agent is moved out of the pool during a turn, so the control oneshot is // the only reachable lever; an idle channel has no such entry. let turn_in_flight = pool .task_map() .values() .any(|m| m.channel_id == Some(channel_id)); let status = if turn_in_flight { // Busy path: deliver over the oneshot. `false` means the oneshot was // already consumed this turn (a prior cancel/interrupt) — the turn is // already ending, so the switch cannot land on it. if signal_in_flight_task( pool, channel_id, ControlSignal::SwitchModel(model_id.to_string()), ) { "sent" } else { "turn_ending" } } else { // Idle path: validate against the cached catalog before invalidating. match pool.switch_idle_agent_model(channel_id, model_id) { IdleSwitchResult::Switched => "switched", IdleSwitchResult::UnsupportedModel => "unsupported_model", IdleSwitchResult::NoIdleAgent => "no_active_turn", } }; if let Some(observer) = observer { observer.emit( "control_result", None, &observer::ObserverContext { channel_id: Some(channel_id.to_string()), session_id: None, turn_id: None, }, serde_json::json!({ "type": "switch_model", "status": status, "modelId": model_id, }), ); } } /// Maximum crashes in a 60-second window before a slot's circuit opens. const CIRCUIT_BREAKER_THRESHOLD: usize = 3; /// Window for circuit-breaker crash counting. const CIRCUIT_BREAKER_WINDOW: Duration = Duration::from_secs(60); /// Cooldown before a tripped circuit breaker allows a probe respawn. const CIRCUIT_BREAKER_COOLDOWN: Duration = Duration::from_secs(300); // 5 minutes /// Base backoff delay for respawn (doubles per recent crash, capped at 30s). const RESPAWN_BASE_DELAY: Duration = Duration::from_secs(1); /// Maximum respawn backoff delay. const RESPAWN_MAX_DELAY: Duration = Duration::from_secs(30); /// Per-slot circuit breaker state. /// /// `crash_times` holds timestamps of recent crashes within `CIRCUIT_BREAKER_WINDOW`. /// `open_until` is set when the threshold is hit; the circuit stays open until that /// instant, then allows one probe respawn (half-open). If the probe crashes, the /// circuit re-opens for another `CIRCUIT_BREAKER_COOLDOWN` period. /// /// All state transitions go through methods on this struct — callers never /// manipulate `crash_times` or `open_until` directly. struct SlotCircuit { crash_times: Vec, open_until: Option, /// True while a background respawn/refill task is in flight for this slot. /// Prevents duplicate spawns from maintenance ticks that fire before the /// previous spawn_and_init completes. respawn_in_flight: bool, } /// Result of [`SlotCircuit::record_crash`]. enum CrashVerdict { /// Respawn is allowed after sleeping for this duration (jittered backoff). Respawn(Duration), /// Circuit is open — do not respawn. CircuitOpen, /// Circuit was open but cooldown has elapsed — one probe respawn is allowed /// (no backoff sleep). If the probe crashes, the next `record_crash` will /// immediately re-open the circuit. HalfOpenProbe, } impl SlotCircuit { /// Record a crash and decide whether to respawn. /// /// This is the **single canonical path** for all crash → respawn decisions. /// Called by `respawn_agent_into`, `recover_panicked_agent`, and slot refill. fn record_crash(&mut self) -> CrashVerdict { let now = std::time::Instant::now(); // Half-open: cooldown elapsed → allow one probe. if let Some(open_until) = self.open_until { if now >= open_until { // Pre-seed crash_times to threshold-1 so that if the probe // itself crashes on the *next* call, the threshold is hit // immediately and the circuit re-opens. This implements a // "prove stability for one full window" policy. self.crash_times.clear(); for _ in 0..(CIRCUIT_BREAKER_THRESHOLD - 1) { self.crash_times.push(now); } self.open_until = None; return CrashVerdict::HalfOpenProbe; } else { return CrashVerdict::CircuitOpen; } } // Record this crash and prune old entries. self.crash_times.push(now); self.crash_times .retain(|&t| now.duration_since(t) < CIRCUIT_BREAKER_WINDOW); let recent = self.crash_times.len(); if recent >= CIRCUIT_BREAKER_THRESHOLD { self.open_until = Some(now + CIRCUIT_BREAKER_COOLDOWN); return CrashVerdict::CircuitOpen; } // Exponential backoff: 1s * 2^(recent-1), capped at 30s, with ±20% jitter. let base = RESPAWN_BASE_DELAY.saturating_mul(1u32 << (recent - 1).min(5)); let capped = base.min(RESPAWN_MAX_DELAY); let jitter = (std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .subsec_nanos() as f64) / 1_000_000_000.0; // 0.0..1.0 let factor = 0.8 + jitter * 0.4; // 0.8..1.2 CrashVerdict::Respawn(capped.mul_f64(factor)) } /// Mark a spawn failure — opens the circuit so the slot isn't retried /// on every heartbeat tick. Uses fresh `Instant::now()` so spawn latency /// doesn't shorten the effective cooldown. fn mark_spawn_failed(&mut self) { self.open_until = Some(std::time::Instant::now() + CIRCUIT_BREAKER_COOLDOWN); } /// Check if an empty slot can be refilled. Unlike `record_crash`, this /// does NOT record a new crash — it only checks whether the circuit /// allows a respawn attempt. /// /// Returns `true` if respawn is allowed. For half-open probes, pre-seeds /// crash_times so the next crash re-opens immediately. For normal refills /// (no circuit was ever opened), crash history is preserved so the breaker /// can still trip if the refilled agent crashes quickly. fn can_refill(&mut self) -> bool { let now = std::time::Instant::now(); match self.open_until { Some(open_until) => { if now >= open_until { // Half-open probe: pre-seed crash_times. self.crash_times.clear(); for _ in 0..(CIRCUIT_BREAKER_THRESHOLD - 1) { self.crash_times.push(now); } self.open_until = None; true } else { false // cooldown not elapsed } } None => true, // no circuit open — normal refill, preserve crash history } } } /// True if any slot has a respawn task in flight. Used to prevent premature /// "all agents dead" exits — a respawning agent may succeed in seconds. fn any_respawn_in_flight(crash_history: &[SlotCircuit]) -> bool { crash_history.iter().any(|s| s.respawn_in_flight) } /// Result of a background respawn task. struct RespawnResult { index: usize, /// Tuple: (initialized client, protocol version, supports_goose_steer). /// The third element is always `true` — the supervisor uses /// try-and-tolerate for the steer extension. result: Result<(AcpClient, u32, String)>, } /// Outcome of a non-cancelling steer attempt, forwarded from a per-attempt /// watcher task (which awaits the `SteerRequest.ack_tx` oneshot) back to /// the main loop's `select!`. The main loop drives queue side-effects from /// this — it cannot await the oneshot itself without blocking the relay /// stream. /// /// Carries enough identity to operate on the right withheld event in /// `EventQueue::withheld_native_steer`: `channel_id` is the routing key, /// `event_id` is the hex id of the single event the steer carried. struct SteerAckEvent { channel_id: Uuid, event_id: String, /// `Ok` if the read loop sent any of the locked `SteerAck` variants. /// `Err` if the oneshot was dropped without a send — should not happen /// under the current read-loop drains, but if it ever does the main /// loop treats it as `PromptCompletedNeutral` (release withheld, no /// fallback signal) to avoid leaking the withheld event. ack: std::result::Result, } /// RAII guard that ensures a `RespawnResult` is sent even if the task panics. /// Without this, a panicked respawn task would leave `respawn_in_flight = true` /// permanently, silently losing the slot forever. struct RespawnGuard { index: usize, tx: mpsc::Sender, sent: bool, } impl RespawnGuard { fn new(index: usize, tx: mpsc::Sender) -> Self { Self { index, tx, sent: false, } } /// Send the result and disarm the guard. Uses `try_send` (sync) so there /// is no await boundary between marking `sent` and actually enqueueing — /// cancellation cannot slip between the two. fn send(mut self, result: Result<(AcpClient, u32, String)>) { // Invariant: try_send succeeds because the channel capacity equals the // slot count, and respawn_in_flight guarantees at most one outstanding // result per slot. If this ever fails, the channel sizing or the // respawn_in_flight guard has drifted — that's a bug, not a transient. match self.tx.try_send(RespawnResult { index: self.index, result, }) { Ok(()) => self.sent = true, Err(e) => { tracing::error!( agent = self.index, "respawn result channel full or closed: {e}" ); // Drop will fire and send a failure result as fallback. } } } } impl Drop for RespawnGuard { fn drop(&mut self) { if !self.sent { tracing::error!( agent = self.index, "respawn task exited without sending result — sending failure" ); // Best-effort: try_send in Drop (can't await). let _ = self.tx.try_send(RespawnResult { index: self.index, result: Err(anyhow::anyhow!("respawn task panicked or was cancelled")), }); } } } // // Sync env-var propagation must run before the tokio runtime starts so that // any child processes inherit the correct environment. This must happen in the // sync entry point — `std::env::set_var` is only safe before tokio spawns // worker threads (Rust 2024 edition safety requirement). pub fn run() -> Result<()> { config::propagate_legacy_env_vars(); tokio_main() } #[tokio::main] async fn tokio_main() -> Result<()> { // Install the ring crypto provider for rustls (required for wss:// connections). rustls::crypto::ring::default_provider() .install_default() .expect("failed to install rustls crypto provider"); if is_subcommand("models") { // Strip the subcommand token so clap doesn't reject it as a positional. // Keeps argv[0] (binary name) and passes everything after the subcommand. let filtered: Vec = std::env::args() .enumerate() .filter(|(i, _)| *i != 1) .map(|(_, a)| a) .collect(); let args = ModelsArgs::parse_from(&filtered); return run_models(args).await; } if is_subcommand("auth-methods") { let filtered: Vec = std::env::args() .enumerate() .filter(|(i, _)| *i != 1) .map(|(_, a)| a) .collect(); let args = AuthMethodsArgs::parse_from(&filtered); return run_auth_methods(args).await; } if is_subcommand("authenticate") { let filtered: Vec = std::env::args() .enumerate() .filter(|(i, _)| *i != 1) .map(|(_, a)| a) .collect(); let args = AuthenticateArgs::parse_from(&filtered); return run_authenticate(args).await; } tracing_subscriber::fmt() .with_env_filter( EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("buzz_acp=info")), ) .compact() .init(); let mut config = Config::from_cli().map_err(|e| anyhow::anyhow!("configuration error: {e}"))?; // ── Setup-mode early branch ─────────────────────────────────────────────── // // When the desktop determines an agent is not ready (missing credentials, // model, or provider), it spawns buzz-acp with BUZZ_ACP_SETUP_PAYLOAD set. // We enter the minimal setup-listener path and never start the agent pool. if let Some(payload) = setup_mode::SetupPayload::from_env() .map_err(|e| anyhow::anyhow!("setup payload error: {e}"))? { tracing::info!("buzz-acp: setup payload present, entering setup-listener mode"); return setup_mode::run_setup_listener(config, payload).await; } tracing::info!("buzz-acp starting: {}", config.summary()); let observer = config .relay_observer .then(observer::ObserverHandle::in_process); if let Some(handle) = &observer { handle.emit( "harness_started", None, &observer::ObserverContext::default(), serde_json::json!({ "relayUrl": config.relay_url, "agentCommand": config.agent_command, "agentArgs": config.agent_args, "parallelism": config.agents, "relayObserver": config.relay_observer, }), ); } // // Finding #10: one agent failing to start must not kill the whole pool. // We attempt each spawn under a 60-second timeout; failures are logged and // skipped. If ALL agents fail we return an error. A partial pool is valid — // the harness continues with reduced capacity and logs a warning. let mut agent_slots: Vec> = Vec::with_capacity(config.agents as usize); for i in 0..config.agents as usize { // Spawn OUTSIDE the timeout so we always own the child for cleanup. // This matches the run_models pattern and prevents zombie leaks on // init timeout (the cancelled future would drop the AcpClient via // Drop which is best-effort only). let spawn_result = AcpClient::spawn( &config.agent_command, &config.agent_args, &config.persona_env_vars, config.has_generated_codex_config, ) .await; match spawn_result { Ok(mut acp) => { acp.set_observer(observer.clone(), i); match tokio::time::timeout(Duration::from_secs(60), acp.initialize()).await { Ok(Ok(init_result)) => { tracing::info!(agent = i, "agent initialized: {init_result}"); let protocol_version = init_result["protocolVersion"].as_u64().unwrap_or(1) as u32; tracing::info!( agent = i, name = init_result .get("agentInfo") .or_else(|| init_result.get("serverInfo")) .and_then(|info| info.get("name")) .and_then(|v| v.as_str()) .unwrap_or("unknown"), "agent initialized — non-cancelling steer enabled (try-and-tolerate)" ); acp.observe( "agent_initialized", serde_json::json!({ "agentIndex": i, "initializeResult": init_result, }), ); let agent_name = normalized_agent_name(&init_result); agent_slots.push(Some(OwnedAgent { index: i, acp, state: SessionState::default(), model_capabilities: None, desired_model: config.model.clone(), model_overridden: false, agent_name, goose_system_prompt_supported: None, protocol_version, })); } Ok(Err(e)) => { tracing::error!(agent = i, "agent initialize failed: {e}"); acp.shutdown().await; agent_slots.push(None); } Err(_) => { tracing::error!(agent = i, "agent timed out during init (60s)"); acp.shutdown().await; agent_slots.push(None); } } } Err(e) => { tracing::error!(agent = i, "agent failed to spawn: {e}"); agent_slots.push(None); } } } let live_count = agent_slots.iter().filter(|s| s.is_some()).count(); if live_count == 0 { return Err(anyhow::anyhow!( "all {} agents failed to start — cannot continue", config.agents )); } if live_count < config.agents as usize { tracing::warn!( "started {}/{} agents — continuing with reduced pool", live_count, config.agents ); } tracing::info!("agent_pool_ready agents={}", live_count); let mut pool = AgentPool::from_slots(agent_slots); // // Finding #22: capture a startup watermark BEFORE connecting to the relay. // This timestamp is used for membership notification replay (via // startup_watermark) and as the initial subscribe_since for channels // discovered at startup. The Subscribe handler falls back to // subscribe_since when last_seen is None, closing the blind spot // between "agents ready" and "first REQ sent". let startup_watermark: u64 = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_secs(); let pubkey_hex = config.keys.public_key().to_hex(); // Parse BUZZ_AUTH_TAG into a nostr::Tag for NIP-OA relay membership delegation. let relay_auth_tag: Option = std::env::var("BUZZ_AUTH_TAG") .ok() .filter(|s| !s.is_empty()) .and_then(|s| buzz_sdk::nip_oa::parse_auth_tag(&s).ok()); let mut relay = HarnessRelay::connect(&config.relay_url, &config.keys, &pubkey_hex, relay_auth_tag) .await .map_err(|e| anyhow::anyhow!("relay connect error: {e}"))?; // Finding #22: tell the relay background task the watermark so it can use // `since = watermark - 5s` on the first REQ instead of `since=now`. // Best-effort: a failure here is non-fatal (we just lose the startup window // protection, which is the same as the pre-fix behaviour). if let Err(e) = relay.set_startup_watermark(startup_watermark).await { tracing::warn!("failed to set startup watermark: {e}"); } tracing::info!("connected to relay at {}", config.relay_url); relay .subscribe_membership_notifications() .await .map_err(|e| anyhow::anyhow!("membership notification subscribe error: {e}"))?; tracing::info!("subscribed to membership notifications"); let presence_publisher = relay.event_publisher(); let presence_keys = config.keys.clone(); // Priority: BUZZ_AUTH_TAG (NIP-OA attestation) → --agent-owner flag. let startup_owner: Option = resolve_agent_owner(&config); if let Some(ref owner) = startup_owner { tracing::info!("agent owner: {owner}"); } else { tracing::info!("no agent owner configured"); } // Warn if owner-dependent mode but no owner resolved yet. if startup_owner.is_none() { match &config.respond_to { RespondTo::OwnerOnly => { tracing::warn!( "respond-to=owner-only but no owner is set — all events will be \ dropped. Set BUZZ_AUTH_TAG or --agent-owner, or use --respond-to=anyone." ); } RespondTo::Allowlist => { tracing::warn!( "respond-to=allowlist but no owner is set — allowlisted pubkeys \ will still be accepted, but owner-based matching is unavailable \ until owner is resolved." ); } _ => {} // anyone/nobody don't depend on owner } } let owner_cache = OwnerCache::new(startup_owner.clone()); let mut relay_observer_control_rx = None; let mut relay_observer_publisher_task = None; let mut relay_observer_publisher = None; if config.relay_observer { if let (Some(observer), Some(owner_pubkey_hex)) = (observer.clone(), owner_cache.pubkey.clone()) { match PublicKey::from_hex(&owner_pubkey_hex) { Ok(owner_pubkey) => { relay_observer_publisher = Some(( observer, relay.event_publisher(), config.keys.clone(), pubkey_hex.clone(), owner_pubkey_hex, owner_pubkey, )); relay .subscribe_observer_controls() .await .map_err(|e| anyhow::anyhow!("observer control subscribe error: {e}"))?; relay_observer_control_rx = relay.take_observer_control_rx(); tracing::info!("relay observer enabled"); } Err(error) => { tracing::warn!("relay observer disabled: invalid owner pubkey: {error}"); } } } else { tracing::warn!( "relay observer requested but no agent owner was resolved at startup; \ observer frames will not be published" ); } } let channel_info_map = relay .discover_channels() .await .map_err(|e| anyhow::anyhow!("channel discovery error: {e}"))?; tracing::info!("discovered {} channel(s)", channel_info_map.len()); let channel_ids: Vec = channel_info_map.keys().copied().collect(); let rules: Vec = match config.subscribe_mode { SubscribeMode::Mentions => { vec![SubscriptionRule { name: "mentions".into(), channels: filter::ChannelScope::All("all".into()), kinds: config.kinds_override.clone().unwrap_or_else(|| { vec![ KIND_STREAM_MESSAGE, KIND_WORKFLOW_APPROVAL_REQUESTED, KIND_STREAM_REMINDER, ] }), require_mention: !config.no_mention_filter, filter: None, compiled_filter: None, consecutive_timeouts: std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0)), prompt_tag: Some("@mention".into()), }] } SubscribeMode::All => { vec![SubscriptionRule { name: "all".into(), channels: filter::ChannelScope::All("all".into()), kinds: config.kinds_override.clone().unwrap_or_default(), require_mention: false, filter: None, compiled_filter: None, consecutive_timeouts: std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0)), prompt_tag: Some("all".into()), }] } SubscribeMode::Config => { // load_rules() already warns if the config file has zero rules. config::load_rules(&config.config_path)? } }; let channel_filters = config::resolve_channel_filters(&config, &channel_ids, &rules); if channel_filters.is_empty() { tracing::warn!("no channel subscriptions resolved — agent will sit idle"); } let mut subscribed_channel_ids = HashSet::with_capacity(channel_filters.len()); for (channel_id, filter) in &channel_filters { if let Err(e) = relay.subscribe_channel(*channel_id, filter.clone()).await { tracing::warn!("failed to subscribe to channel {channel_id}: {e}"); } else { subscribed_channel_ids.insert(*channel_id); tracing::info!("subscribed to channel {channel_id}"); } } if let Some((observer, publisher, keys, agent_pubkey, owner_pubkey, owner)) = relay_observer_publisher.take() { relay_observer_publisher_task = Some(spawn_relay_observer_publisher( observer, publisher, keys, agent_pubkey, owner_pubkey, owner, )); } // Online means the harness can receive work, not merely that its socket is // connected. Publishing after channel subscriptions gives desktop callers // a durable readiness boundary before they send a startup mention. if config.presence_enabled { match publish_presence(&presence_publisher, &presence_keys, "online").await { Ok(_) => tracing::info!("presence set to online"), Err(e) => tracing::warn!("failed to set initial presence: {e}"), } } let dedup_mode = config.dedup_mode; let mut queue = EventQueue::new(dedup_mode).with_in_flight_deadline(config.max_turn_duration_secs); let base_prompt_content = config.base_prompt_content.take(); let ctx = Arc::new(PromptContext { mcp_servers: build_mcp_servers(&config), initial_message: config.initial_message.clone(), idle_timeout: Duration::from_secs(config.idle_timeout_secs), max_turn_duration: Duration::from_secs(config.max_turn_duration_secs), turn_liveness_interval: Duration::from_secs(config.turn_liveness_secs), dedup_mode: config.dedup_mode, system_prompt: config.system_prompt.clone(), team_instructions: config.team_instructions.clone(), base_prompt: if config.no_base_prompt { None } else if let Some(content) = base_prompt_content { Some(Box::leak(content.into_boxed_str())) } else { Some(include_str!("base_prompt.md")) }, heartbeat_prompt: config.heartbeat_prompt.clone(), cwd: std::env::current_dir() .unwrap_or_else(|_| std::path::PathBuf::from("/")) .to_string_lossy() .to_string(), rest_client: relay.rest_client(), channel_info: channel_info_map, context_message_limit: config.context_message_limit, max_turns_per_session: config.max_turns_per_session, permission_mode: config.permission_mode, agent_keys: config.keys.clone(), agent_owner_pubkey: startup_owner .as_deref() .and_then(|hex| nostr::PublicKey::from_hex(hex).ok()), memory_enabled: config.memory_enabled, harness_name: crate::config::normalize_agent_command_identity(&config.agent_command), }); if !config.memory_enabled { tracing::info!( target: "engram::core", "NIP-AE core memory injection disabled (re-enable by removing --no-memory / BUZZ_ACP_NO_MEMORY)" ); } let mut heartbeat = if config.heartbeat_interval_secs > 0 { let interval = Duration::from_secs(config.heartbeat_interval_secs); Some(tokio::time::interval_at( tokio::time::Instant::now() + interval, interval, )) } else { None }; let mut heartbeat_in_flight = false; let mut presence_heartbeat = if config.presence_enabled { let interval = Duration::from_secs(60); Some(tokio::time::interval_at( tokio::time::Instant::now() + interval, interval, )) } else { None }; let mut typing_refresh = if config.typing_enabled { let interval = Duration::from_secs(3); Some(tokio::time::interval_at( tokio::time::Instant::now() + interval, interval, )) } else { None }; let mut typing_channels: HashMap = HashMap::new(); let mut presence_task: Option> = None; // Runs at the TOP of every loop iteration via Instant check — cannot be // starved by the biased select. Slot refill spawns background tasks so // spawn_and_init never blocks the main loop. let maintenance_interval = Duration::from_secs(30); let mut last_maintenance = std::time::Instant::now(); // Channel for background respawn tasks to return completed agents. // Bounded to agent count — at most one respawn per slot in flight. let (respawn_tx, mut respawn_rx) = mpsc::channel::(config.agents as usize); // JoinSet for respawn tasks so shutdown can abort them. let mut respawn_tasks: tokio::task::JoinSet<()> = tokio::task::JoinSet::new(); // Channel for non-cancelling steer ack watchers to forward outcomes back // to the main loop. Each `pool.send_steer(...) == Ok(())` spawns a // short-lived task that awaits the `SteerRequest.ack_tx` oneshot and // forwards a `SteerAckEvent`. Unbounded because: // 1. The producer count is bounded by in-flight goose turns // (`agents` slots, capacity-1 `steer_tx` each), so the channel // cannot legitimately back up under steady state. // 2. We must never drop a steer outcome — losing an ack would leak a // withheld event in `EventQueue::withheld_native_steer` until // `IN_FLIGHT_DEADLINE_SECS` expires. let (steer_ack_tx, mut steer_ack_rx) = mpsc::unbounded_channel::(); // ── Step 7: Shutdown signal ─────────────────────────────────────────────── let (shutdown_tx, mut shutdown_rx) = watch::channel(()); let tx = shutdown_tx.clone(); tokio::spawn(async move { tokio::signal::ctrl_c().await.ok(); let _ = tx.send(()); }); #[cfg(unix)] { let tx = shutdown_tx.clone(); tokio::spawn(async move { use tokio::signal::unix::{signal, SignalKind}; let mut sigterm = signal(SignalKind::terminate()).expect("SIGTERM handler"); sigterm.recv().await; let _ = tx.send(()); }); } // Track the newest membership notification timestamp per channel. // On reconnect the relay replays events newest-first, so the first event // per channel is authoritative. Any later event with ts < newest is stale. // Exact duplicates (same event ID) are caught by seen_membership_ids. // // Uses strict `<` (not `<=`) so that legitimate live events at the same // second are both processed. The seen_membership_ids set handles exact // replays that share the same timestamp. let mut membership_newest_ts: HashMap = HashMap::new(); // Two-generation dedup for membership event replays (bounded, no amnesia). // Rotates at 1000 entries instead of clearing the entire set at 2000. let mut seen_membership_current: HashSet = HashSet::new(); let mut seen_membership_previous: HashSet = HashSet::new(); // Channels the agent has been removed from. When a checked-out agent is // returned to the pool, its sessions for these channels are stripped, and // failed/panicked batches for these channels are dropped instead of requeued. // // Cleared on re-add (KIND_MEMBER_ADDED_NOTIFICATION) so re-joined channels // regain session affinity. // // Known limitation: if a batch is in-flight when the channel is removed AND // re-added before the batch returns, the stale batch may be requeued. This // is acceptable because: (a) the agent is a member again and has access, // (b) the events are from the agent's authorized history, (c) the window // is extremely narrow (membership changes are rare, prompt turns are seconds), // and (d) fixing this would require per-channel epoch tracking on TaskMeta // and PromptResult — significant complexity for a benign edge case. If strict // causal invalidation is needed, add a monotonic epoch counter per channel // and capture it in TaskMeta at dispatch time. let mut removed_channels: HashSet = HashSet::new(); // // One SlotCircuit per agent slot. crash_times entries are pruned to the last // CIRCUIT_BREAKER_WINDOW on each respawn attempt. The Vec is indexed by // agent slot index, so it must be sized to the configured pool capacity // (not the live count, which may be smaller after partial startup). let mut crash_history: Vec = (0..config.agents as usize) .map(|_| SlotCircuit { crash_times: Vec::new(), open_until: None, respawn_in_flight: false, }) .collect(); // // Branches 1 & 2 both need to borrow `pool`, but they access different // fields (result_rx vs join_set). We use `rx_and_join_set()` to split the // borrow, yielding a typed enum so the outer code can dispatch cleanly. enum PoolEvent { Result(Box), Panic(tokio::task::JoinError), SteerAck(SteerAckEvent), } loop { if last_maintenance.elapsed() >= maintenance_interval { last_maintenance = std::time::Instant::now(); queue.compact_expired_state(); // Slot refill: spawn background tasks for empty slots whose // circuit breaker allows it. spawn_and_init runs off the main // loop so it never blocks event processing. for (idx, slot) in crash_history.iter_mut().enumerate() { if pool.slot_alive(idx) || slot.respawn_in_flight { continue; } if !slot.can_refill() { continue; } slot.respawn_in_flight = true; tracing::info!(agent = idx, "slot refill: spawning background respawn"); let cmd = config.agent_command.clone(); let args = config.agent_args.clone(); let env = config.persona_env_vars.clone(); let has_codex = config.has_generated_codex_config; let observer = observer.clone(); let guard = RespawnGuard::new(idx, respawn_tx.clone()); respawn_tasks.spawn(async move { let result = spawn_and_init(&cmd, &args, &env, has_codex, idx, observer).await; guard.send(result); }); } // Flush requeued batches whose retry_after has expired. Without // this, a batch requeued during crash recovery can sit idle // indefinitely on quiet channels — dispatch_pending is only // called on relay events or pool results, neither of which // arrive when the channel is silent. if queue.has_flushable_work() { for (channel_id, thread_tags) in dispatch_pending(&mut pool, &mut queue, &ctx) { typing_channels.insert(channel_id, thread_tags); } } } let mut respawn_collected = false; while let Ok(rr) = respawn_rx.try_recv() { crash_history[rr.index].respawn_in_flight = false; match rr.result { Ok((acp, protocol_version, agent_name)) => { let agent = OwnedAgent { index: rr.index, acp, state: SessionState::default(), model_capabilities: None, desired_model: config.model.clone(), model_overridden: false, agent_name, goose_system_prompt_supported: None, protocol_version, }; pool.return_agent(agent); tracing::info!(agent = rr.index, "respawn complete"); respawn_collected = true; } Err(e) => { crash_history[rr.index].mark_spawn_failed(); tracing::warn!(agent = rr.index, "respawn failed: {e} — circuit re-opened"); } } } // Flush requeued events that were waiting for a live agent. Without // this, batches requeued during crash recovery sit idle until the // next relay event arrives — which can be minutes on quiet channels. if respawn_collected { for (channel_id, thread_tags) in dispatch_pending(&mut pool, &mut queue, &ctx) { typing_channels.insert(channel_id, thread_tags); } } // Borrow result_rx and join_set simultaneously via split-borrow helper. let pool_event: Option = { let (result_rx, join_set) = pool.rx_and_join_set(); tokio::select! { biased; // Finding #24: recv() returning None means all senders dropped // (pool was torn down). Break cleanly instead of panicking. r = result_rx.recv() => match r { Some(result) => Some(PoolEvent::Result(Box::new(result))), None => { tracing::info!("result channel closed — exiting main loop"); break; } }, // Guard: join_next() returns None immediately when JoinSet is // empty, which would cause a tight spin. Only poll when there // are in-flight tasks. Some(Err(e)) = join_set.join_next(), if !join_set.is_empty() => { Some(PoolEvent::Panic(e)) } // Goose-native steer ack from a watcher task. Outcomes drive // queue side-effects (drop / release withheld event) and // optionally the cancel+merge fallback signal. See the // `Some(PoolEvent::SteerAck(...))` match arm below for the // locked semantics (Eva + Max + Perci). Some(ack_event) = steer_ack_rx.recv() => { Some(PoolEvent::SteerAck(ack_event)) } control_event = async { match relay_observer_control_rx.as_mut() { Some(rx) => rx.recv().await, None => std::future::pending().await, } } => { let _ = result_rx; match control_event { Some(event) => { if let Some(ref owner_hex) = owner_cache.pubkey { handle_relay_observer_control_event(&config.keys, event, &mut pool, observer.as_ref(), owner_hex); } else { tracing::warn!("observer control frame received but no owner resolved — dropping"); } } None => { relay_observer_control_rx = None; tracing::warn!("relay observer control channel closed"); } } None } // Remaining branches don't touch pool — evaluated when pool is idle. buzz_event = relay.next_event() => { let _ = result_rx; // end split borrow before relay handling match buzz_event { Some(buzz_event) => { let kind_u32 = buzz_event.event.kind.as_u16() as u32; if kind_u32 == KIND_MEMBER_ADDED_NOTIFICATION || kind_u32 == KIND_MEMBER_REMOVED_NOTIFICATION { let ch = buzz_event.channel_id; let ts = buzz_event.event.created_at.as_secs(); let eid = buzz_event.event.id.to_hex(); // Two-layer membership dedup: // // 1. Exact duplicate rejection (seen_membership_ids): // Catches the same event replayed on reconnect. // // 2. Timestamp watermark (membership_newest_ts): // Uses strict `<` so that older events from reconnect // replay are dropped, but legitimate live events at the // same second are both processed. This is safe because // exact duplicates are already caught by layer 1. // // Why not `<=`? That would suppress legitimate live // add→remove (or remove→add) sequences in the same second, // leaving the harness in the wrong membership state. // Two-generation dedup: check both sets before inserting. if seen_membership_current.contains(&eid) || seen_membership_previous.contains(&eid) { tracing::debug!( channel_id = %ch, kind = kind_u32, "skipping duplicate membership notification (same event_id)" ); continue; } seen_membership_current.insert(eid); // Rotate at 1000: current → previous, no amnesia window. if seen_membership_current.len() >= 1000 { seen_membership_previous = std::mem::take(&mut seen_membership_current); } if let Some(&newest) = membership_newest_ts.get(&ch) { if ts < newest { tracing::debug!( channel_id = %ch, kind = kind_u32, ts, newest, "skipping stale membership notification (older than newest)" ); continue; } } membership_newest_ts.insert(ch, ts); if kind_u32 == KIND_MEMBER_ADDED_NOTIFICATION { // Clear removal tracking so sessions are not // stripped for a legitimately re-added channel. removed_channels.remove(&ch); if subscribed_channel_ids.contains(&ch) { tracing::debug!(channel_id = %ch, "membership notification: channel already subscribed"); } else if let Some(filter) = config::resolve_dynamic_channel_filter(&config, ch, &rules) { tracing::info!(channel_id = %ch, "membership notification: subscribing to new channel"); if let Err(e) = relay.subscribe_channel_from(ch, filter, Some(ts)).await { tracing::warn!("failed to subscribe to new channel {ch}: {e}"); } else { subscribed_channel_ids.insert(ch); } } else { tracing::debug!(channel_id = %ch, "membership notification: no matching rules — skipping"); } } else { subscribed_channel_ids.remove(&ch); tracing::info!(channel_id = %ch, "membership notification: unsubscribing from channel"); if let Err(e) = relay.unsubscribe_channel(ch).await { tracing::warn!("failed to unsubscribe from channel {ch}: {e}"); } // Drain queued events and invalidate sessions for the // removed channel. Events already in-flight will // complete normally (the relay may reject actions if // the agent lost access). let drained_ids = queue.drain_channel(ch); let invalidated = pool.invalidate_channel_sessions(ch); // Track removed channels so checked-out agents get // their sessions stripped when they return to the pool. removed_channels.insert(ch); typing_channels.remove(&ch); // Best-effort: clean up 👀 on drained events. // Note: the relay revokes membership before // emitting the notification, so this DELETE may // 403 on non-open channels. Stale 👀 in that // case is a known limitation — fix belongs in // the relay (clean up bot reactions on removal). if !drained_ids.is_empty() { let rc = ctx.rest_client.clone(); let ids = drained_ids.clone(); tokio::spawn(async move { for eid in &ids { pool::reaction_remove(&rc, eid, "👀").await; } }); } if !drained_ids.is_empty() || invalidated > 0 { tracing::info!( channel_id = %ch, drained = drained_ids.len(), invalidated, "cleaned up after membership removal" ); } } continue; } if config.ignore_self && buzz_event.event.pubkey.to_hex() == pubkey_hex { tracing::debug!(channel_id = %buzz_event.channel_id, "dropping self-authored event"); continue; } // Check: kind:9, content "!shutdown", from owner, mentions THIS agent. let is_shutdown = is_owner_control_command( &buzz_event.event, kind_u32, "!shutdown", &pubkey_hex, ); if is_shutdown { let owner = owner_cache.get(); if let Some(owner) = owner { if buzz_event.event.pubkey.to_hex() == *owner { tracing::info!( channel_id = %buzz_event.channel_id, sender = %buzz_event.event.pubkey.to_hex(), "shutdown command from owner — exiting gracefully" ); let _ = shutdown_tx.send(()); continue; } } // Not from owner — fall through to normal prompt handling. // Don't drop it — it's a regular message that happens to // contain "!shutdown" from a non-owner. } // Mirrors !shutdown: kind:9, content "!cancel", from // owner, mentions THIS agent. Must be BEFORE // queue.push() — the event content is moved by push. // // Mode-independent: !cancel fires regardless of // --multiple-event-handling. It is explicit user // intent, not an automatic policy decision. let is_cancel = is_owner_control_command( &buzz_event.event, kind_u32, "!cancel", &pubkey_hex, ); if is_cancel { if let Some(owner) = owner_cache.get() { if buzz_event.event.pubkey.to_hex() == *owner { let fired = signal_in_flight_task( &mut pool, buzz_event.channel_id, ControlSignal::Cancel, ); if !fired { tracing::warn!( channel_id = %buzz_event.channel_id, "!cancel received but no in-flight task — no-op" ); } continue; // consume event — do NOT push to queue } } // Not from owner — fall through to normal prompt handling. } // Mirrors !shutdown / !cancel: kind:9, content // "!rotate", from owner, mentions THIS agent. // // Rotation is explicit owner intent to start the // next turn in this channel with a fresh ACP // session. It is consumed by the harness and never // forwarded to the agent. If a turn is in-flight, // cancel it, drop its triggering batch, and // invalidate the channel session when the task // returns. If idle, invalidate the cached channel // session immediately. Queued future events remain // queued and will create a fresh session on dispatch. let is_rotate = is_owner_control_command( &buzz_event.event, kind_u32, "!rotate", &pubkey_hex, ); if is_rotate { if let Some(owner) = owner_cache.get() { if buzz_event.event.pubkey.to_hex() == *owner { let fired = signal_in_flight_task( &mut pool, buzz_event.channel_id, ControlSignal::Rotate, ); if fired { tracing::info!( channel_id = %buzz_event.channel_id, "!rotate received — cancelling in-flight turn and rotating session" ); } else { let invalidated = pool.invalidate_channel_sessions(buzz_event.channel_id); tracing::info!( channel_id = %buzz_event.channel_id, invalidated, "!rotate received — invalidated idle channel session(s)" ); } continue; // consume event — do NOT push to queue } } // Not from owner — fall through to normal prompt handling. } // Coarse security policy: drop events from disallowed // authors before they reach subscription rules or the // agent. Must be AFTER !shutdown (owner can always // shut down regardless of gate mode). // // Both OwnerOnly and Allowlist accept events from // "siblings" — pubkeys whose agent_owner_pubkey // matches this agent's owner (e.g. other bots // launched by the same human). Allowlist adds the // explicit pubkey list on top, for external people; // it never revokes same-owner team bots. { let author = buzz_event.event.pubkey.to_hex(); let allowed = author_allowed( &config.respond_to, &config.respond_to_allowlist, &author, &owner_cache, &ctx.rest_client, ) .await; if !allowed { tracing::debug!( channel_id = %buzz_event.channel_id, author = %buzz_event.event.pubkey.to_hex(), mode = %config.respond_to, "inbound author gate — dropping event" ); continue; } } let matched = filter::match_event(&buzz_event.event, buzz_event.channel_id, &rules, &pubkey_hex).await; let prompt_tag = match matched { Some(m) => m.prompt_tag, None => { tracing::debug!(channel_id = %buzz_event.channel_id, kind = buzz_event.event.kind.as_u16(), "event matched no rule — dropping"); continue; } }; // Capture author pubkey before queue.push() moves // buzz_event.event (needed for mode gate below). let author_hex = buzz_event.event.pubkey.to_hex(); let event_id_hex = buzz_event.event.id.to_hex(); // Clone for the non-cancelling steer fork, which // needs the event to render the steer body. The // clone is unconditional because we don't know // yet whether the mode gate will demand a steer // — checking `multiple_event_handling` here // would couple the queueing path to the mode // and break the existing invariant that every // accepted event goes through `queue.push` // first. `nostr::Event::clone` is cheap (Arc- // backed payload) so the cost is negligible. let event_for_steer = buzz_event.event.clone(); let prompt_tag_for_steer = prompt_tag.clone(); let accepted = queue.push(QueuedEvent { channel_id: buzz_event.channel_id, event: buzz_event.event, received_at: std::time::Instant::now(), prompt_tag, }); // 👀 — immediate "seen" reaction, only if the event // was actually queued (not dropped by DedupMode::Drop). // Fire-and-forget: on rare fast-failure paths the // guard's cleanup may race with this add, leaving a // cosmetic stale 👀. Acceptable — see ReactionGuard docs. if accepted { let rc = ctx.rest_client.clone(); let eid = event_id_hex.clone(); tokio::spawn(async move { pool::reaction_add(&rc, &eid, "👀").await; }); } // Event is already queued. If mode requires it AND // the channel has an in-flight task, fire cancel — // OR take the non-cancelling (ACP steer) fork for Steer signals. if accepted && queue.is_channel_in_flight(buzz_event.channel_id) { // Author eligibility (owner ∪ allowlist ∪ siblings) // is already enforced by the inbound author gate // above, so the mid-turn signal fires for every // event that reaches here. let signal = mode_gate_signal( config.multiple_event_handling, &author_hex, owner_cache.get(), ); if let Some(signal) = signal { // Try-and-tolerate fork: when the mode // wants a Steer, attempt the non-cancelling // path first for any agent. On accept, // withhold the queued event and spawn an // ack watcher; the main loop's // `PoolEvent::SteerAck` arm decides // success/release/fallback. On reject // (including `-32601 method_not_found` // from agents that don't implement the // extension), fall through to the universal // cancel+merge `Steer` signal so the event // still reaches the agent. let native_attempted = matches!(signal, ControlSignal::Steer) && try_native_steer( &mut pool, &mut queue, buzz_event.channel_id, event_for_steer, prompt_tag_for_steer, &steer_ack_tx, ); if !native_attempted { signal_in_flight_task( &mut pool, buzz_event.channel_id, signal, ); } } } for (channel_id, thread_tags) in dispatch_pending(&mut pool, &mut queue, &ctx) { typing_channels.insert(channel_id, thread_tags); } } None => { tracing::warn!("relay event stream ended — requesting reconnect"); if let Err(e) = relay.reconnect().await { tracing::error!("relay background task is gone: {e} — exiting"); tokio::time::sleep(Duration::from_secs(1)).await; break; } } } None } _ = async { match heartbeat.as_mut() { Some(hb) => hb.tick().await, None => std::future::pending().await, } } => { let _ = result_rx; if queue.has_flushable_work() { tracing::debug!("heartbeat_skipped_events"); for (channel_id, thread_tags) in dispatch_pending(&mut pool, &mut queue, &ctx) { typing_channels.insert(channel_id, thread_tags); } } else if pool.any_idle() { dispatch_heartbeat(&mut pool, &ctx, &mut heartbeat_in_flight); } else { tracing::debug!("heartbeat_skipped_busy"); } None } _ = async { match presence_heartbeat.as_mut() { Some(t) => t.tick().await, None => std::future::pending().await, } } => { let _ = result_rx; // Abort previous heartbeat if still in flight (prevents race on shutdown). if let Some(h) = presence_task.take() { h.abort(); } let pp = presence_publisher.clone(); let pk = presence_keys.clone(); presence_task = Some(tokio::spawn(async move { if let Err(e) = publish_presence(&pp, &pk, "online").await { tracing::warn!("presence heartbeat failed: {e}"); } })); None } _ = async { match typing_refresh.as_mut() { Some(t) => t.tick().await, None => std::future::pending().await, } } => { let _ = result_rx; // Use try_publish (non-blocking) for typing indicators — // they're ephemeral and must not block the main loop during // relay reconnection (#35). for (&ch, thread_tags) in &typing_channels { if let Ok(event) = relay.build_typing_event( ch, thread_tags.root_event_id.as_deref(), thread_tags.parent_event_id.as_deref(), ) { if let Err(e) = relay.try_publish_event(event) { tracing::debug!("typing indicator dropped for {ch}: {e}"); } } } None } _ = shutdown_rx.changed() => { tracing::info!("shutting down"); break; } } }; match pool_event { Some(PoolEvent::Result(result)) => { // Stop typing indicator for the completed channel. if let PromptSource::Channel(ch) = &result.source { typing_channels.remove(ch); } if handle_prompt_result( &mut pool, &mut queue, &config, *result, &mut heartbeat_in_flight, &removed_channels, &mut crash_history, &respawn_tx, &mut respawn_tasks, observer.clone(), Some(&ctx.rest_client), ) == LoopAction::Exit { break; } if drain_ready_join_results( &mut pool, &mut queue, &config, &mut heartbeat_in_flight, &removed_channels, &mut typing_channels, &mut crash_history, &respawn_tx, &mut respawn_tasks, observer.clone(), ) == LoopAction::Exit { break; } for (channel_id, thread_tags) in dispatch_pending(&mut pool, &mut queue, &ctx) { typing_channels.insert(channel_id, thread_tags); } } Some(PoolEvent::Panic(join_error)) => { tracing::error!("agent task panicked: {join_error}"); recover_panicked_agent( &mut pool, &mut queue, &config, join_error, &mut heartbeat_in_flight, &removed_channels, &mut typing_channels, &mut crash_history, &respawn_tx, &mut respawn_tasks, observer.clone(), ); if pool.live_count() == 0 && !any_respawn_in_flight(&crash_history) { tracing::error!("all agents dead — exiting"); break; } for (channel_id, thread_tags) in dispatch_pending(&mut pool, &mut queue, &ctx) { typing_channels.insert(channel_id, thread_tags); } } Some(PoolEvent::SteerAck(SteerAckEvent { channel_id, event_id, ack, })) => { // Goose-native steer attempt resolved. Locked semantics // (Eva + Max + Perci, unanimous on Option X): // // Success // The agent received the steer via the non-cancelling // path. Drop the withheld event so normal dispatch // never redelivers it. // // Err(_) where the write never landed (Transport / // ExpectedRunIdMissing): // Delivery state of the underlying message is "never // attempted on the wire". Release withheld back to the // queue front AND issue the cancel+merge fallback so // the message still reaches the agent. // // Err(AgentError { code: -32601, .. }) // The agent returned method_not_found — it does not // implement the steer extension. Release withheld AND // fire the cancel+merge fallback so the message still // reaches the agent via the universal path. // // Err(AgentError { code: other, .. }) // The write landed and the agent returned a JSON-RPC // error at the application level (e.g. wrong run id). // The agent's turn is still running (or just completed). // Release withheld for normal dispatch; do NOT fire the // fallback signal — the agent already saw the steer // attempt. If the turn is still running, normal dispatch // re-delivers when it completes. If the turn already // ended, there is nothing to cancel. // // PromptCompletedNeutral // The read loop wrote the steer (or was preparing to) // but the prompt completed before the response landed. // Delivery state is unknown — but the prompt completing // means there is no in-flight turn to signal anymore. // Release withheld for normal dispatch; do NOT fire // the fallback signal (it would target a turn that // just ended; normal dispatch already handles // redelivery via the released queue entry). // // Err(PromptCompleted) // `SteerError::PromptCompleted` is returned synchronously // by `pool::send_steer` when no task is in flight (handled // in `try_native_steer`'s Err branch, which falls through // to cancel+merge). It is never routed through the ack // channel, so this variant never appears in `SteerAckEvent`. // // Watcher Err (oneshot dropped) // Should not happen — the read loop drains // pending_steer on every return path. If it does, // treat as PromptCompletedNeutral to avoid leaking // the withheld event in `withheld_native_steer`. let (release_withheld, drop_withheld, signal_fallback) = match &ack { Ok(pool::SteerAck::Success) => (false, true, false), // -32601 = method_not_found: agent does not implement the // steer extension. Fire cancel+merge so the message still // reaches the agent. Ok(pool::SteerAck::Err(pool::SteerError::AgentError { code, .. })) if *code == -32601 => { (true, false, true) } // AgentError: write landed, agent rejected it at the // application level (e.g. wrong run id). Release for // normal dispatch; no fallback signal (the turn is still // running or just ended — either way there is nothing to // cancel). Ok(pool::SteerAck::Err(pool::SteerError::AgentError { .. })) => { (true, false, false) } // Transport / ExpectedRunIdMissing: write never landed. // Release and fire the cancel+merge fallback so the // message still reaches the agent. Ok(pool::SteerAck::Err(_)) => (true, false, true), Ok(pool::SteerAck::PromptCompletedNeutral) => (true, false, false), Err(_recv_err) => (true, false, false), }; tracing::info!( channel = %channel_id, event_id = %event_id, ?ack, release_withheld, drop_withheld, signal_fallback, "non-cancelling steer ack received" ); if drop_withheld { queue.remove_event(channel_id, &event_id); } if release_withheld { queue.release_native_steer(channel_id, &event_id); } if signal_fallback { // Universal cancel+merge fallback. Note: the // queued event has already been released to the // front of `queues[channel_id]`, so the cancel // will pick it up as part of the merged batch and // re-prompt the agent. signal_in_flight_task(&mut pool, channel_id, ControlSignal::Steer); } // After releasing a withheld event, give dispatch a chance // to re-flush. If the prompt is still in flight, the // channel stays `in_flight_channels` and `flush_next` // skips it — but a Steer fallback signal sent above will // tear down the in-flight task; on its completion the // queue drains. We still try here in case the in-flight // task has already returned. for (channel_id, thread_tags) in dispatch_pending(&mut pool, &mut queue, &ctx) { typing_channels.insert(channel_id, thread_tags); } } None => {} // relay/heartbeat/shutdown branches handled inline above } } tracing::info!("shutdown: waiting for in-flight prompts"); // 30 s is generous for in-flight prompts to be cancelled; using // max_turn_duration here would cause Ctrl+C to hang for up to an hour. let grace = Duration::from_secs(30); // Best-effort drain of both join_set and result_rx during the grace period. // Tasks that finish normally send their OwnedAgent through result_rx — we // explicitly shut them down here to reap child processes. If the grace // period expires, remaining tasks are aborted and fall back to // AcpClient::Drop (start_kill + try_wait — best-effort, not guaranteed). let (rx_ref, js_ref) = pool.rx_and_join_set(); let shutdown_result = tokio::time::timeout(grace, async { loop { tokio::select! { result = js_ref.join_next() => { match result { Some(Err(e)) => tracing::warn!("task error during shutdown: {e}"), Some(Ok(())) => {} None => break, // join_set empty } } maybe_result = rx_ref.recv() => { if let Some(mut pr) = maybe_result { let idx = pr.agent.index; pr.agent.acp.shutdown().await; tracing::debug!(agent = idx, "reaped checked-out agent on shutdown"); } // If None, channel closed — tasks are done. } } } }) .await; if shutdown_result.is_err() { tracing::warn!("grace period expired, aborting remaining tasks"); pool.join_set.shutdown().await; } // Drain any remaining results that arrived after join_set drained but // before tasks were aborted. while let Ok(mut pr) = pool.result_rx_try_recv() { let idx = pr.agent.index; pr.agent.acp.shutdown().await; tracing::debug!(agent = idx, "reaped late-arriving agent on shutdown"); } // Explicitly shut down idle agents still sitting in their slots. for slot in pool.agents_mut().iter_mut() { if let Some(agent) = slot.take() { let idx = agent.index; let mut acp = agent.acp; acp.shutdown().await; tracing::debug!(agent = idx, "reaped idle agent on shutdown"); } } drop(pool); // Abort any in-flight respawn tasks. They may be sleeping in backoff or // running spawn_and_init — either way, we don't want them spawning new // children after the main loop has exited. RespawnGuard::Drop sends a // failure result for aborted tasks, so respawn_in_flight is cleared. respawn_tasks.shutdown().await; // Drain any respawn results that completed before the abort. Explicitly // shut down returned agents instead of relying on AcpClient::Drop. while let Ok(rr) = respawn_rx.try_recv() { if let Ok((mut acp, _, _)) = rr.result { acp.shutdown().await; tracing::debug!(agent = rr.index, "reaped respawned agent on shutdown"); } } // Cancel any in-flight presence heartbeat before sending offline. if let Some(h) = presence_task.take() { h.abort(); } // Best-effort: set presence to offline before exiting. if config.presence_enabled { match tokio::time::timeout( Duration::from_secs(2), publish_presence(&presence_publisher, &presence_keys, "offline"), ) .await { Ok(Ok(_)) => tracing::info!("presence set to offline"), Ok(Err(e)) => tracing::warn!("failed to set offline presence: {e}"), Err(_) => tracing::warn!("offline presence timed out"), } } if let Some(handle) = relay_observer_publisher_task.take() { handle.abort(); } // Graceful relay shutdown — sends WebSocket close frame and waits up to 5s // for the background task to finish, rather than aborting immediately (#40). relay.shutdown().await; tracing::info!("buzz-acp stopped"); Ok(()) } #[derive(PartialEq)] enum LoopAction { Continue, Exit, } fn event_mentions_agent(event: &nostr::Event, agent_pubkey_hex: &str) -> bool { event.tags.iter().any(|t| { t.as_slice().first().map(|s| s.as_str()) == Some("p") && t.as_slice().get(1).map(|s| s.as_str()) == Some(agent_pubkey_hex) }) } fn is_owner_control_command( event: &nostr::Event, kind_u32: u32, command: &str, agent_pubkey_hex: &str, ) -> bool { kind_u32 == KIND_STREAM_MESSAGE && event.content.trim() == command && event_mentions_agent(event, agent_pubkey_hex) } // ── signal_in_flight_task ───────────────────────────────────────────────────── /// Decide which [`ControlSignal`] (if any) to send to an in-flight turn when a /// new, already-author-gated event arrives for that channel. /// /// Returns `None` to leave the in-flight turn untouched (the event waits in the /// queue and is delivered when the turn completes). Author eligibility — owner /// ∪ allowlist ∪ siblings — is enforced upstream by the inbound author gate, so /// `Steer`/`Interrupt` apply to every event that reaches this point; only /// `OwnerInterrupt` re-checks authorship (owner-only) here. /// /// `owner` is the resolved owner pubkey hex, if known. fn mode_gate_signal( handling: MultipleEventHandling, author_hex: &str, owner: Option<&str>, ) -> Option { match handling { MultipleEventHandling::Queue => None, MultipleEventHandling::Steer => Some(ControlSignal::Steer), MultipleEventHandling::Interrupt => Some(ControlSignal::Interrupt), MultipleEventHandling::OwnerInterrupt => match owner { Some(o) if author_hex == o => Some(ControlSignal::Interrupt), _ => None, }, } } /// Send a control signal to the in-flight task for `channel_id`. /// Returns `true` if a signal was sent, `false` if no in-flight task was found. fn signal_in_flight_task( pool: &mut AgentPool, channel_id: uuid::Uuid, mode: ControlSignal, ) -> bool { let entry = pool .task_map_mut() .values_mut() .find(|m| m.channel_id == Some(channel_id)); if let Some(meta) = entry { if let Some(tx) = meta.control_tx.take() { tracing::info!(channel = %channel_id, ?mode, "control signal sent to in-flight task"); let _ = tx.send(mode); return true; } } false } /// Attempt the non-cancelling (ACP) steer for a freshly-queued event. /// /// Caller invariants: /// - `event` has already been pushed into `EventQueue::queues[channel_id]` /// via [`EventQueue::push`] — its `event.id` must still be locatable /// there so [`EventQueue::mark_native_steer_pending`] can move it to the /// side table. /// - `multiple_event_handling` resolved to `ControlSignal::Steer`; this /// function is the non-cancelling fork of that signal. /// /// Returns `true` if the native attempt was accepted by the read loop /// (capacity-1 mpsc `try_send` succeeded, event withheld synchronously, /// ack watcher spawned). On `true` the caller MUST NOT issue the /// universal cancel+merge `ControlSignal::Steer` fallback — the watcher /// will issue it from the ack arm if the native attempt fails. /// /// Returns `false` if `pool.send_steer` failed (no in-flight task, /// `steer_tx` already full from a prior in-flight steer, or read loop /// torn down). The caller MUST fall through to /// `signal_in_flight_task(channel_id, ControlSignal::Steer)` so the /// event still reaches the agent via the universal path. /// /// The withheld event is NOT released here on `false` because no withhold /// was established: `mark_native_steer_pending` only runs on `Ok(())`. fn try_native_steer( pool: &mut AgentPool, queue: &mut EventQueue, channel_id: uuid::Uuid, event: nostr::Event, prompt_tag: String, steer_ack_tx: &mpsc::UnboundedSender, ) -> bool { // Build the steer body: framing strings come from // `queue::native_steer_framing()` (Eva's drift-proof requirement — // native and cancel+merge fallback share these so the agent gets the // same orientation regardless of transport). The single event block // is rendered by `queue::format_event_block`, the same function // `queue::format_prompt` uses internally for `[Buzz event: …]` // sections, so the rendering also cannot drift. // // Passing `None` for `channel_info` / `profile_lookup` is intentional: // native steer is a *delta* into a live turn — the agent already saw // channel context and the actor's profile in the original prompt, // duplicating it here would defeat the point of non-cancelling // steering (which is to inject only what's new). let (header, closing) = queue::native_steer_framing(); let event_id_hex = event.id.to_hex(); let be = queue::BatchEvent { event, prompt_tag: prompt_tag.clone(), received_at: std::time::Instant::now(), }; let event_block = queue::format_event_block(channel_id, None, &be, None); let body = format!("{header}\n\n[Buzz event: {prompt_tag}]\n{event_block}\n\n{closing}"); let (ack_tx, ack_rx) = tokio::sync::oneshot::channel::(); let request = pool::SteerRequest { prompt_blocks: vec![body], ack_tx, }; match pool.send_steer(channel_id, request) { Ok(()) => { // Withhold the queued event synchronously BEFORE spawning // the watcher: this closes the race where `mark_complete` // clears `in_flight_channels` and a stray `flush_next` could // re-deliver the event via normal dispatch. See // `EventQueue::mark_native_steer_pending` docs at queue.rs:606. let withheld = queue.mark_native_steer_pending(channel_id, &event_id_hex); if !withheld { // Race: the event was already drained out of the queue // before we got here (e.g. a concurrent flush picked it // up). The steer is on the wire; if it succeeds the // agent gets it via the native path AND normal // dispatch — duplicate delivery is benign (agent gets // the same message twice). Log so this is visible if it // ever happens in production. tracing::warn!( channel = %channel_id, event_id = %event_id_hex, "native steer accepted by read loop but event was not in queue to withhold \ — possible duplicate delivery if steer succeeds" ); } let ack_tx_clone = steer_ack_tx.clone(); let event_id_for_watcher = event_id_hex.clone(); tokio::spawn(async move { let ack = ack_rx.await; let _ = ack_tx_clone.send(SteerAckEvent { channel_id, event_id: event_id_for_watcher, ack, }); }); true } Err(e) => { tracing::info!( channel = %channel_id, error = ?e, "non-cancelling steer not accepted — falling back to cancel+merge" ); false } } } // ── dispatch_pending ────────────────────────────────────────────────────────── /// Flush queued work to available agents. fn dispatch_pending( pool: &mut AgentPool, queue: &mut EventQueue, ctx: &Arc, ) -> Vec<(Uuid, ThreadTags)> { let mut dispatched_channels = Vec::new(); loop { let batch = match queue.flush_next() { Some(b) => b, None => break, }; let channel_id = batch.channel_id; let typing_scope = batch .events .last() .map(|event| queue::parse_thread_tags(&event.event)) .unwrap_or_default(); let affinity_hit = pool.has_session_for(channel_id); let mut agent = match pool.try_claim(Some(channel_id)) { Some(a) => a, None => { let pending = queue.pending_channels(); tracing::debug!(pending_channels = pending, "pool_exhausted"); queue.requeue_preserve_timestamps(batch); queue.mark_complete(channel_id); break; } }; tracing::debug!(agent = agent.index, channel = %channel_id, affinity_hit, "agent_claimed"); let recoverable_batch = match ctx.dedup_mode { DedupMode::Queue => Some(batch.clone()), DedupMode::Drop => None, }; let result_tx = pool.result_tx(); let ctx_clone = Arc::clone(ctx); let agent_index = agent.index; // Goose-native non-cancelling steer seam: snapshot capability before // the agent moves into `run_prompt_task`, and install the per-turn // steer receiver on the read loop so the main loop's mode-gate fork // (see the `if accepted && queue.is_channel_in_flight(...)` block // in the relay event branch of the main `select!` loop) can drive // it via the matching sender stored in `TaskMeta.steer_tx`. // Install the steer channel for every prompt task — the supervisor // uses try-and-tolerate: it attempts the steer for any agent and // treats `-32601 method_not_found` as "fall back to cancel+merge". let (tx, rx) = tokio::sync::mpsc::channel::(1); agent.acp.install_steer_rx(rx); let steer_tx = Some(tx); // Prompt text is now built inside run_prompt_task (needs async for // context fetching). Pass None for prompt_text; batch carries the data. let (control_tx, control_rx) = tokio::sync::oneshot::channel::(); let abort_handle = pool.join_set.spawn(async move { pool::run_prompt_task( agent, Some(batch), None, ctx_clone, result_tx, Some(control_rx), ) .await; }); pool.task_map_mut().insert( abort_handle.id(), pool::TaskMeta { agent_index, channel_id: Some(channel_id), recoverable_batch, control_tx: Some(control_tx), steer_tx, }, ); dispatched_channels.push((channel_id, typing_scope)); } tracing::debug!( dispatched = dispatched_channels.len(), queue_depth = queue.pending_channels(), "dispatch_pending" ); dispatched_channels } /// Spawn a task that posts a user-visible failure notice to the relay. /// /// Shared by the hard-cap immediate dead-letter path and the retries-exhausted /// dead-letter path so neither duplicates the tokio::spawn block. fn spawn_failure_notice( rest_client: Option<&relay::RestClient>, batch: &FlushBatch, content: String, ) { if let Some(rest) = rest_client { let thread_tags = batch .events .last() .map(|be| queue::parse_thread_tags(&be.event)) .unwrap_or_default(); let rest = rest.clone(); let channel_id = batch.channel_id; tokio::spawn(async move { pool::post_failure_notice(&rest, channel_id, &thread_tags, &content).await; }); } } #[allow(clippy::too_many_arguments)] fn handle_prompt_result( pool: &mut AgentPool, queue: &mut EventQueue, config: &Config, mut result: PromptResult, heartbeat_in_flight: &mut bool, removed_channels: &HashSet, crash_history: &mut [SlotCircuit], respawn_tx: &mpsc::Sender, respawn_tasks: &mut tokio::task::JoinSet<()>, observer: Option, rest_client: Option<&relay::RestClient>, ) -> LoopAction { let before = pool.task_map().len(); let agent_index = result.agent.index; pool.task_map_mut() .retain(|_, meta| meta.agent_index != agent_index); debug_assert_eq!(before, pool.task_map().len() + 1); // Requeue BEFORE mark_complete: requeue() sets retry_after with a future // deadline, and mark_complete() checks for it to decide whether to preserve // retry_counts. If mark_complete runs first, retry_counts is cleared and // every retry starts at attempt 1 — defeating exponential backoff and // dead-letter protection. if let Some(batch) = result.batch.take() { // Don't requeue batches for channels the agent was removed from — // those events are stale and should be silently dropped. if !removed_channels.contains(&batch.channel_id) { if matches!( result.outcome, PromptOutcome::Cancelled | PromptOutcome::CancelDrainTimeout(_) ) { // Cancel re-prompt: store as cancelled events so flush_next() // merges them into the next FlushBatch.cancelled_events, // enabling the annotated merged-prompt format. The batch's // cancel_reason (set by the pool task per the control signal) // selects steer vs interrupt framing. It is always set on this // path; if somehow unset, fall back to the gentler Steer framing // — consistent with MergeFraming::for_reason(None) and the // system default — rather than telling the agent to supersede. // // CancelDrainTimeout shares this path with Cancelled: a failed // 5s drain after a control-signal cancel is a cleanup-deadline // problem, not the deterministic hard-cap death below — the // original batch must survive with no retry/dead-letter // accounting, same as a clean cancel. let reason = batch.cancel_reason.unwrap_or(CancelReason::Steer); queue.requeue_as_cancelled(batch, reason); } else if matches!(result.outcome, PromptOutcome::Timeout(TimeoutKind::Hard)) { // Hard-cap timeout is deterministic: re-running the same task // from a fresh session will reproduce the same death. Dead-letter // immediately without requeueing so the channel isn't subjected to // up to 10 × 1-hour retry cycles. tracing::error!( channel_id = %batch.channel_id, events = batch.events.len(), "dead-lettering batch after hard-cap timeout — discarding {} events", batch.events.len(), ); let content = format!( "⚠️ I couldn't process the last request (the turn exceeded the maximum duration ({}s)). Please re-send if it's still needed.", config.max_turn_duration_secs ); spawn_failure_notice(rest_client, &batch, content); } else if let Some(dead) = queue.requeue(batch) { // Dead-lettered: retries exhausted and the events are gone. // Post a visible notice so the channel isn't left waiting on // a turn that will never happen. let reason = match &result.outcome { PromptOutcome::Timeout(TimeoutKind::Idle) => "the turn timed out".to_string(), // Unreachable today: Timeout(Hard) is consumed by the immediate // dead-letter arm above before requeue() runs. Fail soft rather // than panicking the main loop if that chain is ever reordered. PromptOutcome::Timeout(TimeoutKind::Hard) => { "the turn exceeded the maximum duration".to_string() } PromptOutcome::AgentExited => "the agent process exited".to_string(), PromptOutcome::Error(e) => format!("{e}"), _ => "repeated failures".to_string(), }; let content = format!( "⚠️ I couldn't process the last request after multiple retries ({reason}). Please re-send if it's still needed." ); spawn_failure_notice(rest_client, &dead, content); } } else { tracing::debug!( channel_id = %batch.channel_id, events = batch.events.len(), "dropping failed batch for removed channel" ); } } match &result.source { PromptSource::Channel(ch) => queue.mark_complete(*ch), PromptSource::Heartbeat => *heartbeat_in_flight = false, } // Strip sessions for channels the agent was removed from while this // agent was checked out. This covers the gap where invalidate_channel_sessions // only touches idle agents. for ch in removed_channels { result.agent.state.invalidate_channel(ch); } let outcome_label = match &result.outcome { PromptOutcome::Ok(_) => "ok", PromptOutcome::Error(_) => "error", PromptOutcome::Timeout(TimeoutKind::Idle) => "idle_timeout", PromptOutcome::Timeout(TimeoutKind::Hard) => "hard_timeout", PromptOutcome::AgentExited => "exited", PromptOutcome::Cancelled => "cancelled", PromptOutcome::CancelDrainTimeout(_) => "cancel_drain_timeout", }; let agent_index = result.agent.index; // Capture the spawn-time configured model and our PID before the agent is // moved into match arms below. `desired_model` reflects the config/persona // model at spawn time — it does NOT reflect `session/set_model` overrides, // which live in buzz-agent's session state and are what `llm: (model) …` // errors carry. The two can legitimately differ; `configured_model=` is // still valuable for identifying a stale orphan running an old model. let harness_configured_model = result .agent .desired_model .as_deref() .unwrap_or("") .to_string(); let harness_pid = std::process::id(); let channel_id = match &result.source { PromptSource::Channel(ch) => Some(*ch), PromptSource::Heartbeat => None, }; let emit_turn_error = |error_msg: &str, error_code: Option| { if let Some(ref observer) = observer { let mut payload = serde_json::json!({ "outcome": outcome_label, "error": error_msg, }); if let Some(code) = error_code { payload["code"] = serde_json::json!(code); } observer.emit( "turn_error", Some(agent_index), &observer::context_for(channel_id, None, None), payload, ); } }; match result.outcome { // Successful prompt — return agent to pool. PromptOutcome::Ok(_) => { tracing::debug!( agent = agent_index, outcome = outcome_label, "agent_returned" ); pool.return_agent(result.agent); } // Fatal outcomes: the agent subprocess is dead or poisoned — respawn it. PromptOutcome::AgentExited | PromptOutcome::Timeout(_) => { tracing::warn!( agent = agent_index, outcome = outcome_label, configured_model = %harness_configured_model, pid = harness_pid, "agent_returned — respawning" ); let death_message: String = match outcome_label { "exited" => "Agent process exited unexpectedly".to_string(), "hard_timeout" => format!( "Agent turn exceeded the maximum duration ({}s)", config.max_turn_duration_secs ), _ => "Agent session timed out due to inactivity".to_string(), }; emit_turn_error(&death_message, None); let index = result.agent.index; let slot_history = &mut crash_history[index]; if !spawn_respawn_task( result.agent, config, slot_history, respawn_tx, respawn_tasks, observer.clone(), ) { // Circuit open — slot stays empty until maintenance refill. if pool.live_count() == 0 && !any_respawn_in_flight(crash_history) { tracing::error!("all agents dead — exiting"); return LoopAction::Exit; } } } // Cancel-drain expiry: a control-signal cancel (steer fallback, // interrupt, or explicit stop) did not drain within its bounded // grace window. The process is poisoned/uncertain like a hard // timeout — respawn it — but this is NOT the configured max-turn // cap, so the message must name the actual grace, not // `max_turn_duration_secs`. The triggering batch's fate (preserved // for Steer/Interrupt, dropped for explicit Cancel/Rotate or a // removed channel) is decided above — the message stays fate-neutral // since it must be true in every case. PromptOutcome::CancelDrainTimeout(grace) => { tracing::warn!( agent = agent_index, outcome = outcome_label, configured_model = %harness_configured_model, pid = harness_pid, grace = ?grace, "agent_returned — respawning (cancel-drain timeout)" ); let death_message = format!( "Agent did not stop within {grace:?} after cancellation; the agent process is being replaced." ); emit_turn_error(&death_message, None); let index = result.agent.index; let slot_history = &mut crash_history[index]; if !spawn_respawn_task( result.agent, config, slot_history, respawn_tx, respawn_tasks, observer.clone(), ) { // Circuit open — slot stays empty until maintenance refill. if pool.live_count() == 0 && !any_respawn_in_flight(crash_history) { tracing::error!("all agents dead — exiting"); return LoopAction::Exit; } } } // Errors fall into two categories: // // 1. Transport-class (Io, WriteTimeout, Timeout, Protocol): the stdio // pipe may be corrupted or the agent desynchronized. These are fatal // to the agent regardless of whether they occurred during session // creation or an active prompt — respawn unconditionally. // // 2. Application-class (IdleTimeout, HardTimeout, Json): the pipe is // intact but the prompt failed. Return the agent to the pool so it // can be reused for the next event. // Intentional cancel — agent is healthy, return it to the pool. // No respawn, no retry penalty. The cancelled batch was already stored // via requeue_as_cancelled() above and will be merged into the next // FlushBatch by flush_next(). PromptOutcome::Cancelled => { tracing::debug!( agent = agent_index, outcome = outcome_label, configured_model = %harness_configured_model, pid = harness_pid, "agent_returned (cancelled)" ); pool.return_agent(result.agent); } PromptOutcome::Error(ref e) => { let is_transport_error = matches!( e, acp::AcpError::Io(_) | acp::AcpError::WriteTimeout(_) | acp::AcpError::Timeout(_) | acp::AcpError::Protocol(_) ); let error_code = match &e { acp::AcpError::AgentError { code, .. } => Some(*code), _ => None, }; if is_transport_error { tracing::warn!( agent = agent_index, outcome = outcome_label, configured_model = %harness_configured_model, pid = harness_pid, error = %e, "transport/protocol error — respawning agent" ); emit_turn_error(&e.to_string(), error_code); let index = result.agent.index; let slot_history = &mut crash_history[index]; if !spawn_respawn_task( result.agent, config, slot_history, respawn_tx, respawn_tasks, observer, ) && pool.live_count() == 0 && !any_respawn_in_flight(crash_history) { tracing::error!("all agents dead — exiting"); return LoopAction::Exit; } } else { tracing::warn!( agent = agent_index, outcome = outcome_label, configured_model = %harness_configured_model, pid = harness_pid, error = %e, "agent_returned (application error — pipe intact)" ); emit_turn_error(&e.to_string(), error_code); pool.return_agent(result.agent); } } } LoopAction::Continue } #[allow(clippy::too_many_arguments)] fn recover_panicked_agent( pool: &mut AgentPool, queue: &mut EventQueue, config: &Config, join_error: tokio::task::JoinError, heartbeat_in_flight: &mut bool, removed_channels: &HashSet, typing_channels: &mut HashMap, crash_history: &mut [SlotCircuit], respawn_tx: &mpsc::Sender, respawn_tasks: &mut tokio::task::JoinSet<()>, observer: Option, ) { let task_id = join_error.id(); let Some(meta) = pool.task_map_mut().remove(&task_id) else { tracing::error!("panic for unknown task {task_id:?} — bug"); return; }; let i = meta.agent_index; // Requeue BEFORE mark_complete (same rationale as handle_prompt_result). if let Some(batch) = meta.recoverable_batch { if let Some(ch) = meta.channel_id { if !removed_channels.contains(&ch) { // Dead-letter on exhaustion is logged inside requeue(); a // panic path has no outcome to report, so no notice here. let _ = queue.requeue(batch); tracing::warn!("requeued batch for panicked agent {i}"); } else { tracing::debug!( channel_id = %ch, "dropping panicked batch for removed channel" ); } } } if let Some(ch) = meta.channel_id { queue.mark_complete(ch); typing_channels.remove(&ch); tracing::warn!("cleared wedged in-flight channel {ch} from panicked agent {i}"); } else { *heartbeat_in_flight = false; tracing::warn!("cleared wedged heartbeat_in_flight from panicked agent {i}"); } if let Some(ref observer) = observer { observer.emit( "agent_panic", Some(i), &observer::context_for(meta.channel_id, None, None), serde_json::json!({ "outcome": "panic", "error": format!("Agent task panicked: {join_error}"), }), ); } // Panics count as crashes for the circuit breaker. // The panicked task already dropped the AcpClient, so we just need to // check the circuit and spawn a fresh agent in the background. let slot = &mut crash_history[i]; let delay = match slot.record_crash() { CrashVerdict::CircuitOpen => { tracing::error!(agent = i, "circuit open after panic — not respawning"); return; } CrashVerdict::HalfOpenProbe => { tracing::info!(agent = i, "circuit half-open — probe respawn after panic"); Duration::ZERO } CrashVerdict::Respawn(d) => { tracing::info!( agent = i, delay_ms = d.as_millis(), "respawn backoff after panic" ); d } }; // Spawn respawn work off the main loop. slot.respawn_in_flight = true; let cmd = config.agent_command.clone(); let args = config.agent_args.clone(); let env = config.persona_env_vars.clone(); let has_codex = config.has_generated_codex_config; let guard = RespawnGuard::new(i, respawn_tx.clone()); respawn_tasks.spawn(async move { if !delay.is_zero() { tokio::time::sleep(delay).await; } let result = spawn_and_init(&cmd, &args, &env, has_codex, i, observer).await; guard.send(result); }); } #[allow(clippy::too_many_arguments)] fn drain_ready_join_results( pool: &mut AgentPool, queue: &mut EventQueue, config: &Config, heartbeat_in_flight: &mut bool, removed_channels: &HashSet, typing_channels: &mut HashMap, crash_history: &mut [SlotCircuit], respawn_tx: &mpsc::Sender, respawn_tasks: &mut tokio::task::JoinSet<()>, observer: Option, ) -> LoopAction { while let Some(Some(join_result)) = pool.join_set.join_next().now_or_never() { if let Err(join_error) = join_result { tracing::error!("agent task panicked: {join_error}"); recover_panicked_agent( pool, queue, config, join_error, heartbeat_in_flight, removed_channels, typing_channels, crash_history, respawn_tx, respawn_tasks, observer.clone(), ); if pool.live_count() == 0 && !any_respawn_in_flight(crash_history) { return LoopAction::Exit; } } } LoopAction::Continue } fn dispatch_heartbeat( pool: &mut AgentPool, ctx: &Arc, heartbeat_in_flight: &mut bool, ) { if *heartbeat_in_flight { return; } let agent = match pool.try_claim(None) { Some(a) => a, None => return, }; let prompt_text = ctx .heartbeat_prompt .clone() .unwrap_or_else(default_heartbeat_prompt); let result_tx = pool.result_tx(); let ctx_clone = Arc::clone(ctx); let agent_index = agent.index; let abort_handle = pool.join_set.spawn(async move { pool::run_prompt_task(agent, None, Some(prompt_text), ctx_clone, result_tx, None).await; }); pool.task_map_mut().insert( abort_handle.id(), pool::TaskMeta { agent_index, channel_id: None, recoverable_batch: None, control_tx: None, steer_tx: None, }, ); *heartbeat_in_flight = true; tracing::info!(agent = agent_index, "heartbeat_fired"); } #[cfg(test)] mod agent_draft_prompt_tests { #[test] fn shared_base_prompt_teaches_portable_agent_drafts() { let prompt = include_str!("base_prompt.md"); assert!(prompt.contains("buzz agents draft-create")); assert!(prompt.contains("ask for at most two things")); assert!(prompt.contains("what it should do day-to-day")); assert!(prompt.contains("owner saves it")); assert!(prompt.contains("Do not ask about runtime, provider, model, credentials")); } } fn default_heartbeat_prompt() -> String { let now = chrono::Utc::now().to_rfc3339(); format!( "[System: Heartbeat]\nTime: {now}\n\n\ You have been awakened for a routine heartbeat. You have NO incoming messages or\n\ active channel context for this turn.\n\n\ Your tasks:\n\ 1. Run `buzz feed get --types needs_action` to check for pending workflow approvals or\n\ high-priority requests addressed to you.\n\ 2. Run `buzz feed get --types mentions` to check for unanswered @mentions.\n\ 3. If you find actionable items, address them using the appropriate CLI commands\n\ (e.g., `buzz workflows approve --token `, `buzz messages send`,\n\ `buzz messages send --reply-to `).\n\ 4. If there are no pending actions or mentions, end your turn immediately.\n\n\ Do not run `buzz channels list` or `buzz messages search` unless you have a specific reason.\n\ Do not invent work — only act on items surfaced by the feed commands." ) } /// Spawn a background respawn task for a crashed agent slot. /// /// Does the circuit breaker check synchronously (non-blocking), then spawns /// the actual shutdown + backoff + spawn_and_init work into a background task. /// The result comes back through `respawn_tx` so the main loop stays responsive. /// /// Returns `true` if a respawn task was spawned, `false` if the circuit is open. fn spawn_respawn_task( old_agent: OwnedAgent, config: &Config, slot: &mut SlotCircuit, respawn_tx: &mpsc::Sender, respawn_tasks: &mut tokio::task::JoinSet<()>, observer: Option, ) -> bool { let index = old_agent.index; // Circuit breaker: record crash, decide whether to respawn. let delay = match slot.record_crash() { CrashVerdict::CircuitOpen => { tracing::error!(agent = index, "circuit open — not respawning"); return false; } CrashVerdict::HalfOpenProbe => { tracing::info!(agent = index, "circuit half-open — probe respawn"); Duration::ZERO } CrashVerdict::Respawn(d) => { tracing::info!(agent = index, delay_ms = d.as_millis(), "respawn backoff"); d } }; slot.respawn_in_flight = true; // Spawn the actual work (shutdown + sleep + spawn + init) off the main loop. let cmd = config.agent_command.clone(); let args = config.agent_args.clone(); let env = config.persona_env_vars.clone(); let has_codex = config.has_generated_codex_config; let guard = RespawnGuard::new(index, respawn_tx.clone()); respawn_tasks.spawn(async move { // Shutdown old agent (reap child, prevent zombie). let mut agent = old_agent; agent.acp.shutdown().await; drop(agent); if !delay.is_zero() { tokio::time::sleep(delay).await; } let result = spawn_and_init(&cmd, &args, &env, has_codex, index, observer).await; guard.send(result); }); true } fn normalized_agent_name(init_result: &serde_json::Value) -> String { init_result .get("agentInfo") .or_else(|| init_result.get("serverInfo")) .and_then(|info| info.get("name")) .and_then(|value| value.as_str()) .unwrap_or("unknown") .trim() .to_ascii_lowercase() } // ── spawn_and_init ──────────────────────────────────────────────────────────── /// Spawn an agent subprocess and run the MCP `initialize` handshake. /// /// Takes owned args so it can run in a background `tokio::spawn` task without /// borrowing `Config`. All respawn/refill paths use this. async fn spawn_and_init( command: &str, args: &[String], extra_env: &[(String, String)], has_generated_codex_config: bool, agent_index: usize, observer: Option, ) -> Result<(AcpClient, u32, String)> { let mut acp = AcpClient::spawn(command, args, extra_env, has_generated_codex_config) .await .map_err(|e| anyhow::anyhow!("failed to spawn agent: {e}"))?; acp.set_observer(observer, agent_index); match acp.initialize().await { Ok(init_result) => { tracing::info!("agent initialized: {init_result}"); let protocol_version = init_result["protocolVersion"].as_u64().unwrap_or(1) as u32; acp.observe( "agent_initialized", serde_json::json!({ "agentIndex": agent_index, "initializeResult": init_result, }), ); let agent_name = normalized_agent_name(&init_result); Ok((acp, protocol_version, agent_name)) } Err(e) => { // Explicitly shut down the spawned child to prevent zombie/leak. // Drop only does start_kill + try_wait (best-effort); shutdown() // does start_kill + bounded wait (guaranteed reap). acp.shutdown().await; Err(anyhow::anyhow!("agent initialize failed: {e}")) } } } async fn spawn_auth_client(agent: &AuthAgentArgs) -> Result { let agent_args = config::normalize_agent_args(&agent.agent_command, agent.agent_args.clone()); AcpClient::spawn(&agent.agent_command, &agent_args, &[], false).await } fn extract_auth_methods(init_result: &serde_json::Value) -> Vec { init_result .get("authMethods") .and_then(|methods| methods.as_array()) .cloned() .unwrap_or_default() } /// `buzz-acp auth-methods` — spawn an adapter, initialize it, print authMethods. async fn run_auth_methods(args: AuthMethodsArgs) -> Result<()> { let mut client = match spawn_auth_client(&args.agent).await { Ok(c) => c, Err(e) => { eprintln!("error: failed to spawn agent: {e}"); std::process::exit(1); } }; let init_result = match tokio::time::timeout(MODELS_TIMEOUT, client.initialize()).await { Ok(Ok(result)) => result, Ok(Err(e)) => { client.shutdown().await; eprintln!("error: agent initialize failed: {e}"); std::process::exit(1); } Err(_) => { client.shutdown().await; eprintln!("error: agent timed out ({MODELS_TIMEOUT:?})"); std::process::exit(1); } }; let methods = extract_auth_methods(&init_result); client.shutdown().await; if args.json { let output = serde_json::json!({ "methods": methods }); println!("{}", serde_json::to_string_pretty(&output)?); } else if methods.is_empty() { println!("No auth methods advertised."); } else { for method in methods { let id = method .get("id") .and_then(|value| value.as_str()) .unwrap_or("unknown"); let name = method .get("name") .and_then(|value| value.as_str()) .unwrap_or(id); println!("{id}\t{name}"); } } Ok(()) } /// `buzz-acp authenticate` — invoke one adapter-owned auth method. async fn run_authenticate(args: AuthenticateArgs) -> Result<()> { let mut client = match spawn_auth_client(&args.agent).await { Ok(c) => c, Err(e) => { eprintln!("error: failed to spawn agent: {e}"); std::process::exit(1); } }; let init_result = match tokio::time::timeout(MODELS_TIMEOUT, client.initialize()).await { Ok(Ok(result)) => result, Ok(Err(e)) => { client.shutdown().await; eprintln!("error: agent initialize failed: {e}"); std::process::exit(1); } Err(_) => { client.shutdown().await; eprintln!("error: agent initialize timed out ({MODELS_TIMEOUT:?})"); std::process::exit(1); } }; let supports_method = extract_auth_methods(&init_result) .iter() .any(|method| method.get("id").and_then(|id| id.as_str()) == Some(args.method_id.as_str())); if !supports_method { client.shutdown().await; eprintln!( "error: auth method '{}' is not advertised by this adapter", args.method_id ); std::process::exit(1); } let result = tokio::time::timeout(AUTHENTICATE_TIMEOUT, client.authenticate(&args.method_id)).await; match result { Ok(Ok(_)) => { client.shutdown().await; Ok(()) } Ok(Err(e)) => { client.shutdown().await; eprintln!("error: authenticate failed: {e}"); std::process::exit(1); } Err(_) => { client.shutdown().await; eprintln!("error: authenticate timed out ({AUTHENTICATE_TIMEOUT:?})"); std::process::exit(1); } } } /// Flow: spawn → initialize → session/new → print models → shutdown. /// No relay connection, no MCP servers, no subscriptions. ~2-5s total. async fn run_models(args: ModelsArgs) -> Result<()> { use acp::{extract_model_config_options, extract_model_state}; let agent_args = config::normalize_agent_args(&args.agent.agent_command, args.agent.agent_args); let cwd = std::env::current_dir() .unwrap_or_else(|_| std::path::PathBuf::from("/")) .to_string_lossy() .to_string(); // Spawn outside the timeout so we always own the child for cleanup. // `models` subcommand doesn't use persona packs — no extra env, no codex config. let mut client = match AcpClient::spawn(&args.agent.agent_command, &agent_args, &[], false).await { Ok(c) => c, Err(e) => { eprintln!("error: failed to spawn agent: {e}"); std::process::exit(1); } }; // Initialize + session/new under a timeout. Client is owned above, // so shutdown() runs on all paths (success, error, timeout). let protocol_result = tokio::time::timeout(MODELS_TIMEOUT, async { let init = client.initialize().await?; let session = client.session_new_full(&cwd, vec![], None).await?; Ok::<_, acp::AcpError>((init, session)) }) .await; let (init_result, session_resp) = match protocol_result { Ok(Ok(tuple)) => tuple, Ok(Err(e)) => { client.shutdown().await; eprintln!("error: agent communication failed: {e}"); std::process::exit(1); } Err(_) => { client.shutdown().await; eprintln!("error: agent timed out ({MODELS_TIMEOUT:?})"); std::process::exit(1); } }; // Extract agent info from initialize response. // ACP spec uses "serverInfo" (MCP heritage); some agents may use "agentInfo". let info_obj = init_result .get("serverInfo") .or_else(|| init_result.get("agentInfo")); let agent_name = info_obj .and_then(|ai| ai.get("name")) .and_then(|v| v.as_str()) .unwrap_or("unknown"); let agent_version = info_obj .and_then(|ai| ai.get("version")) .and_then(|v| v.as_str()) .unwrap_or("unknown"); // Extract model info from session/new response. let config_options = extract_model_config_options(&session_resp.raw); let model_state = extract_model_state(&session_resp.raw); if args.json { // Structured JSON output — consumed by Phase 3 `get_agent_models`. let output = serde_json::json!({ "agent": { "name": agent_name, "version": agent_version, }, "stable": { "configOptions": config_options, }, "unstable": model_state.as_ref().map(|ms| serde_json::json!({ "currentModelId": ms.get("currentModelId"), "availableModels": ms.get("availableModels"), })), }); println!("{}", serde_json::to_string_pretty(&output)?); } else { // Human-readable output. println!("Agent: {} v{}", agent_name, agent_version); println!(); let mut has_models = false; if !config_options.is_empty() { println!("Models (stable configOptions):"); for opt in &config_options { let config_id = opt.get("configId").and_then(|v| v.as_str()).unwrap_or("?"); let display = opt .get("displayName") .and_then(|v| v.as_str()) .unwrap_or(config_id); println!(" {display} (configId: {config_id})"); if let Some(options) = opt.get("options").and_then(|v| v.as_array()) { for o in options { let val = o.get("value").and_then(|v| v.as_str()).unwrap_or("?"); let name = o.get("displayName").and_then(|v| v.as_str()).unwrap_or(val); println!(" - {name} (value: {val})"); } } } has_models = true; } if let Some(ref ms) = model_state { let current = ms .get("currentModelId") .and_then(|v| v.as_str()) .unwrap_or("(none)"); println!("Models (unstable SessionModelState):"); println!(" Current: {current}"); if let Some(available) = ms.get("availableModels").and_then(|v| v.as_array()) { println!(" Available:"); for m in available { let id = m.get("modelId").and_then(|v| v.as_str()).unwrap_or("?"); let name = m.get("name").and_then(|v| v.as_str()).unwrap_or(id); let desc = m.get("description").and_then(|v| v.as_str()).unwrap_or(""); if desc.is_empty() { println!(" - {name} (id: {id})"); } else { println!(" - {name} (id: {id}) — {desc}"); } } } has_models = true; } if !has_models { println!("No model information available from this agent."); } } client.shutdown().await; Ok(()) } fn build_mcp_servers(config: &Config) -> Vec { if config.mcp_command.is_empty() { return vec![]; } vec![McpServer { name: std::path::Path::new(&config.mcp_command) .file_stem() .and_then(|s| s.to_str()) .unwrap_or("mcp") .to_string(), command: config.mcp_command.clone(), args: vec![], env: { let mut env = vec![ EnvVar { name: "BUZZ_RELAY_URL".into(), value: config.relay_url.clone(), }, EnvVar { name: "BUZZ_PRIVATE_KEY".into(), // bech32 encoding of a valid secret key is infallible. // Panic here is correct: injecting a bogus secret would cause // delayed, hard-to-diagnose agent failures downstream. value: config .keys .secret_key() .to_bech32() .expect("secret key bech32 encoding should never fail"), }, ]; // Forward BUZZ_AUTH_TAG (NIP-OA owner attestation credential) // so the MCP server can attach it to every signed event. if let Ok(auth_tag) = std::env::var("BUZZ_AUTH_TAG") { if !auth_tag.is_empty() { env.push(EnvVar { name: "BUZZ_AUTH_TAG".into(), value: auth_tag, }); } } env }, }] } #[cfg(test)] mod heartbeat_base_prompt_tests { use super::*; // Pins the heartbeat dispatch path (dispatch_heartbeat, ~line 2359): a // legacy agent WITH a base_prompt must get [Base] prepended to the // heartbeat user message, composed as `[Base]\n{bp}\n\n{prompt}`. This is // the second half of the round-2 regression (the first being initial_message). #[test] fn test_heartbeat_legacy_agent_gets_base_prepended() { // protocol_version 1 + Some(base_prompt): heartbeat prompt is prefixed // with the [Base] section exactly as the legacy session/new path would. let prompt = "[System: Heartbeat]\nrun feed get"; let composed = pool::prepend_base_for_legacy(1, Some("you are a helpful agent"), prompt); assert_eq!( composed, "[Base]\nyou are a helpful agent\n\n[System: Heartbeat]\nrun feed get" ); assert!(composed.starts_with("[Base]\nyou are a helpful agent\n\n")); } #[test] fn test_heartbeat_modern_agent_omits_base() { // protocol_version 2 gets base_prompt via session/new; the heartbeat // prompt is sent verbatim. let prompt = "[System: Heartbeat]\nrun feed get"; let composed = pool::prepend_base_for_legacy(2, Some("you are a helpful agent"), prompt); assert_eq!(composed, prompt); } } #[cfg(test)] mod owner_control_command_tests { use super::*; use nostr::{EventBuilder, Keys, Kind, Tag}; fn make_event(kind: u32, content: &str, p_hex: Option<&str>) -> nostr::Event { let keys = Keys::generate(); let tags = match p_hex { Some(hex) => vec![Tag::parse(["p", hex]).expect("p tag")], None => vec![], }; EventBuilder::new(Kind::Custom(kind as u16), content) .tags(tags) .sign_with_keys(&keys) .unwrap() } #[test] fn owner_control_command_requires_kind_content_and_agent_mention() { let agent = "ab".repeat(32); let event = make_event(KIND_STREAM_MESSAGE, " !rotate ", Some(&agent)); assert!(is_owner_control_command( &event, KIND_STREAM_MESSAGE, "!rotate", &agent )); let wrong_kind = make_event(1, "!rotate", Some(&agent)); assert!(!is_owner_control_command(&wrong_kind, 1, "!rotate", &agent)); let wrong_content = make_event(KIND_STREAM_MESSAGE, "!cancel", Some(&agent)); assert!(!is_owner_control_command( &wrong_content, KIND_STREAM_MESSAGE, "!rotate", &agent )); let no_mention = make_event(KIND_STREAM_MESSAGE, "!rotate", None); assert!(!is_owner_control_command( &no_mention, KIND_STREAM_MESSAGE, "!rotate", &agent )); } #[test] fn mode_gate_signal_maps_handling_to_control_signal() { let owner = "a".repeat(64); let other = "b".repeat(64); // Queue: never signals — events wait for the turn to finish. assert!(mode_gate_signal(MultipleEventHandling::Queue, &owner, Some(&owner)).is_none()); // Steer: always steers (eligibility already enforced upstream). assert!(matches!( mode_gate_signal(MultipleEventHandling::Steer, &other, Some(&owner)), Some(ControlSignal::Steer) )); // Steer even when owner is unknown — gate doesn't re-check authorship. assert!(matches!( mode_gate_signal(MultipleEventHandling::Steer, &other, None), Some(ControlSignal::Steer) )); // Interrupt: always interrupts for any eligible author. assert!(matches!( mode_gate_signal(MultipleEventHandling::Interrupt, &other, Some(&owner)), Some(ControlSignal::Interrupt) )); // OwnerInterrupt: interrupts only for the owner. assert!(matches!( mode_gate_signal(MultipleEventHandling::OwnerInterrupt, &owner, Some(&owner)), Some(ControlSignal::Interrupt) )); assert!( mode_gate_signal(MultipleEventHandling::OwnerInterrupt, &other, Some(&owner)).is_none(), "owner-interrupt must not fire for a non-owner author" ); assert!( mode_gate_signal(MultipleEventHandling::OwnerInterrupt, &owner, None).is_none(), "owner-interrupt must not fire when the owner is unknown" ); } #[tokio::test] async fn signal_in_flight_task_sends_rotate_once() { let mut pool = AgentPool::from_slots(vec![]); let channel_id = Uuid::new_v4(); let other_channel_id = Uuid::new_v4(); let (control_tx, control_rx) = tokio::sync::oneshot::channel(); let abort_handle = pool.join_set.spawn(async {}); pool.task_map_mut().insert( abort_handle.id(), pool::TaskMeta { agent_index: 0, channel_id: Some(channel_id), recoverable_batch: None, control_tx: Some(control_tx), steer_tx: None, }, ); assert!(!signal_in_flight_task( &mut pool, other_channel_id, ControlSignal::Rotate )); assert!(signal_in_flight_task( &mut pool, channel_id, ControlSignal::Rotate )); assert_eq!(control_rx.await.unwrap(), ControlSignal::Rotate); assert!(!signal_in_flight_task( &mut pool, channel_id, ControlSignal::Rotate )); } } #[cfg(test)] mod owner_cache_tests { use super::*; #[test] fn new_with_some_caches_immediately() { let cache = OwnerCache::new(Some("abcd".into())); assert_eq!(cache.get(), Some("abcd")); } #[test] fn new_with_none_returns_none() { let cache = OwnerCache::new(None); assert!(cache.get().is_none()); } #[test] fn get_returns_cached_value() { let cache = OwnerCache::new(Some("ab".repeat(32))); assert_eq!(cache.get(), Some("ab".repeat(32)).as_deref()); } } #[cfg(test)] mod author_gate_tests { use super::*; /// A `RestClient` for tests. The author-gate decisions exercised here all /// resolve from the owner pubkey or sibling cache before any HTTP call, so /// this client is never actually used to make a request. fn dummy_rest_client() -> relay::RestClient { relay::RestClient { http: reqwest::Client::new(), base_url: "http://localhost:0".into(), keys: nostr::Keys::generate(), auth_tag_json: None, } } const OWNER: &str = "00"; const SIBLING: &str = "11"; const EXTERNAL: &str = "22"; const STRANGER: &str = "33"; /// Owner + a known sibling, none of them on the explicit allowlist. fn cache_with_sibling() -> OwnerCache { let cache = OwnerCache::new(Some(OWNER.into())); cache.cache_sibling(SIBLING.into(), true); cache.cache_sibling(STRANGER.into(), false); cache } #[tokio::test] async fn test_allowlist_accepts_sibling_not_in_allowlist() { let cache = cache_with_sibling(); let allowlist = HashSet::from([EXTERNAL.to_string()]); assert!( author_allowed( &RespondTo::Allowlist, &allowlist, SIBLING, &cache, &dummy_rest_client() ) .await, "a same-owner sibling must fire a turn under Allowlist even when not listed" ); } #[tokio::test] async fn test_allowlist_accepts_explicit_external_pubkey() { let cache = cache_with_sibling(); let allowlist = HashSet::from([EXTERNAL.to_string()]); assert!( author_allowed( &RespondTo::Allowlist, &allowlist, EXTERNAL, &cache, &dummy_rest_client() ) .await, "an explicitly allowlisted external pubkey must still be accepted" ); } #[tokio::test] async fn test_allowlist_rejects_non_sibling_not_in_allowlist() { let cache = cache_with_sibling(); let allowlist = HashSet::from([EXTERNAL.to_string()]); assert!( !author_allowed( &RespondTo::Allowlist, &allowlist, STRANGER, &cache, &dummy_rest_client() ) .await, "a non-sibling absent from the allowlist must be dropped" ); } #[tokio::test] async fn test_allowlist_accepts_owner() { let cache = cache_with_sibling(); let allowlist = HashSet::new(); assert!( author_allowed( &RespondTo::Allowlist, &allowlist, OWNER, &cache, &dummy_rest_client() ) .await, "the owner must always be accepted under Allowlist" ); } // The default `respond-to` is OwnerOnly. Under steering, "an ineligible // author must NOT steer" is enforced *here* — author_allowed drops the // event before it reaches the mode gate — not in the gate itself. These // pin that invariant against the default mode. #[tokio::test] async fn test_owner_only_rejects_stranger_so_no_steer() { let cache = cache_with_sibling(); assert!( !author_allowed( &RespondTo::OwnerOnly, &HashSet::new(), STRANGER, &cache, &dummy_rest_client() ) .await, "under the default OwnerOnly, a stranger must be dropped — so it can never reach the mode gate to steer" ); } #[tokio::test] async fn test_owner_only_admits_owner_and_sibling_to_steer() { let cache = cache_with_sibling(); for (who, label) in [(OWNER, "owner"), (SIBLING, "sibling")] { assert!( author_allowed( &RespondTo::OwnerOnly, &HashSet::new(), who, &cache, &dummy_rest_client() ) .await, "under default OwnerOnly, the {label} must be admitted so steering can fire" ); } } } #[cfg(test)] mod observer_chunk_coalescer_tests { use super::*; fn chunk_event( seq: u64, update_type: &str, message_id: &str, text: &str, ) -> observer::ObserverEvent { observer::ObserverEvent { seq, timestamp: format!("2026-04-29T04:00:0{seq}Z"), kind: "acp_read".to_string(), agent_index: Some(0), channel_id: Some("channel-1".to_string()), session_id: Some("session-1".to_string()), turn_id: Some("turn-1".to_string()), payload: serde_json::json!({ "jsonrpc": "2.0", "method": "session/update", "params": { "sessionId": "session-1", "update": { "sessionUpdate": update_type, "messageId": message_id, "content": { "type": "text", "text": text, }, }, }, }), } } fn non_chunk_event(seq: u64) -> observer::ObserverEvent { observer::ObserverEvent { seq, timestamp: format!("2026-04-29T04:00:0{seq}Z"), kind: "turn_started".to_string(), agent_index: Some(0), channel_id: Some("channel-1".to_string()), session_id: Some("session-1".to_string()), turn_id: Some("turn-1".to_string()), payload: serde_json::json!({ "type": "turn_started" }), } } fn chunk_text(event: &observer::ObserverEvent) -> &str { event.payload["params"]["update"]["content"]["text"] .as_str() .expect("chunk text") } #[test] fn coalesces_chunks_until_non_chunk_event() { let mut coalescer = ObserverChunkCoalescer::default(); assert!(coalescer .ingest(chunk_event(1, "agent_message_chunk", "message-1", "hello ")) .is_empty()); assert!(coalescer .ingest(chunk_event(2, "agent_message_chunk", "message-1", "world")) .is_empty()); let events = coalescer.ingest(non_chunk_event(3)); assert_eq!(events.len(), 2); assert_eq!(events[0].seq, 2); assert_eq!(chunk_text(&events[0]), "hello world"); assert_eq!(events[1].kind, "turn_started"); } #[test] fn keeps_independent_chunk_streams_separate() { let mut coalescer = ObserverChunkCoalescer::default(); assert!(coalescer .ingest(chunk_event(1, "agent_message_chunk", "message-1", "answer")) .is_empty()); assert!(coalescer .ingest(chunk_event( 2, "agent_thought_chunk", "thought-1", "thinking" )) .is_empty()); let events = coalescer.flush(); assert_eq!(events.len(), 2); assert_eq!(chunk_text(&events[0]), "answer"); assert_eq!(chunk_text(&events[1]), "thinking"); } } #[cfg(test)] mod build_mcp_servers_tests { use super::*; use std::sync::Mutex; /// Env-var-touching tests must run serially — env vars are process-global. static ENV_LOCK: Mutex<()> = Mutex::new(()); fn test_config() -> Config { Config { keys: nostr::Keys::generate(), relay_url: "ws://localhost:3000".into(), agent_command: "goose".into(), agent_args: vec!["acp".into()], mcp_command: "test-mcp-server".into(), idle_timeout_secs: config::DEFAULT_IDLE_TIMEOUT_SECS, max_turn_duration_secs: config::DEFAULT_MAX_TURN_DURATION_SECS, agents: 1, heartbeat_interval_secs: 0, turn_liveness_secs: 10, heartbeat_prompt: None, system_prompt: None, team_instructions: None, initial_message: None, subscribe_mode: config::SubscribeMode::All, dedup_mode: config::DedupMode::Queue, multiple_event_handling: config::MultipleEventHandling::Queue, ignore_self: true, kinds_override: None, channels_override: None, no_mention_filter: false, config_path: std::path::PathBuf::from("./buzz-acp.toml"), context_message_limit: 12, max_turns_per_session: 0, presence_enabled: true, typing_enabled: true, memory_enabled: false, model: None, permission_mode: config::PermissionMode::BypassPermissions, respond_to: config::RespondTo::Anyone, respond_to_allowlist: std::collections::HashSet::new(), allowed_respond_to: vec![], persona_env_vars: vec![], has_generated_codex_config: false, relay_observer: false, agent_owner: None, no_base_prompt: false, base_prompt_content: None, } } #[test] fn session_new_mcp_server_has_required_fields() { let config = test_config(); let servers = build_mcp_servers(&config); assert_eq!(servers.len(), 1); let server = &servers[0]; assert_eq!(server.name, "test-mcp-server"); let names: Vec<&str> = server.env.iter().map(|e| e.name.as_str()).collect(); assert!( names.contains(&"BUZZ_RELAY_URL"), "missing BUZZ_RELAY_URL; got {names:?}" ); assert!( names.contains(&"BUZZ_PRIVATE_KEY"), "missing BUZZ_PRIVATE_KEY; got {names:?}" ); } #[test] fn session_new_mcp_server_forwards_buzz_auth_tag() { let _guard = ENV_LOCK.lock().unwrap(); std::env::set_var("BUZZ_AUTH_TAG", "test-attestation-tag"); let config = test_config(); let servers = build_mcp_servers(&config); std::env::remove_var("BUZZ_AUTH_TAG"); let server = &servers[0]; let auth_tag_env = server.env.iter().find(|e| e.name == "BUZZ_AUTH_TAG"); assert!( auth_tag_env.is_some(), "BUZZ_AUTH_TAG should be forwarded when set" ); assert_eq!(auth_tag_env.unwrap().value, "test-attestation-tag"); } #[test] fn session_new_mcp_server_skips_empty_buzz_auth_tag() { let _guard = ENV_LOCK.lock().unwrap(); std::env::set_var("BUZZ_AUTH_TAG", ""); let config = test_config(); let servers = build_mcp_servers(&config); std::env::remove_var("BUZZ_AUTH_TAG"); let server = &servers[0]; let has_auth_tag = server.env.iter().any(|e| e.name == "BUZZ_AUTH_TAG"); assert!(!has_auth_tag, "empty BUZZ_AUTH_TAG should not be forwarded"); } #[test] fn empty_mcp_command_returns_no_servers() { let mut config = test_config(); config.mcp_command = "".into(); let servers = build_mcp_servers(&config); assert!( servers.is_empty(), "empty mcp_command should produce no MCP servers" ); } #[test] fn absolute_path_mcp_command_uses_file_stem_as_name() { let mut config = test_config(); config.mcp_command = "/opt/bin/my-mcp-server".into(); let servers = build_mcp_servers(&config); assert_eq!(servers.len(), 1); assert_eq!(servers[0].name, "my-mcp-server"); } #[test] fn mcp_command_with_no_stem_falls_back_to_mcp() { // Path::new("").file_stem() returns None — exercises the unwrap_or("mcp") path. let mut config = test_config(); config.mcp_command = "".into(); // Empty command returns no servers; test the stem logic directly. assert_eq!( std::path::Path::new("") .file_stem() .and_then(|s| s.to_str()) .unwrap_or("mcp"), "mcp" ); // Confirm a non-empty command with no stem (e.g. just a dot) also falls back. config.mcp_command = ".".into(); let servers = build_mcp_servers(&config); assert_eq!(servers.len(), 1); assert_eq!( servers[0].name, "mcp", "Path::new(\".\").file_stem() is None — should fall back to \"mcp\"" ); } } #[cfg(test)] mod error_outcome_emission_tests { //! Pins the policy that error-class outcomes surface to the activity feed //! and never to the channel: //! //! - Channel silence is enforced *structurally* — `handle_prompt_result` //! takes no relay handle, so it has no way to post a channel message. A //! future re-introduction of channel notices would have to add the relay //! parameter back, which these tests' construction would then refuse to //! compile against. //! - Feed coverage is the regression-prone half and is asserted at runtime: //! each error outcome must emit exactly one `turn_error` observer event. //! If any branch drops its `emit_turn_error` call, the matching test goes //! red. use super::*; use crate::acp::{AcpClient, AcpError}; use crate::observer::ObserverHandle; use crate::pool::{ AgentPool, OwnedAgent, PromptOutcome, PromptResult, PromptSource, TimeoutKind, }; use crate::queue::{BatchEvent, FlushBatch}; use nostr::{EventBuilder, Keys, Kind}; use std::collections::HashSet; fn test_config() -> Config { Config { keys: nostr::Keys::generate(), relay_url: "ws://localhost:3000".into(), // `true` exits cleanly, so the async respawn fails fast and // harmlessly off the JoinSet — irrelevant to the synchronous // feed emission under test. agent_command: "true".into(), agent_args: vec![], mcp_command: "test-mcp-server".into(), idle_timeout_secs: config::DEFAULT_IDLE_TIMEOUT_SECS, max_turn_duration_secs: config::DEFAULT_MAX_TURN_DURATION_SECS, agents: 1, heartbeat_interval_secs: 0, turn_liveness_secs: 10, heartbeat_prompt: None, system_prompt: None, team_instructions: None, initial_message: None, subscribe_mode: config::SubscribeMode::All, dedup_mode: config::DedupMode::Queue, multiple_event_handling: config::MultipleEventHandling::Queue, ignore_self: true, kinds_override: None, channels_override: None, no_mention_filter: false, config_path: std::path::PathBuf::from("./buzz-acp.toml"), context_message_limit: 12, max_turns_per_session: 0, presence_enabled: true, typing_enabled: true, memory_enabled: false, model: None, permission_mode: config::PermissionMode::BypassPermissions, respond_to: config::RespondTo::Anyone, respond_to_allowlist: HashSet::new(), allowed_respond_to: vec![], persona_env_vars: vec![], has_generated_codex_config: false, relay_observer: false, agent_owner: None, no_base_prompt: false, base_prompt_content: None, } } #[test] fn normalizes_agent_name_from_initialize_result() { assert_eq!( normalized_agent_name(&serde_json::json!({ "agentInfo": { "name": " Goose ", "version": "1.43.0" } })), "goose" ); assert_eq!( normalized_agent_name(&serde_json::json!({ "serverInfo": { "name": "buzz-agent" } })), "buzz-agent" ); } /// Spawn a real but inert agent subprocess (`cat`) so the error paths have /// an `OwnedAgent` to move into respawn or return to the pool. The error /// branches never talk to the subprocess. async fn dummy_agent(index: usize) -> OwnedAgent { OwnedAgent { index, acp: AcpClient::spawn("cat", &[], &[], false) .await .expect("spawn cat as inert agent"), state: Default::default(), model_capabilities: None, desired_model: None, model_overridden: false, agent_name: "unknown".into(), goose_system_prompt_supported: None, // Error branches under test never read this; 1 is the legacy // non-systemPrompt path, the simplest valid value. protocol_version: 1, } } /// Drive one error outcome through `handle_prompt_result` and return how /// many `turn_error` events it emitted to the observer feed. async fn turn_errors_emitted_for(outcome: PromptOutcome) -> usize { let agent = dummy_agent(0).await; let mut pool = AgentPool::from_slots(vec![None]); // `handle_prompt_result` asserts it removes exactly one in-flight task // for the completing agent (the slot was checked out, not idle). Mirror // the real dispatch path by registering a TaskMeta keyed on a genuine // `task::Id` — only obtainable from inside a spawned task. let task_id = pool.join_set.spawn(async {}).id(); pool.task_map_mut().insert( task_id, crate::pool::TaskMeta { agent_index: 0, channel_id: None, recoverable_batch: None, control_tx: None, steer_tx: None, }, ); let mut queue = EventQueue::new(config::DedupMode::Queue); let config = test_config(); let mut heartbeat_in_flight = false; let removed_channels = HashSet::new(); let mut crash_history = vec![SlotCircuit { crash_times: Vec::new(), open_until: None, respawn_in_flight: false, }]; let (respawn_tx, _respawn_rx) = mpsc::channel(8); let mut respawn_tasks = tokio::task::JoinSet::new(); let observer = ObserverHandle::in_process(); let result = PromptResult { agent, source: PromptSource::Channel(Uuid::new_v4()), outcome, batch: None, }; handle_prompt_result( &mut pool, &mut queue, &config, result, &mut heartbeat_in_flight, &removed_channels, &mut crash_history, &respawn_tx, &mut respawn_tasks, Some(observer.clone()), None, ); observer .snapshot() .iter() .filter(|e| e.kind == "turn_error") .count() } #[tokio::test] async fn agent_exited_emits_exactly_one_feed_event() { assert_eq!(turn_errors_emitted_for(PromptOutcome::AgentExited).await, 1); } #[tokio::test] async fn idle_timeout_emits_exactly_one_feed_event() { assert_eq!( turn_errors_emitted_for(PromptOutcome::Timeout(TimeoutKind::Idle)).await, 1 ); } #[tokio::test] async fn hard_timeout_emits_exactly_one_feed_event() { assert_eq!( turn_errors_emitted_for(PromptOutcome::Timeout(TimeoutKind::Hard)).await, 1 ); } #[tokio::test] async fn cancel_drain_timeout_emits_exactly_one_feed_event() { assert_eq!( turn_errors_emitted_for(PromptOutcome::CancelDrainTimeout( std::time::Duration::from_secs(5) )) .await, 1 ); } /// idle_timeout outcome_label is "idle_timeout"; hard_timeout is "hard_timeout". #[tokio::test] async fn timeout_outcome_labels_differ() { let check_label = |outcome: PromptOutcome, expected_label: &'static str| async move { let agent = dummy_agent(0).await; let mut pool = AgentPool::from_slots(vec![None]); let task_id = pool.join_set.spawn(async {}).id(); pool.task_map_mut().insert( task_id, crate::pool::TaskMeta { agent_index: 0, channel_id: None, recoverable_batch: None, control_tx: None, steer_tx: None, }, ); let mut queue = EventQueue::new(config::DedupMode::Queue); let config = test_config(); let mut heartbeat_in_flight = false; let removed_channels = HashSet::new(); let mut crash_history = vec![SlotCircuit { crash_times: Vec::new(), open_until: None, respawn_in_flight: false, }]; let (respawn_tx, _respawn_rx) = mpsc::channel(8); let mut respawn_tasks = tokio::task::JoinSet::new(); let observer = ObserverHandle::in_process(); let result = PromptResult { agent, source: PromptSource::Channel(Uuid::new_v4()), outcome, batch: None, }; handle_prompt_result( &mut pool, &mut queue, &config, result, &mut heartbeat_in_flight, &removed_channels, &mut crash_history, &respawn_tx, &mut respawn_tasks, Some(observer.clone()), None, ); let events = observer.snapshot(); let turn_error = events.iter().find(|e| e.kind == "turn_error").unwrap(); assert_eq!( turn_error.payload["outcome"].as_str().unwrap(), expected_label ); }; check_label(PromptOutcome::Timeout(TimeoutKind::Idle), "idle_timeout").await; check_label(PromptOutcome::Timeout(TimeoutKind::Hard), "hard_timeout").await; check_label( PromptOutcome::CancelDrainTimeout(std::time::Duration::from_secs(5)), "cancel_drain_timeout", ) .await; } /// hard-cap timeout dead-letters immediately (no requeue); idle timeout is requeued. #[tokio::test] async fn hard_timeout_not_requeued_idle_timeout_is_requeued() { let make_batch = || { let keys = Keys::generate(); let event = EventBuilder::new(Kind::Custom(9), "test") .sign_with_keys(&keys) .unwrap(); FlushBatch { channel_id: Uuid::new_v4(), events: vec![BatchEvent { event, prompt_tag: "test".into(), received_at: std::time::Instant::now(), }], cancelled_events: vec![], cancel_reason: None, } }; // Returns (pending_channels, queued_event_count_for_channel). let run = |outcome: PromptOutcome, batch: FlushBatch| async move { let channel_id = batch.channel_id; let agent = dummy_agent(0).await; let mut pool = AgentPool::from_slots(vec![None]); let task_id = pool.join_set.spawn(async {}).id(); pool.task_map_mut().insert( task_id, crate::pool::TaskMeta { agent_index: 0, channel_id: None, recoverable_batch: None, control_tx: None, steer_tx: None, }, ); let mut queue = EventQueue::new(config::DedupMode::Queue); let config = test_config(); let mut heartbeat_in_flight = false; let removed_channels = HashSet::new(); let mut crash_history = vec![SlotCircuit { crash_times: Vec::new(), open_until: None, respawn_in_flight: false, }]; let (respawn_tx, _respawn_rx) = mpsc::channel(8); let mut respawn_tasks = tokio::task::JoinSet::new(); let result = PromptResult { agent, source: PromptSource::Channel(channel_id), outcome, batch: Some(batch), }; handle_prompt_result( &mut pool, &mut queue, &config, result, &mut heartbeat_in_flight, &removed_channels, &mut crash_history, &respawn_tx, &mut respawn_tasks, None, None, ); ( queue.pending_channels(), queue.queued_event_count(&channel_id), ) }; // Hard timeout: batch must NOT be requeued (dead-lettered immediately). let hard_batch = make_batch(); let (hard_channels, hard_events) = run(PromptOutcome::Timeout(TimeoutKind::Hard), hard_batch).await; assert_eq!( hard_channels, 0, "hard-cap timeout must not requeue the batch" ); assert_eq!(hard_events, 0, "hard-cap timeout must drop all events"); // Idle timeout: batch IS requeued (first attempt, not yet dead-lettered). let idle_batch = make_batch(); let (idle_channels, idle_events) = run(PromptOutcome::Timeout(TimeoutKind::Idle), idle_batch).await; assert_eq!( idle_channels, 1, "idle timeout must requeue the batch for retry" ); assert_eq!( idle_events, 1, "idle timeout must preserve the event for retry" ); } /// Cancel-drain-timeout batches are requeued as cancelled (merge into the /// next flush, `CancelReason` preserved) — never dead-lettered like a real /// hard-cap. The agent itself is NOT returned to the idle pool: it is /// handed to `spawn_respawn_task` instead, mirroring a fatal `Timeout`. /// /// This reproduces the full steer-fallback incident, not just the /// original batch in isolation: the steer ack handler already released /// the new triggering event back to `queue` (`lib.rs`'s /// `ExpectedRunIdMissing` path) before the cancel-drain expiry fires. The /// next `flush_next()` must merge the surviving original event (via /// `cancelled_events`) with that already-queued new event (via `events`) /// exactly once each — proving no loss and no duplication. #[tokio::test] async fn cancel_drain_timeout_requeues_batch_and_does_not_return_agent() { let keys = Keys::generate(); let original_event = EventBuilder::new(Kind::Custom(9), "original") .sign_with_keys(&keys) .unwrap(); let new_event = EventBuilder::new(Kind::Custom(9), "new") .sign_with_keys(&keys) .unwrap(); assert_ne!( original_event.id, new_event.id, "test fixture must use two distinct events" ); let channel_id = Uuid::new_v4(); let batch = FlushBatch { channel_id, events: vec![BatchEvent { event: original_event.clone(), prompt_tag: "test".into(), received_at: std::time::Instant::now(), }], cancelled_events: vec![], cancel_reason: Some(CancelReason::Steer), }; let agent = dummy_agent(0).await; let mut pool = AgentPool::from_slots(vec![None]); let task_id = pool.join_set.spawn(async {}).id(); pool.task_map_mut().insert( task_id, crate::pool::TaskMeta { agent_index: 0, channel_id: None, recoverable_batch: None, control_tx: None, steer_tx: None, }, ); let mut queue = EventQueue::new(config::DedupMode::Queue); // The steer ack handler releases the new event to the queue BEFORE // signaling the fallback ControlSignal::Steer that ultimately times // out on drain — so it is already queued by the time // handle_prompt_result runs. queue.push(QueuedEvent { channel_id, event: new_event.clone(), received_at: std::time::Instant::now(), prompt_tag: "test".into(), }); let config = test_config(); let mut heartbeat_in_flight = false; let removed_channels = HashSet::new(); let mut crash_history = vec![SlotCircuit { crash_times: Vec::new(), open_until: None, respawn_in_flight: false, }]; let (respawn_tx, _respawn_rx) = mpsc::channel(8); let mut respawn_tasks = tokio::task::JoinSet::new(); let observer = ObserverHandle::in_process(); let grace = std::time::Duration::from_secs(5); let result = PromptResult { agent, source: PromptSource::Channel(channel_id), outcome: PromptOutcome::CancelDrainTimeout(grace), batch: Some(batch), }; handle_prompt_result( &mut pool, &mut queue, &config, result, &mut heartbeat_in_flight, &removed_channels, &mut crash_history, &respawn_tx, &mut respawn_tasks, Some(observer.clone()), None, ); // Batch preserved as a cancelled merge, not dead-lettered — same // treatment as a normal `Cancelled` outcome. `handle_prompt_result` // already called `mark_complete` internally, releasing the channel. // `flush_next()` must merge the already-queued new event with the // preserved original: each exactly once, in the correct bucket. let requeued = queue.flush_next().expect("batch must be requeued"); assert_eq!( requeued.events.len(), 1, "exactly one new event must be in the regular events bucket" ); assert_eq!( requeued.events[0].event.id, new_event.id, "the regular events bucket must hold the new (already-queued) event" ); assert_eq!( requeued.cancelled_events.len(), 1, "exactly one original event must be in the cancelled_events bucket" ); assert_eq!( requeued.cancelled_events[0].event.id, original_event.id, "the cancelled_events bucket must hold the original (interrupted) event" ); assert_ne!( requeued.events[0].event.id, requeued.cancelled_events[0].event.id, "the new and original events must not be the same event" ); assert_eq!( requeued.cancel_reason, Some(CancelReason::Steer), "CancelReason must ride through to the requeued batch" ); // Agent must NOT be back in the idle pool — it was handed to respawn. assert_eq!( pool.live_count(), 0, "agent must not be returned to the pool after a cancel-drain timeout" ); assert_eq!( respawn_tasks.len(), 1, "a respawn task must be spawned for the poisoned agent" ); // The observer payload must be fate-neutral: it names the grace and // the process replacement, and must NOT claim the batch was // preserved — that claim is false for explicit Stop/removed-channel // drops (see the sibling dropped-Stop test below), so the same // wording is used regardless of fate. let events = observer.snapshot(); let turn_error = events .iter() .find(|e| e.kind == "turn_error") .expect("exactly one turn_error event must be emitted"); assert_eq!( turn_error.payload["outcome"].as_str().unwrap(), "cancel_drain_timeout" ); assert_eq!( turn_error.payload["error"].as_str().unwrap(), format!("Agent did not stop within {grace:?} after cancellation; the agent process is being replaced."), "observer message must name the actual grace and must not claim preservation" ); assert_eq!( events.iter().filter(|e| e.kind == "turn_error").count(), 1, "exactly one turn_error event must be emitted" ); } /// Explicit Stop (`ControlSignal::Cancel`) on cancel-drain expiry drops /// the triggering batch — `requeue_cancelled_batch` returns `None` for /// `Cancel`/`Rotate`. The observer payload must be the SAME fate-neutral /// text as the preserved-Steer case above: it must never claim work was /// preserved when it was intentionally discarded. The poisoned agent is /// still respawned exactly as in the preserved case. #[tokio::test] async fn cancel_drain_timeout_dropped_stop_batch_none_same_neutral_payload() { let agent = dummy_agent(0).await; let mut pool = AgentPool::from_slots(vec![None]); let task_id = pool.join_set.spawn(async {}).id(); pool.task_map_mut().insert( task_id, crate::pool::TaskMeta { agent_index: 0, channel_id: None, recoverable_batch: None, control_tx: None, steer_tx: None, }, ); let mut queue = EventQueue::new(config::DedupMode::Queue); let config = test_config(); let mut heartbeat_in_flight = false; let removed_channels = HashSet::new(); let mut crash_history = vec![SlotCircuit { crash_times: Vec::new(), open_until: None, respawn_in_flight: false, }]; let (respawn_tx, _respawn_rx) = mpsc::channel(8); let mut respawn_tasks = tokio::task::JoinSet::new(); let observer = ObserverHandle::in_process(); let grace = std::time::Duration::from_secs(5); let result = PromptResult { agent, source: PromptSource::Channel(Uuid::new_v4()), outcome: PromptOutcome::CancelDrainTimeout(grace), // Explicit Stop already dropped the batch upstream in // `classify_control_cancel_failure` — `handle_prompt_result` // never sees one to requeue. batch: None, }; handle_prompt_result( &mut pool, &mut queue, &config, result, &mut heartbeat_in_flight, &removed_channels, &mut crash_history, &respawn_tx, &mut respawn_tasks, Some(observer.clone()), None, ); // No batch to merge — the queue has nothing pending for any channel. assert_eq!( queue.pending_channels(), 0, "a dropped Stop batch must not leave anything queued" ); // Same respawn treatment as the preserved case: never returned idle. assert_eq!( pool.live_count(), 0, "agent must not be returned to the pool after a cancel-drain timeout" ); assert_eq!( respawn_tasks.len(), 1, "a respawn task must be spawned for the poisoned agent" ); // The observer payload is byte-identical to the preserved-Steer case: // fate-neutral, naming the grace, with no preservation claim. let events = observer.snapshot(); let turn_error = events .iter() .find(|e| e.kind == "turn_error") .expect("exactly one turn_error event must be emitted"); assert_eq!( turn_error.payload["outcome"].as_str().unwrap(), "cancel_drain_timeout" ); assert_eq!( turn_error.payload["error"].as_str().unwrap(), format!("Agent did not stop within {grace:?} after cancellation; the agent process is being replaced."), "observer message must be fate-neutral even though the batch was dropped" ); assert_eq!( events.iter().filter(|e| e.kind == "turn_error").count(), 1, "exactly one turn_error event must be emitted" ); } #[tokio::test] async fn transport_error_emits_exactly_one_feed_event() { let io = AcpError::Io(std::io::Error::other("pipe broke")); assert_eq!(turn_errors_emitted_for(PromptOutcome::Error(io)).await, 1); } #[tokio::test] async fn application_error_emits_exactly_one_feed_event() { let app = AcpError::IdleTimeout(std::time::Duration::from_secs(1)); assert_eq!(turn_errors_emitted_for(PromptOutcome::Error(app)).await, 1); } } #[cfg(test)] mod observer_payload_trim_tests { use super::*; fn event_with_payload(kind: &str, payload: serde_json::Value) -> observer::ObserverEvent { observer::ObserverEvent { seq: 1, timestamp: "2026-06-16T00:00:00Z".to_string(), kind: kind.to_string(), agent_index: Some(0), channel_id: Some("11111111-1111-1111-1111-111111111111".to_string()), session_id: Some("sess-1".to_string()), turn_id: Some("turn-1".to_string()), payload, } } fn serialized(event: &observer::ObserverEvent) -> String { serde_json::to_string(event).unwrap() } #[test] fn test_under_budget_frame_passes_through_byte_identical() { let mut event = event_with_payload("acp_read", serde_json::json!({ "body": "small" })); let before = serialized(&event); fit_observer_event_to_budget(&mut event); assert_eq!( serialized(&event), before, "under-budget frame must not be mutated" ); } #[test] fn test_single_giant_leaf_is_elided_to_fit_with_envelope_intact() { let big = "x".repeat(100_000); let mut event = event_with_payload("acp_read", serde_json::json!({ "body": big })); fit_observer_event_to_budget(&mut event); assert!( serialized(&event).len() <= OBSERVER_MAX_PLAINTEXT_LEN, "frame must fit after trimming" ); // Envelope intact. assert_eq!(event.kind, "acp_read"); assert_eq!(event.turn_id.as_deref(), Some("turn-1")); assert_eq!( event.channel_id.as_deref(), Some("11111111-1111-1111-1111-111111111111") ); assert_eq!(event.seq, 1); let leaf = event.payload["body"].as_str().unwrap(); assert!( leaf.starts_with(&"x".repeat(OBSERVER_LEAF_RETAIN_BYTES)), "head retained" ); assert!( leaf.ends_with(&"x".repeat(OBSERVER_LEAF_RETAIN_BYTES)), "tail retained" ); // N in the marker is RAW bytes removed: original len minus retained len. let removed = 100_000 - leaf.chars().filter(|c| *c == 'x').count(); assert!( leaf.contains(&format!("…[elided {removed} bytes]…")), "marker reports raw bytes removed" ); } #[test] fn test_multi_block_prompt_retains_every_section_header_after_elision() { // The real session/prompt fix: format_prompt now emits one block per // section, so the observer payload is params.prompt = [{text: "[Base]…"}, // {text: "[Agent Memory — core]…"}, … {text: "[Buzz event: …]…"}]. // An oversized section is its own leaf, so eliding its body keeps the // leaf's head-3000 (which begins with the section's [Header] line) — every // header survives, so the desktop "Prompt context" panel counts them all. // This is the regression the single-fat-leaf shape caused (the trailing // [Buzz event] header fell into the elided middle and the count collapsed // to 1). let sections = [ "[Base]\nyou are a helpful agent".to_string(), "[System]\npersona text".to_string(), "[Agent Memory — core]\nremember this".to_string(), "[Context]\nScope: thread".to_string(), // The triggering event body, oversized on its own. format!("[Buzz event: @mention]\nContent: {}", "E".repeat(90_000)), ]; let block_refs: Vec<&str> = sections.iter().map(String::as_str).collect(); // Mirror the wire shape build_prompt_params produces: each block is its // own {type:"text", text} leaf under params.prompt. let prompt_blocks: Vec = block_refs .iter() .map(|text| serde_json::json!({ "type": "text", "text": text })) .collect(); let mut event = event_with_payload( "acp_write", serde_json::json!({ "method": "session/prompt", "params": { "sessionId": "sess-1", "prompt": prompt_blocks }, }), ); assert!( serialized(&event).len() > OBSERVER_MAX_PLAINTEXT_LEN, "precondition: oversized event body pushes the frame over the cap" ); fit_observer_event_to_budget(&mut event); assert!( serialized(&event).len() <= OBSERVER_MAX_PLAINTEXT_LEN, "frame must fit after trimming" ); let blocks = event.payload["params"]["prompt"] .as_array() .expect("prompt array survives"); let texts: Vec<&str> = blocks.iter().map(|b| b["text"].as_str().unwrap()).collect(); for header in [ "[Base]", "[System]", "[Agent Memory — core]", "[Context]", "[Buzz event: @mention]", ] { assert!( texts.iter().any(|t| t.starts_with(header)), "section header {header} must survive at the head of its own block" ); } // The oversized event body was elided in place (header kept, middle cut). let event_block = texts .iter() .find(|t| t.starts_with("[Buzz event: @mention]")) .unwrap(); assert!( event_block.contains("…[elided"), "the oversized event body is elided, not dropped" ); } #[test] fn test_multi_leaf_elides_largest_shrinkable_first_and_stops_when_it_fits() { // One leaf alone over the cap; a second smaller-but-still-large leaf. // Eliding the biggest should suffice, leaving the smaller intact. let mut event = event_with_payload( "acp_write", serde_json::json!({ "huge": "a".repeat(90_000), "medium": "b".repeat(20_000), }), ); fit_observer_event_to_budget(&mut event); assert!(serialized(&event).len() <= OBSERVER_MAX_PLAINTEXT_LEN); assert!( event.payload["huge"].as_str().unwrap().contains("…[elided"), "the largest leaf is elided" ); assert_eq!( event.payload["medium"].as_str().unwrap().len(), 20_000, "the smaller leaf is left untouched once the frame fits" ); } #[test] fn test_coalesced_chunk_nested_leaf_is_reached_by_recursive_walk() { // The coalesced-chunk big leaf lives at params.update.content.text, // not a top-level field — the walk must recurse to reach it. let big = "z".repeat(80_000); let mut event = event_with_payload( "session_update", serde_json::json!({ "params": { "update": { "sessionUpdate": "agent_message_chunk", "content": { "text": big } } } }), ); fit_observer_event_to_budget(&mut event); assert!(serialized(&event).len() <= OBSERVER_MAX_PLAINTEXT_LEN); let text = event.payload["params"]["update"]["content"]["text"] .as_str() .unwrap(); assert!(text.contains("…[elided"), "nested leaf was elided"); } #[test] fn test_many_medium_leaves_terminate_via_stub() { // Many leaves each too small to shrink on their own (below 2x retain), // collectively over the cap. No leaf can strictly shrink, so the trimmer // must terminate via the stub rather than loop forever. let leaf = "m".repeat(OBSERVER_LEAF_RETAIN_BYTES); // shorter than head+tail → cannot shrink let items: Vec = (0..40) .map(|_| serde_json::Value::String(leaf.clone())) .collect(); let mut event = event_with_payload("acp_read", serde_json::json!({ "items": items })); assert!( serialized(&event).len() > OBSERVER_MAX_PLAINTEXT_LEN, "precondition: frame is over the cap" ); fit_observer_event_to_budget(&mut event); assert!(serialized(&event).len() <= OBSERVER_MAX_PLAINTEXT_LEN); assert_eq!( event.payload["elided"].as_str().unwrap(), "acp_read payload too large", "fell back to the stub" ); assert!(event.payload.get("originalBytes").is_some()); } #[test] fn test_leaf_too_small_to_shrink_is_not_mutated() { // A frame already under budget whose only leaf is below the shrink floor: // nothing should change. (Under-budget short-circuits, and even if forced, // leaf_shrinks would reject it.) let short = "s".repeat(OBSERVER_LEAF_RETAIN_BYTES); // == head; cannot strictly shrink assert!( !leaf_shrinks(&short), "a leaf at the retain floor must not shrink" ); let longer = "L".repeat(OBSERVER_LEAF_RETAIN_BYTES * 2 + 100); assert!(leaf_shrinks(&longer), "a clearly larger leaf must shrink"); } #[test] fn test_utf8_multibyte_leaf_elides_on_char_boundary() { // A leaf of 3-byte chars (… = U+2026) — eliding must land on char // boundaries and never panic or produce invalid UTF-8. let big: String = "…".repeat(40_000); // 120_000 bytes let mut event = event_with_payload("acp_read", serde_json::json!({ "body": big })); fit_observer_event_to_budget(&mut event); assert!(serialized(&event).len() <= OBSERVER_MAX_PLAINTEXT_LEN); let leaf = event.payload["body"].as_str().unwrap(); // Valid UTF-8 by construction (it's a &str); confirm head/tail are whole // multi-byte chars and the marker is present. assert!(leaf.starts_with('…')); assert!(leaf.ends_with('…')); assert!(leaf.contains("[elided")); } }