From 41974bc57b372f280b651caf1dbda9bb18092a64 Mon Sep 17 00:00:00 2001 From: Michael Neale Date: Tue, 28 Jul 2026 10:34:43 +1000 Subject: [PATCH] fix(buzz-agent): catch mesh-llm's 502 MoA failure path, not just the 503 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Yesterday I documented a "known blind spot" and left it. Checking mesh-llm's actual source shows that was the wrong call: the gap covers the COMMON failure, not a rare one. mesh-llm has two failure paths: 503, gateway level — mesh too small to start MoA at all. Plain message, no JSON. (moa_gateway/mod.rs:56) 502, MoA level — workers or reducers died mid-turn. Body carries error.message AND error.type=moa_failure. (mesh-mixture-of-agents/src/lib.rs:1168) Only the 503 was handled. The 502 is the one that actually fires in a running lab — a worker dropping out mid-turn — and because goose reduces a payload to error.message when that field exists (http_status.rs:186-197), the moa_failure type never reached us. Those turns failed outright instead of retrying on auto. Fixed by matching the messages themselves: MOA_FAILURE_MESSAGES lists every error_response call site in mesh-llm. The JSON-shaped check stays as a fallback for moa_failure with no message field. Message matching is brittle if mesh-llm rewords these, but the alternative is an HTTP-level seam goose does not expose, and silently losing the fallback is worse than a string that needs updating. 49 tests green, fmt + clippy clean. Signed-off-by: Michael Neale --- crates/buzz-agent/src/mesh.rs | 90 +++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 40 deletions(-) diff --git a/crates/buzz-agent/src/mesh.rs b/crates/buzz-agent/src/mesh.rs index dc045d5cc5..2ff88268b1 100644 --- a/crates/buzz-agent/src/mesh.rs +++ b/crates/buzz-agent/src/mesh.rs @@ -61,9 +61,25 @@ const COOLDOWN: Duration = Duration::from_secs(30); /// Consecutive positive observations required before enabling MoA. const ENABLE_OBSERVATIONS: u8 = 2; -/// The 503 body mesh-llm returns when MoA is configured but under-provisioned. +/// The 503 body mesh-llm returns when MoA is configured but under-provisioned +/// (`mesh-llm-host-runtime/src/network/openai/moa_gateway/mod.rs:56`). const MOA_UNAVAILABLE_MESSAGE: &str = "MoA requires ≥2 models available in the mesh"; +/// Messages from mesh-llm's *other* failure path: `error_response` +/// (`mesh-mixture-of-agents/src/lib.rs:1168`) returns a 502 whose body carries +/// both `error.message` and `error.type = "moa_failure"`. +/// +/// Goose keeps only the message (`http_status.rs:186-197`), so the type never +/// reaches us and we have to match the messages themselves. These are every +/// call site of `error_response` in mesh-llm. +const MOA_FAILURE_MESSAGES: &[&str] = &[ + "All MoA workers failed", + "All MoA reducers failed", + "MoA could not produce a usable answer", + "MoA reducer returned no usable answer", + "Reducer failed", +]; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Observation { Available, @@ -110,45 +126,37 @@ fn catalog_supports_collective(catalog: &Value) -> Option { /// Did this error mean "the mesh shrank", as opposed to a generic failure? /// -/// The old loop inspected the raw HTTP body (`llm.rs:1362-1388`) and accepted -/// two shapes: a 503 whose `error.message` is exactly -/// [`MOA_UNAVAILABLE_MESSAGE`], or any 5xx whose `error.type` is -/// `moa_failure`. +/// mesh-llm has two failure paths and both must be caught: /// -/// **We only see what goose leaves us.** `extract_message` -/// (`goose-providers/src/http_status.rs:186-197`) reduces the payload to -/// `error.message` when that field exists, and only falls back to the whole -/// JSON when it does not. So: +/// * **503, gateway level** — under-provisioned mesh. Plain message, no JSON +/// (`moa_gateway/mod.rs:56`). +/// * **502, MoA level** — workers or reducers failed. Body carries +/// `error.message` *and* `error.type = "moa_failure"` +/// (`mesh-mixture-of-agents/src/lib.rs:1168`). /// -/// * message shape — always detected (the message survives verbatim); -/// * `moa_failure` with no `message` — detected (whole JSON survives); -/// * `moa_failure` *alongside* a `message` — **not** detected, because the -/// type is discarded before it reaches us. Such a request fails the turn -/// instead of retrying on `auto`. +/// Goose reduces a payload to `error.message` when that field exists +/// (`goose-providers/src/http_status.rs:186-197`), so `moa_failure` never +/// survives — we match [`MOA_FAILURE_MESSAGES`] instead. That second path is +/// the *common* one at runtime (a worker dying mid-turn); the 503 only fires +/// when the mesh is too small to start with. /// -/// That last case is the one behaviour the provider-level wrapper cannot -/// reproduce; recovering it needs an HTTP-level seam goose does not expose. -/// mesh-llm's under-provisioned path sends the message shape, which is why the -/// common case is covered. -/// -/// Anything else is a real error and must surface — silently retrying every -/// 5xx as `auto` would mask outages. +/// Anything else is a real error and must surface — retrying every 5xx as +/// `auto` would mask outages. fn is_mesh_contraction(err: &ProviderError) -> bool { let text = err.to_string(); - if text.contains(MOA_UNAVAILABLE_MESSAGE) { + if text.contains(MOA_UNAVAILABLE_MESSAGE) + || MOA_FAILURE_MESSAGES.iter().any(|m| text.contains(m)) + { return true; } - // Provider errors carry the body inline; find the JSON and inspect it. + // Fallback for the shape goose passes through whole: `moa_failure` with no + // `error.message` field at all. let Some(start) = text.find('{') else { return false; }; serde_json::from_str::(&text[start..]) .ok() - .is_some_and(|v| { - let message = v.pointer("/error/message").and_then(Value::as_str); - let kind = v.pointer("/error/type").and_then(Value::as_str); - message == Some(MOA_UNAVAILABLE_MESSAGE) || kind == Some("moa_failure") - }) + .is_some_and(|v| v.pointer("/error/type").and_then(Value::as_str) == Some("moa_failure")) } /// Wraps a provider and applies Buzz's relay-mesh `auto` policy. @@ -486,18 +494,20 @@ mod tests { } #[test] - fn moa_failure_alongside_a_message_is_a_known_blind_spot() { - // goose keeps only `error.message`, so `type` never reaches us. This - // pins the gap rather than pretending parity: if goose ever preserves - // the body, this test flips and we can delete the caveat in the docs. - let err = ProviderError::ServerError( - "Server error (500) at http://x: upstream exploded".to_string(), - ); - assert!( - !is_mesh_contraction(&err), - "if this now passes, goose stopped stripping the body -- update the \ - doc comment on is_mesh_contraction" - ); + fn recognises_moa_worker_and_reducer_failures() { + // mesh-llm's 502 path always sends message AND type; goose strips the + // type, so these must be caught by message. This is the common runtime + // failure — a worker dying mid-turn. + for message in [ + "All MoA workers failed", + "MoA could not produce a usable answer", + "MoA reducer returned no usable answer", + "Reducer failed (tried 3): timeout", + ] { + let err = + ProviderError::ServerError(format!("Server error (502) at http://x: {message}")); + assert!(is_mesh_contraction(&err), "missed: {message}"); + } } #[test]