From ad13ed8411c4e958c0e326a3f76231e5293c12b6 Mon Sep 17 00:00:00 2001 From: Michael Neale Date: Thu, 30 Jul 2026 10:47:44 +1000 Subject: [PATCH] fix(mesh): leave thinking effort to the model's own template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shared compute pinned BUZZ_AGENT_THINKING_EFFORT. That is not a sampling hint: mesh translates `reasoning_effort` into the chat template's `enable_thinking` flag, so any value Buzz sends overrides the default the model authors shipped — and the right value is model-specific. Measured against a request captured from the real agent (full ~43k-token system prompt, real dev-mcp toolset, tool_choice=auto): effort gemma-4-E4B Qwen3-8B Qwen3-8B output tokens none 0/8 8/8 ~121 low 4/4 7/8 ~470 unset 6/6 8/8 ~500 `none` (the previous default, chosen when shared compute was Qwen3-only) stops gemma tool-calling outright, so its replies were never published. Enabling thinking instead makes Qwen3 spend ~4x the output budget, which risks the 4096 cap this preset sets. Omitting the field leaves `enable_thinking` as None, so mesh passes no override and each model uses its own template default: 3/3 delivery on gemma-4-E4B, gemma-4-26B-A4B, gemma-4-31B and Qwen3-8B, and 3/3 replies through the real buzz-acp + buzz-agent path against a live relay. Thinking levels are unaffected: an explicit agent/persona/global value is still preserved and applied. Signed-off-by: Michael Neale --- crates/buzz-relay/examples/mesh_agent_e2e.rs | 7 ++--- .../src/managed_agents/relay_mesh.rs | 30 +++++++++---------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/crates/buzz-relay/examples/mesh_agent_e2e.rs b/crates/buzz-relay/examples/mesh_agent_e2e.rs index 35aa3f1b70..7e55d07b5d 100644 --- a/crates/buzz-relay/examples/mesh_agent_e2e.rs +++ b/crates/buzz-relay/examples/mesh_agent_e2e.rs @@ -471,10 +471,9 @@ async fn agent_chat_in_isolated_home( .env("OPENAI_COMPAT_API_KEY", "buzz-mesh-local") .env("OPENAI_COMPAT_API", "chat") .env("BUZZ_AGENT_MAX_OUTPUT_TOKENS", "4096") - // Must track apply_relay_mesh_env()'s default. `none` suppresses tool - // calling on local models, so pinning it here would test a config the - // product no longer ships and hide the very regression P5/P6 exist for. - .env("BUZZ_AGENT_THINKING_EFFORT", "low") + // No BUZZ_AGENT_THINKING_EFFORT: apply_relay_mesh_env() deliberately + // leaves it unset so each model's chat template picks its own default. + // Pinning a value here would test a config the product does not ship. .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::null()); diff --git a/desktop/src-tauri/src/managed_agents/relay_mesh.rs b/desktop/src-tauri/src/managed_agents/relay_mesh.rs index 972ce51c3a..327c106bc8 100644 --- a/desktop/src-tauri/src/managed_agents/relay_mesh.rs +++ b/desktop/src-tauri/src/managed_agents/relay_mesh.rs @@ -47,14 +47,15 @@ pub fn apply_relay_mesh_env( // may deliberately choose a smaller cap or a different effort. This function // runs after those layers during readiness, so never clobber their values. insert_default_if_unset(env, "BUZZ_AGENT_MAX_OUTPUT_TOKENS", "4096"); - // `none` suppresses tool calling outright on local models: llama.cpp's - // Gemma grammar is lazy under `tool_choice: auto`, and `reasoning_effort: - // "none"` keeps the model on its prose path so it never emits the tool-call - // token. Measured with the real prompt and toolset, gemma-4-E4B delivered a - // reply 0/8 times at `none` and 8/8 at `low`; `minimal` was 3/4. `low` buys - // just enough deliberation to pick a tool without spending the output - // budget on hidden reasoning. - insert_default_if_unset(env, "BUZZ_AGENT_THINKING_EFFORT", "low"); + // Deliberately no BUZZ_AGENT_THINKING_EFFORT default: mesh translates + // `reasoning_effort` into the chat template's `enable_thinking` flag, so any + // value we pick overrides each model's own template default — and the right + // value is model-specific. Measured with the real prompt and toolset: + // gemma-4-E4B delivers 0/8 at `none` but 6/6 with the field absent, while + // Qwen3-8B delivers 8/8 either way and burns ~4x the output tokens once + // thinking is on (121 -> ~470), risking the 4096 cap. Omitting the field + // lets every model use its own default; explicit agent/persona/global + // values still apply. } #[cfg(feature = "mesh-llm")] @@ -107,12 +108,11 @@ mod tests { env.get("BUZZ_AGENT_MAX_OUTPUT_TOKENS").map(String::as_str), Some("4096") ); - // Must not be "none": that suppresses tool calling on local models, so - // the agent answers in prose and the reply is never published. - assert_eq!( - env.get("BUZZ_AGENT_THINKING_EFFORT").map(String::as_str), - Some("low") - ); + // Must stay unset: any value we pick overrides the model's own chat + // template default, and the right value is model-specific ("none" + // stops gemma tool-calling; enabling thinking makes Qwen3 burn ~4x the + // output budget). + assert_eq!(env.get("BUZZ_AGENT_THINKING_EFFORT"), None); assert_eq!( env.get(RELAY_MESH_PREFER_MESH_FOR_AUTO_ENV) .map(String::as_str), @@ -127,8 +127,6 @@ mod tests { "BUZZ_AGENT_MAX_OUTPUT_TOKENS".to_string(), "2048".to_string(), ), - // Deliberately not the default ("low"), so this asserts - // preservation rather than coinciding with the default. ("BUZZ_AGENT_THINKING_EFFORT".to_string(), "high".to_string()), ]); apply_relay_mesh_env(