From 0b754688b709aad08e38ea0d9ab487ef70ae227c Mon Sep 17 00:00:00 2001 From: Diem Nguyen Date: Wed, 9 Sep 2026 12:04:57 -0700 Subject: [PATCH] fix(agent): fail fast and name the model on model-not-found errors A Databricks v2 agent configured with the bare id gpt-6-astra got a 404 (the endpoint is databricks-gpt-6-astra). The harness retried the batch 10 times over ~40 minutes before posting the raw error, and the picker could not offer the right id because the v2 gateway listing lags v1. Dead-letter -32002 immediately with a notice that names the model, supplement v2 discovery with v1 databricks-* endpoints, add a gpt-6 capability rule, and name the rejected model in the desktop copy. Signed-off-by: Diem Nguyen --- crates/buzz-acp/src/lib.rs | 111 ++++++++++++++++++ crates/buzz-agent/src/catalog.rs | 67 ++++++++++- crates/buzz-agent/src/model_capabilities.rs | 2 +- .../lib/friendlyAgentLastError.test.mjs | 10 +- .../agents/lib/friendlyAgentLastError.ts | 10 +- scripts/model-capabilities.json | 24 ++++ scripts/normative-corpus.json | 11 +- 7 files changed, 224 insertions(+), 11 deletions(-) diff --git a/crates/buzz-acp/src/lib.rs b/crates/buzz-acp/src/lib.rs index cc2952a2f8..ef6bf04cf4 100644 --- a/crates/buzz-acp/src/lib.rs +++ b/crates/buzz-acp/src/lib.rs @@ -4798,6 +4798,31 @@ fn handle_prompt_result( and then re-send." .to_string(); spawn_failure_notice(rest_client, &batch, content); + } else if let PromptOutcome::Error(acp::AcpError::AgentError { + code: -32002, + message, + }) = &result.outcome + { + // Model-not-found (-32002) is non-retryable like auth errors: + // the configured model won't appear between retries, so tell + // the owner now which model failed and where to change it. + tracing::warn!( + channel_id = %batch.channel_id, + events = batch.events.len(), + "dead-lettering batch immediately — configured model not found (non-retryable)" + ); + // buzz-agent stamps the rejected id: `llm model not found: () 404 …` + let model = message + .split("llm model not found: (") + .nth(1) + .and_then(|rest| rest.split(')').next()) + .filter(|m| !m.is_empty()) + .unwrap_or(""); + let content = format!( + "⚠️ I couldn't process the last request: my configured model `{model}` was not found by the LLM provider (404). \ + The agent owner needs to open this agent's settings and select a different model, then re-send." + ); + spawn_failure_notice(rest_client, &batch, content); } else if let Some(dead) = queue.requeue(batch) { let reason = match &result.outcome { PromptOutcome::Timeout(TimeoutKind::Idle) => "the turn timed out".to_string(), @@ -11012,6 +11037,92 @@ mod error_outcome_emission_tests { ); } + /// A model-not-found (`-32002`) error must dead-letter immediately: the + /// configured model id won't start existing between retries, and the + /// owner needs to be told which model to change right away. + #[tokio::test] + async fn model_not_found_error_dead_letters_immediately_without_requeueing() { + let keys = nostr::Keys::generate(); + let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "test") + .sign_with_keys(&keys) + .unwrap(); + let channel_id = uuid::Uuid::new_v4(); + let batch = FlushBatch { + channel_id, + scope: scope::SessionScope::Conversation { channel_id }, + events: vec![BatchEvent { + event, + prompt_tag: "test".into(), + received_at: std::time::Instant::now(), + }], + cancelled_events: vec![], + cancel_reason: None, + }; + let not_found = acp::AcpError::AgentError { + code: -32002, + message: "llm model not found: (gpt-6-astra) 404 Not Found".to_string(), + }; + + let agent = dummy_agent(0).await; + let mut pool = AgentPool::from_slots(vec![None]); + let task_id = pool.join_set.spawn(async {}).id(); + pool.task_map_mut().insert( + task_id, + crate::pool::TaskMeta { + agent_index: 0, + channel_id: None, + scope: None, + turn_id: "test-turn-id".to_string(), + recoverable_batch: None, + control_tx: None, + steer_tx: None, + successful_steer_deliveries: HashSet::new(), + }, + ); + let mut queue = EventQueue::new(config::DedupMode::Queue); + let config = test_config(); + let mut heartbeat_in_flight = false; + let removed_channels = std::collections::HashSet::new(); + let mut crash_history = vec![SlotCircuit { + crash_times: Vec::new(), + open_until: None, + respawn_in_flight: false, + }]; + let (respawn_tx, _respawn_rx) = mpsc::channel(8); + let mut respawn_tasks = tokio::task::JoinSet::new(); + let result = PromptResult { + agent, + source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id }), + turn_id: "test-turn-id".to_string(), + outcome: PromptOutcome::Error(not_found), + batch: Some(batch), + }; + handle_prompt_result( + &mut pool, + &mut queue, + &config, + result, + &mut heartbeat_in_flight, + &removed_channels, + &mut crash_history, + &respawn_tx, + &mut respawn_tasks, + None, + None, + ); + + assert_eq!( + queue.pending_channels(), + 0, + "model-not-found must dead-letter immediately — batch must not be requeued" + ); + assert_eq!( + queue.queued_event_count(channel_id), + 0, + "model-not-found must dead-letter immediately — no events should be pending" + ); + } + /// A non-auth application error (e.g. usage credits) must still follow the /// standard requeue path so today's behavior is unchanged. #[tokio::test] diff --git a/crates/buzz-agent/src/catalog.rs b/crates/buzz-agent/src/catalog.rs index f2cda834fd..ef54b68061 100644 --- a/crates/buzz-agent/src/catalog.rs +++ b/crates/buzz-agent/src/catalog.rs @@ -289,6 +289,23 @@ pub(crate) fn parse_v1_endpoints(json: &Value) -> Result, AgentE Ok(models) } +/// Append v1 `databricks-*` (Databricks-managed) chat endpoints the v2 listing +/// does not know yet. Workspace-created endpoints are left out: the gap this +/// fills is new frontier models, not every team's custom endpoint. +fn merge_v1_managed_endpoints(v2: &mut Vec, v1: Vec) { + let known: HashSet = v2.iter().map(|e| e.entry.id.to_ascii_lowercase()).collect(); + v2.extend( + v1.into_iter() + .filter(|m| { + m.id.starts_with("databricks-") && !known.contains(&m.id.to_ascii_lowercase()) + }) + .map(|entry| V2Endpoint { + entry, + created_ms: None, + }), + ); +} + // --------------------------------------------------------------------------- // v2 — api/ai-gateway/v2/endpoints + Unity Catalog model-services // --------------------------------------------------------------------------- @@ -340,9 +357,14 @@ async fn fetch_v2_models_with_policy( fetch_catalog_pages_with_policy(http, host, bearer, WORKSPACE_CATALOG_DESCRIPTOR, policy); let unity_catalog = fetch_catalog_pages_with_policy(http, host, bearer, UNITY_CATALOG_DESCRIPTOR, policy); + // The v2 listing lags the serving-endpoints registry: a Databricks-managed + // model can be READY and callable for days before v2 lists it (2026-09-09: + // `databricks-gpt-6-astra` was v1-only). Fill the gap from v1 so the + // picker offers it instead of forcing a hand-typed id. Best-effort. + let v1 = fetch_v1_models(http, host, bearer); - let (workspace, unity_catalog) = tokio::join!(workspace, unity_catalog); - let (workspace, unity_catalog, both_succeeded) = match (workspace, unity_catalog) { + let (workspace, unity_catalog, v1) = tokio::join!(workspace, unity_catalog, v1); + let (mut workspace, unity_catalog, both_succeeded) = match (workspace, unity_catalog) { (Ok(workspace), Ok(unity_catalog)) => (workspace, unity_catalog, true), (Ok(workspace), Err(error)) => { if matches!(&error, AgentError::LlmAuth(_)) && !allow_partial_auth_failure { @@ -370,6 +392,14 @@ async fn fetch_v2_models_with_policy( return Err(combined_catalog_error(workspace_error, unity_catalog_error)); } }; + match v1 { + Ok(v1) => merge_v1_managed_endpoints(&mut workspace, v1), + Err(error) => tracing::warn!( + catalog = "workspace serving-endpoints v1", + error_kind = catalog_error_kind(&error), + "Databricks v1 catalog supplement unavailable; using v2 listing only" + ), + } Ok(merge_v2_models( workspace, @@ -1571,6 +1601,39 @@ mod tests { assert_eq!(ids, vec!["my-llm", "my-completions", "no-state", "no-task"]); } + #[test] + fn v1_managed_endpoints_supplement_v2_listing_without_duplicates() { + let entry = |id: &str| ModelEntry { + id: id.into(), + name: id.into(), + }; + let mut v2 = vec![ + V2Endpoint { + entry: entry("databricks-gpt-5-5"), + created_ms: Some(2), + }, + V2Endpoint { + entry: entry("goose-claude-opus-5"), + created_ms: Some(1), + }, + ]; + let v1 = vec![ + entry("databricks-gpt-6-astra"), // new managed model → added + entry("Databricks-GPT-5-5"), // already listed (case-insensitive) → skipped + entry("kgoose-claude-fable-5"), // workspace-created endpoint → skipped + ]; + merge_v1_managed_endpoints(&mut v2, v1); + let ids: Vec<&str> = v2.iter().map(|e| e.entry.id.as_str()).collect(); + assert_eq!( + ids, + vec![ + "databricks-gpt-5-5", + "goose-claude-opus-5", + "databricks-gpt-6-astra" + ] + ); + } + #[test] fn v1_parse_errors_on_missing_endpoints_array() { let json = serde_json::json!({"data": []}); diff --git a/crates/buzz-agent/src/model_capabilities.rs b/crates/buzz-agent/src/model_capabilities.rs index 18a4726012..d2f37f6ee5 100644 --- a/crates/buzz-agent/src/model_capabilities.rs +++ b/crates/buzz-agent/src/model_capabilities.rs @@ -730,7 +730,7 @@ mod tests { Q::Vector { id: "collision-gpt-5-6-probe", provider: "openai", raw_model_id: "gpt-5.6", note: Some("Probes a dotted minor version against the gpt-5 stem.") }, Q::Vector { id: "collision-gpt-5-1-probe", provider: "openai", raw_model_id: "gpt-5.1", note: Some("Probes the gpt-5.1 prefix.") }, Q::Section { group: "Uncurated DBv2 token probes", note: None }, - Q::Vector { id: "uncurated-dbv2-gpt-6-probe", provider: "databricks_v2", raw_model_id: "databricks-gpt-6", note: Some("Probes a non-5 gpt version with no exact record or prefix rule.") }, + Q::Vector { id: "uncurated-dbv2-gpt-6-probe", provider: "databricks_v2", raw_model_id: "databricks-gpt-6", note: Some("Probes the bare gpt-6 stem: no exact record, binds to the gpt-6 prefix rule.") }, Q::Vector { id: "uncurated-dbv2-gpt-4o-probe", provider: "databricks_v2", raw_model_id: "databricks-gpt-4o", note: Some("Probes an uncatalogued gpt-4o databricks_v2 id.") }, Q::Vector { id: "uncurated-dbv2-opus-5-bare-probe", provider: "databricks_v2", raw_model_id: "opus-5", note: Some("Probes a bare Claude code-name segment with no leading claude.") }, Q::Vector { id: "uncurated-dbv2-sol-bare-probe", provider: "databricks_v2", raw_model_id: "sol", note: Some("Probes a bare OpenAI code name.") }, diff --git a/desktop/src/features/agents/lib/friendlyAgentLastError.test.mjs b/desktop/src/features/agents/lib/friendlyAgentLastError.test.mjs index 597b1b9323..df691c6c06 100644 --- a/desktop/src/features/agents/lib/friendlyAgentLastError.test.mjs +++ b/desktop/src/features/agents/lib/friendlyAgentLastError.test.mjs @@ -78,11 +78,19 @@ test("non-auth Agent reported error stays generic", () => { ); }); -test("code -32002 → model-not-found copy (severity: denied)", () => { +test("code -32002 → model-not-found copy naming the stamped model (severity: denied)", () => { const result = friendlyAgentLastError( "Agent reported error: llm model not found: (goose-claude-opus-4-8) 404 Not Found: ...", -32002, ); + assert.deepEqual(result, { + severity: "denied", + copy: 'Model "goose-claude-opus-4-8" was not found by the LLM provider (404) — open agent settings and select a different model from the dropdown.', + }); +}); + +test("code -32002 without a stamped model falls back to the generic copy", () => { + const result = friendlyAgentLastError("llm model not found", -32002); assert.deepEqual(result, { severity: "denied", copy: MODEL_NOT_FOUND_COPY, diff --git a/desktop/src/features/agents/lib/friendlyAgentLastError.ts b/desktop/src/features/agents/lib/friendlyAgentLastError.ts index 60c77bb04c..054a79413a 100644 --- a/desktop/src/features/agents/lib/friendlyAgentLastError.ts +++ b/desktop/src/features/agents/lib/friendlyAgentLastError.ts @@ -42,6 +42,14 @@ export const RELAY_MESH_DENIED_COPY = export const MODEL_NOT_FOUND_COPY = "The configured model is not available — open agent settings and select a different one from the dropdown."; +/** Name the rejected model when buzz-agent stamped it: `llm model not found: () 404 …`. */ +function modelNotFoundCopy(raw: string): string { + const model = /llm model not found: \(([^)]+)\)/.exec(raw)?.[1]; + return model + ? `Model "${model}" was not found by the LLM provider (404) — open agent settings and select a different model from the dropdown.` + : MODEL_NOT_FOUND_COPY; +} + export const CLI_ACP_INTERNAL_ERROR_COPY = "The agent's harness reported an internal error. For Codex agents this can mean the configured model isn't supported by your installed codex-acp — check the model in `~/.codex/config.toml` or upgrade the adapter (`brew upgrade codex-acp`)."; @@ -80,7 +88,7 @@ export function friendlyAgentLastError( case -32001: return { severity: "denied", copy: RELAY_MESH_DENIED_COPY }; case -32002: - return { severity: "denied", copy: MODEL_NOT_FOUND_COPY }; + return { severity: "denied", copy: modelNotFoundCopy(trimmed) }; case -32603: { // Standard JSON-RPC "Internal error" — emitted by external harnesses // (e.g. codex-acp) when the configured model is unsupported. Only diff --git a/scripts/model-capabilities.json b/scripts/model-capabilities.json index edcf25a783..8d416101cf 100644 --- a/scripts/model-capabilities.json +++ b/scripts/model-capabilities.json @@ -259,6 +259,30 @@ "databricks_v2_wire_route": "openai-responses", "normalization_policy": "openai-standard" }, + { + "id": "openai-gpt6", + "match_kind": "prefix", + "match_value": "gpt-6", + "match_aliases": [ + "gpt6" + ], + "providers": [ + "openai", + "databricks", + "databricks_v2" + ], + "thinking_mode": "none", + "supported_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "openai-responses", + "normalization_policy": "openai-standard" + }, { "id": "openai-gpt5-6", "match_kind": "prefix", diff --git a/scripts/normative-corpus.json b/scripts/normative-corpus.json index cb0ff4ae27..6efa84b0f9 100644 --- a/scripts/normative-corpus.json +++ b/scripts/normative-corpus.json @@ -2035,20 +2035,19 @@ "id": "uncurated-dbv2-gpt-6-probe", "provider": "databricks_v2", "raw_model_id": "databricks-gpt-6", - "_note": "Probes a non-5 gpt version with no exact record or prefix rule.", + "_note": "Probes the bare gpt-6 stem: no exact record, binds to the gpt-6 prefix rule.", "expect": { "thinking_mode": "none", "supported_efforts": [ - "none", - "minimal", "low", "medium", "high", - "xhigh" + "xhigh", + "max" ], "default_effort": "medium", - "databricks_v2_wire_route": "mlflow-chat", - "normalization_policy": "openai-clamp-max-to-xhigh", + "databricks_v2_wire_route": "openai-responses", + "normalization_policy": "openai-standard", "registry_label": null } },