From a075619f47784f5408eb7a77e02c19a9297790f9 Mon Sep 17 00:00:00 2001 From: Atish Patel Date: Tue, 18 Aug 2026 13:31:24 -0500 Subject: [PATCH] fix: preserve DeepSeek max effort on Databricks --- Justfile | 2 +- crates/buzz-agent/src/model_capabilities.rs | 5 ++-- .../ui/modelCapabilitiesCorpus.test.mjs | 4 ++-- scripts/model-capabilities.json | 24 +++++++++++++++++-- scripts/normative-corpus.json | 18 ++++++++++++++ scripts/run-tests.sh | 2 +- 6 files changed, 47 insertions(+), 8 deletions(-) diff --git a/Justfile b/Justfile index ce8647cf77..213869aff7 100644 --- a/Justfile +++ b/Justfile @@ -335,7 +335,7 @@ test-unit: # buzz-agent model-capabilities corpus: the Rust half of the # cross-language drift guard. `model_capabilities.rs` embeds # scripts/model-capabilities.json + scripts/normative-corpus.json via - # include_str! and replays all 103 vectors as pure in-process tests (no + # include_str! and replays all 104 vectors as pure in-process tests (no # infra). Enumerated explicitly because nothing in CI runs # `cargo test --workspace`; without this step a manifest edit that # diverges Rust from the corpus ships green. diff --git a/crates/buzz-agent/src/model_capabilities.rs b/crates/buzz-agent/src/model_capabilities.rs index 81f4b4e3b6..7e06b9bf68 100644 --- a/crates/buzz-agent/src/model_capabilities.rs +++ b/crates/buzz-agent/src/model_capabilities.rs @@ -608,6 +608,7 @@ mod tests { Q::Vector { id: "anthropic-opus-5-prefix-probe", provider: "anthropic", raw_model_id: "claude-opus-5-20270101", note: Some("Probes the claude-opus-5 prefix rule.") }, Q::Vector { id: "dbv2-gpt-5-6-sol-normalization-probe", provider: "databricks_v2", raw_model_id: "databricks-gpt-5-6-sol", note: Some("Probes the sol exact record's normalization and effort axes.") }, Q::Vector { id: "dbv2-gpt-5-6-luna-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-gpt-5-6-luna", note: Some("Probes the luna exact record against its family rule.") }, + Q::Vector { id: "dbv2-deepseek-v4-flash-max-probe", provider: "databricks_v2", raw_model_id: "databricks-deepseek-v4-flash-0731", note: Some("Probes that the validated DeepSeek endpoint preserves max instead of inheriting the concrete-unknown max-to-xhigh clamp.") }, Q::Vector { id: "dbv2-gpt-5-6-terra-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-gpt-5-6-terra", note: Some("Probes the terra exact record against its family rule.") }, Q::Vector { id: "dbv2-gpt-5-4-nano-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-gpt-5-4-nano", note: Some("Probes the gpt-5-4-nano exact record and its label.") }, Q::Vector { id: "openrouter-concrete-unknown-probe", provider: "openrouter", raw_model_id: "some-model-xyz", note: Some("Probes an uncatalogued openrouter model id.") }, @@ -746,7 +747,7 @@ mod tests { } #[test] - fn corpus_has_exactly_103_executable_vectors() { + fn corpus_has_exactly_104_executable_vectors() { // Locks the vector count so a silent INPUTS edit can't quietly drop // coverage; must equal the gate in the TS harness // (modelCapabilitiesCorpus.test.mjs). @@ -755,7 +756,7 @@ mod tests { .filter(|q| matches!(q, Q::Vector { .. })) .count(); assert_eq!( - vectors, 103, + vectors, 104, "corpus executable-vector count changed; update this gate deliberately" ); } diff --git a/desktop/src/features/agents/ui/modelCapabilitiesCorpus.test.mjs b/desktop/src/features/agents/ui/modelCapabilitiesCorpus.test.mjs index 52f1f0ecf0..9446219493 100644 --- a/desktop/src/features/agents/ui/modelCapabilitiesCorpus.test.mjs +++ b/desktop/src/features/agents/ui/modelCapabilitiesCorpus.test.mjs @@ -23,10 +23,10 @@ const corpus = JSON.parse(readFileSync(fileURLToPath(corpusUrl), "utf8")); // (`_group`) are skipped. Mirrors the Rust corpus filter. const executable = corpus.filter((entry) => entry.expect != null); -test("corpus has exactly 103 executable vectors", () => { +test("corpus has exactly 104 executable vectors", () => { // Locks the vector count so a silent corpus edit can't quietly drop coverage; // must equal the gate in the Rust suite (model_capabilities.rs). - assert.equal(executable.length, 103); + assert.equal(executable.length, 104); }); test("every executable corpus vector resolves to its expected six-axis profile", () => { diff --git a/scripts/model-capabilities.json b/scripts/model-capabilities.json index 2c78867f13..c556f4596b 100644 --- a/scripts/model-capabilities.json +++ b/scripts/model-capabilities.json @@ -392,12 +392,32 @@ "normalization_policy": "none" } ], - "_comment_databricks_v2_known_models": "Curated fallback catalog for Databricks v2: the model IDs offered when an authenticated api/ai-gateway/v2/endpoints call succeeds but returns an empty list (see catalog.rs authenticated_empty_v2_catalog). Both IDs also have full exact_records, so once offered they classify precisely. Uniqueness enforced by validate_manifest().", + "_comment_databricks_v2_known_models": "Curated fallback catalog for Databricks v2: the model IDs offered when an authenticated api/ai-gateway/v2/endpoints call succeeds but returns an empty list (see catalog.rs authenticated_empty_v2_catalog). Every ID also has a full exact_record, so once offered it classifies precisely. Uniqueness enforced by validate_manifest().", "databricks_v2_known_models": [ "databricks-gpt-5-5", - "databricks-claude-opus-4-7" + "databricks-claude-opus-4-7", + "databricks-deepseek-v4-flash-0731" ], "exact_records": [ + { + "provider": "databricks_v2", + "raw_model_id": "databricks-deepseek-v4-flash-0731", + "registry_label": "DeepSeek V4 Flash 0731", + "thinking_mode": "none", + "supported_efforts": [ + "low", + "high", + "max" + ], + "default_effort": "high", + "databricks_v2_wire_route": "mlflow-chat", + "normalization_policy": "openai-standard", + "_provenance": "live Databricks staging endpoint validation plus provider-advertised effort set", + "source": "Direct staging invocation with reasoning_effort=max returned HTTP 200 and reasoning content on 2026-08-18; model metadata advertises low|high|max", + "_reconciliation": "adopt", + "_reconciliation_note": "The concrete-unknown DatabricksV2 fallback clamps max to xhigh. This endpoint accepts max, so an exact record is required to preserve the requested leaderboard condition.", + "_reconciliation_doc": "Live POST /serving-endpoints/databricks-deepseek-v4-flash-0731/invocations on 2026-08-18 with reasoning_effort=max" + }, { "provider": "databricks_v2", "raw_model_id": "databricks-gpt-5-4-mini", diff --git a/scripts/normative-corpus.json b/scripts/normative-corpus.json index 2543fd6cbe..8d8d686d48 100644 --- a/scripts/normative-corpus.json +++ b/scripts/normative-corpus.json @@ -1312,6 +1312,24 @@ "registry_label": "GPT-5.6 Luna" } }, + { + "id": "dbv2-deepseek-v4-flash-max-probe", + "provider": "databricks_v2", + "raw_model_id": "databricks-deepseek-v4-flash-0731", + "_note": "Probes that the validated DeepSeek endpoint preserves max instead of inheriting the concrete-unknown max-to-xhigh clamp.", + "expect": { + "thinking_mode": "none", + "supported_efforts": [ + "low", + "high", + "max" + ], + "default_effort": "high", + "databricks_v2_wire_route": "mlflow-chat", + "normalization_policy": "openai-standard", + "registry_label": "DeepSeek V4 Flash 0731" + } + }, { "id": "dbv2-gpt-5-6-terra-exact-record-probe", "provider": "databricks_v2", diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 0bbdfca6a4..4a1d3147a9 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -115,7 +115,7 @@ run_unit_tests() { # buzz-agent model-capabilities corpus: the Rust half of the cross-language # drift guard. model_capabilities.rs embeds scripts/model-capabilities.json + - # scripts/normative-corpus.json via include_str! and replays all 103 vectors + # scripts/normative-corpus.json via include_str! and replays all 104 vectors # as pure in-process tests (no infra). Mirrors the nextest path in # `just test-unit` — the two lists must stay in step. run_test_step "buzz-agent unit tests" \