mirror of
https://github.com/block/buzz.git
synced 2026-10-31 06:21:34 +00:00
fix: preserve DeepSeek max effort on Databricks
This commit is contained in:
2
Justfile
2
Justfile
@@ -335,7 +335,7 @@ test-unit:
|
||||
# buzz-agent model-capabilities corpus: the Rust half of the
|
||||
# cross-language drift guard. `model_capabilities.rs` embeds
|
||||
# scripts/model-capabilities.json + scripts/normative-corpus.json via
|
||||
# include_str! and replays all 103 vectors as pure in-process tests (no
|
||||
# include_str! and replays all 104 vectors as pure in-process tests (no
|
||||
# infra). Enumerated explicitly because nothing in CI runs
|
||||
# `cargo test --workspace`; without this step a manifest edit that
|
||||
# diverges Rust from the corpus ships green.
|
||||
|
||||
@@ -608,6 +608,7 @@ mod tests {
|
||||
Q::Vector { id: "anthropic-opus-5-prefix-probe", provider: "anthropic", raw_model_id: "claude-opus-5-20270101", note: Some("Probes the claude-opus-5 prefix rule.") },
|
||||
Q::Vector { id: "dbv2-gpt-5-6-sol-normalization-probe", provider: "databricks_v2", raw_model_id: "databricks-gpt-5-6-sol", note: Some("Probes the sol exact record's normalization and effort axes.") },
|
||||
Q::Vector { id: "dbv2-gpt-5-6-luna-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-gpt-5-6-luna", note: Some("Probes the luna exact record against its family rule.") },
|
||||
Q::Vector { id: "dbv2-deepseek-v4-flash-max-probe", provider: "databricks_v2", raw_model_id: "databricks-deepseek-v4-flash-0731", note: Some("Probes that the validated DeepSeek endpoint preserves max instead of inheriting the concrete-unknown max-to-xhigh clamp.") },
|
||||
Q::Vector { id: "dbv2-gpt-5-6-terra-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-gpt-5-6-terra", note: Some("Probes the terra exact record against its family rule.") },
|
||||
Q::Vector { id: "dbv2-gpt-5-4-nano-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-gpt-5-4-nano", note: Some("Probes the gpt-5-4-nano exact record and its label.") },
|
||||
Q::Vector { id: "openrouter-concrete-unknown-probe", provider: "openrouter", raw_model_id: "some-model-xyz", note: Some("Probes an uncatalogued openrouter model id.") },
|
||||
@@ -746,7 +747,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corpus_has_exactly_103_executable_vectors() {
|
||||
fn corpus_has_exactly_104_executable_vectors() {
|
||||
// Locks the vector count so a silent INPUTS edit can't quietly drop
|
||||
// coverage; must equal the gate in the TS harness
|
||||
// (modelCapabilitiesCorpus.test.mjs).
|
||||
@@ -755,7 +756,7 @@ mod tests {
|
||||
.filter(|q| matches!(q, Q::Vector { .. }))
|
||||
.count();
|
||||
assert_eq!(
|
||||
vectors, 103,
|
||||
vectors, 104,
|
||||
"corpus executable-vector count changed; update this gate deliberately"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -23,10 +23,10 @@ const corpus = JSON.parse(readFileSync(fileURLToPath(corpusUrl), "utf8"));
|
||||
// (`_group`) are skipped. Mirrors the Rust corpus filter.
|
||||
const executable = corpus.filter((entry) => entry.expect != null);
|
||||
|
||||
test("corpus has exactly 103 executable vectors", () => {
|
||||
test("corpus has exactly 104 executable vectors", () => {
|
||||
// Locks the vector count so a silent corpus edit can't quietly drop coverage;
|
||||
// must equal the gate in the Rust suite (model_capabilities.rs).
|
||||
assert.equal(executable.length, 103);
|
||||
assert.equal(executable.length, 104);
|
||||
});
|
||||
|
||||
test("every executable corpus vector resolves to its expected six-axis profile", () => {
|
||||
|
||||
@@ -392,12 +392,32 @@
|
||||
"normalization_policy": "none"
|
||||
}
|
||||
],
|
||||
"_comment_databricks_v2_known_models": "Curated fallback catalog for Databricks v2: the model IDs offered when an authenticated api/ai-gateway/v2/endpoints call succeeds but returns an empty list (see catalog.rs authenticated_empty_v2_catalog). Both IDs also have full exact_records, so once offered they classify precisely. Uniqueness enforced by validate_manifest().",
|
||||
"_comment_databricks_v2_known_models": "Curated fallback catalog for Databricks v2: the model IDs offered when an authenticated api/ai-gateway/v2/endpoints call succeeds but returns an empty list (see catalog.rs authenticated_empty_v2_catalog). Every ID also has a full exact_record, so once offered it classifies precisely. Uniqueness enforced by validate_manifest().",
|
||||
"databricks_v2_known_models": [
|
||||
"databricks-gpt-5-5",
|
||||
"databricks-claude-opus-4-7"
|
||||
"databricks-claude-opus-4-7",
|
||||
"databricks-deepseek-v4-flash-0731"
|
||||
],
|
||||
"exact_records": [
|
||||
{
|
||||
"provider": "databricks_v2",
|
||||
"raw_model_id": "databricks-deepseek-v4-flash-0731",
|
||||
"registry_label": "DeepSeek V4 Flash 0731",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"high",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "mlflow-chat",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": "live Databricks staging endpoint validation plus provider-advertised effort set",
|
||||
"source": "Direct staging invocation with reasoning_effort=max returned HTTP 200 and reasoning content on 2026-08-18; model metadata advertises low|high|max",
|
||||
"_reconciliation": "adopt",
|
||||
"_reconciliation_note": "The concrete-unknown DatabricksV2 fallback clamps max to xhigh. This endpoint accepts max, so an exact record is required to preserve the requested leaderboard condition.",
|
||||
"_reconciliation_doc": "Live POST /serving-endpoints/databricks-deepseek-v4-flash-0731/invocations on 2026-08-18 with reasoning_effort=max"
|
||||
},
|
||||
{
|
||||
"provider": "databricks_v2",
|
||||
"raw_model_id": "databricks-gpt-5-4-mini",
|
||||
|
||||
@@ -1312,6 +1312,24 @@
|
||||
"registry_label": "GPT-5.6 Luna"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "dbv2-deepseek-v4-flash-max-probe",
|
||||
"provider": "databricks_v2",
|
||||
"raw_model_id": "databricks-deepseek-v4-flash-0731",
|
||||
"_note": "Probes that the validated DeepSeek endpoint preserves max instead of inheriting the concrete-unknown max-to-xhigh clamp.",
|
||||
"expect": {
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"high",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "mlflow-chat",
|
||||
"normalization_policy": "openai-standard",
|
||||
"registry_label": "DeepSeek V4 Flash 0731"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "dbv2-gpt-5-6-terra-exact-record-probe",
|
||||
"provider": "databricks_v2",
|
||||
|
||||
@@ -115,7 +115,7 @@ run_unit_tests() {
|
||||
|
||||
# buzz-agent model-capabilities corpus: the Rust half of the cross-language
|
||||
# drift guard. model_capabilities.rs embeds scripts/model-capabilities.json +
|
||||
# scripts/normative-corpus.json via include_str! and replays all 103 vectors
|
||||
# scripts/normative-corpus.json via include_str! and replays all 104 vectors
|
||||
# as pure in-process tests (no infra). Mirrors the nextest path in
|
||||
# `just test-unit` — the two lists must stay in step.
|
||||
run_test_step "buzz-agent unit tests" \
|
||||
|
||||
Reference in New Issue
Block a user