mirror of
https://github.com/block/buzz.git
synced 2026-10-27 04:30:52 +00:00
feat(model-capabilities): humanize Databricks UC model families (#6955)
Chat-eligible Unity Catalog models from the Gemini, DeepSeek, GLM, Grok, Llama, Qwen, Gemma, and Inkling families were rendering as raw FQNs such as `system.ai.gemini-3-5-flash` in the agent model picker. #6918 surfaces these models, but the capability manifest carried no records for most of them and the registry-label path could not strip their Unity Catalog or `goose-` prefixes. I added a display-only `label_family_tokens` list and `databricks_v2` exact records for the sixteen previously unlabeled endpoint stems. All 26 target FQNs and their `goose-` aliases now resolve to curated labels through `databricks_registry_label`, while capability resolution continues to use only the existing `family_tokens` (`claude-`, `gpt-`, and `kimi-`). This keeps label discovery from changing capability profiles for unrelated model IDs. Each new record includes reconciliation metadata against the pinned [models.dev catalog](https://models.dev/api.json). Nine records adopt first-party effort evidence that the Databricks MLflow Chat transport can express: Gemini 3.5 Flash, Gemini 3.5 Flash Lite, Gemini 3.6 Flash, Gemini 3 Pro Image, DeepSeek V4 Flash, DeepSeek V4 Pro, GLM-5.3 Flash, Grok 4.6, and Inkling. The remaining seven deliberately retain the `databricks_v2/concrete_unknown` fallback because upstream evidence is absent, identifies a non-reasoning model, or provides only toggle/token-budget controls that `reasoning_effort` cannot represent. The normative corpus adds the sixteen exact records, four Unity Catalog alias probes, and two capability-isolation probes. Existing base vectors remain unchanged, and Rust and TypeScript consume the same strict manifest and corpus. Related: #6918 --------- Signed-off-by: Will Pfleger <pfleger.will@gmail.com> Signed-off-by: Thufir <7ebdb0b67dab08a570b9faf7bbada97535673b4ccaba2cbd546ad3ba84c87fa6@buzz.block.builderlab.xyz> Co-authored-by: Duncan <dcfd242e557282d7a1e2cf2e6877522682f1e5c6156dc92ca7d90eaedd3b0f95@buzz.block.builderlab.xyz> Co-authored-by: Thufir <7ebdb0b67dab08a570b9faf7bbada97535673b4ccaba2cbd546ad3ba84c87fa6@buzz.block.builderlab.xyz>
This commit is contained in:
Will Pfleger
@@ -192,6 +192,7 @@ impl ProviderFallbacks {
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Manifest {
|
||||
family_tokens: Vec<String>,
|
||||
label_family_tokens: Vec<String>,
|
||||
family_rules: Vec<FamilyRule>,
|
||||
databricks_v2_known_models: Vec<String>,
|
||||
exact_records: Vec<ExactRecord>,
|
||||
@@ -200,6 +201,9 @@ struct Manifest {
|
||||
#[serde(rename = "_comment", default)]
|
||||
#[allow(dead_code)]
|
||||
comment: Option<String>,
|
||||
#[serde(rename = "_comment_label_family_tokens", default)]
|
||||
#[allow(dead_code)]
|
||||
comment_label_family_tokens: Option<String>,
|
||||
#[serde(rename = "_comment_databricks_v2_known_models", default)]
|
||||
#[allow(dead_code)]
|
||||
comment_known_models: Option<String>,
|
||||
@@ -410,7 +414,7 @@ pub fn databricks_v2_known_models() -> &'static [String] {
|
||||
/// record label contract.
|
||||
pub fn databricks_registry_label(raw_model_id: &str) -> Option<&'static str> {
|
||||
let m = manifest();
|
||||
registry_label_for_databricks_records(raw_model_id, &m.exact_records, &m.family_tokens)
|
||||
registry_label_for_databricks_records(raw_model_id, &m.exact_records, &m.label_family_tokens)
|
||||
}
|
||||
|
||||
fn registry_label_for_databricks_records<'a>(
|
||||
@@ -452,6 +456,9 @@ fn validate_manifest(m: &Manifest) -> Result<(), String> {
|
||||
if m.family_tokens.is_empty() {
|
||||
return Err("family_tokens must be non-empty".to_string());
|
||||
}
|
||||
if m.label_family_tokens.is_empty() {
|
||||
return Err("label_family_tokens must be non-empty".to_string());
|
||||
}
|
||||
|
||||
let check_efforts = |ctx: &str,
|
||||
efforts: &[ThinkingEffort],
|
||||
@@ -713,6 +720,30 @@ mod tests {
|
||||
Q::Vector { id: "boundary-claude-3-digit-run-anthropic-probe", provider: "anthropic", raw_model_id: "claude-35", note: Some("Probes whether the claude-3 prefix binds a longer digit run ('35').") },
|
||||
Q::Vector { id: "boundary-claude-opus-4-70-anthropic-probe", provider: "anthropic", raw_model_id: "claude-opus-4-70", note: Some("Probes whether the claude-opus-4-7 prefix binds a longer digit run ('70').") },
|
||||
Q::Vector { id: "boundary-gpt-5-1234-openai-probe", provider: "openai", raw_model_id: "gpt-5-1234", note: Some("Probes a 4-digit run after the gpt-5 stem.") },
|
||||
Q::Section { group: "Databricks UC model-family humanization probes (#6918 follow-up)", note: Some("Exact-record and UC-FQN strip probes for the Gemini/DeepSeek/GLM/Grok/Llama/Qwen/Gemma/Inkling families surfaced by UC discovery.") },
|
||||
Q::Vector { id: "dbv2-gemini-3-1-flash-image-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-gemini-3-1-flash-image", note: Some("Probes the Gemini 3.1 Flash Image endpoint record and label.") },
|
||||
Q::Vector { id: "dbv2-gemini-3-5-flash-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-gemini-3-5-flash", note: Some("Probes the Gemini 3.5 Flash endpoint record and label.") },
|
||||
Q::Vector { id: "dbv2-gemini-3-5-flash-lite-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-gemini-3-5-flash-lite", note: Some("Probes the Gemini 3.5 Flash Lite endpoint record and label.") },
|
||||
Q::Vector { id: "dbv2-gemini-3-6-flash-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-gemini-3-6-flash", note: Some("Probes the Gemini 3.6 Flash endpoint record and label.") },
|
||||
Q::Vector { id: "dbv2-gemini-3-pro-image-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-gemini-3-pro-image", note: Some("Probes the Gemini 3 Pro Image endpoint record and label.") },
|
||||
Q::Vector { id: "dbv2-deepseek-v4-flash-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-deepseek-v4-flash-0731", note: Some("Probes the DeepSeek V4 Flash endpoint record and label.") },
|
||||
Q::Vector { id: "dbv2-deepseek-v4-pro-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-deepseek-v4-pro-0813", note: Some("Probes the DeepSeek V4 Pro endpoint record and label.") },
|
||||
Q::Vector { id: "dbv2-glm-5-3-flash-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-glm-5-3-flash", note: Some("Probes the GLM-5.3 Flash endpoint record and label.") },
|
||||
Q::Vector { id: "dbv2-grok-4-6-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-grok-4-6", note: Some("Probes the Grok 4.6 endpoint record and label.") },
|
||||
Q::Vector { id: "dbv2-llama-4-maverick-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-llama-4-maverick", note: Some("Probes the Llama 4 Maverick endpoint record and label.") },
|
||||
Q::Vector { id: "dbv2-meta-llama-3-1-8b-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-meta-llama-3-1-8b-instruct", note: Some("Probes the meta-llama record; the llama- token strips the meta- prefix identically for record and query.") },
|
||||
Q::Vector { id: "dbv2-meta-llama-3-3-70b-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-meta-llama-3-3-70b-instruct", note: Some("Probes the meta-llama 3.3 70B record and label.") },
|
||||
Q::Vector { id: "dbv2-qwen3-next-80b-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-qwen3-next-80b-a3b-instruct", note: Some("Probes the Qwen3 Next 80B record; the bare qwen token strips on a hyphen boundary.") },
|
||||
Q::Vector { id: "dbv2-qwen35-122b-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-qwen35-122b-a10b", note: Some("Probes the Qwen3.5 122B record; the bare qwen token strips a qwen35 stem with no separator.") },
|
||||
Q::Vector { id: "dbv2-gemma-3-12b-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-gemma-3-12b", note: Some("Probes the Gemma 3 12B endpoint record and label.") },
|
||||
Q::Vector { id: "dbv2-inkling-exact-record-probe", provider: "databricks_v2", raw_model_id: "databricks-inkling", note: Some("Probes the Inkling endpoint record and label.") },
|
||||
Q::Vector { id: "dbv2-uc-fqn-gemini-3-5-flash-strip-probe", provider: "databricks_v2", raw_model_id: "system.ai.gemini-3-5-flash", note: Some("Probes strip parity on a system.ai. UC FQN carrying the gemini- token (resolve carries no label; the alias label path is unit-tested).") },
|
||||
Q::Vector { id: "dbv2-uc-fqn-meta-llama-strip-probe", provider: "databricks_v2", raw_model_id: "system.ai.meta-llama-3-3-70b-instruct", note: Some("Probes strip parity on a UC FQN where the llama- token strips through meta-.") },
|
||||
Q::Vector { id: "dbv2-uc-goose-deepseek-strip-probe", provider: "databricks_v2", raw_model_id: "data_workflow_tools.goose.goose-deepseek-v4-pro-0813", note: Some("Probes strip parity on a goose- prefixed UC FQN carrying the deepseek- token.") },
|
||||
Q::Vector { id: "dbv2-uc-fqn-inkling-strip-probe", provider: "databricks_v2", raw_model_id: "system.ai.inkling", note: Some("Probes strip parity on a UC FQN carrying the bare inkling token.") },
|
||||
Q::Section { group: "Label/capability token isolation probes (#6955 review pass 1)", note: Some("Pins that label_family_tokens (the UC-humanization superset) never leaks into capability resolve(): capability stripping still uses only claude-/gpt-/kimi-, so a label token appearing before a gpt- marker must NOT displace the gpt-5-pro exact profile.") },
|
||||
Q::Vector { id: "isolation-openai-gemini-gpt-5-pro-probe", provider: "openai", raw_model_id: "tenant-gemini-gpt-5-pro", note: Some("The gemini- label token must not strip here; capability resolve keeps the gpt-5-pro high-only profile.") },
|
||||
Q::Vector { id: "isolation-openai-qwenchanted-gpt-5-pro-probe", provider: "openai", raw_model_id: "tenant-qwenchanted-gpt-5-pro", note: Some("The bare qwen label token must not fire mid-segment; capability resolve keeps the gpt-5-pro high-only profile.") },
|
||||
];
|
||||
|
||||
/// A section marker in the generated corpus (`_group` + optional `_note`).
|
||||
@@ -808,7 +839,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corpus_has_exactly_113_executable_vectors() {
|
||||
fn corpus_has_exactly_135_executable_vectors() {
|
||||
// Locks the vector count so a silent INPUTS edit can't quietly drop
|
||||
// coverage; must equal the gate in the TS harness
|
||||
// (modelCapabilitiesCorpus.test.mjs).
|
||||
@@ -817,7 +848,7 @@ mod tests {
|
||||
.filter(|q| matches!(q, Q::Vector { .. }))
|
||||
.count();
|
||||
assert_eq!(
|
||||
vectors, 113,
|
||||
vectors, 135,
|
||||
"corpus executable-vector count changed; update this gate deliberately"
|
||||
);
|
||||
}
|
||||
@@ -998,6 +1029,38 @@ mod tests {
|
||||
"alias={alias}"
|
||||
);
|
||||
}
|
||||
// UC-family humanization (#6918 follow-up): the new family tokens let the
|
||||
// shared UC-FQN and goose- alias forms resolve onto their base records.
|
||||
for (fqn, label) in [
|
||||
("system.ai.gemini-3-5-flash", "Gemini 3.5 Flash"),
|
||||
("system.ai.gemini-3-pro-image", "Gemini 3 Pro Image"),
|
||||
("system.ai.deepseek-v4-pro-0813", "DeepSeek V4 Pro"),
|
||||
("system.ai.glm-5-3-flash", "GLM-5.3 Flash"),
|
||||
("system.ai.grok-4-6", "Grok 4.6"),
|
||||
("system.ai.llama-4-maverick", "Llama 4 Maverick"),
|
||||
(
|
||||
"system.ai.meta-llama-3-3-70b-instruct",
|
||||
"Llama 3.3 70B Instruct",
|
||||
),
|
||||
(
|
||||
"system.ai.qwen3-next-80b-a3b-instruct",
|
||||
"Qwen3 Next 80B A3B Instruct",
|
||||
),
|
||||
("system.ai.qwen35-122b-a10b", "Qwen3.5 122B A10B"),
|
||||
("system.ai.gemma-3-12b", "Gemma 3 12B"),
|
||||
("system.ai.inkling", "Inkling"),
|
||||
(
|
||||
"data_workflow_tools.goose.goose-deepseek-v4-flash-0731",
|
||||
"DeepSeek V4 Flash",
|
||||
),
|
||||
(
|
||||
"data_workflow_tools.goose.goose-glm-5-3-flash",
|
||||
"GLM-5.3 Flash",
|
||||
),
|
||||
("data_workflow_tools.goose.goose-grok-4-6", "Grok 4.6"),
|
||||
] {
|
||||
assert_eq!(databricks_registry_label(fqn), Some(label), "fqn={fqn}");
|
||||
}
|
||||
// Unknown ids, bare family ids, and blanks remain uncurated.
|
||||
assert_eq!(databricks_registry_label("custom-unlisted-endpoint"), None);
|
||||
assert_eq!(databricks_registry_label("gpt-5"), None);
|
||||
|
||||
Reference in New Issue
Block a user