mirror of
https://github.com/block/buzz.git
synced 2026-10-31 22:41:12 +00:00
feat(agent): Phase 2 — wire Rust and TS consumers to generated model-capabilities module (#3958)
## What Phase 2 consumer cutover targeting the `duncan/databricks-model-label-registry` umbrella branch. Wires `crates/**` and `desktop/**` consumers to the generated capability module introduced in Phase 1 (#3821), while keeping old and new paths both live for differential testing. Phase 3 removes the old paths. ## Commits (boundary-separated) ### feat(agent): Phase 2a — wire Rust consumers to generated capability module (`crates/**`, `scripts/**`) - `catalog.rs`: `DATABRICKS_V2_KNOWN_MODELS` re-exported from the generated module — single source of truth. - `llm.rs`: `databricks_v2_route_for_model` delegates to `resolve_model_capabilities("databricks_v2", model)`. Old segment-based classifier preserved as `#[cfg(test)] _old_*` for the differential harness. New `databricks_v2_route_differential_old_vs_new` test confirms 100% agreement on all 20 route vectors. - `config.rs`: new `effort_table_fixture_differential_old_vs_new` test runs `resolve_model_capabilities` over the 36-entry `effortTable.fixture.json` and asserts old/new agree modulo a doc-cited allowlist (4 F1 corrections). - `scripts/run-differential.mjs`: JS differential harness over effortTable fixture + normative corpus + catalog-sample fixture. 85 checks, 0 unexpected divergences (5 allowlisted: 4 F1 corrections + goose-opus-5 anthropic route correction). - `scripts/MODELS_DEV_RECONCILIATION.md`: deferred MINOR from Phase 1 — 8 trailing-double-space line breaks replaced with `<br>`. ### feat(desktop): Phase 2b — cut TS consumers to generated model-capabilities module (`desktop/**`) - `buzzAgentConfig.ts`: adds `getProviderEffortConfigFromManifest(provider, model?)` — thin wrapper over `resolveModelCapabilities()` from `modelCapabilities.ts`. Maps `supportedEfforts → validValues` and `defaultEffort → defaultValue` (null preserved for manual-budget/Inherit). Old `getProviderEffortConfig()` and all hand-tables stay live for the differential harness; Phase 3 retires them. - `formatAgentModelLabel.ts`: registry-label lookup re-pointed from hand-maintained `databricksModelNames.ts` import to generated `DATABRICKS_MODEL_NAMES` exported from `modelCapabilities.ts`. Same Map shape, identical contents, behavior unchanged. ### fix(scripts): add ts-esm-loader and fix allowlist coverage in run-differential (`scripts/**`) - `scripts/ts-esm-loader.mjs`: minimal ESM custom loader that resolves extensionless relative TS imports. Required because Phase 2b's `buzzAgentConfig.ts` imports `modelCapabilities` without `.ts` extension — which Node's `--experimental-strip-types` runner cannot resolve without a hook. - `scripts/run-differential.mjs`: shebang updated to self-bootstrap with the loader; fixes the `totalAllowlisted` counter (was declared but never incremented — always printed `0 allowlisted`). Replaced with per-axis hit tracking: reports exercised slot count (`N/total`) in summary; fails with `STALE_ALLOWLIST` if any declared entry fires zero divergences, preventing stale entries from silently masking future regressions. ## Verification - `cargo test -p buzz-agent --lib`: 426/426 - Corpus: 45/45 · schema-negative: 24/24 · `--check` byte-clean - Differential: 85 checks, 0 unexpected divergences, 6/6 allowlist slots exercised - Desktop: 3847/3847 · typecheck clean · biome clean - Mobile: 1019 pass, 1 skipped — same 5 flaky tests in `mobile/test/features/channels/` that reproduce at the umbrella base; zero mobile files in this branch range - `git diff --check`: clean ## What Remains (Phase 3) Remove old hand-maintained paths: `_old_*` functions in `llm.rs`/`config.rs`, old `getProviderEffortConfig` tables in `buzzAgentConfig.ts`, old `databricksModelNames.ts` import in `formatAgentModelLabel.ts`, old `databricks_model_names.rs` module. --------- Signed-off-by: Will Pfleger <pfleger.will@gmail.com> Signed-off-by: npub1g8493u0xfsjrvflg4n08ezd7vec99mnwzlv0qgwpr9d7gvjwhuzqx59rhw <41ea58f1e64c243627e8acde7c89be667052ee6e17d8f021c1195be4324ebf04@buzz.block.builderlab.xyz> Signed-off-by: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 <dcfd242e557282d7a1e2cf2e6877522682f1e5c6156dc92ca7d90eaedd3b0f95@buzz.block.builderlab.xyz> Co-authored-by: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 <dcfd242e557282d7a1e2cf2e6877522682f1e5c6156dc92ca7d90eaedd3b0f95@buzz.block.builderlab.xyz> Co-authored-by: npub1g8493u0xfsjrvflg4n08ezd7vec99mnwzlv0qgwpr9d7gvjwhuzqx59rhw <41ea58f1e64c243627e8acde7c89be667052ee6e17d8f021c1195be4324ebf04@buzz.block.builderlab.xyz>
This commit is contained in:
Will Pfleger
@@ -9,6 +9,14 @@ on:
|
||||
- 'desktop/src/features/agents/ui/modelCapabilities.ts'
|
||||
- 'scripts/generated-model-capabilities-coverage.json'
|
||||
- '.github/workflows/model-capability-regen-diff.yml'
|
||||
# Differential harness and fixtures — any change to old/new side or inputs re-runs.
|
||||
- 'scripts/run-differential.mjs'
|
||||
- 'scripts/normative-corpus.json'
|
||||
- 'scripts/catalog-sample-fixture.json'
|
||||
- 'desktop/src/features/agents/ui/effortTable.fixture.json'
|
||||
- 'desktop/src/features/agents/ui/buzzAgentConfig.ts'
|
||||
- 'crates/buzz-agent/src/config.rs'
|
||||
- 'crates/buzz-agent/src/llm.rs'
|
||||
push:
|
||||
branches: [main, release, 'duncan/databricks-model-label-registry']
|
||||
|
||||
@@ -51,3 +59,19 @@ jobs:
|
||||
|
||||
- name: Validate manifest (schema-negative tests)
|
||||
run: node --test scripts/test-manifest-validator.mjs
|
||||
|
||||
- name: Run differential harness (old vs new, all input sets)
|
||||
run: node --experimental-strip-types scripts/run-differential.mjs
|
||||
|
||||
rust-unit-tests:
|
||||
name: buzz-agent unit tests (normative corpus + behavioral differential)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
||||
|
||||
- name: Run buzz-agent unit tests
|
||||
run: cargo test -p buzz-agent --lib
|
||||
|
||||
@@ -47,8 +47,10 @@ pub struct ModelEntry {
|
||||
/// Known Databricks AI Gateway v2 models — used as a fallback when the
|
||||
/// `api/ai-gateway/v2/endpoints` call returns an empty list.
|
||||
/// Mirrors goose's `DATABRICKS_V2_KNOWN_MODELS`.
|
||||
pub const DATABRICKS_V2_KNOWN_MODELS: &[&str] =
|
||||
&["databricks-gpt-5-5", "databricks-claude-opus-4-7"];
|
||||
///
|
||||
/// Phase 2 cutover: this is now a re-export of the generated constant in
|
||||
/// `generated_model_capabilities`. Phase 3 removes the old hand-maintained list.
|
||||
pub use crate::generated_model_capabilities::DATABRICKS_V2_KNOWN_MODELS;
|
||||
|
||||
/// Returns the discovery-failure fallback catalog for a Databricks provider.
|
||||
///
|
||||
|
||||
@@ -86,7 +86,7 @@ impl ThinkingEffort {
|
||||
/// - `llama-3` → `llama-3` (no family token, returned unchanged)
|
||||
///
|
||||
/// If no family token is present the name is returned unchanged.
|
||||
fn strip_catalog_prefix(model: &str) -> &str {
|
||||
pub(crate) fn strip_catalog_prefix(model: &str) -> &str {
|
||||
const FAMILY_TOKENS: &[&str] = &["claude-", "gpt-"];
|
||||
let lower = model.to_ascii_lowercase();
|
||||
let first_idx = FAMILY_TOKENS.iter().filter_map(|tok| lower.find(tok)).min();
|
||||
@@ -576,6 +576,195 @@ pub fn normalize_effort_for_anthropic_route(effort: ThinkingEffort) -> Option<Th
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Generated-backed production helpers (Phase 2 cutover)
|
||||
//
|
||||
// One adapter per request path — resolves the effective provider/model ONCE
|
||||
// from the generated manifest and applies every manifest-owned axis.
|
||||
// Old hand-table implementations are renamed `_old_*` and used only in tests
|
||||
// and the behavioral differential shims — they have no production callers.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Normalize the effort value for any OpenAI-shaped request (pure OpenAI or legacy Databricks).
|
||||
///
|
||||
/// Resolves the generated capability record for the actual provider/model and
|
||||
/// applies `resolve_openai_effort` over its `supported_efforts`. Uses the same
|
||||
/// clamping/peer-fallback semantics as the old hand-table lookup but is driven by
|
||||
/// the generated manifest, so exact-record F1 corrections (e.g. `databricks-gpt-5-5`
|
||||
/// → `[low,medium,high]`) are enforced in production.
|
||||
///
|
||||
/// This is the single production authority for `Provider::OpenAi` and
|
||||
/// `Provider::Databricks` effort normalization. The old helper
|
||||
/// `normalize_effort_for_openai_route` exists only as a test/differential shim.
|
||||
pub fn normalize_effort_for_provider(
|
||||
provider: &str,
|
||||
raw_model: &str,
|
||||
effort: ThinkingEffort,
|
||||
) -> ThinkingEffort {
|
||||
use crate::generated_model_capabilities::resolve_model_capabilities;
|
||||
let cap = resolve_model_capabilities(provider, raw_model);
|
||||
resolve_openai_effort(raw_model, effort, cap.supported_efforts.as_ref())
|
||||
}
|
||||
|
||||
/// Normalize the effort value for a DatabricksV2 request.
|
||||
///
|
||||
/// Reads `normalization_policy` from the generated capability record for this
|
||||
/// raw model ID (provider = "databricks_v2") and applies it:
|
||||
/// - `OpenAiStandard` → per-family table lookup (GPT-5.x, etc.)
|
||||
/// - `OpenAiClampMaxToXHigh` → clamp max → xhigh, pass others unchanged
|
||||
/// - `None` → pass effort through unchanged (Anthropic path)
|
||||
///
|
||||
/// This is the production authority for DatabricksV2 effort normalization.
|
||||
/// `normalize_effort_for_provider` is the authority for pure OpenAI and legacy
|
||||
/// Databricks; `normalize_effort_for_openai_route` is a test/differential shim only.
|
||||
pub fn normalize_effort_for_databricks_v2(
|
||||
effort: ThinkingEffort,
|
||||
raw_model: &str,
|
||||
) -> ThinkingEffort {
|
||||
use crate::generated_model_capabilities::{resolve_model_capabilities, NormalizationPolicy};
|
||||
let cap = resolve_model_capabilities("databricks_v2", raw_model);
|
||||
match cap.normalization_policy {
|
||||
NormalizationPolicy::OpenAiStandard => {
|
||||
// Resolve against the generated `supported_efforts` — this is the axis that
|
||||
// carries exact-record corrections (e.g. databricks-gpt-5-5 → [low,medium,high]).
|
||||
// Uses the same clamping/peer-fallback semantics as the old hand-table lookup.
|
||||
resolve_openai_effort(raw_model, effort, cap.supported_efforts.as_ref())
|
||||
}
|
||||
NormalizationPolicy::OpenAiClampMaxToXHigh => {
|
||||
// Only `max` is out-of-range; all other values pass through if supported.
|
||||
// Resolve against supported_efforts so that unsupported values are clamped
|
||||
// consistently (not just `max`).
|
||||
if effort == ThinkingEffort::Max {
|
||||
tracing::warn!(
|
||||
requested = "max",
|
||||
resolved = "xhigh",
|
||||
model = raw_model,
|
||||
"BUZZ_AGENT_THINKING_EFFORT=max not confirmed for this DatabricksV2 model; clamping to xhigh"
|
||||
);
|
||||
ThinkingEffort::XHigh
|
||||
} else {
|
||||
resolve_openai_effort(raw_model, effort, cap.supported_efforts.as_ref())
|
||||
}
|
||||
}
|
||||
NormalizationPolicy::None => effort,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the Anthropic thinking/effort request fields for any manifest-owned provider/model.
|
||||
///
|
||||
/// Resolves `thinking_mode` and `supported_efforts` from the generated capability record
|
||||
/// for the effective provider/model and applies them:
|
||||
/// - `ManualBudget` → `thinking:{type:"enabled", budget_tokens}` shape
|
||||
/// - `Adaptive` → `thinking:{type:"adaptive"} + output_config:{effort}` shape,
|
||||
/// with effort clamped down to the highest supported level
|
||||
/// - `OmitFields` / `None` / `NotApplicable` → omit both fields
|
||||
///
|
||||
/// This is the single production authority for all providers' Anthropic thinking.
|
||||
/// The old `anthropic_thinking_config_for_databricks_v2` is a test-only shim.
|
||||
pub fn anthropic_thinking_config_generated(
|
||||
provider: &str,
|
||||
raw_model: &str,
|
||||
effort: ThinkingEffort,
|
||||
max_output_tokens: u32,
|
||||
) -> (Option<serde_json::Value>, Option<serde_json::Value>) {
|
||||
use crate::generated_model_capabilities::{resolve_model_capabilities, ThinkingMode};
|
||||
use serde_json::json;
|
||||
|
||||
let cap = resolve_model_capabilities(provider, raw_model);
|
||||
match cap.thinking_mode {
|
||||
ThinkingMode::ManualBudget => {
|
||||
// Manual-budget shape (claude-3*, claude-opus-4-5): budget_tokens clamped
|
||||
// to fit within max_output_tokens.
|
||||
const MIN_ANSWER_TOKENS: u32 = 1024;
|
||||
let level_budget = effort.anthropic_budget_tokens();
|
||||
let headroom = max_output_tokens.saturating_sub(MIN_ANSWER_TOKENS);
|
||||
let budget = level_budget.min(headroom);
|
||||
if budget < MIN_ANSWER_TOKENS {
|
||||
tracing::warn!(
|
||||
max_output_tokens,
|
||||
level_budget,
|
||||
headroom,
|
||||
model = raw_model,
|
||||
"BUZZ_AGENT_THINKING_EFFORT: max_output_tokens too small to fit thinking budget + answer headroom; omitting thinking fields"
|
||||
);
|
||||
return (None, None);
|
||||
}
|
||||
(
|
||||
Some(json!({ "type": "enabled", "budget_tokens": budget })),
|
||||
None,
|
||||
)
|
||||
}
|
||||
ThinkingMode::Adaptive => {
|
||||
// Adaptive shape: clamp effort downward to the highest supported level.
|
||||
// Uses the generated supported_efforts (the manifest-owned authority) rather
|
||||
// than the legacy clamp_adaptive_effort hand table.
|
||||
let clamped = cap
|
||||
.supported_efforts
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|&&e| e <= effort)
|
||||
.copied()
|
||||
.unwrap_or(effort); // effort is below the lowest supported; pass through (rare)
|
||||
if clamped != effort {
|
||||
tracing::warn!(
|
||||
model = raw_model,
|
||||
requested = effort.openai_effort_str(),
|
||||
clamped = clamped.openai_effort_str(),
|
||||
"BUZZ_AGENT_THINKING_EFFORT is not available for this model; clamping to highest supported level"
|
||||
);
|
||||
}
|
||||
(
|
||||
Some(json!({ "type": "adaptive" })),
|
||||
Some(json!({ "effort": clamped.anthropic_effort_str() })),
|
||||
)
|
||||
}
|
||||
ThinkingMode::OmitFields | ThinkingMode::None | ThinkingMode::NotApplicable => {
|
||||
// Unknown Anthropic model, non-Anthropic-routed, or not applicable:
|
||||
// omit thinking fields rather than guess.
|
||||
(None, None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Old DatabricksV2-scoped Anthropic thinking config — kept as a differential shim.
|
||||
///
|
||||
/// Production code uses `anthropic_thinking_config_generated` instead.
|
||||
/// This hard-codes `"databricks_v2"` and uses the legacy `clamp_adaptive_effort` hand table.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn _old_anthropic_thinking_config_for_databricks_v2(
|
||||
raw_model: &str,
|
||||
effort: ThinkingEffort,
|
||||
max_output_tokens: u32,
|
||||
) -> (Option<serde_json::Value>, Option<serde_json::Value>) {
|
||||
use crate::generated_model_capabilities::{resolve_model_capabilities, ThinkingMode};
|
||||
use serde_json::json;
|
||||
|
||||
match resolve_model_capabilities("databricks_v2", raw_model).thinking_mode {
|
||||
ThinkingMode::ManualBudget => {
|
||||
const MIN_ANSWER_TOKENS: u32 = 1024;
|
||||
let level_budget = effort.anthropic_budget_tokens();
|
||||
let headroom = max_output_tokens.saturating_sub(MIN_ANSWER_TOKENS);
|
||||
let budget = level_budget.min(headroom);
|
||||
if budget < MIN_ANSWER_TOKENS {
|
||||
return (None, None);
|
||||
}
|
||||
(
|
||||
Some(json!({ "type": "enabled", "budget_tokens": budget })),
|
||||
None,
|
||||
)
|
||||
}
|
||||
ThinkingMode::Adaptive => {
|
||||
let model = strip_catalog_prefix(raw_model);
|
||||
let clamped = clamp_adaptive_effort(model, effort);
|
||||
(
|
||||
Some(json!({ "type": "adaptive" })),
|
||||
Some(json!({ "effort": clamped.anthropic_effort_str() })),
|
||||
)
|
||||
}
|
||||
ThinkingMode::OmitFields | ThinkingMode::None | ThinkingMode::NotApplicable => (None, None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true for Claude model families that use manual thinking budgets (doc-verified, July 2025).
|
||||
///
|
||||
/// Source: https://platform.claude.com/docs/en/build-with-claude/extended-thinking (support table)
|
||||
@@ -1151,6 +1340,32 @@ fn parse_hook_servers(raw: Option<&str>) -> HookServers {
|
||||
HookServers::Only(names)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test-only re-exports: let llm.rs tests call private classifiers without
|
||||
// duplicating them. These wrappers are cfg(test)-only and intentionally thin.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn is_manual_budget_model_for_test(model: &str) -> bool {
|
||||
is_manual_budget_model(model)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn is_adaptive_thinking_model_for_test(model: &str) -> bool {
|
||||
is_adaptive_thinking_model(model)
|
||||
}
|
||||
|
||||
/// Mirror of the `tests::valid_effort_values_for_provider_model` helper in config's
|
||||
/// own test module, promoted to a module-level cfg(test) function so llm.rs tests
|
||||
/// can call it without re-implementing the logic.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn valid_effort_values_for_provider_model_for_test(
|
||||
provider: &str,
|
||||
model: &str,
|
||||
) -> (Vec<&'static str>, Option<&'static str>) {
|
||||
tests::valid_effort_values_for_provider_model(provider, model)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -2604,7 +2819,7 @@ mod tests {
|
||||
/// Returns `(valid_values, default_value)` where `default_value` is `None`
|
||||
/// for Anthropic manual-budget models (TS `defaultValue: null`), otherwise
|
||||
/// `Some("medium")` or `Some("high")`.
|
||||
fn valid_effort_values_for_provider_model(
|
||||
pub(super) fn valid_effort_values_for_provider_model(
|
||||
provider: &str,
|
||||
model: &str,
|
||||
) -> (Vec<&'static str>, Option<&'static str>) {
|
||||
@@ -2614,6 +2829,13 @@ mod tests {
|
||||
const GPT5_1: &[&str] = &["none", "low", "medium", "high"];
|
||||
|
||||
let p = provider.to_ascii_lowercase();
|
||||
// Canonicalize provider aliases — mirrors the production path and TS
|
||||
// PROVIDER_ALIASES so this shim stays in sync with the fixture.
|
||||
let p = match p.as_str() {
|
||||
"openai-compat" => "openai".to_owned(),
|
||||
"databricks-v2" => "databricks_v2".to_owned(),
|
||||
_ => p,
|
||||
};
|
||||
// Strip arbitrary endpoint-naming prefix before model matching, mirroring TS and
|
||||
// strip_catalog_prefix: find the first known family token (claude-, gpt-) and
|
||||
// drop everything before it. Handles any catalog naming convention.
|
||||
@@ -2698,7 +2920,7 @@ mod tests {
|
||||
if p == "openrouter" {
|
||||
return (ALL_7.to_vec(), Some("medium"));
|
||||
}
|
||||
// openai-compat, unknown, empty → all-7, default medium.
|
||||
// Unknown/empty provider → all-7, default medium.
|
||||
(ALL_7.to_vec(), Some("medium"))
|
||||
}
|
||||
|
||||
@@ -2750,6 +2972,59 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// ---- normalize_effort_for_databricks_v2 regression tests (F1 corrections) ----
|
||||
// These pin the exact behavior Paul's pre-review probes checked. The key invariant:
|
||||
// normalize_effort_for_databricks_v2 must resolve against the generated supported_efforts
|
||||
// (which carries exact-record F1 corrections), NOT the old hand table.
|
||||
|
||||
#[test]
|
||||
fn normalize_effort_for_databricks_v2_gpt_5_5_xhigh_clamps_to_high() {
|
||||
// F1 correction: databricks-gpt-5-5 generated supported_efforts = [low, medium, high].
|
||||
// XHigh is outside the supported set → nearest supported is High.
|
||||
assert_eq!(
|
||||
normalize_effort_for_databricks_v2(ThinkingEffort::XHigh, "databricks-gpt-5-5"),
|
||||
ThinkingEffort::High,
|
||||
"databricks-gpt-5-5 XHigh must clamp to High (F1 correction: supported=[low,medium,high])"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_effort_for_databricks_v2_gpt_5_5_none_clamps_to_low() {
|
||||
// F1 correction: databricks-gpt-5-5 supported_efforts = [low, medium, high].
|
||||
// None is outside the set → nearest supported is Low.
|
||||
assert_eq!(
|
||||
normalize_effort_for_databricks_v2(ThinkingEffort::None, "databricks-gpt-5-5"),
|
||||
ThinkingEffort::Low,
|
||||
"databricks-gpt-5-5 None must clamp to Low (F1 correction: supported=[low,medium,high])"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_effort_for_databricks_v2_gpt_5_5_in_range_passes_through() {
|
||||
// Values within the corrected set must pass through unchanged.
|
||||
for effort in [
|
||||
ThinkingEffort::Low,
|
||||
ThinkingEffort::Medium,
|
||||
ThinkingEffort::High,
|
||||
] {
|
||||
assert_eq!(
|
||||
normalize_effort_for_databricks_v2(effort, "databricks-gpt-5-5"),
|
||||
effort,
|
||||
"databricks-gpt-5-5 {effort:?} is in supported set, must pass through"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_effort_for_databricks_v2_gpt_5_6_sol_max_passes_through() {
|
||||
// databricks-gpt-5-6-sol F1 adoption: [low, medium, high, max] — max is supported.
|
||||
assert_eq!(
|
||||
normalize_effort_for_databricks_v2(ThinkingEffort::Max, "databricks-gpt-5-6-sol"),
|
||||
ThinkingEffort::Max,
|
||||
"databricks-gpt-5-6-sol Max must pass through (F1: supported includes max)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_provider_openrouter_with_key() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -253,6 +253,19 @@ pub fn lookup_by_family_rules(provider: &str, normalized: &str) -> Option<Capabi
|
||||
normalization_policy: NormalizationPolicy::OpenAiStandard,
|
||||
});
|
||||
}
|
||||
// rule: openai-gpt5-pro, provider: databricks, priority: 20
|
||||
if provider == "databricks"
|
||||
&& (gpt5_token_matches_rs(lower, "gpt-5-pro") || gpt5_token_matches_rs(lower, "gpt5-pro"))
|
||||
{
|
||||
return Some(CapabilityResult {
|
||||
registry_label: Some("GPT-5 Pro"),
|
||||
thinking_mode: ThinkingMode::None,
|
||||
supported_efforts: Cow::Borrowed(&[ThinkingEffort::High]),
|
||||
default_effort: Some(ThinkingEffort::High),
|
||||
databricks_v2_wire_route: DatabricksV2Route::NotApplicable,
|
||||
normalization_policy: NormalizationPolicy::OpenAiStandard,
|
||||
});
|
||||
}
|
||||
// rule: openai-gpt5-pro, provider: databricks_v2, priority: 20
|
||||
if provider == "databricks_v2"
|
||||
&& (gpt5_token_matches_rs(lower, "gpt-5-pro") || gpt5_token_matches_rs(lower, "gpt5-pro"))
|
||||
@@ -289,6 +302,29 @@ pub fn lookup_by_family_rules(provider: &str, normalized: &str) -> Option<Capabi
|
||||
normalization_policy: NormalizationPolicy::OpenAiStandard,
|
||||
});
|
||||
}
|
||||
// rule: openai-gpt5-6, provider: databricks, priority: 15
|
||||
if provider == "databricks"
|
||||
&& (gpt5_token_matches_rs(lower, "gpt-5.6")
|
||||
|| gpt5_token_matches_rs(lower, "gpt5.6")
|
||||
|| gpt5_token_matches_rs(lower, "gpt-5-6")
|
||||
|| gpt5_token_matches_rs(lower, "gpt5-6"))
|
||||
{
|
||||
return Some(CapabilityResult {
|
||||
registry_label: Some("GPT-5.6"),
|
||||
thinking_mode: ThinkingMode::None,
|
||||
supported_efforts: Cow::Borrowed(&[
|
||||
ThinkingEffort::None,
|
||||
ThinkingEffort::Low,
|
||||
ThinkingEffort::Medium,
|
||||
ThinkingEffort::High,
|
||||
ThinkingEffort::XHigh,
|
||||
ThinkingEffort::Max,
|
||||
]),
|
||||
default_effort: Some(ThinkingEffort::Medium),
|
||||
databricks_v2_wire_route: DatabricksV2Route::NotApplicable,
|
||||
normalization_policy: NormalizationPolicy::OpenAiStandard,
|
||||
});
|
||||
}
|
||||
// rule: openai-gpt5-6, provider: databricks_v2, priority: 15
|
||||
if provider == "databricks_v2"
|
||||
&& (gpt5_token_matches_rs(lower, "gpt-5.6")
|
||||
@@ -334,6 +370,28 @@ pub fn lookup_by_family_rules(provider: &str, normalized: &str) -> Option<Capabi
|
||||
normalization_policy: NormalizationPolicy::OpenAiStandard,
|
||||
});
|
||||
}
|
||||
// rule: openai-gpt5-5, provider: databricks, priority: 15
|
||||
if provider == "databricks"
|
||||
&& (gpt5_token_matches_rs(lower, "gpt-5.5")
|
||||
|| gpt5_token_matches_rs(lower, "gpt5.5")
|
||||
|| gpt5_token_matches_rs(lower, "gpt-5-5")
|
||||
|| gpt5_token_matches_rs(lower, "gpt5-5"))
|
||||
{
|
||||
return Some(CapabilityResult {
|
||||
registry_label: Some("GPT-5.5"),
|
||||
thinking_mode: ThinkingMode::None,
|
||||
supported_efforts: Cow::Borrowed(&[
|
||||
ThinkingEffort::None,
|
||||
ThinkingEffort::Low,
|
||||
ThinkingEffort::Medium,
|
||||
ThinkingEffort::High,
|
||||
ThinkingEffort::XHigh,
|
||||
]),
|
||||
default_effort: Some(ThinkingEffort::Medium),
|
||||
databricks_v2_wire_route: DatabricksV2Route::NotApplicable,
|
||||
normalization_policy: NormalizationPolicy::OpenAiStandard,
|
||||
});
|
||||
}
|
||||
// rule: openai-gpt5-5, provider: databricks_v2, priority: 15
|
||||
if provider == "databricks_v2"
|
||||
&& (gpt5_token_matches_rs(lower, "gpt-5.5")
|
||||
@@ -378,6 +436,28 @@ pub fn lookup_by_family_rules(provider: &str, normalized: &str) -> Option<Capabi
|
||||
normalization_policy: NormalizationPolicy::OpenAiStandard,
|
||||
});
|
||||
}
|
||||
// rule: openai-gpt5-4, provider: databricks, priority: 15
|
||||
if provider == "databricks"
|
||||
&& (gpt5_token_matches_rs(lower, "gpt-5.4")
|
||||
|| gpt5_token_matches_rs(lower, "gpt5.4")
|
||||
|| gpt5_token_matches_rs(lower, "gpt-5-4")
|
||||
|| gpt5_token_matches_rs(lower, "gpt5-4"))
|
||||
{
|
||||
return Some(CapabilityResult {
|
||||
registry_label: Some("GPT-5.4"),
|
||||
thinking_mode: ThinkingMode::None,
|
||||
supported_efforts: Cow::Borrowed(&[
|
||||
ThinkingEffort::None,
|
||||
ThinkingEffort::Low,
|
||||
ThinkingEffort::Medium,
|
||||
ThinkingEffort::High,
|
||||
ThinkingEffort::XHigh,
|
||||
]),
|
||||
default_effort: Some(ThinkingEffort::Medium),
|
||||
databricks_v2_wire_route: DatabricksV2Route::NotApplicable,
|
||||
normalization_policy: NormalizationPolicy::OpenAiStandard,
|
||||
});
|
||||
}
|
||||
// rule: openai-gpt5-4, provider: databricks_v2, priority: 15
|
||||
if provider == "databricks_v2"
|
||||
&& (gpt5_token_matches_rs(lower, "gpt-5.4")
|
||||
@@ -421,6 +501,27 @@ pub fn lookup_by_family_rules(provider: &str, normalized: &str) -> Option<Capabi
|
||||
normalization_policy: NormalizationPolicy::OpenAiStandard,
|
||||
});
|
||||
}
|
||||
// rule: openai-gpt5-1, provider: databricks, priority: 15
|
||||
if provider == "databricks"
|
||||
&& (gpt5_token_matches_rs(lower, "gpt-5.1")
|
||||
|| gpt5_token_matches_rs(lower, "gpt5.1")
|
||||
|| gpt5_token_matches_rs(lower, "gpt-5-1")
|
||||
|| gpt5_token_matches_rs(lower, "gpt5-1"))
|
||||
{
|
||||
return Some(CapabilityResult {
|
||||
registry_label: Some("GPT-5.1"),
|
||||
thinking_mode: ThinkingMode::None,
|
||||
supported_efforts: Cow::Borrowed(&[
|
||||
ThinkingEffort::None,
|
||||
ThinkingEffort::Low,
|
||||
ThinkingEffort::Medium,
|
||||
ThinkingEffort::High,
|
||||
]),
|
||||
default_effort: Some(ThinkingEffort::None),
|
||||
databricks_v2_wire_route: DatabricksV2Route::NotApplicable,
|
||||
normalization_policy: NormalizationPolicy::OpenAiStandard,
|
||||
});
|
||||
}
|
||||
// rule: openai-gpt5-1, provider: databricks_v2, priority: 15
|
||||
if provider == "databricks_v2"
|
||||
&& (gpt5_token_matches_rs(lower, "gpt-5.1")
|
||||
@@ -820,6 +921,24 @@ pub fn lookup_by_family_rules(provider: &str, normalized: &str) -> Option<Capabi
|
||||
normalization_policy: NormalizationPolicy::OpenAiStandard,
|
||||
});
|
||||
}
|
||||
// rule: openai-gpt5-base, provider: databricks, priority: 10
|
||||
if provider == "databricks"
|
||||
&& (gpt5_base_matches_rs(lower, "gpt-5") || gpt5_base_matches_rs(lower, "gpt5"))
|
||||
{
|
||||
return Some(CapabilityResult {
|
||||
registry_label: Some("GPT-5"),
|
||||
thinking_mode: ThinkingMode::None,
|
||||
supported_efforts: Cow::Borrowed(&[
|
||||
ThinkingEffort::Minimal,
|
||||
ThinkingEffort::Low,
|
||||
ThinkingEffort::Medium,
|
||||
ThinkingEffort::High,
|
||||
]),
|
||||
default_effort: Some(ThinkingEffort::Medium),
|
||||
databricks_v2_wire_route: DatabricksV2Route::NotApplicable,
|
||||
normalization_policy: NormalizationPolicy::OpenAiStandard,
|
||||
});
|
||||
}
|
||||
// rule: openai-gpt5-base, provider: databricks_v2, priority: 10
|
||||
if provider == "databricks_v2"
|
||||
&& (gpt5_base_matches_rs(lower, "gpt-5") || gpt5_base_matches_rs(lower, "gpt5"))
|
||||
|
||||
@@ -135,7 +135,16 @@ mod shared_corpus_tests {
|
||||
.as_ref()
|
||||
.unwrap_or_else(|| panic!("corpus vector {id} missing expect"));
|
||||
|
||||
let result = resolve_model_capabilities(provider, raw_model_id);
|
||||
// Canonicalize provider aliases before resolving — mirrors the
|
||||
// production path where Rust normalizes "openai-compat" → Provider::OpenAi
|
||||
// (config.rs) and the TS canonicalizeProvider() resolves "databricks-v2"
|
||||
// → "databricks_v2" before generated lookups.
|
||||
let canonical_provider = match provider {
|
||||
"openai-compat" => "openai",
|
||||
"databricks-v2" => "databricks_v2",
|
||||
other => other,
|
||||
};
|
||||
let result = resolve_model_capabilities(canonical_provider, raw_model_id);
|
||||
ran += 1;
|
||||
|
||||
// Check thinking_mode if present in expect
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -22,8 +22,10 @@ import type { ManagedAgent } from "@/shared/api/types";
|
||||
* than falling through to "inherited" for lack of an instance.
|
||||
*/
|
||||
export function resolveAgentCardModelLabel(input: {
|
||||
agent: Pick<ManagedAgent, "modelSource" | "model"> | undefined;
|
||||
agent: Pick<ManagedAgent, "modelSource" | "model" | "provider"> | undefined;
|
||||
personaModel: string | null | undefined;
|
||||
/** Inference provider for the persona/agent — threads provider-qualified label lookup. */
|
||||
provider?: string | null | undefined;
|
||||
defaultModel: string;
|
||||
}): string {
|
||||
if (input.agent) {
|
||||
@@ -33,11 +35,11 @@ export function resolveAgentCardModelLabel(input: {
|
||||
return formatDefaultModelLabel(input.defaultModel);
|
||||
}
|
||||
return input.agent.model?.trim()
|
||||
? formatAgentModelLabel(input.agent.model)
|
||||
? formatAgentModelLabel(input.agent.model, input.agent.provider)
|
||||
: formatDefaultModelLabel(input.defaultModel);
|
||||
}
|
||||
return input.personaModel?.trim()
|
||||
? formatAgentModelLabel(input.personaModel)
|
||||
? formatAgentModelLabel(input.personaModel, input.provider)
|
||||
: formatDefaultModelLabel(input.defaultModel);
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import { DATABRICKS_MODEL_NAMES } from "./databricksModelNames.ts";
|
||||
import {
|
||||
resolveModelLabel,
|
||||
formatAgentModelLabel,
|
||||
canonicalizeProvider,
|
||||
} from "./formatAgentModelLabel.ts";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -198,7 +199,80 @@ test("ModelPicker — discovered rows render through resolveModelLabel", () => {
|
||||
|
||||
assert.match(
|
||||
source,
|
||||
/<DropdownMenuRadioItem[\s\S]*?\{resolveModelLabel\(model\.id, model\.name\)\}/,
|
||||
"dropdown rows must resolve labels through resolveModelLabel, not raw model.name/model.id",
|
||||
/<DropdownMenuRadioItem[\s\S]*?\{resolveModelLabel\(model\.id, model\.name, agent\.provider\)\}/,
|
||||
"dropdown rows must resolve labels through resolveModelLabel with provider, not raw model.name/model.id",
|
||||
);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// P3-B: provider-scoped label resolution (Thufir pass-2 finding 2)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("resolveModelLabel — provider-scoped miss returns raw ID, not Databricks registry name", () => {
|
||||
// "databricks-gemini-3-pro" is in DATABRICKS_MODEL_NAMES as "Gemini 3 Pro Preview".
|
||||
// When provider="anthropic", the generated lookup misses → must return raw ID.
|
||||
const result = resolveModelLabel(
|
||||
"databricks-gemini-3-pro",
|
||||
null,
|
||||
"anthropic",
|
||||
);
|
||||
assert.notEqual(
|
||||
result,
|
||||
"Gemini 3 Pro Preview",
|
||||
"must not leak Databricks registry label",
|
||||
);
|
||||
assert.equal(result, "databricks-gemini-3-pro");
|
||||
});
|
||||
|
||||
test("resolveModelLabel — openai provider scoped miss returns raw ID", () => {
|
||||
const result = resolveModelLabel("databricks-gemini-3-pro", null, "openai");
|
||||
assert.equal(result, "databricks-gemini-3-pro");
|
||||
});
|
||||
|
||||
test("resolveModelLabel — providerless call still returns unscoped registry label", () => {
|
||||
// No provider: the unscoped DATABRICKS_MODEL_NAMES map is reachable.
|
||||
const result = resolveModelLabel("databricks-gemini-3-pro", null, undefined);
|
||||
// The unscoped map should have a curated name for this ID.
|
||||
assert.ok(
|
||||
DATABRICKS_MODEL_NAMES.has("databricks-gemini-3-pro"),
|
||||
"databricks-gemini-3-pro must be in DATABRICKS_MODEL_NAMES for this test to be valid",
|
||||
);
|
||||
assert.equal(result, DATABRICKS_MODEL_NAMES.get("databricks-gemini-3-pro"));
|
||||
assert.notEqual(result, "databricks-gemini-3-pro");
|
||||
});
|
||||
|
||||
test("resolveModelLabel — null provider treated as providerless (uses unscoped registry)", () => {
|
||||
const result = resolveModelLabel("databricks-gemini-3-pro", null, null);
|
||||
assert.equal(result, DATABRICKS_MODEL_NAMES.get("databricks-gemini-3-pro"));
|
||||
});
|
||||
|
||||
test("resolveModelLabel — provider case-insensitivity: 'Anthropic' same as 'anthropic'", () => {
|
||||
const lower = resolveModelLabel("databricks-gemini-3-pro", null, "anthropic");
|
||||
const upper = resolveModelLabel("databricks-gemini-3-pro", null, "Anthropic");
|
||||
assert.equal(lower, upper);
|
||||
assert.equal(lower, "databricks-gemini-3-pro");
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// P3-B: canonicalizeProvider — alias normalization
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("canonicalizeProvider — databricks-v2 (hyphen) maps to databricks_v2 (underscore)", () => {
|
||||
assert.equal(canonicalizeProvider("databricks-v2"), "databricks_v2");
|
||||
});
|
||||
|
||||
test("canonicalizeProvider — uppercase DATABRICKS-V2 also normalizes to databricks_v2", () => {
|
||||
assert.equal(canonicalizeProvider("DATABRICKS-V2"), "databricks_v2");
|
||||
});
|
||||
|
||||
test("canonicalizeProvider — databricks_v2 passes through unchanged", () => {
|
||||
assert.equal(canonicalizeProvider("databricks_v2"), "databricks_v2");
|
||||
});
|
||||
|
||||
test("canonicalizeProvider — anthropic lowercases and trims", () => {
|
||||
assert.equal(canonicalizeProvider(" Anthropic "), "anthropic");
|
||||
});
|
||||
|
||||
test("canonicalizeProvider — unknown alias passes through lowercased", () => {
|
||||
assert.equal(canonicalizeProvider("OpenAI"), "openai");
|
||||
});
|
||||
|
||||
@@ -1,11 +1,45 @@
|
||||
import { DATABRICKS_MODEL_NAMES } from "./databricksModelNames";
|
||||
import {
|
||||
DATABRICKS_MODEL_NAMES,
|
||||
resolveModelCapabilities,
|
||||
} from "../ui/modelCapabilities.ts";
|
||||
|
||||
/**
|
||||
* Known provider-id aliases that must be normalized before generated lookups.
|
||||
*
|
||||
* - "databricks-v2" (hyphen form) → "databricks_v2" (underscore form used in manifest).
|
||||
* - "openai-compat" → "openai": Rust already accepts "openai-compat" as Provider::OpenAi
|
||||
* (crates/buzz-agent/src/config.rs); TS must canonicalize identically so the UI shows
|
||||
* the same effort table that the Rust request path will apply.
|
||||
*/
|
||||
const PROVIDER_ALIASES: Readonly<Record<string, string>> = {
|
||||
"databricks-v2": "databricks_v2",
|
||||
"openai-compat": "openai",
|
||||
};
|
||||
|
||||
/**
|
||||
* Normalizes a provider id to the canonical form expected by the generated
|
||||
* manifest: lowercases, trims, and applies the known alias table.
|
||||
*
|
||||
* Used as the single canonicalization point before every generated lookup
|
||||
* in both resolveModelLabel() and getProviderEffortConfig().
|
||||
*/
|
||||
export function canonicalizeProvider(provider: string): string {
|
||||
const normalized = provider.trim().toLowerCase();
|
||||
return PROVIDER_ALIASES[normalized] ?? normalized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a human-readable label for a model, following the three-tier
|
||||
* precedence documented in AGENTS.md:
|
||||
*
|
||||
* 1. Nonblank discovered/API name (e.g. from AgentModelInfo.name)
|
||||
* 2. Registry lookup by ID (models.dev-seeded Databricks table)
|
||||
* 2. Registry lookup by ID:
|
||||
* - When `provider` is supplied: provider-qualified exact record only.
|
||||
* If the generated lookup returns no registryLabel, return the raw ID.
|
||||
* The unscoped DATABRICKS_MODEL_NAMES registry is NOT consulted for a
|
||||
* known provider — this prevents Databricks names from leaking through
|
||||
* anthropic/openai provider contexts.
|
||||
* - When `provider` is absent/null: unscoped DATABRICKS_MODEL_NAMES map.
|
||||
* 3. Raw ID unchanged
|
||||
*
|
||||
* Returns the empty string when both id and discoveredName are blank.
|
||||
@@ -14,11 +48,24 @@ import { DATABRICKS_MODEL_NAMES } from "./databricksModelNames";
|
||||
export function resolveModelLabel(
|
||||
id: string,
|
||||
discoveredName?: string | null | undefined,
|
||||
provider?: string | null | undefined,
|
||||
): string {
|
||||
const trimmedName = discoveredName?.trim();
|
||||
if (trimmedName) return trimmedName;
|
||||
const trimmedId = id.trim();
|
||||
if (!trimmedId) return "";
|
||||
// Provider-qualified exact record (registry_label tier, provider-scoped).
|
||||
// When a provider is known, do NOT fall through to the unscoped registry —
|
||||
// return the raw ID directly on a miss (P3-B contract).
|
||||
if (provider?.trim()) {
|
||||
const canonical = canonicalizeProvider(provider);
|
||||
const registryLabel = resolveModelCapabilities(
|
||||
canonical,
|
||||
trimmedId,
|
||||
).registryLabel;
|
||||
return registryLabel ?? trimmedId;
|
||||
}
|
||||
// Providerless path only: unscoped registry map for legacy/inherited IDs.
|
||||
return DATABRICKS_MODEL_NAMES.get(trimmedId) ?? trimmedId;
|
||||
}
|
||||
|
||||
@@ -29,9 +76,15 @@ export function resolveModelLabel(
|
||||
* For known Databricks managed endpoints the registry-curated name is returned
|
||||
* (e.g. "databricks-gpt-5-5" → "GPT-5.5"). Unknown or custom endpoint IDs are
|
||||
* returned unchanged — no heuristic string mangling.
|
||||
*
|
||||
* Pass `provider` when the inference provider is known to get a provider-qualified
|
||||
* registry label (e.g. exact records in the generated manifest take priority).
|
||||
*/
|
||||
export function formatAgentModelLabel(model: string | null | undefined) {
|
||||
export function formatAgentModelLabel(
|
||||
model: string | null | undefined,
|
||||
provider?: string | null | undefined,
|
||||
) {
|
||||
const trimmed = model?.trim();
|
||||
if (!trimmed) return "Auto";
|
||||
return resolveModelLabel(trimmed);
|
||||
return resolveModelLabel(trimmed, null, provider);
|
||||
}
|
||||
|
||||
@@ -411,7 +411,9 @@ function RuntimeBlock({
|
||||
{runtimeSource || agent.model ? (
|
||||
<div className="flex flex-wrap items-center gap-x-3 gap-y-1 text-xs text-muted-foreground">
|
||||
{runtimeSource ? <span>{runtimeSource}</span> : null}
|
||||
{agent.model ? <span>{resolveModelLabel(agent.model)}</span> : null}
|
||||
{agent.model ? (
|
||||
<span>{resolveModelLabel(agent.model, null, agent.provider)}</span>
|
||||
) : null}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
@@ -84,9 +84,9 @@ export function ModelPicker({
|
||||
|
||||
const currentValue = agent.model ?? modelsData?.agentDefaultModel ?? "";
|
||||
const displayLabel = agent.model
|
||||
? resolveModelLabel(agent.model)
|
||||
? resolveModelLabel(agent.model, null, agent.provider)
|
||||
: modelsData?.agentDefaultModel
|
||||
? `${resolveModelLabel(modelsData.agentDefaultModel)} (default)`
|
||||
? `${resolveModelLabel(modelsData.agentDefaultModel, null, agent.provider)} (default)`
|
||||
: hasRequestedModels && loading
|
||||
? "Loading..."
|
||||
: "Auto";
|
||||
@@ -223,7 +223,7 @@ export function ModelPicker({
|
||||
{agent.model ? (
|
||||
<>
|
||||
<p className="font-medium text-foreground">
|
||||
{resolveModelLabel(agent.model)}
|
||||
{resolveModelLabel(agent.model, null, agent.provider)}
|
||||
</p>
|
||||
<p className="mt-0.5 text-xs">
|
||||
This runtime does not support switching models.
|
||||
@@ -240,7 +240,7 @@ export function ModelPicker({
|
||||
>
|
||||
{modelsData.models.map((model) => (
|
||||
<DropdownMenuRadioItem key={model.id} value={model.id}>
|
||||
{resolveModelLabel(model.id, model.name)}
|
||||
{resolveModelLabel(model.id, model.name, agent.provider)}
|
||||
</DropdownMenuRadioItem>
|
||||
))}
|
||||
</DropdownMenuRadioGroup>
|
||||
|
||||
@@ -203,7 +203,7 @@ function TeamAvatarItem({
|
||||
|
||||
function getTeamFooterModelLabel(personas: AgentPersona[]) {
|
||||
const modelLabels = personas
|
||||
.map((persona) => formatAgentModelLabel(persona.model))
|
||||
.map((persona) => formatAgentModelLabel(persona.model, persona.provider))
|
||||
.filter((model): model is string => Boolean(model));
|
||||
|
||||
if (modelLabels.length === 0) return "Auto";
|
||||
|
||||
@@ -276,6 +276,7 @@ function AgentPersonaCard({
|
||||
const modelLabel = resolveAgentCardModelLabel({
|
||||
agent,
|
||||
personaModel: persona.model,
|
||||
provider: persona.provider,
|
||||
defaultModel,
|
||||
});
|
||||
const isActive = agent ? isManagedAgentActive(agent) : false;
|
||||
|
||||
@@ -541,13 +541,19 @@ test("databricks v1 routes like openai unknown (no gpt-5 model)", () => {
|
||||
assert.equal(defaultValue, "medium");
|
||||
});
|
||||
|
||||
test("openai-compat returns all-7 with medium default", () => {
|
||||
const { validValues, defaultValue } = getProviderEffortConfig(
|
||||
"openai-compat",
|
||||
"",
|
||||
test("openai-compat canonicalizes to openai: empty model returns all-except-max with medium default", () => {
|
||||
// Regression: without canonicalization, openai-compat hit the unknown-provider fallback
|
||||
// (all 7 values, default medium). After alias, it must resolve identically to openai.
|
||||
const compat = getProviderEffortConfig("openai-compat", "");
|
||||
const openai = getProviderEffortConfig("openai", "");
|
||||
assert.deepEqual([...compat.validValues], [...openai.validValues]);
|
||||
assert.equal(compat.defaultValue, openai.defaultValue);
|
||||
// Concrete assertion: same as openai unknown model, not the all-7 fallback.
|
||||
assert.deepEqual(
|
||||
[...compat.validValues],
|
||||
["none", "minimal", "low", "medium", "high", "xhigh"],
|
||||
);
|
||||
assert.equal(validValues.length, 7);
|
||||
assert.equal(defaultValue, "medium");
|
||||
assert.equal(compat.defaultValue, "medium");
|
||||
});
|
||||
|
||||
test("empty provider returns all-7 with medium default", () => {
|
||||
@@ -621,3 +627,79 @@ test("effort none is invalid for anthropic manual-budget (should trigger auto-cl
|
||||
"none must not be in manual-budget set",
|
||||
);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// getProviderEffortConfig — databricks-v2 hyphen alias (P3-B regression)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("databricks-v2 hyphen alias canonicalizes to databricks_v2 underscore records", () => {
|
||||
// The persisted alias "databricks-v2" must hit the same canonical records as "databricks_v2".
|
||||
const hyphen = getProviderEffortConfig("databricks-v2", "databricks-gpt-5-5");
|
||||
const underscore = getProviderEffortConfig(
|
||||
"databricks_v2",
|
||||
"databricks-gpt-5-5",
|
||||
);
|
||||
assert.deepEqual([...hyphen.validValues], [...underscore.validValues]);
|
||||
assert.equal(hyphen.defaultValue, underscore.defaultValue);
|
||||
});
|
||||
|
||||
test("databricks-v2 hyphen alias with databricks-gpt-5-5 returns [low,medium,high]", () => {
|
||||
// Regression: without canonicalization this returned the all-7 unknown-provider fallback.
|
||||
const { validValues, defaultValue } = getProviderEffortConfig(
|
||||
"databricks-v2",
|
||||
"databricks-gpt-5-5",
|
||||
);
|
||||
assert.deepEqual([...validValues], ["low", "medium", "high"]);
|
||||
assert.equal(defaultValue, "medium");
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// getProviderEffortConfig — openai-compat alias (Thufir P3 corrective action 1)
|
||||
// ---------------------------------------------------------------------------
|
||||
// Rust normalizes "openai-compat" → Provider::OpenAi at config.rs:1218.
|
||||
// TS PROVIDER_ALIASES must match so UI effort table = Rust request behavior.
|
||||
|
||||
test("openai-compat/gpt-5-pro resolves identically to openai/gpt-5-pro", () => {
|
||||
const compat = getProviderEffortConfig("openai-compat", "gpt-5-pro");
|
||||
const openai = getProviderEffortConfig("openai", "gpt-5-pro");
|
||||
assert.deepEqual([...compat.validValues], [...openai.validValues]);
|
||||
assert.equal(compat.defaultValue, openai.defaultValue);
|
||||
// Concrete: gpt-5-pro is [high] only.
|
||||
assert.deepEqual([...compat.validValues], ["high"]);
|
||||
assert.equal(compat.defaultValue, "high");
|
||||
});
|
||||
|
||||
test("openai-compat/gpt-5.5 resolves identically to openai/gpt-5.5", () => {
|
||||
const compat = getProviderEffortConfig("openai-compat", "gpt-5.5");
|
||||
const openai = getProviderEffortConfig("openai", "gpt-5.5");
|
||||
assert.deepEqual([...compat.validValues], [...openai.validValues]);
|
||||
assert.equal(compat.defaultValue, openai.defaultValue);
|
||||
});
|
||||
|
||||
test("openai-compat/gpt-5 base resolves identically to openai/gpt-5", () => {
|
||||
const compat = getProviderEffortConfig("openai-compat", "gpt-5");
|
||||
const openai = getProviderEffortConfig("openai", "gpt-5");
|
||||
assert.deepEqual([...compat.validValues], [...openai.validValues]);
|
||||
assert.equal(compat.defaultValue, openai.defaultValue);
|
||||
});
|
||||
|
||||
test("openai-compat alias is case-insensitive: OpenAI-Compat canonicalizes correctly", () => {
|
||||
const mixed = getProviderEffortConfig("OpenAI-Compat", "gpt-5-pro");
|
||||
const lower = getProviderEffortConfig("openai-compat", "gpt-5-pro");
|
||||
assert.deepEqual([...mixed.validValues], [...lower.validValues]);
|
||||
assert.equal(mixed.defaultValue, lower.defaultValue);
|
||||
});
|
||||
|
||||
test("openai-compat alias handles surrounding whitespace: ' openai-compat ' canonicalizes correctly", () => {
|
||||
const padded = getProviderEffortConfig(" openai-compat ", "gpt-5-pro");
|
||||
const clean = getProviderEffortConfig("openai-compat", "gpt-5-pro");
|
||||
assert.deepEqual([...padded.validValues], [...clean.validValues]);
|
||||
assert.equal(padded.defaultValue, clean.defaultValue);
|
||||
});
|
||||
|
||||
test("openai-compat alias handles mixed case + whitespace: ' OpenAI-Compat ' canonicalizes correctly", () => {
|
||||
const messy = getProviderEffortConfig(" OpenAI-Compat ", "gpt-5");
|
||||
const canonical = getProviderEffortConfig("openai", "gpt-5");
|
||||
assert.deepEqual([...messy.validValues], [...canonical.validValues]);
|
||||
assert.equal(messy.defaultValue, canonical.defaultValue);
|
||||
});
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
/**
|
||||
* Source-of-truth constants for buzz-agent model-tuning configuration knobs.
|
||||
*
|
||||
* Values must stay in sync with `crates/buzz-agent/src/config.rs`
|
||||
* `parse_thinking_effort` — that function is the authoritative list.
|
||||
* Phase 2b pass 2: getProviderEffortConfig() is now generated-backed (thin
|
||||
* wrapper over resolveModelCapabilities()). The legacy hand-table implementation
|
||||
* is preserved as getProviderEffortConfig_oldHandTable() for the differential
|
||||
* harness only — nothing user-facing imports the _old shim. Phase 3 retires it.
|
||||
*/
|
||||
import { canonicalizeProvider } from "../lib/formatAgentModelLabel.ts";
|
||||
import { resolveModelCapabilities } from "./modelCapabilities.ts";
|
||||
|
||||
/** Env var key for the thinking/effort level sent to the LLM. */
|
||||
export const BUZZ_AGENT_THINKING_EFFORT = "BUZZ_AGENT_THINKING_EFFORT";
|
||||
@@ -61,25 +65,43 @@ const ALL_VALUES = BUZZ_AGENT_THINKING_EFFORT_VALUES;
|
||||
|
||||
/**
|
||||
* Returns the valid thinking-effort values and semantic default for the
|
||||
* given provider and optional model string.
|
||||
* given provider and optional model string, resolved from the generated
|
||||
* model-capabilities manifest (modelCapabilities.ts).
|
||||
*
|
||||
* Model matching mirrors the Rust backend:
|
||||
* - Anthropic: strip any endpoint-naming prefix, then test `is_manual_budget_model`
|
||||
* / `is_adaptive_thinking_model` / `clamp_adaptive_effort` family checks.
|
||||
* - OpenAI: strip any endpoint-naming prefix, then test `openai_efforts_for_model`
|
||||
* family checks (boundary-aware: -pro before -5.x, digit/letter boundary).
|
||||
* - DatabricksV2: strip prefix and route by model family.
|
||||
* - Unknown/empty: all 7 values, default medium.
|
||||
* This is the production entry point for all UI consumers. Resolution order:
|
||||
* 1. Provider-qualified raw exact lookup
|
||||
* 2. Provider-scoped family rules on normalized alias
|
||||
* 3. Per-provider fallback
|
||||
*
|
||||
* Prefix stripping: finds the first occurrence of a known model-family token
|
||||
* (`claude-`, `gpt-`) and drops everything before it. This handles any
|
||||
* endpoint-naming convention (e.g. `databricks-`, `goose-`, `team-x-`) without
|
||||
* maintaining an allowlist of known prefixes. If no family token is found, the
|
||||
* raw model name is used as-is.
|
||||
* The manifest's `supportedEfforts` maps to `validValues`; `defaultEffort`
|
||||
* (which may be null for manual-budget models — "Inherit" is the natural
|
||||
* default) maps to `defaultValue`.
|
||||
*/
|
||||
export function getProviderEffortConfig(
|
||||
providerId: string,
|
||||
model?: string,
|
||||
): ProviderEffortConfig {
|
||||
const cap = resolveModelCapabilities(
|
||||
canonicalizeProvider(providerId),
|
||||
model ?? "",
|
||||
);
|
||||
return {
|
||||
validValues: cap.supportedEfforts,
|
||||
defaultValue: cap.defaultEffort,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy hand-table implementation — differential harness shim only.
|
||||
*
|
||||
* Preserved for the run-differential.mjs old-vs-new comparison until Phase 3
|
||||
* retires it. Nothing user-facing should import this name.
|
||||
*
|
||||
* @deprecated Use getProviderEffortConfig() (generated-backed) instead.
|
||||
*/
|
||||
export function getProviderEffortConfig_oldHandTable(
|
||||
providerId: string,
|
||||
model?: string,
|
||||
): ProviderEffortConfig {
|
||||
const provider = providerId.toLowerCase();
|
||||
// Strip arbitrary endpoint-naming prefix before model-family matching.
|
||||
@@ -307,3 +329,7 @@ function openaiConfig(m: string): ProviderEffortConfig {
|
||||
export function isBuzzAgentRuntime(runtimeId: string): boolean {
|
||||
return runtimeId === "buzz-agent";
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Differential harness support
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -203,12 +203,19 @@
|
||||
"defaultValue": "medium"
|
||||
},
|
||||
{
|
||||
"note": "openai-compat: all-7 with medium default",
|
||||
"note": "openai-compat: canonicalizes to openai, empty model → all-except-max with medium default",
|
||||
"provider": "openai-compat",
|
||||
"model": "",
|
||||
"validValues": ["none", "minimal", "low", "medium", "high", "xhigh", "max"],
|
||||
"validValues": ["none", "minimal", "low", "medium", "high", "xhigh"],
|
||||
"defaultValue": "medium"
|
||||
},
|
||||
{
|
||||
"note": "openai-compat/gpt-5-pro: canonicalizes to openai, gpt-5-pro → [high] only",
|
||||
"provider": "openai-compat",
|
||||
"model": "gpt-5-pro",
|
||||
"validValues": ["high"],
|
||||
"defaultValue": "high"
|
||||
},
|
||||
{
|
||||
"note": "openrouter: all-7 with medium default",
|
||||
"provider": "openrouter",
|
||||
|
||||
@@ -308,6 +308,17 @@ function lookupByFamilyRules(provider: string, normalized: string): CapabilityRe
|
||||
normalizationPolicy: "openai-standard",
|
||||
};
|
||||
}
|
||||
// rule: openai-gpt5-pro, provider: databricks, priority: 20
|
||||
if (provider === "databricks" && (gpt5TokenMatchesGenerated(lower, "gpt-5-pro") || gpt5TokenMatchesGenerated(lower, "gpt5-pro"))) {
|
||||
return {
|
||||
registryLabel: "GPT-5 Pro",
|
||||
thinkingMode: "none",
|
||||
supportedEfforts: ["high"] as const,
|
||||
defaultEffort: "high",
|
||||
databricksV2WireRoute: "not-applicable",
|
||||
normalizationPolicy: "openai-standard",
|
||||
};
|
||||
}
|
||||
// rule: openai-gpt5-pro, provider: databricks_v2, priority: 20
|
||||
if (provider === "databricks_v2" && (gpt5TokenMatchesGenerated(lower, "gpt-5-pro") || gpt5TokenMatchesGenerated(lower, "gpt5-pro"))) {
|
||||
return {
|
||||
@@ -330,6 +341,17 @@ function lookupByFamilyRules(provider: string, normalized: string): CapabilityRe
|
||||
normalizationPolicy: "openai-standard",
|
||||
};
|
||||
}
|
||||
// rule: openai-gpt5-6, provider: databricks, priority: 15
|
||||
if (provider === "databricks" && (gpt5TokenMatchesGenerated(lower, "gpt-5.6") || gpt5TokenMatchesGenerated(lower, "gpt5.6") || gpt5TokenMatchesGenerated(lower, "gpt-5-6") || gpt5TokenMatchesGenerated(lower, "gpt5-6"))) {
|
||||
return {
|
||||
registryLabel: "GPT-5.6",
|
||||
thinkingMode: "none",
|
||||
supportedEfforts: ["none", "low", "medium", "high", "xhigh", "max"] as const,
|
||||
defaultEffort: "medium",
|
||||
databricksV2WireRoute: "not-applicable",
|
||||
normalizationPolicy: "openai-standard",
|
||||
};
|
||||
}
|
||||
// rule: openai-gpt5-6, provider: databricks_v2, priority: 15
|
||||
if (provider === "databricks_v2" && (gpt5TokenMatchesGenerated(lower, "gpt-5.6") || gpt5TokenMatchesGenerated(lower, "gpt5.6") || gpt5TokenMatchesGenerated(lower, "gpt-5-6") || gpt5TokenMatchesGenerated(lower, "gpt5-6"))) {
|
||||
return {
|
||||
@@ -352,6 +374,17 @@ function lookupByFamilyRules(provider: string, normalized: string): CapabilityRe
|
||||
normalizationPolicy: "openai-standard",
|
||||
};
|
||||
}
|
||||
// rule: openai-gpt5-5, provider: databricks, priority: 15
|
||||
if (provider === "databricks" && (gpt5TokenMatchesGenerated(lower, "gpt-5.5") || gpt5TokenMatchesGenerated(lower, "gpt5.5") || gpt5TokenMatchesGenerated(lower, "gpt-5-5") || gpt5TokenMatchesGenerated(lower, "gpt5-5"))) {
|
||||
return {
|
||||
registryLabel: "GPT-5.5",
|
||||
thinkingMode: "none",
|
||||
supportedEfforts: ["none", "low", "medium", "high", "xhigh"] as const,
|
||||
defaultEffort: "medium",
|
||||
databricksV2WireRoute: "not-applicable",
|
||||
normalizationPolicy: "openai-standard",
|
||||
};
|
||||
}
|
||||
// rule: openai-gpt5-5, provider: databricks_v2, priority: 15
|
||||
if (provider === "databricks_v2" && (gpt5TokenMatchesGenerated(lower, "gpt-5.5") || gpt5TokenMatchesGenerated(lower, "gpt5.5") || gpt5TokenMatchesGenerated(lower, "gpt-5-5") || gpt5TokenMatchesGenerated(lower, "gpt5-5"))) {
|
||||
return {
|
||||
@@ -374,6 +407,17 @@ function lookupByFamilyRules(provider: string, normalized: string): CapabilityRe
|
||||
normalizationPolicy: "openai-standard",
|
||||
};
|
||||
}
|
||||
// rule: openai-gpt5-4, provider: databricks, priority: 15
|
||||
if (provider === "databricks" && (gpt5TokenMatchesGenerated(lower, "gpt-5.4") || gpt5TokenMatchesGenerated(lower, "gpt5.4") || gpt5TokenMatchesGenerated(lower, "gpt-5-4") || gpt5TokenMatchesGenerated(lower, "gpt5-4"))) {
|
||||
return {
|
||||
registryLabel: "GPT-5.4",
|
||||
thinkingMode: "none",
|
||||
supportedEfforts: ["none", "low", "medium", "high", "xhigh"] as const,
|
||||
defaultEffort: "medium",
|
||||
databricksV2WireRoute: "not-applicable",
|
||||
normalizationPolicy: "openai-standard",
|
||||
};
|
||||
}
|
||||
// rule: openai-gpt5-4, provider: databricks_v2, priority: 15
|
||||
if (provider === "databricks_v2" && (gpt5TokenMatchesGenerated(lower, "gpt-5.4") || gpt5TokenMatchesGenerated(lower, "gpt5.4") || gpt5TokenMatchesGenerated(lower, "gpt-5-4") || gpt5TokenMatchesGenerated(lower, "gpt5-4"))) {
|
||||
return {
|
||||
@@ -396,6 +440,17 @@ function lookupByFamilyRules(provider: string, normalized: string): CapabilityRe
|
||||
normalizationPolicy: "openai-standard",
|
||||
};
|
||||
}
|
||||
// rule: openai-gpt5-1, provider: databricks, priority: 15
|
||||
if (provider === "databricks" && (gpt5TokenMatchesGenerated(lower, "gpt-5.1") || gpt5TokenMatchesGenerated(lower, "gpt5.1") || gpt5TokenMatchesGenerated(lower, "gpt-5-1") || gpt5TokenMatchesGenerated(lower, "gpt5-1"))) {
|
||||
return {
|
||||
registryLabel: "GPT-5.1",
|
||||
thinkingMode: "none",
|
||||
supportedEfforts: ["none", "low", "medium", "high"] as const,
|
||||
defaultEffort: "none",
|
||||
databricksV2WireRoute: "not-applicable",
|
||||
normalizationPolicy: "openai-standard",
|
||||
};
|
||||
}
|
||||
// rule: openai-gpt5-1, provider: databricks_v2, priority: 15
|
||||
if (provider === "databricks_v2" && (gpt5TokenMatchesGenerated(lower, "gpt-5.1") || gpt5TokenMatchesGenerated(lower, "gpt5.1") || gpt5TokenMatchesGenerated(lower, "gpt-5-1") || gpt5TokenMatchesGenerated(lower, "gpt5-1"))) {
|
||||
return {
|
||||
@@ -660,6 +715,17 @@ function lookupByFamilyRules(provider: string, normalized: string): CapabilityRe
|
||||
normalizationPolicy: "openai-standard",
|
||||
};
|
||||
}
|
||||
// rule: openai-gpt5-base, provider: databricks, priority: 10
|
||||
if (provider === "databricks" && (gpt5BaseMatchesGenerated(lower, "gpt-5") || gpt5BaseMatchesGenerated(lower, "gpt5"))) {
|
||||
return {
|
||||
registryLabel: "GPT-5",
|
||||
thinkingMode: "none",
|
||||
supportedEfforts: ["minimal", "low", "medium", "high"] as const,
|
||||
defaultEffort: "medium",
|
||||
databricksV2WireRoute: "not-applicable",
|
||||
normalizationPolicy: "openai-standard",
|
||||
};
|
||||
}
|
||||
// rule: openai-gpt5-base, provider: databricks_v2, priority: 10
|
||||
if (provider === "databricks_v2" && (gpt5BaseMatchesGenerated(lower, "gpt-5") || gpt5BaseMatchesGenerated(lower, "gpt5"))) {
|
||||
return {
|
||||
|
||||
@@ -65,7 +65,7 @@ export function getDiscoveredPersonaModelOptions(
|
||||
provider === "relay-mesh"
|
||||
? "Default (auto)"
|
||||
: agentDefaultModel
|
||||
? `Default model (${resolveModelLabel(agentDefaultModel)})`
|
||||
? `Default model (${resolveModelLabel(agentDefaultModel, null, provider)})`
|
||||
: "Default model",
|
||||
},
|
||||
];
|
||||
@@ -78,7 +78,7 @@ export function getDiscoveredPersonaModelOptions(
|
||||
...defaultModelOption,
|
||||
...explicitModels.map((model) => ({
|
||||
id: model.id,
|
||||
label: resolveModelLabel(model.id, model.name),
|
||||
label: resolveModelLabel(model.id, model.name, provider),
|
||||
})),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -620,7 +620,13 @@ export function UserProfilePopover({
|
||||
<InfoBadge>{runtimeLabel(relayAgent.agentType)}</InfoBadge>
|
||||
) : null}
|
||||
{managedAgent?.model ? (
|
||||
<InfoBadge>{resolveModelLabel(managedAgent.model)}</InfoBadge>
|
||||
<InfoBadge>
|
||||
{resolveModelLabel(
|
||||
managedAgent.model,
|
||||
null,
|
||||
managedAgent.provider,
|
||||
)}
|
||||
</InfoBadge>
|
||||
) : null}
|
||||
{managedAgent?.acpCommand ? (
|
||||
<InfoBadge>ACP: {managedAgent.acpCommand}</InfoBadge>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# models.dev Reasoning Options Reconciliation Table
|
||||
|
||||
**Source queried**: https://models.dev/api.json (2026-07-31)
|
||||
**Payload SHA-256**: `d5a4974cd69f19b0f67713acaa6bb3b16e920defdc07ecbdf6b0a936181bb0e0`
|
||||
**Source queried**: https://models.dev/api.json (2026-07-31)<br>
|
||||
**Payload SHA-256**: `d5a4974cd69f19b0f67713acaa6bb3b16e920defdc07ecbdf6b0a936181bb0e0`<br>
|
||||
**Policy (plan v4 §Behavior policy)**: models.dev `reasoning_options` become exact overrides.
|
||||
Each divergence from the current family rule result is reconciled here: either (a) adopted as an
|
||||
intentional correction or (b) rejected with a curation note.
|
||||
@@ -23,8 +23,8 @@ advertises only `[low, medium, high]` in its `reasoning_options`. The family rul
|
||||
`xhigh` are derived from the upstream OpenAI GPT-5.4 spec, which this Databricks endpoint does
|
||||
not expose. Provider-advertised wins per plan F1 policy.
|
||||
|
||||
**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-4-mini"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]`
|
||||
**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-4-mini"`
|
||||
**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-4-mini"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]`<br>
|
||||
**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-4-mini"`<br>
|
||||
**Test vector**: `resolver-exact-raw-id-hit` in `scripts/normative-corpus.json`
|
||||
|
||||
---
|
||||
@@ -38,7 +38,7 @@ not expose. Provider-advertised wins per plan F1 policy.
|
||||
**Rationale**: Same as `databricks-gpt-5-4-mini`. The nano variant exposes the same restricted
|
||||
effort set. Provider-advertised wins.
|
||||
|
||||
**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-4-nano"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]`
|
||||
**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-4-nano"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]`<br>
|
||||
**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-4-nano"`
|
||||
|
||||
---
|
||||
@@ -54,7 +54,7 @@ effort set. Provider-advertised wins.
|
||||
derived from the upstream OpenAI GPT-5.6 spec, which this Databricks endpoint does not expose.
|
||||
Provider-advertised wins per plan F1 policy.
|
||||
|
||||
**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-6-sol"].reasoning_options = [{"type":"effort","values":["low","medium","high","max"]}]`
|
||||
**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-6-sol"].reasoning_options = [{"type":"effort","values":["low","medium","high","max"]}]`<br>
|
||||
**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-6-sol"`
|
||||
|
||||
---
|
||||
@@ -70,7 +70,7 @@ Provider-advertised wins per plan F1 policy.
|
||||
derived from the upstream OpenAI GPT-5.5 spec, which this Databricks endpoint does not expose.
|
||||
Provider-advertised wins per plan F1 policy.
|
||||
|
||||
**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-5"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]`
|
||||
**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-5"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]`<br>
|
||||
**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-5"`
|
||||
|
||||
---
|
||||
@@ -86,7 +86,7 @@ a different capability axis (extended thinking token budget), not an effort-leve
|
||||
There is no effort divergence to reconcile. The effort capabilities for this model come from the
|
||||
`anthropic-adaptive-xhigh-opus-4-7` family rule (Anthropic extended-thinking support table).
|
||||
|
||||
**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-claude-opus-4-7"].reasoning_options = [{"type":"budget_tokens","min":1024}]`
|
||||
**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-claude-opus-4-7"].reasoning_options = [{"type":"budget_tokens","min":1024}]`<br>
|
||||
**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-claude-opus-4-7"`
|
||||
|
||||
---
|
||||
|
||||
@@ -601,6 +601,50 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-pro / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt-5-pro",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5 Pro",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"high"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-pro",
|
||||
"rule_priority": 20,
|
||||
"normalized_alias": "gpt-5-pro",
|
||||
"raw_model_id": "gpt-5-pro"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-pro alias gpt5-pro / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt5-pro",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5 Pro",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"high"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-pro",
|
||||
"rule_priority": 20,
|
||||
"normalized_alias": "gpt5-pro",
|
||||
"raw_model_id": "gpt5-pro"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-pro / provider databricks_v2",
|
||||
"provider": "databricks_v2",
|
||||
@@ -753,6 +797,114 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-6 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt-5.6",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.6",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-6",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt-5.6",
|
||||
"raw_model_id": "gpt-5.6"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-6 alias gpt5.6 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt5.6",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.6",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-6",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt5.6",
|
||||
"raw_model_id": "gpt5.6"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-6 alias gpt-5-6 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt-5-6",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.6",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-6",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt-5-6",
|
||||
"raw_model_id": "gpt-5-6"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-6 alias gpt5-6 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt5-6",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.6",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-6",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt5-6",
|
||||
"raw_model_id": "gpt5-6"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-6 / provider databricks_v2",
|
||||
"provider": "databricks_v2",
|
||||
@@ -965,6 +1117,110 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-5 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt-5.5",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.5",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-5",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt-5.5",
|
||||
"raw_model_id": "gpt-5.5"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-5 alias gpt5.5 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt5.5",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.5",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-5",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt5.5",
|
||||
"raw_model_id": "gpt5.5"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-5 alias gpt-5-5 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt-5-5",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.5",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-5",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt-5-5",
|
||||
"raw_model_id": "gpt-5-5"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-5 alias gpt5-5 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt5-5",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.5",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-5",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt5-5",
|
||||
"raw_model_id": "gpt5-5"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-5 / provider databricks_v2",
|
||||
"provider": "databricks_v2",
|
||||
@@ -1173,6 +1429,110 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-4 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt-5.4",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.4",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-4",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt-5.4",
|
||||
"raw_model_id": "gpt-5.4"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-4 alias gpt5.4 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt5.4",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.4",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-4",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt5.4",
|
||||
"raw_model_id": "gpt5.4"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-4 alias gpt-5-4 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt-5-4",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.4",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-4",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt-5-4",
|
||||
"raw_model_id": "gpt-5-4"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-4 alias gpt5-4 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt5-4",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.4",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-4",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt5-4",
|
||||
"raw_model_id": "gpt5-4"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-4 / provider databricks_v2",
|
||||
"provider": "databricks_v2",
|
||||
@@ -1377,6 +1737,106 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-1 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt-5.1",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.1",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"default_effort": "none",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-1",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt-5.1",
|
||||
"raw_model_id": "gpt-5.1"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-1 alias gpt5.1 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt5.1",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.1",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"default_effort": "none",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-1",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt5.1",
|
||||
"raw_model_id": "gpt5.1"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-1 alias gpt-5-1 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt-5-1",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.1",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"default_effort": "none",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-1",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt-5-1",
|
||||
"raw_model_id": "gpt-5-1"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-1 alias gpt5-1 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt5-1",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5.1",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"default_effort": "none",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-1",
|
||||
"rule_priority": 15,
|
||||
"normalized_alias": "gpt5-1",
|
||||
"raw_model_id": "gpt5-1"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-1 / provider databricks_v2",
|
||||
"provider": "databricks_v2",
|
||||
@@ -1527,6 +1987,56 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-base / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt-5",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-base",
|
||||
"rule_priority": 10,
|
||||
"normalized_alias": "gpt-5",
|
||||
"raw_model_id": "gpt-5"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-base alias gpt5 / provider databricks",
|
||||
"provider": "databricks",
|
||||
"model": "gpt5",
|
||||
"resolved": {
|
||||
"registry_label": "GPT-5",
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"normalization_policy": "openai-standard",
|
||||
"_provenance": {
|
||||
"source": "family",
|
||||
"rule_id": "openai-gpt5-base",
|
||||
"rule_priority": 10,
|
||||
"normalized_alias": "gpt5",
|
||||
"raw_model_id": "gpt5"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"note": "family rule openai-gpt5-base / provider databricks_v2",
|
||||
"provider": "databricks_v2",
|
||||
|
||||
@@ -257,6 +257,7 @@
|
||||
],
|
||||
"providers": [
|
||||
"openai",
|
||||
"databricks",
|
||||
"databricks_v2"
|
||||
],
|
||||
"match_priority": 20,
|
||||
@@ -280,6 +281,7 @@
|
||||
],
|
||||
"providers": [
|
||||
"openai",
|
||||
"databricks",
|
||||
"databricks_v2"
|
||||
],
|
||||
"match_priority": 15,
|
||||
@@ -308,6 +310,7 @@
|
||||
],
|
||||
"providers": [
|
||||
"openai",
|
||||
"databricks",
|
||||
"databricks_v2"
|
||||
],
|
||||
"match_priority": 15,
|
||||
@@ -335,6 +338,7 @@
|
||||
],
|
||||
"providers": [
|
||||
"openai",
|
||||
"databricks",
|
||||
"databricks_v2"
|
||||
],
|
||||
"match_priority": 15,
|
||||
@@ -362,6 +366,7 @@
|
||||
],
|
||||
"providers": [
|
||||
"openai",
|
||||
"databricks",
|
||||
"databricks_v2"
|
||||
],
|
||||
"match_priority": 15,
|
||||
@@ -386,6 +391,7 @@
|
||||
],
|
||||
"providers": [
|
||||
"openai",
|
||||
"databricks",
|
||||
"databricks_v2"
|
||||
],
|
||||
"match_priority": 10,
|
||||
|
||||
@@ -9,7 +9,11 @@
|
||||
"raw_model_id": "claude-3-7-sonnet-20250219",
|
||||
"expect": {
|
||||
"thinking_mode": "manual-budget",
|
||||
"supported_efforts": ["low", "medium", "high"],
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"default_effort": null,
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -20,7 +24,11 @@
|
||||
"raw_model_id": "claude-opus-4-5",
|
||||
"expect": {
|
||||
"thinking_mode": "manual-budget",
|
||||
"supported_efforts": ["low", "medium", "high"],
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"default_effort": null,
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -31,7 +39,13 @@
|
||||
"raw_model_id": "claude-opus-4-7",
|
||||
"expect": {
|
||||
"thinking_mode": "adaptive",
|
||||
"supported_efforts": ["low", "medium", "high", "xhigh", "max"],
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -42,7 +56,13 @@
|
||||
"raw_model_id": "claude-opus-4-8",
|
||||
"expect": {
|
||||
"thinking_mode": "adaptive",
|
||||
"supported_efforts": ["low", "medium", "high", "xhigh", "max"],
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -53,7 +73,13 @@
|
||||
"raw_model_id": "claude-sonnet-5-20260101",
|
||||
"expect": {
|
||||
"thinking_mode": "adaptive",
|
||||
"supported_efforts": ["low", "medium", "high", "xhigh", "max"],
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -64,7 +90,13 @@
|
||||
"raw_model_id": "claude-fable-5",
|
||||
"expect": {
|
||||
"thinking_mode": "adaptive",
|
||||
"supported_efforts": ["low", "medium", "high", "xhigh", "max"],
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -75,7 +107,13 @@
|
||||
"raw_model_id": "claude-mythos-5",
|
||||
"expect": {
|
||||
"thinking_mode": "adaptive",
|
||||
"supported_efforts": ["low", "medium", "high", "xhigh", "max"],
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -86,7 +124,12 @@
|
||||
"raw_model_id": "claude-opus-4-6",
|
||||
"expect": {
|
||||
"thinking_mode": "adaptive",
|
||||
"supported_efforts": ["low", "medium", "high", "max"],
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -97,7 +140,12 @@
|
||||
"raw_model_id": "claude-sonnet-4-6",
|
||||
"expect": {
|
||||
"thinking_mode": "adaptive",
|
||||
"supported_efforts": ["low", "medium", "high", "max"],
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -108,7 +156,12 @@
|
||||
"raw_model_id": "claude-mythos-preview",
|
||||
"expect": {
|
||||
"thinking_mode": "adaptive",
|
||||
"supported_efforts": ["low", "medium", "high", "max"],
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -122,7 +175,13 @@
|
||||
"raw_model_id": "",
|
||||
"expect": {
|
||||
"thinking_mode": "adaptive",
|
||||
"supported_efforts": ["low", "medium", "high", "xhigh", "max"],
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -133,7 +192,13 @@
|
||||
"raw_model_id": "claude-ultra-9000",
|
||||
"expect": {
|
||||
"thinking_mode": "omit-fields",
|
||||
"supported_efforts": ["low", "medium", "high", "xhigh", "max"],
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -147,7 +212,9 @@
|
||||
"raw_model_id": "gpt-5-pro",
|
||||
"expect": {
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": ["high"],
|
||||
"supported_efforts": [
|
||||
"high"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -158,7 +225,14 @@
|
||||
"raw_model_id": "gpt-5.6",
|
||||
"expect": {
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": ["none", "low", "medium", "high", "xhigh", "max"],
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -169,7 +243,14 @@
|
||||
"raw_model_id": "gpt-5-6",
|
||||
"expect": {
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": ["none", "low", "medium", "high", "xhigh", "max"],
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -180,7 +261,13 @@
|
||||
"raw_model_id": "gpt-5.5",
|
||||
"expect": {
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": ["none", "low", "medium", "high", "xhigh"],
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -191,7 +278,13 @@
|
||||
"raw_model_id": "gpt-5.4",
|
||||
"expect": {
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": ["none", "low", "medium", "high", "xhigh"],
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -202,7 +295,12 @@
|
||||
"raw_model_id": "gpt-5.1",
|
||||
"expect": {
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": ["none", "low", "medium", "high"],
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"default_effort": "none",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
@@ -213,13 +311,18 @@
|
||||
"raw_model_id": "gpt-5",
|
||||
"expect": {
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": ["minimal", "low", "medium", "high"],
|
||||
"supported_efforts": [
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
},
|
||||
{
|
||||
"_group": "OpenAI adversarial — gpt5 boundary-aware matching (ported from config.rs tests)"
|
||||
"_group": "OpenAI adversarial \u2014 gpt5 boundary-aware matching (ported from config.rs tests)"
|
||||
},
|
||||
{
|
||||
"id": "openai-gpt5-1106-should-not-match-base",
|
||||
@@ -227,7 +330,12 @@
|
||||
"raw_model_id": "gpt-5-1106",
|
||||
"_note": "gpt-5-1106: '-1106' is a 4-digit date segment, NOT a short version (gpt5-base rejects only 1-3 digit suffixes). Must match base table [minimal,low,medium,high], NOT fall through to unknown.",
|
||||
"expect": {
|
||||
"supported_efforts": ["minimal", "low", "medium", "high"]
|
||||
"supported_efforts": [
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -236,7 +344,12 @@
|
||||
"raw_model_id": "gpt-5-4o",
|
||||
"_note": "gpt-5-4o: '4o' after '-' is NOT a short numeric suffix (it contains a letter). Must match gpt5-base. Crucially, must NOT match gpt-5.4 (the '4' is followed by 'o', not boundary char).",
|
||||
"expect": {
|
||||
"supported_efforts": ["minimal", "low", "medium", "high"]
|
||||
"supported_efforts": [
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -245,7 +358,9 @@
|
||||
"raw_model_id": "gpt-5-pro",
|
||||
"_note": "gpt-5-pro should hit gpt5-pro rule (priority 20), NOT gpt-5 base.",
|
||||
"expect": {
|
||||
"supported_efforts": ["high"],
|
||||
"supported_efforts": [
|
||||
"high"
|
||||
],
|
||||
"default_effort": "high"
|
||||
}
|
||||
},
|
||||
@@ -253,22 +368,34 @@
|
||||
"id": "openai-multi-digit-version-gpt5-10",
|
||||
"provider": "openai",
|
||||
"raw_model_id": "gpt-5-10",
|
||||
"_note": "gpt-5-10 — two-digit suffix prevents gpt5-base match. Falls through to unknown.",
|
||||
"_note": "gpt-5-10 \u2014 two-digit suffix prevents gpt5-base match. Falls through to unknown.",
|
||||
"expect": {
|
||||
"supported_efforts": ["none", "minimal", "low", "medium", "high", "xhigh"]
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "openai-gpt5-date-suffix",
|
||||
"provider": "openai",
|
||||
"raw_model_id": "gpt-5-20260101",
|
||||
"_note": "gpt-5-20260101 — long numeric suffix after base: '20260101' is 8 digits, beyond 1-3 digit reject, should hit gpt5-base.",
|
||||
"_note": "gpt-5-20260101 \u2014 long numeric suffix after base: '20260101' is 8 digits, beyond 1-3 digit reject, should hit gpt5-base.",
|
||||
"expect": {
|
||||
"supported_efforts": ["minimal", "low", "medium", "high"]
|
||||
"supported_efforts": [
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"_group": "DatabricksV2 — segment-based routing (ported from llm.rs tests)"
|
||||
"_group": "DatabricksV2 \u2014 segment-based routing (ported from llm.rs tests)"
|
||||
},
|
||||
{
|
||||
"id": "dbv2-gpt5-route-openai-responses",
|
||||
@@ -276,7 +403,13 @@
|
||||
"raw_model_id": "gpt-5.5",
|
||||
"expect": {
|
||||
"databricks_v2_wire_route": "openai-responses",
|
||||
"supported_efforts": ["none", "low", "medium", "high", "xhigh"]
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -286,14 +419,20 @@
|
||||
"expect": {
|
||||
"databricks_v2_wire_route": "anthropic-messages",
|
||||
"thinking_mode": "adaptive",
|
||||
"supported_efforts": ["low", "medium", "high", "xhigh", "max"]
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "dbv2-claude-prefix-stripped",
|
||||
"provider": "databricks_v2",
|
||||
"raw_model_id": "databricks-claude-opus-4-7",
|
||||
"_note": "databricks- prefix stripped → claude-opus-4-7 → Anthropic route",
|
||||
"_note": "databricks- prefix stripped \u2192 claude-opus-4-7 \u2192 Anthropic route",
|
||||
"expect": {
|
||||
"databricks_v2_wire_route": "anthropic-messages",
|
||||
"thinking_mode": "adaptive"
|
||||
@@ -303,29 +442,41 @@
|
||||
"id": "dbv2-goose-claude-prefix-stripped",
|
||||
"provider": "databricks_v2",
|
||||
"raw_model_id": "goose-claude-fable-5",
|
||||
"_note": "goose- prefix stripped → claude-fable-5 → Anthropic adaptive+xhigh",
|
||||
"_note": "goose- prefix stripped \u2192 claude-fable-5 \u2192 Anthropic adaptive+xhigh",
|
||||
"expect": {
|
||||
"databricks_v2_wire_route": "anthropic-messages",
|
||||
"thinking_mode": "adaptive",
|
||||
"supported_efforts": ["low", "medium", "high", "xhigh", "max"]
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "dbv2-team-prefix-stripped",
|
||||
"provider": "databricks_v2",
|
||||
"raw_model_id": "team-x-claude-opus-4-7",
|
||||
"_note": "team-x- prefix stripped → claude-opus-4-7 → Anthropic route",
|
||||
"_note": "team-x- prefix stripped \u2192 claude-opus-4-7 \u2192 Anthropic route",
|
||||
"expect": {
|
||||
"databricks_v2_wire_route": "anthropic-messages",
|
||||
"thinking_mode": "adaptive",
|
||||
"supported_efforts": ["low", "medium", "high", "xhigh", "max"]
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "dbv2-consolidated-llama-not-sol",
|
||||
"provider": "databricks_v2",
|
||||
"raw_model_id": "consolidated-llama",
|
||||
"_note": "segment test: 'sol' is a SUBSTRING of 'consolidated' — must NOT match DATABRICKS_V2_OPENAI_CODE_NAMES 'sol'. Falls through to mlflow-chat.",
|
||||
"_note": "segment test: 'sol' is a SUBSTRING of 'consolidated' \u2014 must NOT match DATABRICKS_V2_OPENAI_CODE_NAMES 'sol'. Falls through to mlflow-chat.",
|
||||
"expect": {
|
||||
"databricks_v2_wire_route": "mlflow-chat"
|
||||
}
|
||||
@@ -334,7 +485,7 @@
|
||||
"id": "dbv2-terraform-coder-not-terra",
|
||||
"provider": "databricks_v2",
|
||||
"raw_model_id": "terraform-coder",
|
||||
"_note": "segment test: 'terra' is a prefix of 'terraform' — must NOT match 'terra' code name. Falls through to mlflow-chat.",
|
||||
"_note": "segment test: 'terra' is a prefix of 'terraform' \u2014 must NOT match 'terra' code name. Falls through to mlflow-chat.",
|
||||
"expect": {
|
||||
"databricks_v2_wire_route": "mlflow-chat"
|
||||
}
|
||||
@@ -367,7 +518,7 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"_group": "P2-A resolver-contract vectors (plan v4 §Resolver contract)"
|
||||
"_group": "P2-A resolver-contract vectors (plan v4 \u00a7Resolver contract)"
|
||||
},
|
||||
{
|
||||
"id": "resolver-exact-raw-id-hit",
|
||||
@@ -375,16 +526,26 @@
|
||||
"raw_model_id": "databricks-gpt-5-4-mini",
|
||||
"_note": "Exact record exists. Must return exact Databricks override: low|medium|high (not family's none+xhigh).",
|
||||
"expect": {
|
||||
"supported_efforts": ["low", "medium", "high"]
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "resolver-prefixed-alias-misses-exact",
|
||||
"provider": "databricks_v2",
|
||||
"raw_model_id": "team-x-databricks-gpt-5-4-mini",
|
||||
"_note": "Prefixed alias of an exact ID. Raw exact lookup MUST miss (key is team-x-..., not databricks-...). Falls to family rules (gpt5-4 family → none+xhigh).",
|
||||
"_note": "Prefixed alias of an exact ID. Raw exact lookup MUST miss (key is team-x-..., not databricks-...). Falls to family rules (gpt5-4 family \u2192 none+xhigh).",
|
||||
"expect": {
|
||||
"supported_efforts": ["none", "low", "medium", "high", "xhigh"]
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -400,9 +561,14 @@
|
||||
"id": "resolver-exact-efforts-plus-family-route",
|
||||
"provider": "databricks_v2",
|
||||
"raw_model_id": "databricks-gpt-5-6-sol",
|
||||
"_note": "Exact record with efforts from models.dev (low|medium|high|max — provider-advertised, no none/xhigh). Route materialized from gpt5-6 family rule (openai-responses). Must return both, complete.",
|
||||
"_note": "Exact record with efforts from models.dev (low|medium|high|max \u2014 provider-advertised, no none/xhigh). Route materialized from gpt5-6 family rule (openai-responses). Must return both, complete.",
|
||||
"expect": {
|
||||
"supported_efforts": ["low", "medium", "high", "max"],
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"max"
|
||||
],
|
||||
"databricks_v2_wire_route": "openai-responses"
|
||||
}
|
||||
},
|
||||
@@ -410,9 +576,13 @@
|
||||
"id": "dbv2-gpt5-5-exact-override",
|
||||
"provider": "databricks_v2",
|
||||
"raw_model_id": "databricks-gpt-5-5",
|
||||
"_note": "Exact record adopts models.dev advertised set [low,medium,high]. Family rule (gpt5-5) has none+xhigh — provider-advertised wins per plan F1.",
|
||||
"_note": "Exact record adopts models.dev advertised set [low,medium,high]. Family rule (gpt5-5) has none+xhigh \u2014 provider-advertised wins per plan F1.",
|
||||
"expect": {
|
||||
"supported_efforts": ["low", "medium", "high"],
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"databricks_v2_wire_route": "openai-responses"
|
||||
}
|
||||
},
|
||||
@@ -426,7 +596,15 @@
|
||||
"_note": "DBv2 blank: route-unknown, all 7 efforts, default medium.",
|
||||
"expect": {
|
||||
"databricks_v2_wire_route": "route-unknown",
|
||||
"supported_efforts": ["none", "minimal", "low", "medium", "high", "xhigh", "max"],
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "medium"
|
||||
}
|
||||
},
|
||||
@@ -437,7 +615,14 @@
|
||||
"_note": "DBv2 concrete-unknown: mlflow-chat, all-except-max (6 efforts).",
|
||||
"expect": {
|
||||
"databricks_v2_wire_route": "mlflow-chat",
|
||||
"supported_efforts": ["none", "minimal", "low", "medium", "high", "xhigh"]
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -447,7 +632,14 @@
|
||||
"_note": "OpenAI blank: not-applicable route, all-except-max, medium default.",
|
||||
"expect": {
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"supported_efforts": ["none", "minimal", "low", "medium", "high", "xhigh"],
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
],
|
||||
"default_effort": "medium"
|
||||
}
|
||||
},
|
||||
@@ -458,7 +650,14 @@
|
||||
"_note": "OpenAI concrete unknown (unverified family): not-applicable route, all-except-max, medium default.",
|
||||
"expect": {
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"supported_efforts": ["none", "minimal", "low", "medium", "high", "xhigh"],
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
],
|
||||
"default_effort": "medium"
|
||||
}
|
||||
},
|
||||
@@ -469,7 +668,13 @@
|
||||
"_note": "Anthropic blank: assume adaptive with full support (incl. xhigh).",
|
||||
"expect": {
|
||||
"thinking_mode": "adaptive",
|
||||
"supported_efforts": ["low", "medium", "high", "xhigh", "max"],
|
||||
"supported_efforts": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "high"
|
||||
}
|
||||
},
|
||||
@@ -481,5 +686,110 @@
|
||||
"expect": {
|
||||
"thinking_mode": "omit-fields"
|
||||
}
|
||||
},
|
||||
{
|
||||
"_group": "Legacy Databricks provider (P3 effort rules)"
|
||||
},
|
||||
{
|
||||
"id": "databricks-gpt5-pro-effort",
|
||||
"provider": "databricks",
|
||||
"raw_model_id": "databricks-gpt-5-pro",
|
||||
"_note": "Legacy databricks GPT-5 Pro: same effort set as openai/gpt-5-pro \u2014 only [high], default high. Wire route not-applicable.",
|
||||
"expect": {
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"supported_efforts": [
|
||||
"high"
|
||||
],
|
||||
"default_effort": "high"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "databricks-gpt5-6-effort",
|
||||
"provider": "databricks",
|
||||
"raw_model_id": "databricks-gpt-5.6",
|
||||
"_note": "Legacy databricks GPT-5.6: same effort set as openai/gpt-5.6 \u2014 [none,low,medium,high,xhigh,max], default medium. Wire route not-applicable.",
|
||||
"expect": {
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"default_effort": "medium"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "databricks-gpt5-1-effort",
|
||||
"provider": "databricks",
|
||||
"raw_model_id": "databricks-gpt-5.1",
|
||||
"_note": "Legacy databricks GPT-5.1: same effort set as openai/gpt-5.1 \u2014 [none,low,medium,high], default none. Wire route not-applicable.",
|
||||
"expect": {
|
||||
"databricks_v2_wire_route": "not-applicable",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"default_effort": "none"
|
||||
}
|
||||
},
|
||||
{
|
||||
"_group": "openai-compat alias canonicalization (Thufir P3 corrective action 1)",
|
||||
"_note": "Rust normalizes openai-compat \u2192 Provider::OpenAi before reaching normalize_effort_for_provider. TS PROVIDER_ALIASES must match so the UI effort table equals the Rust request behavior. Interpreters must canonicalize openai-compat \u2192 openai before resolving; the expected values are identical to the corresponding openai vectors."
|
||||
},
|
||||
{
|
||||
"id": "openai-compat-gpt-5-pro",
|
||||
"provider": "openai-compat",
|
||||
"raw_model_id": "gpt-5-pro",
|
||||
"_note": "openai-compat/gpt-5-pro must resolve identically to openai/gpt-5-pro: [high] only, default high.",
|
||||
"expect": {
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"high"
|
||||
],
|
||||
"default_effort": "high",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "openai-compat-gpt-5-5",
|
||||
"provider": "openai-compat",
|
||||
"raw_model_id": "gpt-5.5",
|
||||
"_note": "openai-compat/gpt-5.5 must resolve identically to openai/gpt-5.5: [none,low,medium,high,xhigh], default medium.",
|
||||
"expect": {
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "openai-compat-empty-model",
|
||||
"provider": "openai-compat",
|
||||
"raw_model_id": "",
|
||||
"_note": "openai-compat with blank model: resolves identically to openai unknown \u2014 all-except-max, default medium.",
|
||||
"expect": {
|
||||
"thinking_mode": "none",
|
||||
"supported_efforts": [
|
||||
"none",
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
],
|
||||
"default_effort": "medium",
|
||||
"databricks_v2_wire_route": "not-applicable"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -30,6 +30,19 @@ const corpus = JSON.parse(
|
||||
readFileSync(join(repoRoot, "scripts", "normative-corpus.json"), "utf8"),
|
||||
);
|
||||
|
||||
// ----- Provider alias canonicalization -----
|
||||
// Mirrors production canonicalizeProvider() in desktop/src/features/agents/lib/formatAgentModelLabel.ts.
|
||||
// Applied before every generated lookup so alias vectors (e.g. "openai-compat") pass both interpreters.
|
||||
const PROVIDER_ALIASES = {
|
||||
"databricks-v2": "databricks_v2",
|
||||
"openai-compat": "openai",
|
||||
};
|
||||
|
||||
function canonicalizeProvider(provider) {
|
||||
const normalized = (provider ?? "").trim().toLowerCase();
|
||||
return PROVIDER_ALIASES[normalized] ?? normalized;
|
||||
}
|
||||
|
||||
// ----- Run corpus -----
|
||||
|
||||
let passed = 0;
|
||||
@@ -41,7 +54,7 @@ for (const entry of corpus) {
|
||||
if (!entry.expect) continue;
|
||||
|
||||
// resolveModelCapabilities returns camelCase keys (registryLabel, thinkingMode, etc.)
|
||||
const result = resolveModelCapabilities(entry.provider, entry.raw_model_id);
|
||||
const result = resolveModelCapabilities(canonicalizeProvider(entry.provider), entry.raw_model_id);
|
||||
const expect = entry.expect;
|
||||
|
||||
const failures = [];
|
||||
|
||||
239
scripts/run-differential.mjs
Executable file
239
scripts/run-differential.mjs
Executable file
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Phase-2 differential harness — compare old buzzAgentConfig.ts effort logic with
|
||||
* the new generated modelCapabilities.ts interpreter over:
|
||||
* 1. The 36-entry effortTable.fixture.json (cross-boundary Rust/TS fixture)
|
||||
* 2. The 45-vector normative corpus (scripts/normative-corpus.json)
|
||||
* 3. The catalog-sample fixture (scripts/catalog-sample-fixture.json)
|
||||
*
|
||||
* Equality is required except for entries in the committed allowlist of intentional
|
||||
* F1 corrections (models.dev provider-capability reconciliations).
|
||||
*
|
||||
* Usage: node --experimental-strip-types scripts/run-differential.mjs [--verbose]
|
||||
* Exits 0 on all-pass (modulo allowlist), 1 on unexpected divergence or unexercised allowlist entry.
|
||||
*/
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const repoRoot = join(__dirname, "..");
|
||||
const VERBOSE = process.argv.includes("--verbose");
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Import both interpreters
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// NEW: generated capability module
|
||||
const { resolveModelCapabilities: resolveNew } = await import(
|
||||
join(repoRoot, "desktop", "src", "features", "agents", "ui", "modelCapabilities.ts")
|
||||
);
|
||||
|
||||
// OLD: buzzAgentConfig.ts effort config
|
||||
const { getProviderEffortConfig_oldHandTable: getOldEffortConfig } = await import(
|
||||
join(repoRoot, "desktop", "src", "features", "agents", "ui", "buzzAgentConfig.ts")
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Intentional corrections allowlist (Phase 1 F1 reconciliations)
|
||||
// Each entry: { provider, raw_model_id, reason }
|
||||
// ---------------------------------------------------------------------------
|
||||
const ALLOWLIST = [
|
||||
{
|
||||
provider: "databricks_v2",
|
||||
raw_model_id: "databricks-gpt-5-5",
|
||||
axes: ["supported_efforts"],
|
||||
reason: "Phase 1 ADOPT: models.dev d5a4974c advertises [low,medium,high]; old returns [none,low,medium,high,xhigh]",
|
||||
},
|
||||
{
|
||||
provider: "databricks_v2",
|
||||
raw_model_id: "databricks-gpt-5-4-mini",
|
||||
axes: ["supported_efforts"],
|
||||
reason: "Phase 1 ADOPT: models.dev advertises [low,medium,high]; old returns [none,low,medium,high,xhigh]",
|
||||
},
|
||||
{
|
||||
provider: "databricks_v2",
|
||||
raw_model_id: "databricks-gpt-5-4-nano",
|
||||
axes: ["supported_efforts"],
|
||||
reason: "Phase 1 ADOPT: models.dev advertises [low,medium,high]; old returns [none,low,medium,high,xhigh]",
|
||||
},
|
||||
{
|
||||
provider: "databricks_v2",
|
||||
raw_model_id: "databricks-gpt-5-6-sol",
|
||||
axes: ["supported_efforts"],
|
||||
reason: "Phase 1 ADOPT: models.dev advertises [low,medium,high,max]; old returns [none,low,medium,high,xhigh,max]",
|
||||
},
|
||||
{
|
||||
provider: "databricks_v2",
|
||||
raw_model_id: "goose-opus-5",
|
||||
axes: ["supported_efforts", "default_effort"],
|
||||
reason: "Phase 1 correction: 'opus' is a named DBv2 segment → anthropic-messages route; old config.rs disagreed with llm.rs (corpus note dbv2-goose-opus-5-is-anthropic). Generated adopts anthropic adaptive-xhigh capabilities consistent with the wire route.",
|
||||
},
|
||||
];
|
||||
|
||||
// Track which allowlist entries are actually exercised (suppressed a divergence).
|
||||
// Keyed as "provider:raw_model_id:axis".
|
||||
const allowlistHits = new Set();
|
||||
|
||||
function isAllowlisted(provider, rawModelId, axis) {
|
||||
const entry = ALLOWLIST.find(
|
||||
(e) =>
|
||||
e.provider === provider &&
|
||||
e.raw_model_id === rawModelId &&
|
||||
e.axes.includes(axis),
|
||||
);
|
||||
if (entry) {
|
||||
allowlistHits.add(`${provider}:${rawModelId}:${axis}`);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Comparison helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Compare effort axes from both interpreters for one (provider, model) pair.
|
||||
* Returns array of divergence objects.
|
||||
*/
|
||||
function compareEffortAxes(provider, model) {
|
||||
const newResult = resolveNew(provider, model);
|
||||
const oldResult = getOldEffortConfig(provider, model);
|
||||
|
||||
const divergences = [];
|
||||
|
||||
// supported_efforts
|
||||
const newEfforts = newResult.supportedEfforts ?? [];
|
||||
const oldEfforts = oldResult?.validValues ?? [];
|
||||
if (JSON.stringify(newEfforts) !== JSON.stringify(oldEfforts)) {
|
||||
if (!isAllowlisted(provider, model, "supported_efforts")) {
|
||||
divergences.push({
|
||||
axis: "supported_efforts",
|
||||
old: oldEfforts,
|
||||
new: newEfforts,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// default_effort
|
||||
const newDefault = newResult.defaultEffort ?? null;
|
||||
const oldDefault = oldResult?.defaultValue ?? null;
|
||||
if (newDefault !== oldDefault) {
|
||||
if (!isAllowlisted(provider, model, "default_effort")) {
|
||||
divergences.push({
|
||||
axis: "default_effort",
|
||||
old: oldDefault,
|
||||
new: newDefault,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return divergences;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test suites
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let totalChecks = 0;
|
||||
let totalDivergences = 0;
|
||||
|
||||
function runCheck(label, provider, model) {
|
||||
totalChecks++;
|
||||
const divs = compareEffortAxes(provider, model);
|
||||
if (divs.length > 0) {
|
||||
totalDivergences += divs.length;
|
||||
for (const d of divs) {
|
||||
console.error(
|
||||
`DIVERGE [${label}] provider=${provider} model=${model} axis=${d.axis}\n` +
|
||||
` old: ${JSON.stringify(d.old)}\n` +
|
||||
` new: ${JSON.stringify(d.new)}`,
|
||||
);
|
||||
}
|
||||
} else if (VERBOSE) {
|
||||
console.log(`OK [${label}] provider=${provider} model=${model}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 1. effortTable.fixture.json
|
||||
console.log("--- effortTable.fixture.json ---");
|
||||
const fixture = JSON.parse(
|
||||
readFileSync(
|
||||
join(repoRoot, "desktop", "src", "features", "agents", "ui", "effortTable.fixture.json"),
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
for (const entry of fixture) {
|
||||
if (!entry.provider) continue;
|
||||
runCheck("fixture", entry.provider, entry.model ?? "");
|
||||
}
|
||||
|
||||
// 2. normative-corpus.json (effort axes only)
|
||||
console.log("--- normative-corpus.json ---");
|
||||
const corpus = JSON.parse(
|
||||
readFileSync(join(repoRoot, "scripts", "normative-corpus.json"), "utf8"),
|
||||
);
|
||||
for (const entry of corpus) {
|
||||
if (entry._group) continue;
|
||||
if (!entry.provider || !entry.expect) continue;
|
||||
if (!entry.expect.supported_efforts && !entry.expect.default_effort) continue;
|
||||
runCheck("corpus", entry.provider, entry.raw_model_id ?? "");
|
||||
}
|
||||
|
||||
// 3. catalog-sample-fixture.json (exact records from pinned models.dev payload)
|
||||
console.log("--- catalog-sample-fixture.json ---");
|
||||
const catalogFixture = JSON.parse(
|
||||
readFileSync(join(repoRoot, "scripts", "catalog-sample-fixture.json"), "utf8"),
|
||||
);
|
||||
for (const ep of catalogFixture.endpoints ?? []) {
|
||||
if (!ep.name) continue;
|
||||
// All catalog endpoints are databricks_v2 provider
|
||||
runCheck("catalog-sample", "databricks_v2", ep.name);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Summary
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Count total allowlist axis slots expected to be hit
|
||||
const totalAllowlistSlots = ALLOWLIST.reduce((n, e) => n + e.axes.length, 0);
|
||||
const allowlistHitCount = allowlistHits.size;
|
||||
|
||||
// Detect stale allowlist entries (declared but never actually suppressed a divergence)
|
||||
const staleEntries = [];
|
||||
for (const entry of ALLOWLIST) {
|
||||
for (const axis of entry.axes) {
|
||||
const key = `${entry.provider}:${entry.raw_model_id}:${axis}`;
|
||||
if (!allowlistHits.has(key)) {
|
||||
staleEntries.push({ ...entry, axis });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`\nDifferential: ${totalChecks} checks, ${totalDivergences} unexpected divergences, ${allowlistHitCount}/${totalAllowlistSlots} allowlist slots exercised`,
|
||||
);
|
||||
|
||||
if (staleEntries.length > 0) {
|
||||
for (const e of staleEntries) {
|
||||
console.error(
|
||||
`STALE_ALLOWLIST provider=${e.provider} model=${e.raw_model_id} axis=${e.axis} — entry never fired; remove or update it`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (totalDivergences > 0) {
|
||||
console.error(
|
||||
`FAIL: ${totalDivergences} unexpected divergence(s) — see output above`,
|
||||
);
|
||||
process.exit(1);
|
||||
} else if (staleEntries.length > 0) {
|
||||
console.error(
|
||||
`FAIL: ${staleEntries.length} stale allowlist entry(ies) — entries that never suppress a divergence mask future regressions`,
|
||||
);
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.log("PASS: old and new effort logic agree on all non-allowlisted entries");
|
||||
}
|
||||
Reference in New Issue
Block a user