[ { "_group": "Anthropic exact family rules", "_note": "All require thinking_mode=manual-budget or adaptive, correct supported_efforts, databricks_v2_wire_route=not-applicable" }, { "id": "anthropic-claude-3-family", "provider": "anthropic", "raw_model_id": "claude-3-7-sonnet-20250219", "expect": { "thinking_mode": "manual-budget", "supported_efforts": [ "low", "medium", "high" ], "default_effort": null, "databricks_v2_wire_route": "not-applicable" } }, { "id": "anthropic-claude-opus-4-5", "provider": "anthropic", "raw_model_id": "claude-opus-4-5", "expect": { "thinking_mode": "manual-budget", "supported_efforts": [ "low", "medium", "high" ], "default_effort": null, "databricks_v2_wire_route": "not-applicable" } }, { "id": "anthropic-claude-opus-4-7", "provider": "anthropic", "raw_model_id": "claude-opus-4-7", "expect": { "thinking_mode": "adaptive", "supported_efforts": [ "low", "medium", "high", "xhigh", "max" ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } }, { "id": "anthropic-claude-opus-4-8", "provider": "anthropic", "raw_model_id": "claude-opus-4-8", "expect": { "thinking_mode": "adaptive", "supported_efforts": [ "low", "medium", "high", "xhigh", "max" ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } }, { "id": "anthropic-claude-sonnet-5", "provider": "anthropic", "raw_model_id": "claude-sonnet-5-20260101", "expect": { "thinking_mode": "adaptive", "supported_efforts": [ "low", "medium", "high", "xhigh", "max" ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } }, { "id": "anthropic-claude-fable-5", "provider": "anthropic", "raw_model_id": "claude-fable-5", "expect": { "thinking_mode": "adaptive", "supported_efforts": [ "low", "medium", "high", "xhigh", "max" ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } }, { "id": "anthropic-claude-mythos-5", "provider": "anthropic", "raw_model_id": "claude-mythos-5", "expect": { "thinking_mode": "adaptive", "supported_efforts": [ "low", "medium", "high", "xhigh", "max" ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } }, { "id": "anthropic-claude-opus-4-6", "provider": "anthropic", "raw_model_id": "claude-opus-4-6", "expect": { "thinking_mode": "adaptive", "supported_efforts": [ "low", "medium", "high", "max" ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } }, { "id": "anthropic-claude-sonnet-4-6", "provider": "anthropic", "raw_model_id": "claude-sonnet-4-6", "expect": { "thinking_mode": "adaptive", "supported_efforts": [ "low", "medium", "high", "max" ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } }, { "id": "anthropic-claude-mythos-preview", "provider": "anthropic", "raw_model_id": "claude-mythos-preview", "expect": { "thinking_mode": "adaptive", "supported_efforts": [ "low", "medium", "high", "max" ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } }, { "_group": "Anthropic unknowns and fallbacks" }, { "id": "anthropic-unknown-blank", "provider": "anthropic", "raw_model_id": "", "expect": { "thinking_mode": "adaptive", "supported_efforts": [ "low", "medium", "high", "xhigh", "max" ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } }, { "id": "anthropic-unknown-concrete", "provider": "anthropic", "raw_model_id": "claude-ultra-9000", "expect": { "thinking_mode": "omit-fields", "supported_efforts": [ "low", "medium", "high", "xhigh", "max" ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } }, { "_group": "OpenAI exact family rules" }, { "id": "openai-gpt5-pro", "provider": "openai", "raw_model_id": "gpt-5-pro", "expect": { "thinking_mode": "none", "supported_efforts": [ "high" ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } }, { "id": "openai-gpt5.6", "provider": "openai", "raw_model_id": "gpt-5.6", "expect": { "thinking_mode": "none", "supported_efforts": [ "none", "low", "medium", "high", "xhigh", "max" ], "default_effort": "medium", "databricks_v2_wire_route": "not-applicable" } }, { "id": "openai-gpt5-6-dashed", "provider": "openai", "raw_model_id": "gpt-5-6", "expect": { "thinking_mode": "none", "supported_efforts": [ "none", "low", "medium", "high", "xhigh", "max" ], "default_effort": "medium", "databricks_v2_wire_route": "not-applicable" } }, { "id": "openai-gpt5.5", "provider": "openai", "raw_model_id": "gpt-5.5", "expect": { "thinking_mode": "none", "supported_efforts": [ "none", "low", "medium", "high", "xhigh" ], "default_effort": "medium", "databricks_v2_wire_route": "not-applicable" } }, { "id": "openai-gpt5.4", "provider": "openai", "raw_model_id": "gpt-5.4", "expect": { "thinking_mode": "none", "supported_efforts": [ "none", "low", "medium", "high", "xhigh" ], "default_effort": "medium", "databricks_v2_wire_route": "not-applicable" } }, { "id": "openai-gpt5.1", "provider": "openai", "raw_model_id": "gpt-5.1", "expect": { "thinking_mode": "none", "supported_efforts": [ "none", "low", "medium", "high" ], "default_effort": "none", "databricks_v2_wire_route": "not-applicable" } }, { "id": "openai-gpt5-base", "provider": "openai", "raw_model_id": "gpt-5", "expect": { "thinking_mode": "none", "supported_efforts": [ "minimal", "low", "medium", "high" ], "default_effort": "medium", "databricks_v2_wire_route": "not-applicable" } }, { "_group": "OpenAI adversarial \u2014 gpt5 boundary-aware matching (ported from config.rs tests)" }, { "id": "openai-gpt5-1106-should-not-match-base", "provider": "openai", "raw_model_id": "gpt-5-1106", "_note": "gpt-5-1106: '-1106' is a 4-digit date segment, NOT a short version (gpt5-base rejects only 1-3 digit suffixes). Must match base table [minimal,low,medium,high], NOT fall through to unknown.", "expect": { "supported_efforts": [ "minimal", "low", "medium", "high" ] } }, { "id": "openai-gpt5-4o-matches-base", "provider": "openai", "raw_model_id": "gpt-5-4o", "_note": "gpt-5-4o: '4o' after '-' is NOT a short numeric suffix (it contains a letter). Must match gpt5-base. Crucially, must NOT match gpt-5.4 (the '4' is followed by 'o', not boundary char).", "expect": { "supported_efforts": [ "minimal", "low", "medium", "high" ] } }, { "id": "openai-gpt5-pro-not-matching-gpt5-base", "provider": "openai", "raw_model_id": "gpt-5-pro", "_note": "gpt-5-pro should hit gpt5-pro rule (priority 20), NOT gpt-5 base.", "expect": { "supported_efforts": [ "high" ], "default_effort": "high" } }, { "id": "openai-multi-digit-version-gpt5-10", "provider": "openai", "raw_model_id": "gpt-5-10", "_note": "gpt-5-10 \u2014 two-digit suffix prevents gpt5-base match. Falls through to unknown.", "expect": { "supported_efforts": [ "none", "minimal", "low", "medium", "high", "xhigh" ] } }, { "id": "openai-gpt5-date-suffix", "provider": "openai", "raw_model_id": "gpt-5-20260101", "_note": "gpt-5-20260101 \u2014 long numeric suffix after base: '20260101' is 8 digits, beyond 1-3 digit reject, should hit gpt5-base.", "expect": { "supported_efforts": [ "minimal", "low", "medium", "high" ] } }, { "_group": "DatabricksV2 \u2014 segment-based routing (ported from llm.rs tests)" }, { "id": "dbv2-gpt5-route-openai-responses", "provider": "databricks_v2", "raw_model_id": "gpt-5.5", "expect": { "databricks_v2_wire_route": "openai-responses", "supported_efforts": [ "none", "low", "medium", "high", "xhigh" ] } }, { "id": "dbv2-claude-route-anthropic-messages", "provider": "databricks_v2", "raw_model_id": "claude-opus-4-7", "expect": { "databricks_v2_wire_route": "anthropic-messages", "thinking_mode": "adaptive", "supported_efforts": [ "low", "medium", "high", "xhigh", "max" ] } }, { "id": "dbv2-claude-prefix-stripped", "provider": "databricks_v2", "raw_model_id": "databricks-claude-opus-4-7", "_note": "databricks- prefix stripped \u2192 claude-opus-4-7 \u2192 Anthropic route", "expect": { "databricks_v2_wire_route": "anthropic-messages", "thinking_mode": "adaptive" } }, { "id": "dbv2-goose-claude-prefix-stripped", "provider": "databricks_v2", "raw_model_id": "goose-claude-fable-5", "_note": "goose- prefix stripped \u2192 claude-fable-5 \u2192 Anthropic adaptive+xhigh", "expect": { "databricks_v2_wire_route": "anthropic-messages", "thinking_mode": "adaptive", "supported_efforts": [ "low", "medium", "high", "xhigh", "max" ] } }, { "id": "dbv2-team-prefix-stripped", "provider": "databricks_v2", "raw_model_id": "team-x-claude-opus-4-7", "_note": "team-x- prefix stripped \u2192 claude-opus-4-7 \u2192 Anthropic route", "expect": { "databricks_v2_wire_route": "anthropic-messages", "thinking_mode": "adaptive", "supported_efforts": [ "low", "medium", "high", "xhigh", "max" ] } }, { "id": "dbv2-consolidated-llama-not-sol", "provider": "databricks_v2", "raw_model_id": "consolidated-llama", "_note": "segment test: 'sol' is a SUBSTRING of 'consolidated' \u2014 must NOT match DATABRICKS_V2_OPENAI_CODE_NAMES 'sol'. Falls through to mlflow-chat.", "expect": { "databricks_v2_wire_route": "mlflow-chat" } }, { "id": "dbv2-terraform-coder-not-terra", "provider": "databricks_v2", "raw_model_id": "terraform-coder", "_note": "segment test: 'terra' is a prefix of 'terraform' \u2014 must NOT match 'terra' code name. Falls through to mlflow-chat.", "expect": { "databricks_v2_wire_route": "mlflow-chat" } }, { "id": "dbv2-corpus-reranker-not-opus", "provider": "databricks_v2", "raw_model_id": "corpus-reranker", "_note": "segment test: 'opus' is NOT a segment of corpus-reranker (segments: corpus, reranker). mlflow-chat.", "expect": { "databricks_v2_wire_route": "mlflow-chat" } }, { "id": "dbv2-octopus-model-not-opus", "provider": "databricks_v2", "raw_model_id": "octopus-model", "_note": "segment test: 'opus' is not a segment of octopus-model (segments: octopus, model). mlflow-chat.", "expect": { "databricks_v2_wire_route": "mlflow-chat" } }, { "id": "dbv2-goose-opus-5-is-anthropic", "provider": "databricks_v2", "raw_model_id": "goose-opus-5", "_note": "'opus' IS a named segment of goose-opus-5 (segments: goose, opus, 5). Routes Anthropic. Key test: agrees with llm.rs but disagreed with old config.rs.", "expect": { "databricks_v2_wire_route": "anthropic-messages" } }, { "_group": "P2-A resolver-contract vectors (plan v4 \u00a7Resolver contract)" }, { "id": "resolver-exact-raw-id-hit", "provider": "databricks_v2", "raw_model_id": "databricks-gpt-5-4-mini", "_note": "Exact record exists. Must return exact Databricks override: low|medium|high (not family's none+xhigh).", "expect": { "supported_efforts": [ "low", "medium", "high" ] } }, { "id": "resolver-prefixed-alias-misses-exact", "provider": "databricks_v2", "raw_model_id": "team-x-databricks-gpt-5-4-mini", "_note": "Prefixed alias of an exact ID. Raw exact lookup MUST miss (key is team-x-..., not databricks-...). Falls to family rules (gpt5-4 family \u2192 none+xhigh).", "expect": { "supported_efforts": [ "none", "low", "medium", "high", "xhigh" ] } }, { "id": "resolver-cross-provider-misses-exact", "provider": "openai", "raw_model_id": "databricks-gpt-5-4-mini", "_note": "Same raw ID but different provider. Exact record is databricks_v2-scoped; must miss. Falls to openai family rules.", "expect": { "databricks_v2_wire_route": "not-applicable" } }, { "id": "resolver-exact-efforts-plus-family-route", "provider": "databricks_v2", "raw_model_id": "databricks-gpt-5-6-sol", "_note": "Exact record with efforts from models.dev (low|medium|high|max \u2014 provider-advertised, no none/xhigh). Route materialized from gpt5-6 family rule (openai-responses). Must return both, complete.", "expect": { "supported_efforts": [ "low", "medium", "high", "max" ], "databricks_v2_wire_route": "openai-responses" } }, { "id": "dbv2-gpt5-5-exact-override", "provider": "databricks_v2", "raw_model_id": "databricks-gpt-5-5", "_note": "Exact record adopts models.dev advertised set [low,medium,high]. Family rule (gpt5-5) has none+xhigh \u2014 provider-advertised wins per plan F1.", "expect": { "supported_efforts": [ "low", "medium", "high" ], "databricks_v2_wire_route": "openai-responses" } }, { "_group": "Blank vs concrete-unknown per provider (P2-B fallback vectors)" }, { "id": "dbv2-blank-all7-route-unknown", "provider": "databricks_v2", "raw_model_id": "", "_note": "DBv2 blank: route-unknown, all 7 efforts, default medium.", "expect": { "databricks_v2_wire_route": "route-unknown", "supported_efforts": [ "none", "minimal", "low", "medium", "high", "xhigh", "max" ], "default_effort": "medium" } }, { "id": "dbv2-concrete-unknown-mlflow-no-max", "provider": "databricks_v2", "raw_model_id": "some-unknown-model-xyz", "_note": "DBv2 concrete-unknown: mlflow-chat, all-except-max (6 efforts).", "expect": { "databricks_v2_wire_route": "mlflow-chat", "supported_efforts": [ "none", "minimal", "low", "medium", "high", "xhigh" ] } }, { "id": "openai-blank-all-except-max", "provider": "openai", "raw_model_id": "", "_note": "OpenAI blank: not-applicable route, all-except-max, medium default.", "expect": { "databricks_v2_wire_route": "not-applicable", "supported_efforts": [ "none", "minimal", "low", "medium", "high", "xhigh" ], "default_effort": "medium" } }, { "id": "openai-concrete-unknown-all-except-max", "provider": "openai", "raw_model_id": "gpt-4o", "_note": "OpenAI concrete unknown (unverified family): not-applicable route, all-except-max, medium default.", "expect": { "databricks_v2_wire_route": "not-applicable", "supported_efforts": [ "none", "minimal", "low", "medium", "high", "xhigh" ], "default_effort": "medium" } }, { "id": "anthropic-blank-adaptive-full", "provider": "anthropic", "raw_model_id": "", "_note": "Anthropic blank: assume adaptive with full support (incl. xhigh).", "expect": { "thinking_mode": "adaptive", "supported_efforts": [ "low", "medium", "high", "xhigh", "max" ], "default_effort": "high" } }, { "id": "anthropic-concrete-unknown-omit-fields", "provider": "anthropic", "raw_model_id": "claude-ultra-9000", "_note": "Anthropic concrete-unknown: omit-fields (never guess request shape).", "expect": { "thinking_mode": "omit-fields" } }, { "_group": "Legacy Databricks provider (P3 effort rules)" }, { "id": "databricks-gpt5-pro-effort", "provider": "databricks", "raw_model_id": "databricks-gpt-5-pro", "_note": "Legacy databricks GPT-5 Pro: same effort set as openai/gpt-5-pro \u2014 only [high], default high. Wire route not-applicable.", "expect": { "databricks_v2_wire_route": "not-applicable", "supported_efforts": [ "high" ], "default_effort": "high" } }, { "id": "databricks-gpt5-6-effort", "provider": "databricks", "raw_model_id": "databricks-gpt-5.6", "_note": "Legacy databricks GPT-5.6: same effort set as openai/gpt-5.6 \u2014 [none,low,medium,high,xhigh,max], default medium. Wire route not-applicable.", "expect": { "databricks_v2_wire_route": "not-applicable", "supported_efforts": [ "none", "low", "medium", "high", "xhigh", "max" ], "default_effort": "medium" } }, { "id": "databricks-gpt5-1-effort", "provider": "databricks", "raw_model_id": "databricks-gpt-5.1", "_note": "Legacy databricks GPT-5.1: same effort set as openai/gpt-5.1 \u2014 [none,low,medium,high], default none. Wire route not-applicable.", "expect": { "databricks_v2_wire_route": "not-applicable", "supported_efforts": [ "none", "low", "medium", "high" ], "default_effort": "none" } }, { "_group": "openai-compat alias canonicalization (Thufir P3 corrective action 1)", "_note": "Rust normalizes openai-compat \u2192 Provider::OpenAi before reaching normalize_effort_for_provider. TS PROVIDER_ALIASES must match so the UI effort table equals the Rust request behavior. Interpreters must canonicalize openai-compat \u2192 openai before resolving; the expected values are identical to the corresponding openai vectors." }, { "id": "openai-compat-gpt-5-pro", "provider": "openai-compat", "raw_model_id": "gpt-5-pro", "_note": "openai-compat/gpt-5-pro must resolve identically to openai/gpt-5-pro: [high] only, default high.", "expect": { "thinking_mode": "none", "supported_efforts": [ "high" ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } }, { "id": "openai-compat-gpt-5-5", "provider": "openai-compat", "raw_model_id": "gpt-5.5", "_note": "openai-compat/gpt-5.5 must resolve identically to openai/gpt-5.5: [none,low,medium,high,xhigh], default medium.", "expect": { "thinking_mode": "none", "supported_efforts": [ "none", "low", "medium", "high", "xhigh" ], "default_effort": "medium", "databricks_v2_wire_route": "not-applicable" } }, { "id": "openai-compat-empty-model", "provider": "openai-compat", "raw_model_id": "", "_note": "openai-compat with blank model: resolves identically to openai unknown \u2014 all-except-max, default medium.", "expect": { "thinking_mode": "none", "supported_efforts": [ "none", "minimal", "low", "medium", "high", "xhigh" ], "default_effort": "medium", "databricks_v2_wire_route": "not-applicable" } } ]