fix(desktop): preserve model authentication error details

Signed-off-by: Brad Seiler <seiler@squareup.com>
This commit is contained in:
Brad Seiler
2026-10-09 15:57:18 -04:00
parent 394d4a434b
commit 7b649cf407
5 changed files with 131 additions and 21 deletions

View File

@@ -6,7 +6,7 @@ import {
friendlyTurnErrorCopy,
CLI_ACP_INTERNAL_ERROR_COPY,
MODEL_NOT_FOUND_COPY,
RELAY_MESH_DENIED_COPY,
MODEL_AUTH_DENIED_COPY,
} from "./friendlyAgentLastError.ts";
test("null lastError → null", () => {
@@ -24,7 +24,7 @@ test("buzz-acp wrapped auth failure → denied copy", () => {
);
assert.deepEqual(result, {
severity: "denied",
copy: RELAY_MESH_DENIED_COPY,
copy: `${MODEL_AUTH_DENIED_COPY} 401 unauthorized: ...`,
});
});
@@ -35,7 +35,7 @@ test("unwrapped buzz-agent prefix → denied copy", () => {
const result = friendlyAgentLastError("llm auth: 403 forbidden");
assert.deepEqual(result, {
severity: "denied",
copy: RELAY_MESH_DENIED_COPY,
copy: `${MODEL_AUTH_DENIED_COPY} 403 forbidden`,
});
});
@@ -52,7 +52,7 @@ test("trims whitespace before matching", () => {
" Agent reported error: llm auth: nope\n",
);
assert.equal(result?.severity, "denied");
assert.equal(result?.copy, RELAY_MESH_DENIED_COPY);
assert.equal(result?.copy, `${MODEL_AUTH_DENIED_COPY} nope`);
});
test("substring 'llm auth:' that isn't at start is NOT treated as denial", () => {
@@ -89,11 +89,11 @@ test("code -32002 → model-not-found copy (severity: denied)", () => {
});
});
test("code -32001 → Buzz shared compute denied copy (structured path)", () => {
test("code -32001 → model-provider authentication copy (structured path)", () => {
const result = friendlyAgentLastError("any error text", -32001);
assert.deepEqual(result, {
severity: "denied",
copy: RELAY_MESH_DENIED_COPY,
copy: `${MODEL_AUTH_DENIED_COPY} any error text`,
});
});
@@ -104,7 +104,7 @@ test("code null falls through to legacy string matching", () => {
);
assert.deepEqual(result, {
severity: "denied",
copy: RELAY_MESH_DENIED_COPY,
copy: `${MODEL_AUTH_DENIED_COPY} 401 unauthorized`,
});
});
@@ -115,7 +115,7 @@ test("code undefined falls through to legacy string matching", () => {
);
assert.deepEqual(result, {
severity: "denied",
copy: RELAY_MESH_DENIED_COPY,
copy: `${MODEL_AUTH_DENIED_COPY} 403 forbidden`,
});
});
@@ -137,7 +137,7 @@ test("friendlyTurnErrorCopy: numeric code -32002 → model-not-found copy", () =
test("friendlyTurnErrorCopy: string-encoded code coerces to number", () => {
assert.equal(
friendlyTurnErrorCopy("raw error", "-32001"),
RELAY_MESH_DENIED_COPY,
`${MODEL_AUTH_DENIED_COPY} raw error`,
);
});
@@ -170,7 +170,7 @@ test("NaN code param treated as absent — string path applies", () => {
const result = friendlyAgentLastError("llm auth: denied", NaN);
assert.deepEqual(result, {
severity: "denied",
copy: RELAY_MESH_DENIED_COPY,
copy: `${MODEL_AUTH_DENIED_COPY} denied`,
});
});
@@ -181,7 +181,7 @@ test("embedded code -32001 recovered from message when code param is null", () =
);
assert.deepEqual(result, {
severity: "denied",
copy: RELAY_MESH_DENIED_COPY,
copy: `${MODEL_AUTH_DENIED_COPY} 401`,
});
});
@@ -211,7 +211,7 @@ test("friendlyTurnErrorCopy: garbage string code coerces to NaN → string path"
// "garbage" → NaN → not finite → null → string prefix matches "llm auth:".
assert.equal(
friendlyTurnErrorCopy("llm auth: denied", "garbage"),
RELAY_MESH_DENIED_COPY,
`${MODEL_AUTH_DENIED_COPY} denied`,
);
});
@@ -309,10 +309,32 @@ test("friendlyTurnErrorCopy: code -32603 bare Internal error → cli-acp interna
test("-32603 does not affect -32001/-32002 classification (regression)", () => {
assert.deepEqual(friendlyAgentLastError("any", -32001), {
severity: "denied",
copy: RELAY_MESH_DENIED_COPY,
copy: `${MODEL_AUTH_DENIED_COPY} any`,
});
assert.deepEqual(friendlyAgentLastError("any", -32002), {
severity: "denied",
copy: MODEL_NOT_FOUND_COPY,
});
});
const databricksError = "Databricks rejected the refresh token; sign in again";
for (const [raw, code] of [
[`Agent reported error (code -32001): llm auth: ${databricksError}`, -32001],
[`Agent reported error (code -32001): llm auth: ${databricksError}`, null],
[`Agent reported error: llm auth: ${databricksError}`, undefined],
[`llm auth: ${databricksError}`, undefined],
]) {
test(`Databricks auth failure preserves its cause and recovery: ${raw} (code ${code})`, () => {
assert.deepEqual(friendlyAgentLastError(raw, code), {
severity: "denied",
copy: `${MODEL_AUTH_DENIED_COPY} ${databricksError}`,
});
});
}
test("auth failure without provider detail gets generic sign-in guidance", () => {
assert.deepEqual(friendlyAgentLastError("llm auth:", -32001), {
severity: "denied",
copy: MODEL_AUTH_DENIED_COPY,
});
});

View File

@@ -33,11 +33,11 @@ export type FriendlyAgentLastError =
| { severity: "generic"; copy: string };
/**
* The exact copy for the relay-mesh denial. Centralized as a constant so the
* The fallback copy for a model-provider authentication failure. Centralized so the
* test asserts the user-facing string verbatim rather than a fuzzy pattern.
*/
export const RELAY_MESH_DENIED_COPY =
"Community access denied this agent — check its community membership.";
export const MODEL_AUTH_DENIED_COPY =
"The model provider rejected authentication — check its credentials or sign in again.";
export const MODEL_NOT_FOUND_COPY =
"The configured model is not available — open agent settings and select a different one from the dropdown.";
@@ -61,6 +61,16 @@ function recoverEmbeddedCode(trimmed: string): {
};
}
function modelAuthFailureCopy(message: string): string {
const detail = message
.replace(/^Agent reported error: /, "")
.replace(/^llm auth:\s*/, "")
.trim();
return detail
? `${MODEL_AUTH_DENIED_COPY} ${detail}`
: MODEL_AUTH_DENIED_COPY;
}
export function friendlyAgentLastError(
raw: string | null,
code?: number | null,
@@ -78,7 +88,10 @@ export function friendlyAgentLastError(
if (effectiveCode != null) {
switch (effectiveCode) {
case -32001:
return { severity: "denied", copy: RELAY_MESH_DENIED_COPY };
return {
severity: "denied",
copy: modelAuthFailureCopy(embedded?.remainder ?? trimmed),
};
case -32002:
return { severity: "denied", copy: MODEL_NOT_FOUND_COPY };
case -32603: {
@@ -110,7 +123,7 @@ export function friendlyAgentLastError(
trimmed.startsWith("Agent reported error: llm auth:") ||
trimmed.startsWith("llm auth:")
) {
return { severity: "denied", copy: RELAY_MESH_DENIED_COPY };
return { severity: "denied", copy: modelAuthFailureCopy(trimmed) };
}
return { severity: "generic", copy: trimmed };

View File

@@ -83,9 +83,8 @@ export function ManagedAgentRow({
// When the harness recovered a meaningful error string from the agent's
// log tail (Max's seam in `managed_agents/storage.rs`), promote it to
// user-visible copy below the process detail. Specifically renders the
// friendly "Community access denied this agent — check its community membership."
// for auth failures so the user knows it's a membership thing, not a
// crash. Generic exits stay verbatim so we don't lie about other failures.
// model-provider authentication guidance while preserving the provider's
// reason. Generic exits stay verbatim so we don't lie about other failures.
const friendlyError = friendlyAgentLastError(
agent.lastError,
agent.lastErrorCode,

View File

@@ -768,6 +768,27 @@ test("buildTranscript separates repeated lifecycle text", () => {
assert.equal(item.text, "recovered: first\nrecovered: second");
});
test("buildTranscript keeps the Databricks sign-in failure instead of blaming membership", () => {
const [item] = buildTranscript([
{
...baseEvent,
kind: "turn_error",
payload: {
outcome: "error",
code: -32001,
error:
"Agent reported error (code -32001): llm auth: Databricks rejected the refresh token; sign in again",
},
},
]);
assert.equal(item.type, "lifecycle");
assert.equal(item.title, "Turn error");
assert.equal(
item.text,
"error: The model provider rejected authentication — check its credentials or sign in again. Databricks rejected the refresh token; sign in again",
);
});
// --- permission outcome (Fix #3) ---
function makePermissionRequest(seq, requestId, turnId = "turn-1") {

View File

@@ -114,6 +114,61 @@ async function clickStop(page: Page) {
test.describe("agent control browser regressions", () => {
test.use({ viewport: { width: 1280, height: 720 } });
test("Activity preserves the model provider's authentication failure", async ({
page,
}) => {
await installMockBridge(page, {
managedAgents: [
{
name: "Charlie",
pubkey: AGENT_PUBKEY,
status: "running",
channelNames: ["agents"],
},
],
});
const panel = await openAgentActivity(page, CHANNEL_AGENTS);
await page.evaluate(
({ agentPubkey, channelId }) => {
const seed = window.__BUZZ_E2E_SEED_OBSERVER_EVENTS__;
if (!seed) throw new Error("Observer event seed is unavailable.");
seed({
agentPubkey,
events: [
{
seq: 100,
timestamp: new Date().toISOString(),
kind: "turn_error",
agentIndex: 0,
channelId,
sessionId: "auth-error-session",
turnId: "auth-error-turn",
payload: {
outcome: "error",
code: -32001,
error:
"Agent reported error (code -32001): llm auth: Databricks rejected the refresh token; sign in again",
},
},
],
});
},
{ agentPubkey: AGENT_PUBKEY, channelId: CHANNEL_AGENTS },
);
await expect(panel.getByText("Turn error", { exact: true })).toBeVisible();
if (process.env.BUZZ_AUTH_SCREENSHOT_PATH) {
await waitForAnimations(page);
await panel.screenshot({ path: process.env.BUZZ_AUTH_SCREENSHOT_PATH });
}
const transcript = panel.getByRole("log", { name: "Live ACP transcript" });
await expect(transcript).toContainText(
"Turn error · error: The model provider rejected authentication — check its credentials or sign in again. Databricks rejected the refresh token; sign in again",
);
await expect(transcript).not.toContainText(
"Community access denied this agent",
);
});
test("Stop uses the channelId-only activity scope and carries a requestId", async ({
page,
}) => {