mirror of
https://github.com/block/buzz.git
synced 2026-10-27 12:40:30 +00:00
fix(desktop): preserve model authentication error details
Signed-off-by: Brad Seiler <seiler@squareup.com>
This commit is contained in:
@@ -6,7 +6,7 @@ import {
|
||||
friendlyTurnErrorCopy,
|
||||
CLI_ACP_INTERNAL_ERROR_COPY,
|
||||
MODEL_NOT_FOUND_COPY,
|
||||
RELAY_MESH_DENIED_COPY,
|
||||
MODEL_AUTH_DENIED_COPY,
|
||||
} from "./friendlyAgentLastError.ts";
|
||||
|
||||
test("null lastError → null", () => {
|
||||
@@ -24,7 +24,7 @@ test("buzz-acp wrapped auth failure → denied copy", () => {
|
||||
);
|
||||
assert.deepEqual(result, {
|
||||
severity: "denied",
|
||||
copy: RELAY_MESH_DENIED_COPY,
|
||||
copy: `${MODEL_AUTH_DENIED_COPY} 401 unauthorized: ...`,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -35,7 +35,7 @@ test("unwrapped buzz-agent prefix → denied copy", () => {
|
||||
const result = friendlyAgentLastError("llm auth: 403 forbidden");
|
||||
assert.deepEqual(result, {
|
||||
severity: "denied",
|
||||
copy: RELAY_MESH_DENIED_COPY,
|
||||
copy: `${MODEL_AUTH_DENIED_COPY} 403 forbidden`,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -52,7 +52,7 @@ test("trims whitespace before matching", () => {
|
||||
" Agent reported error: llm auth: nope\n",
|
||||
);
|
||||
assert.equal(result?.severity, "denied");
|
||||
assert.equal(result?.copy, RELAY_MESH_DENIED_COPY);
|
||||
assert.equal(result?.copy, `${MODEL_AUTH_DENIED_COPY} nope`);
|
||||
});
|
||||
|
||||
test("substring 'llm auth:' that isn't at start is NOT treated as denial", () => {
|
||||
@@ -89,11 +89,11 @@ test("code -32002 → model-not-found copy (severity: denied)", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("code -32001 → Buzz shared compute denied copy (structured path)", () => {
|
||||
test("code -32001 → model-provider authentication copy (structured path)", () => {
|
||||
const result = friendlyAgentLastError("any error text", -32001);
|
||||
assert.deepEqual(result, {
|
||||
severity: "denied",
|
||||
copy: RELAY_MESH_DENIED_COPY,
|
||||
copy: `${MODEL_AUTH_DENIED_COPY} any error text`,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -104,7 +104,7 @@ test("code null falls through to legacy string matching", () => {
|
||||
);
|
||||
assert.deepEqual(result, {
|
||||
severity: "denied",
|
||||
copy: RELAY_MESH_DENIED_COPY,
|
||||
copy: `${MODEL_AUTH_DENIED_COPY} 401 unauthorized`,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -115,7 +115,7 @@ test("code undefined falls through to legacy string matching", () => {
|
||||
);
|
||||
assert.deepEqual(result, {
|
||||
severity: "denied",
|
||||
copy: RELAY_MESH_DENIED_COPY,
|
||||
copy: `${MODEL_AUTH_DENIED_COPY} 403 forbidden`,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -137,7 +137,7 @@ test("friendlyTurnErrorCopy: numeric code -32002 → model-not-found copy", () =
|
||||
test("friendlyTurnErrorCopy: string-encoded code coerces to number", () => {
|
||||
assert.equal(
|
||||
friendlyTurnErrorCopy("raw error", "-32001"),
|
||||
RELAY_MESH_DENIED_COPY,
|
||||
`${MODEL_AUTH_DENIED_COPY} raw error`,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -170,7 +170,7 @@ test("NaN code param treated as absent — string path applies", () => {
|
||||
const result = friendlyAgentLastError("llm auth: denied", NaN);
|
||||
assert.deepEqual(result, {
|
||||
severity: "denied",
|
||||
copy: RELAY_MESH_DENIED_COPY,
|
||||
copy: `${MODEL_AUTH_DENIED_COPY} denied`,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -181,7 +181,7 @@ test("embedded code -32001 recovered from message when code param is null", () =
|
||||
);
|
||||
assert.deepEqual(result, {
|
||||
severity: "denied",
|
||||
copy: RELAY_MESH_DENIED_COPY,
|
||||
copy: `${MODEL_AUTH_DENIED_COPY} 401`,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -211,7 +211,7 @@ test("friendlyTurnErrorCopy: garbage string code coerces to NaN → string path"
|
||||
// "garbage" → NaN → not finite → null → string prefix matches "llm auth:".
|
||||
assert.equal(
|
||||
friendlyTurnErrorCopy("llm auth: denied", "garbage"),
|
||||
RELAY_MESH_DENIED_COPY,
|
||||
`${MODEL_AUTH_DENIED_COPY} denied`,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -309,10 +309,32 @@ test("friendlyTurnErrorCopy: code -32603 bare Internal error → cli-acp interna
|
||||
test("-32603 does not affect -32001/-32002 classification (regression)", () => {
|
||||
assert.deepEqual(friendlyAgentLastError("any", -32001), {
|
||||
severity: "denied",
|
||||
copy: RELAY_MESH_DENIED_COPY,
|
||||
copy: `${MODEL_AUTH_DENIED_COPY} any`,
|
||||
});
|
||||
assert.deepEqual(friendlyAgentLastError("any", -32002), {
|
||||
severity: "denied",
|
||||
copy: MODEL_NOT_FOUND_COPY,
|
||||
});
|
||||
});
|
||||
|
||||
const databricksError = "Databricks rejected the refresh token; sign in again";
|
||||
for (const [raw, code] of [
|
||||
[`Agent reported error (code -32001): llm auth: ${databricksError}`, -32001],
|
||||
[`Agent reported error (code -32001): llm auth: ${databricksError}`, null],
|
||||
[`Agent reported error: llm auth: ${databricksError}`, undefined],
|
||||
[`llm auth: ${databricksError}`, undefined],
|
||||
]) {
|
||||
test(`Databricks auth failure preserves its cause and recovery: ${raw} (code ${code})`, () => {
|
||||
assert.deepEqual(friendlyAgentLastError(raw, code), {
|
||||
severity: "denied",
|
||||
copy: `${MODEL_AUTH_DENIED_COPY} ${databricksError}`,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test("auth failure without provider detail gets generic sign-in guidance", () => {
|
||||
assert.deepEqual(friendlyAgentLastError("llm auth:", -32001), {
|
||||
severity: "denied",
|
||||
copy: MODEL_AUTH_DENIED_COPY,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -33,11 +33,11 @@ export type FriendlyAgentLastError =
|
||||
| { severity: "generic"; copy: string };
|
||||
|
||||
/**
|
||||
* The exact copy for the relay-mesh denial. Centralized as a constant so the
|
||||
* The fallback copy for a model-provider authentication failure. Centralized so the
|
||||
* test asserts the user-facing string verbatim rather than a fuzzy pattern.
|
||||
*/
|
||||
export const RELAY_MESH_DENIED_COPY =
|
||||
"Community access denied this agent — check its community membership.";
|
||||
export const MODEL_AUTH_DENIED_COPY =
|
||||
"The model provider rejected authentication — check its credentials or sign in again.";
|
||||
|
||||
export const MODEL_NOT_FOUND_COPY =
|
||||
"The configured model is not available — open agent settings and select a different one from the dropdown.";
|
||||
@@ -61,6 +61,16 @@ function recoverEmbeddedCode(trimmed: string): {
|
||||
};
|
||||
}
|
||||
|
||||
function modelAuthFailureCopy(message: string): string {
|
||||
const detail = message
|
||||
.replace(/^Agent reported error: /, "")
|
||||
.replace(/^llm auth:\s*/, "")
|
||||
.trim();
|
||||
return detail
|
||||
? `${MODEL_AUTH_DENIED_COPY} ${detail}`
|
||||
: MODEL_AUTH_DENIED_COPY;
|
||||
}
|
||||
|
||||
export function friendlyAgentLastError(
|
||||
raw: string | null,
|
||||
code?: number | null,
|
||||
@@ -78,7 +88,10 @@ export function friendlyAgentLastError(
|
||||
if (effectiveCode != null) {
|
||||
switch (effectiveCode) {
|
||||
case -32001:
|
||||
return { severity: "denied", copy: RELAY_MESH_DENIED_COPY };
|
||||
return {
|
||||
severity: "denied",
|
||||
copy: modelAuthFailureCopy(embedded?.remainder ?? trimmed),
|
||||
};
|
||||
case -32002:
|
||||
return { severity: "denied", copy: MODEL_NOT_FOUND_COPY };
|
||||
case -32603: {
|
||||
@@ -110,7 +123,7 @@ export function friendlyAgentLastError(
|
||||
trimmed.startsWith("Agent reported error: llm auth:") ||
|
||||
trimmed.startsWith("llm auth:")
|
||||
) {
|
||||
return { severity: "denied", copy: RELAY_MESH_DENIED_COPY };
|
||||
return { severity: "denied", copy: modelAuthFailureCopy(trimmed) };
|
||||
}
|
||||
|
||||
return { severity: "generic", copy: trimmed };
|
||||
|
||||
@@ -83,9 +83,8 @@ export function ManagedAgentRow({
|
||||
// When the harness recovered a meaningful error string from the agent's
|
||||
// log tail (Max's seam in `managed_agents/storage.rs`), promote it to
|
||||
// user-visible copy below the process detail. Specifically renders the
|
||||
// friendly "Community access denied this agent — check its community membership."
|
||||
// for auth failures so the user knows it's a membership thing, not a
|
||||
// crash. Generic exits stay verbatim so we don't lie about other failures.
|
||||
// model-provider authentication guidance while preserving the provider's
|
||||
// reason. Generic exits stay verbatim so we don't lie about other failures.
|
||||
const friendlyError = friendlyAgentLastError(
|
||||
agent.lastError,
|
||||
agent.lastErrorCode,
|
||||
|
||||
@@ -768,6 +768,27 @@ test("buildTranscript separates repeated lifecycle text", () => {
|
||||
assert.equal(item.text, "recovered: first\nrecovered: second");
|
||||
});
|
||||
|
||||
test("buildTranscript keeps the Databricks sign-in failure instead of blaming membership", () => {
|
||||
const [item] = buildTranscript([
|
||||
{
|
||||
...baseEvent,
|
||||
kind: "turn_error",
|
||||
payload: {
|
||||
outcome: "error",
|
||||
code: -32001,
|
||||
error:
|
||||
"Agent reported error (code -32001): llm auth: Databricks rejected the refresh token; sign in again",
|
||||
},
|
||||
},
|
||||
]);
|
||||
assert.equal(item.type, "lifecycle");
|
||||
assert.equal(item.title, "Turn error");
|
||||
assert.equal(
|
||||
item.text,
|
||||
"error: The model provider rejected authentication — check its credentials or sign in again. Databricks rejected the refresh token; sign in again",
|
||||
);
|
||||
});
|
||||
|
||||
// --- permission outcome (Fix #3) ---
|
||||
|
||||
function makePermissionRequest(seq, requestId, turnId = "turn-1") {
|
||||
|
||||
@@ -114,6 +114,61 @@ async function clickStop(page: Page) {
|
||||
test.describe("agent control browser regressions", () => {
|
||||
test.use({ viewport: { width: 1280, height: 720 } });
|
||||
|
||||
test("Activity preserves the model provider's authentication failure", async ({
|
||||
page,
|
||||
}) => {
|
||||
await installMockBridge(page, {
|
||||
managedAgents: [
|
||||
{
|
||||
name: "Charlie",
|
||||
pubkey: AGENT_PUBKEY,
|
||||
status: "running",
|
||||
channelNames: ["agents"],
|
||||
},
|
||||
],
|
||||
});
|
||||
const panel = await openAgentActivity(page, CHANNEL_AGENTS);
|
||||
await page.evaluate(
|
||||
({ agentPubkey, channelId }) => {
|
||||
const seed = window.__BUZZ_E2E_SEED_OBSERVER_EVENTS__;
|
||||
if (!seed) throw new Error("Observer event seed is unavailable.");
|
||||
seed({
|
||||
agentPubkey,
|
||||
events: [
|
||||
{
|
||||
seq: 100,
|
||||
timestamp: new Date().toISOString(),
|
||||
kind: "turn_error",
|
||||
agentIndex: 0,
|
||||
channelId,
|
||||
sessionId: "auth-error-session",
|
||||
turnId: "auth-error-turn",
|
||||
payload: {
|
||||
outcome: "error",
|
||||
code: -32001,
|
||||
error:
|
||||
"Agent reported error (code -32001): llm auth: Databricks rejected the refresh token; sign in again",
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
},
|
||||
{ agentPubkey: AGENT_PUBKEY, channelId: CHANNEL_AGENTS },
|
||||
);
|
||||
await expect(panel.getByText("Turn error", { exact: true })).toBeVisible();
|
||||
if (process.env.BUZZ_AUTH_SCREENSHOT_PATH) {
|
||||
await waitForAnimations(page);
|
||||
await panel.screenshot({ path: process.env.BUZZ_AUTH_SCREENSHOT_PATH });
|
||||
}
|
||||
const transcript = panel.getByRole("log", { name: "Live ACP transcript" });
|
||||
await expect(transcript).toContainText(
|
||||
"Turn error · error: The model provider rejected authentication — check its credentials or sign in again. Databricks rejected the refresh token; sign in again",
|
||||
);
|
||||
await expect(transcript).not.toContainText(
|
||||
"Community access denied this agent",
|
||||
);
|
||||
});
|
||||
|
||||
test("Stop uses the channelId-only activity scope and carries a requestId", async ({
|
||||
page,
|
||||
}) => {
|
||||
|
||||
Reference in New Issue
Block a user