diff --git a/src/tests/acp-scenarios-v2.test.ts b/src/tests/acp-scenarios-v2.test.ts new file mode 100644 index 00000000..6a6f0852 --- /dev/null +++ b/src/tests/acp-scenarios-v2.test.ts @@ -0,0 +1,194 @@ +/** + * Every scenario of `acp-scenarios/scenarios.ts` on the experimental ACP v2 + * surface: through the protocol router, with the SDK's v2 client. + * + * - Status: each scenario either passes, every prompt turn ending with a stop + * reason, or stops for the reason in {@link V2_STATUS}. A step of the v2 + * support that translates more of the agent's messages turns scenarios from + * stopped into passing, and updates the table. + * - Schema: every message the agent sends, also in a scenario that stops, is + * valid against the draft v2 schema of the SDK. + * - Golden files: `acp-scenarios/__snapshots__/v2/.jsonl` for each + * passing scenario. Run `npx vitest run src/tests/acp-scenarios-v2.test.ts -u` + * to update them. + */ +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { ClaudeAcpAgent } from "../acp-agent.js"; +import { + resetIds, + runScenarioV2, + type ScenarioRunV2, + type WireRecorded, +} from "./acp-scenarios/harness.js"; +import { SCENARIOS } from "./acp-scenarios/scenarios.js"; +import { canonical } from "./acp-scenarios/compare.js"; +import { validateV2Message } from "./acp-scenarios/schema.js"; + +vi.mock("@anthropic-ai/claude-agent-sdk", async (importOriginal) => { + const actual = await importOriginal(); + const harness = await import("./acp-scenarios/harness.js"); + return { + ...actual, + query: harness.mockedQuery, + getSessionMessages: harness.mockedSessionMessages, + }; +}); + +// The recordings replace only the ids that the run generated, so the harness +// learns each id that `randomUUID` returns. +vi.mock("node:crypto", async (importOriginal) => { + const actual = await importOriginal(); + const harness = await import("./acp-scenarios/harness.js"); + return { + ...actual, + randomUUID: (...args: Parameters) => { + const id = actual.randomUUID(...args); + harness.noteGeneratedId(id); + return id; + }, + }; +}); + +const here = path.dirname(fileURLToPath(import.meta.url)); + +const PASSES = "passes"; +/** Step 4 of `docs/acp-v2.md`. */ +const TOOL_CALLS = "The ACP v2 surface does not translate tool_call session updates yet"; +const REPLAY = "Invalid params: The ACP v2 surface does not replay session history yet"; +/** Step 5 of `docs/acp-v2.md`. */ +const PLANS = "The ACP v2 surface does not translate plan session updates yet"; + +/** + * How each scenario runs on the v2 surface: it passes, or stops for a reason + * that names the step of `docs/acp-v2.md` that removes it. + */ +const V2_STATUS: Record = { + "session-setup": PASSES, + "text-and-thinking": PASSES, + "bash-foreground": TOOL_CALLS, + "bash-error": TOOL_CALLS, + "bash-background": TOOL_CALLS, + "async-task-held-until-tool-id": PASSES, + "async-task-ends-before-tool-id": PASSES, + "async-task-released-at-turn-end": PASSES, + read: TOOL_CALLS, + "write-new": TOOL_CALLS, + "write-existing": TOOL_CALLS, + "edit-with-permission": TOOL_CALLS, + "edit-rejected": TOOL_CALLS, + "notebook-edit": TOOL_CALLS, + "grep-and-glob": TOOL_CALLS, + "web-fetch-and-search": TOOL_CALLS, + "subagent-task-legacy": TOOL_CALLS, + "subagent-agent-async": TOOL_CALLS, + "subagent-native-sessions": TOOL_CALLS, + "subagent-nested": TOOL_CALLS, + "subagent-late-child-update": TOOL_CALLS, + "subagent-transcript-extension": TOOL_CALLS, + "todo-write": PLANS, + "task-create-update": PLANS, + goal: PASSES, + "network-permission": TOOL_CALLS, + "exit-plan-approve": TOOL_CALLS, + "exit-plan-reject": TOOL_CALLS, + "ask-user-question": TOOL_CALLS, + skill: TOOL_CALLS, + "mcp-tool": TOOL_CALLS, + "task-output-and-stop": TOOL_CALLS, + "memory-recall": TOOL_CALLS, + "permission-denied": TOOL_CALLS, + "tool-progress": TOOL_CALLS, + "rate-limit-and-origin": PASSES, + "compaction-legacy": TOOL_CALLS, + "compaction-update": TOOL_CALLS, + "compaction-failed-legacy": TOOL_CALLS, + "session-load-replay": REPLAY, +}; + +const runs = new Map(); +let configDir: string; + +beforeAll(async () => { + // A run must not depend on the machine: no remote login, no Claude CLI, no + // user settings, and the bypass mode also for root. + for (const name of ["SSH_CONNECTION", "SSH_CLIENT", "SSH_TTY", "NO_BROWSER"]) { + vi.stubEnv(name, ""); + } + vi.stubEnv("CLAUDE_CODE_REMOTE", ""); + vi.stubEnv("ANTHROPIC_MODEL", ""); + vi.stubEnv("IS_SANDBOX", "1"); + vi.stubEnv("CLAUDE_CODE_EXECUTABLE", "/usr/bin/false"); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "acp-scenario-config-")); + vi.stubEnv("CLAUDE_CONFIG_DIR", configDir); + const { acpProtocolRouter } = await import("../serve.js"); + for (const scenario of SCENARIOS) { + resetIds(); + const agents: ClaudeAcpAgent[] = []; + const quiet = { log: () => {}, error: () => {} }; + runs.set( + scenario.name, + await runScenarioV2( + (stream) => acpProtocolRouter(quiet, (agent) => agents.push(agent)).connect(stream), + scenario, + ), + ); + await Promise.all(agents.map((agent) => agent.dispose())); + } +}, 120_000); + +afterAll(() => { + vi.unstubAllEnvs(); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +function run(scenario: string): ScenarioRunV2 { + const recorded = runs.get(scenario); + if (!recorded) throw new Error(`no v2 run of ${scenario}`); + return recorded; +} + +function toJsonLines(recorded: WireRecorded[]): string { + return recorded.map((record) => `${canonical(record)}\n`).join(""); +} + +describe("ACP v2 scenarios", () => { + it("run as the status table says", () => { + expect( + Object.fromEntries(SCENARIOS.map((scenario) => [scenario.name, run(scenario.name).status])), + ).toEqual(V2_STATUS); + }); + + describe("schema check", () => { + const chunk = (update: Record): WireRecorded => ({ + kind: "notification", + method: "session/update", + payload: { sessionId: "s", update: { sessionUpdate: "agent_message_chunk", ...update } }, + }); + + it("rejects a message that breaks the v2 schema", () => { + const content = { type: "text", text: "hi" }; + expect(validateV2Message(chunk({ messageId: "m", content }))).toEqual([]); + // v2 requires the id of the message a chunk belongs to. + expect(validateV2Message(chunk({ content }))).not.toEqual([]); + }); + + it.each(SCENARIOS.map((scenario) => scenario.name))("%s", (scenario) => { + expect(run(scenario).raw.flatMap((message) => validateV2Message(message))).toEqual([]); + }); + }); + + describe("v2 golden files", () => { + const passing = SCENARIOS.map((scenario) => scenario.name).filter( + (scenario) => V2_STATUS[scenario] === PASSES, + ); + it.each(passing)("%s", async (scenario) => { + await expect(toJsonLines(run(scenario).normalized)).toMatchFileSnapshot( + path.join(here, "acp-scenarios", "__snapshots__", "v2", `${scenario}.jsonl`), + ); + }); + }); +}); diff --git a/src/tests/acp-scenarios/__snapshots__/v2/async-task-ends-before-tool-id.jsonl b/src/tests/acp-scenarios/__snapshots__/v2/async-task-ends-before-tool-id.jsonl new file mode 100644 index 00000000..f91bc5ba --- /dev/null +++ b/src/tests/acp-scenarios/__snapshots__/v2/async-task-ends-before-tool-id.jsonl @@ -0,0 +1,7 @@ +{"kind":"response","method":"initialize","payload":{"_meta":{"steering":{"supported":true}},"authMethods":[],"capabilities":{"_meta":{"authStatus":{},"claudeCode":{"promptQueueing":true}}},"info":{"name":"@agentclientprotocol/claude-agent-acp","title":"Claude Agent","version":""},"protocolVersion":2}} +{"kind":"response","method":"session/new","payload":{"configOptions":[{"category":"mode","configId":"mode","currentValue":"default","description":"Session permission mode","name":"Mode","options":[{"description":"Always ask before making changes","name":"Manual","value":"default"},{"description":"Automatically accept all file edits","name":"Accept edits","value":"acceptEdits"},{"description":"Create a plan before making changes","name":"Plan","value":"plan"},{"description":"Claude handles permission decisions","name":"Auto","value":"auto"},{"description":"Accepts all permissions","name":"Bypass permissions","value":"bypassPermissions"}],"type":"select"},{"category":"model","configId":"model","currentValue":"claude-sonnet-4-6","description":"AI model to use","name":"Model","options":[{"description":"Fast","name":"Claude Sonnet 4.6","value":"claude-sonnet-4-6"}],"type":"select"}],"sessionId":""}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"availableCommands":[{"description":"Show the MCP servers and their status, or reconnect, enable, or disable a server","input":{"hint":"[reconnect|enable|disable [|all]]","type":"text"},"name":"mcp"}],"sessionUpdate":"available_commands_update"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"content":[{"text":"prompt 1","type":"text"}],"messageId":"","sessionUpdate":"user_message"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"sessionUpdate":"state_update","state":"running"}}} +{"kind":"response","method":"session/prompt","payload":{"messageId":""}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"_meta":{"quota":{"model_usage":[],"token_count":{"cachedInputTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"reasoningOutputTokens":0,"totalTokens":0}}},"sessionUpdate":"state_update","state":"idle","stopReason":"end_turn","usage":{"cachedReadTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"totalTokens":0}}}} diff --git a/src/tests/acp-scenarios/__snapshots__/v2/async-task-held-until-tool-id.jsonl b/src/tests/acp-scenarios/__snapshots__/v2/async-task-held-until-tool-id.jsonl new file mode 100644 index 00000000..f91bc5ba --- /dev/null +++ b/src/tests/acp-scenarios/__snapshots__/v2/async-task-held-until-tool-id.jsonl @@ -0,0 +1,7 @@ +{"kind":"response","method":"initialize","payload":{"_meta":{"steering":{"supported":true}},"authMethods":[],"capabilities":{"_meta":{"authStatus":{},"claudeCode":{"promptQueueing":true}}},"info":{"name":"@agentclientprotocol/claude-agent-acp","title":"Claude Agent","version":""},"protocolVersion":2}} +{"kind":"response","method":"session/new","payload":{"configOptions":[{"category":"mode","configId":"mode","currentValue":"default","description":"Session permission mode","name":"Mode","options":[{"description":"Always ask before making changes","name":"Manual","value":"default"},{"description":"Automatically accept all file edits","name":"Accept edits","value":"acceptEdits"},{"description":"Create a plan before making changes","name":"Plan","value":"plan"},{"description":"Claude handles permission decisions","name":"Auto","value":"auto"},{"description":"Accepts all permissions","name":"Bypass permissions","value":"bypassPermissions"}],"type":"select"},{"category":"model","configId":"model","currentValue":"claude-sonnet-4-6","description":"AI model to use","name":"Model","options":[{"description":"Fast","name":"Claude Sonnet 4.6","value":"claude-sonnet-4-6"}],"type":"select"}],"sessionId":""}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"availableCommands":[{"description":"Show the MCP servers and their status, or reconnect, enable, or disable a server","input":{"hint":"[reconnect|enable|disable [|all]]","type":"text"},"name":"mcp"}],"sessionUpdate":"available_commands_update"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"content":[{"text":"prompt 1","type":"text"}],"messageId":"","sessionUpdate":"user_message"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"sessionUpdate":"state_update","state":"running"}}} +{"kind":"response","method":"session/prompt","payload":{"messageId":""}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"_meta":{"quota":{"model_usage":[],"token_count":{"cachedInputTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"reasoningOutputTokens":0,"totalTokens":0}}},"sessionUpdate":"state_update","state":"idle","stopReason":"end_turn","usage":{"cachedReadTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"totalTokens":0}}}} diff --git a/src/tests/acp-scenarios/__snapshots__/v2/async-task-released-at-turn-end.jsonl b/src/tests/acp-scenarios/__snapshots__/v2/async-task-released-at-turn-end.jsonl new file mode 100644 index 00000000..f91bc5ba --- /dev/null +++ b/src/tests/acp-scenarios/__snapshots__/v2/async-task-released-at-turn-end.jsonl @@ -0,0 +1,7 @@ +{"kind":"response","method":"initialize","payload":{"_meta":{"steering":{"supported":true}},"authMethods":[],"capabilities":{"_meta":{"authStatus":{},"claudeCode":{"promptQueueing":true}}},"info":{"name":"@agentclientprotocol/claude-agent-acp","title":"Claude Agent","version":""},"protocolVersion":2}} +{"kind":"response","method":"session/new","payload":{"configOptions":[{"category":"mode","configId":"mode","currentValue":"default","description":"Session permission mode","name":"Mode","options":[{"description":"Always ask before making changes","name":"Manual","value":"default"},{"description":"Automatically accept all file edits","name":"Accept edits","value":"acceptEdits"},{"description":"Create a plan before making changes","name":"Plan","value":"plan"},{"description":"Claude handles permission decisions","name":"Auto","value":"auto"},{"description":"Accepts all permissions","name":"Bypass permissions","value":"bypassPermissions"}],"type":"select"},{"category":"model","configId":"model","currentValue":"claude-sonnet-4-6","description":"AI model to use","name":"Model","options":[{"description":"Fast","name":"Claude Sonnet 4.6","value":"claude-sonnet-4-6"}],"type":"select"}],"sessionId":""}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"availableCommands":[{"description":"Show the MCP servers and their status, or reconnect, enable, or disable a server","input":{"hint":"[reconnect|enable|disable [|all]]","type":"text"},"name":"mcp"}],"sessionUpdate":"available_commands_update"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"content":[{"text":"prompt 1","type":"text"}],"messageId":"","sessionUpdate":"user_message"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"sessionUpdate":"state_update","state":"running"}}} +{"kind":"response","method":"session/prompt","payload":{"messageId":""}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"_meta":{"quota":{"model_usage":[],"token_count":{"cachedInputTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"reasoningOutputTokens":0,"totalTokens":0}}},"sessionUpdate":"state_update","state":"idle","stopReason":"end_turn","usage":{"cachedReadTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"totalTokens":0}}}} diff --git a/src/tests/acp-scenarios/__snapshots__/v2/goal.jsonl b/src/tests/acp-scenarios/__snapshots__/v2/goal.jsonl new file mode 100644 index 00000000..f8adce26 --- /dev/null +++ b/src/tests/acp-scenarios/__snapshots__/v2/goal.jsonl @@ -0,0 +1,7 @@ +{"kind":"response","method":"initialize","payload":{"_meta":{"steering":{"supported":true}},"authMethods":[],"capabilities":{"_meta":{"authStatus":{},"claudeCode":{"promptQueueing":true}}},"info":{"name":"@agentclientprotocol/claude-agent-acp","title":"Claude Agent","version":""},"protocolVersion":2}} +{"kind":"response","method":"session/new","payload":{"configOptions":[{"category":"mode","configId":"mode","currentValue":"default","description":"Session permission mode","name":"Mode","options":[{"description":"Always ask before making changes","name":"Manual","value":"default"},{"description":"Automatically accept all file edits","name":"Accept edits","value":"acceptEdits"},{"description":"Create a plan before making changes","name":"Plan","value":"plan"},{"description":"Claude handles permission decisions","name":"Auto","value":"auto"},{"description":"Accepts all permissions","name":"Bypass permissions","value":"bypassPermissions"}],"type":"select"},{"category":"model","configId":"model","currentValue":"claude-sonnet-4-6","description":"AI model to use","name":"Model","options":[{"description":"Fast","name":"Claude Sonnet 4.6","value":"claude-sonnet-4-6"}],"type":"select"}],"sessionId":""}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"availableCommands":[{"description":"Show the MCP servers and their status, or reconnect, enable, or disable a server","input":{"hint":"[reconnect|enable|disable [|all]]","type":"text"},"name":"mcp"}],"sessionUpdate":"available_commands_update"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"content":[{"text":"/goal Ship the feature","type":"text"}],"messageId":"","sessionUpdate":"user_message"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"sessionUpdate":"state_update","state":"running"}}} +{"kind":"response","method":"session/prompt","payload":{"messageId":""}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"_meta":{"quota":{"model_usage":[],"token_count":{"cachedInputTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"reasoningOutputTokens":0,"totalTokens":0}}},"sessionUpdate":"state_update","state":"idle","stopReason":"end_turn","usage":{"cachedReadTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"totalTokens":0}}}} diff --git a/src/tests/acp-scenarios/__snapshots__/v2/rate-limit-and-origin.jsonl b/src/tests/acp-scenarios/__snapshots__/v2/rate-limit-and-origin.jsonl new file mode 100644 index 00000000..27cc6aab --- /dev/null +++ b/src/tests/acp-scenarios/__snapshots__/v2/rate-limit-and-origin.jsonl @@ -0,0 +1,13 @@ +{"kind":"response","method":"initialize","payload":{"_meta":{"steering":{"supported":true}},"authMethods":[],"capabilities":{"_meta":{"authStatus":{},"claudeCode":{"promptQueueing":true}}},"info":{"name":"@agentclientprotocol/claude-agent-acp","title":"Claude Agent","version":""},"protocolVersion":2}} +{"kind":"response","method":"session/new","payload":{"configOptions":[{"category":"mode","configId":"mode","currentValue":"default","description":"Session permission mode","name":"Mode","options":[{"description":"Always ask before making changes","name":"Manual","value":"default"},{"description":"Automatically accept all file edits","name":"Accept edits","value":"acceptEdits"},{"description":"Create a plan before making changes","name":"Plan","value":"plan"},{"description":"Claude handles permission decisions","name":"Auto","value":"auto"},{"description":"Accepts all permissions","name":"Bypass permissions","value":"bypassPermissions"}],"type":"select"},{"category":"model","configId":"model","currentValue":"claude-sonnet-4-6","description":"AI model to use","name":"Model","options":[{"description":"Fast","name":"Claude Sonnet 4.6","value":"claude-sonnet-4-6"}],"type":"select"}],"sessionId":""}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"availableCommands":[{"description":"Show the MCP servers and their status, or reconnect, enable, or disable a server","input":{"hint":"[reconnect|enable|disable [|all]]","type":"text"},"name":"mcp"}],"sessionUpdate":"available_commands_update"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"content":[{"text":"prompt 1","type":"text"}],"messageId":"","sessionUpdate":"user_message"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"sessionUpdate":"state_update","state":"running"}}} +{"kind":"response","method":"session/prompt","payload":{"messageId":""}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"_meta":{"_claude/model":"claude-sonnet-4-6"},"sessionUpdate":"usage_update","size":200000,"used":2}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"content":{"text":"Work","type":"text"},"messageId":"msg_rate","sessionUpdate":"agent_message_chunk"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"content":{"text":"ing.","type":"text"},"messageId":"msg_rate","sessionUpdate":"agent_message_chunk"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"_meta":{"_claude/model":"claude-sonnet-4-6"},"sessionUpdate":"usage_update","size":200000,"used":3}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"_meta":{"_claude/model":"claude-sonnet-4-6","_claude/rateLimit":{"resetsAt":1700000000,"status":"allowed_warning","utilization":0.9}},"sessionUpdate":"usage_update","size":200000,"used":2}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"_meta":{"_claude/model":"claude-sonnet-4-6","_claude/origin":{"kind":"human"}},"cost":{"amount":0,"currency":"USD"},"sessionUpdate":"usage_update","size":200000,"used":2}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"_meta":{"quota":{"model_usage":[],"token_count":{"cachedInputTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"reasoningOutputTokens":0,"totalTokens":0}}},"sessionUpdate":"state_update","state":"idle","stopReason":"end_turn","usage":{"cachedReadTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"totalTokens":0}}}} diff --git a/src/tests/acp-scenarios/__snapshots__/v2/session-setup.jsonl b/src/tests/acp-scenarios/__snapshots__/v2/session-setup.jsonl new file mode 100644 index 00000000..37dc221b --- /dev/null +++ b/src/tests/acp-scenarios/__snapshots__/v2/session-setup.jsonl @@ -0,0 +1,3 @@ +{"kind":"response","method":"initialize","payload":{"_meta":{"steering":{"supported":true}},"authMethods":[],"capabilities":{"_meta":{"authStatus":{},"claudeCode":{"promptQueueing":true}}},"info":{"name":"@agentclientprotocol/claude-agent-acp","title":"Claude Agent","version":""},"protocolVersion":2}} +{"kind":"response","method":"session/new","payload":{"configOptions":[{"category":"mode","configId":"mode","currentValue":"default","description":"Session permission mode","name":"Mode","options":[{"description":"Always ask before making changes","name":"Manual","value":"default"},{"description":"Automatically accept all file edits","name":"Accept edits","value":"acceptEdits"},{"description":"Create a plan before making changes","name":"Plan","value":"plan"},{"description":"Claude handles permission decisions","name":"Auto","value":"auto"},{"description":"Accepts all permissions","name":"Bypass permissions","value":"bypassPermissions"}],"type":"select"},{"category":"model","configId":"model","currentValue":"claude-sonnet-4-6","description":"AI model to use","name":"Model","options":[{"description":"Fast","name":"Claude Sonnet 4.6","value":"claude-sonnet-4-6"}],"type":"select"}],"sessionId":""}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"availableCommands":[{"description":"Show the MCP servers and their status, or reconnect, enable, or disable a server","input":{"hint":"[reconnect|enable|disable [|all]]","type":"text"},"name":"mcp"}],"sessionUpdate":"available_commands_update"}}} diff --git a/src/tests/acp-scenarios/__snapshots__/v2/text-and-thinking.jsonl b/src/tests/acp-scenarios/__snapshots__/v2/text-and-thinking.jsonl new file mode 100644 index 00000000..93e5a1e4 --- /dev/null +++ b/src/tests/acp-scenarios/__snapshots__/v2/text-and-thinking.jsonl @@ -0,0 +1,14 @@ +{"kind":"response","method":"initialize","payload":{"_meta":{"steering":{"supported":true}},"authMethods":[],"capabilities":{"_meta":{"authStatus":{},"claudeCode":{"promptQueueing":true}}},"info":{"name":"@agentclientprotocol/claude-agent-acp","title":"Claude Agent","version":""},"protocolVersion":2}} +{"kind":"response","method":"session/new","payload":{"configOptions":[{"category":"mode","configId":"mode","currentValue":"default","description":"Session permission mode","name":"Mode","options":[{"description":"Always ask before making changes","name":"Manual","value":"default"},{"description":"Automatically accept all file edits","name":"Accept edits","value":"acceptEdits"},{"description":"Create a plan before making changes","name":"Plan","value":"plan"},{"description":"Claude handles permission decisions","name":"Auto","value":"auto"},{"description":"Accepts all permissions","name":"Bypass permissions","value":"bypassPermissions"}],"type":"select"},{"category":"model","configId":"model","currentValue":"claude-sonnet-4-6","description":"AI model to use","name":"Model","options":[{"description":"Fast","name":"Claude Sonnet 4.6","value":"claude-sonnet-4-6"}],"type":"select"}],"sessionId":""}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"availableCommands":[{"description":"Show the MCP servers and their status, or reconnect, enable, or disable a server","input":{"hint":"[reconnect|enable|disable [|all]]","type":"text"},"name":"mcp"}],"sessionUpdate":"available_commands_update"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"content":[{"text":"prompt 1","type":"text"}],"messageId":"","sessionUpdate":"user_message"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"sessionUpdate":"state_update","state":"running"}}} +{"kind":"response","method":"session/prompt","payload":{"messageId":""}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"_meta":{"_claude/model":"claude-sonnet-4-6"},"sessionUpdate":"usage_update","size":200000,"used":2}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"content":{"text":"Let me thin","type":"text"},"messageId":"msg_text","sessionUpdate":"agent_thought_chunk"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"content":{"text":"k about it.","type":"text"},"messageId":"msg_text","sessionUpdate":"agent_thought_chunk"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"content":{"text":"Here is t","type":"text"},"messageId":"msg_text","sessionUpdate":"agent_message_chunk"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"content":{"text":"he answer.","type":"text"},"messageId":"msg_text","sessionUpdate":"agent_message_chunk"}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"_meta":{"_claude/model":"claude-sonnet-4-6"},"sessionUpdate":"usage_update","size":200000,"used":3}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"_meta":{"_claude/model":"claude-sonnet-4-6"},"cost":{"amount":0,"currency":"USD"},"sessionUpdate":"usage_update","size":200000,"used":2}}} +{"kind":"notification","method":"session/update","payload":{"sessionId":"","update":{"_meta":{"quota":{"model_usage":[],"token_count":{"cachedInputTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"reasoningOutputTokens":0,"totalTokens":0}}},"sessionUpdate":"state_update","state":"idle","stopReason":"end_turn","usage":{"cachedReadTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"totalTokens":0}}}} diff --git a/src/tests/acp-scenarios/harness.ts b/src/tests/acp-scenarios/harness.ts index 4ab7cf96..02b173b0 100644 --- a/src/tests/acp-scenarios/harness.ts +++ b/src/tests/acp-scenarios/harness.ts @@ -8,6 +8,10 @@ * order that Claude Code uses. The harness records every message that the * agent sends to the client. * + * {@link runScenarioV2} runs the same scenarios on the experimental ACP v2 + * surface, through the protocol router and the SDK's v2 client, and records + * the agent's JSON-RPC messages off the wire. + * * The harness is vitest-free. The test file mocks `query` and * `getSessionMessages` of the SDK with {@link mockedQuery} and * {@link mockedSessionMessages}. @@ -18,6 +22,7 @@ import type { RequestPermissionRequest, SessionNotification, } from "@agentclientprotocol/sdk"; +import * as v2 from "@agentclientprotocol/sdk/experimental/v2"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; @@ -265,18 +270,39 @@ export interface ScenarioRun { normalized: Recorded[]; } -/** Runs one scenario for one profile and records every outbound message. */ -export async function runScenario( - Agent: AgentClass, - profile: Profile, - scenario: Scenario, -): Promise { +/** + * Prepares a run of `scenario`: its working directory with the scenario's + * files, and the script that {@link mockedQuery} runs. Returns the directory. + */ +function startRun(scenario: Scenario): string { const cwd = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "acp-scenario-"))); for (const [file, text] of Object.entries(scenario.files ?? {})) { const target = path.join(cwd, file); fs.mkdirSync(path.dirname(target), { recursive: true }); fs.writeFileSync(target, text); } + generatedIds.clear(); + activeScript = { + scenario, + cwd, + transcript: scenario.transcript?.({ cwd, sessionId: SESSION_ID }) ?? [], + }; + return cwd; +} + +function endRun(cwd: string): void { + activeScript = undefined; + sdkSessionId = undefined; + fs.rmSync(cwd, { recursive: true, force: true }); +} + +/** Runs one scenario for one profile and records every outbound message. */ +export async function runScenario( + Agent: AgentClass, + profile: Profile, + scenario: Scenario, +): Promise { + const cwd = startRun(scenario); const recorded: Recorded[] = []; const record = (kind: Recorded["kind"], payload: unknown) => recorded.push({ kind, payload: structuredClone(payload) }); @@ -295,14 +321,7 @@ export async function runScenario( }, async createElicitation(request: CreateElicitationRequest) { record("createElicitation", request); - const schema = (request as any).requestedSchema as - { properties?: Record } | undefined; - const content: Record = {}; - for (const [key, property] of Object.entries(schema?.properties ?? {})) { - const values = property.enum ?? property.oneOf?.map((o: any) => o.const); - if (Array.isArray(values) && values.length > 0) content[key] = values[0]; - } - return { action: "accept", content }; + return elicitationAnswer(request); }, async completeElicitation(params: unknown) { record("completeElicitation", params); @@ -325,12 +344,6 @@ export async function runScenario( ? { _meta: { ...profile.capabilities._meta, ...scenario.capabilities?._meta } } : {}), }; - generatedIds.clear(); - activeScript = { - scenario, - cwd, - transcript: scenario.transcript?.({ cwd, sessionId: SESSION_ID }) ?? [], - }; const logger = { log: () => {}, error: () => {}, warn: () => {}, debug: () => {} }; const agent = new Agent(client, logger); let sessionId: string | undefined; @@ -364,17 +377,220 @@ export async function runScenario( normalized: normalize(raw, cwd, acpSessionId, generatedIds) as Recorded[], }; } finally { - activeScript = undefined; - sdkSessionId = undefined; try { if (sessionId) await agent.unstable_closeSession?.({ sessionId }); } catch { // The session is already gone. } - fs.rmSync(cwd, { recursive: true, force: true }); + endRun(cwd); } } +/** The client's answer to an elicitation: it accepts, with the first choice of each field. */ +function elicitationAnswer(request: unknown): { + action: "accept"; + content: Record; +} { + const schema = (request as { requestedSchema?: { properties?: Record } }) + .requestedSchema; + const content: Record = {}; + for (const [key, property] of Object.entries(schema?.properties ?? {})) { + const values = property.enum ?? property.oneOf?.map((o: any) => o.const); + if (Array.isArray(values) && values.length > 0) content[key] = values[0]; + } + return { action: "accept", content }; +} + +/** One JSON-RPC message that the agent sent to a v2 client. */ +export interface WireRecorded { + /** A request or notification to the client, or the answer to one of its requests. */ + kind: "request" | "notification" | "response" | "error"; + /** The method of the request or notification; for an answer, of the client's request. */ + method: string; + /** The params, the result, or the error. */ + payload: unknown; +} + +/** How a scenario ran on the ACP v2 surface. */ +export interface ScenarioRunV2 { + /** The messages as the agent sent them. */ + raw: WireRecorded[]; + /** The messages with the run-specific values replaced (see {@link normalize}). */ + normalized: WireRecorded[]; + /** + * `passes` when every prompt turn ended with a stop reason other than + * `_error`, or else why the scenario stopped: the error of a rejected + * request, or the notice of a turn that failed. + */ + status: string; +} + +/** + * The v2 counterpart of the plain profile: v2 has no client file system or + * terminals, so a v2 client declares only elicitation. + */ +const V2_CAPABILITIES = { elicitation: { form: {} } }; + +/** + * Runs one scenario on the ACP v2 surface. `connect` serves the agent side + * of the stream, as `serveAcp` does. The SDK's v2 client drives `initialize`, + * `session/new` (or `session/resume` with `replayFrom` for a transcript), and + * one `session/prompt` per turn, and waits for the turn's `idle`. Every + * message that the agent sends is recorded off the wire, so a message that the + * client rejects is still recorded. + */ +export async function runScenarioV2( + connect: (stream: v2.Stream) => void, + scenario: Scenario, +): Promise { + const cwd = startRun(scenario); + const raw: WireRecorded[] = []; + const methodsOfClientRequests = new Map(); + let recording = true; + let sessionId: string | undefined; + // The stop reasons of the session's idle state updates, and its notices. + const idles: unknown[] = []; + const notices: string[] = []; + + const each = (message: v2.AnyWireMessage) => + (Array.isArray(message) ? message : [message]) as Record[]; + const toAgent = new TransformStream({ + transform(message, controller) { + for (const m of each(message)) { + if ("method" in m && "id" in m) methodsOfClientRequests.set(m.id, m.method); + } + controller.enqueue(message); + }, + }); + const toClient = new TransformStream({ + transform(message, controller) { + for (const m of each(message)) { + if (recording) raw.push(wireRecord(m, methodsOfClientRequests)); + } + controller.enqueue(message); + }, + }); + connect({ readable: toAgent.readable, writable: toClient.writable }); + + let status = "passes"; + const stop = (reason: string) => { + if (status === "passes") status = reason; + }; + const client = v2 + .client({ name: "v2-scenario-client" }) + .onNotification(v2.methods.client.session.update, ({ params }) => { + if (params.sessionId !== sessionId) return; + const update = params.update as { + sessionUpdate: string; + state?: string; + stopReason?: unknown; + title?: string; + }; + if (update.sessionUpdate === "state_update" && update.state === "idle") { + idles.push(update.stopReason); + } + if (update.sessionUpdate === "notice" && update.title) notices.push(update.title); + }) + .onRequest(v2.methods.client.elicitation.create, ({ params }) => elicitationAnswer(params)) + .onNotification(v2.methods.client.elicitation.complete, () => {}) + .onNotification( + "_auth/status_update", + (params) => params, + () => {}, + ); + + try { + await client.connectWith( + { readable: toClient.readable, writable: toAgent.writable }, + async (agent) => { + await agent.request(v2.methods.agent.initialize, { + protocolVersion: v2.PROTOCOL_VERSION, + info: { name: "v2-scenario-client", version: "1.0.0" }, + capabilities: { + ...V2_CAPABILITIES, + ...(scenario.capabilities?._meta ? { _meta: scenario.capabilities._meta } : {}), + }, + }); + try { + if (scenario.transcript) { + sessionId = SESSION_ID; + await agent.request(v2.methods.agent.session.resume, { + sessionId: SESSION_ID, + cwd, + replayFrom: { type: "start" }, + }); + } else { + sessionId = (await agent.request(v2.methods.agent.session.new, { cwd })).sessionId; + } + } catch (error) { + stop(errorText(error)); + return; + } + await settle(); + for (let turn = 0; turn < scenario.turns.length; turn++) { + const idlesBefore = idles.length; + try { + await agent.request(v2.methods.agent.session.prompt, { + sessionId, + prompt: [{ type: "text", text: scenario.prompts?.[turn] ?? `prompt ${turn + 1}` }], + }); + } catch (error) { + stop(errorText(error)); + return; + } + if (!(await until(() => idles.length > idlesBefore))) { + stop("the turn reported no idle state"); + return; + } + if (idles[idlesBefore] === "_error") { + stop(notices.at(-1) ?? "the turn failed"); + return; + } + await settle(); + } + recording = false; + await agent.request(v2.methods.agent.session.close, { sessionId }).catch(() => {}); + }, + ); + return { + raw, + normalized: normalize(raw, cwd, sessionId ?? "", generatedIds) as WireRecorded[], + status, + }; + } finally { + endRun(cwd); + } +} + +function wireRecord( + message: Record, + methodsOfClientRequests: ReadonlyMap, +): WireRecorded { + if ("method" in message) { + return { + kind: "id" in message ? "request" : "notification", + method: message.method, + payload: structuredClone(message.params), + }; + } + const method = methodsOfClientRequests.get(message.id) ?? ""; + return "error" in message + ? { kind: "error", method, payload: structuredClone(message.error) } + : { kind: "response", method, payload: structuredClone(message.result) }; +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** Waits until `condition` holds, for at most two seconds. */ +async function until(condition: () => boolean): Promise { + for (let i = 0; i < 1000 && !condition(); i++) { + await new Promise((resolve) => setTimeout(resolve, 2)); + } + return condition(); +} + async function settle() { for (let i = 0; i < 5; i++) await new Promise((resolve) => setTimeout(resolve, 2)); } diff --git a/src/tests/acp-scenarios/schema.ts b/src/tests/acp-scenarios/schema.ts index 4f7be172..7993dadb 100644 --- a/src/tests/acp-scenarios/schema.ts +++ b/src/tests/acp-scenarios/schema.ts @@ -1,6 +1,7 @@ /** - * Validates recorded outbound messages against the ACP JSON schema that - * `@agentclientprotocol/sdk` ships. + * Validates recorded outbound messages against the ACP JSON schemas that + * `@agentclientprotocol/sdk` ships: v1 ({@link validateRecorded}) and the + * draft v2 ({@link validateV2Message}). * * The session updates of the AIR extensions (`async_task_*`, `subagent_*`) * are not in the ACP schema. {@link validateRecorded} accepts them only for a @@ -15,10 +16,13 @@ import { createRequire } from "node:module"; // exports of these CommonJS modules are not constructable or callable there. import { Ajv2020, type ValidateFunction } from "ajv/dist/2020.js"; import { fullFormats } from "ajv-formats/dist/formats.js"; -import type { Recorded } from "./harness.js"; +import type { Recorded, WireRecorded } from "./harness.js"; const require = createRequire(import.meta.url); const schema = require("@agentclientprotocol/sdk/schema/schema.json") as Record; +const v2Schema = require("@agentclientprotocol/sdk/schema/v2/schema.unstable.json") as { + $defs: Record; +}; /** The session update kinds of the AIR extensions. They are not in the ACP schema. */ export const EXTENSION_SESSION_UPDATES = new Set([ @@ -29,23 +33,32 @@ export const EXTENSION_SESSION_UPDATES = new Set([ "subagent_state_update", ]); -const ajv = new Ajv2020({ strict: false, allErrors: true }); -for (const [name, format] of Object.entries(fullFormats)) ajv.addFormat(name, format); -for (const [name, max] of [ - ["uint16", 2 ** 16 - 1], - ["uint32", 2 ** 32 - 1], - ["uint64", 2 ** 64 - 1], -] as const) { - ajv.addFormat(name, { - type: "number", - validate: (value: number) => Number.isInteger(value) && value >= 0 && value <= max, - }); +/** A validator of one ACP schema, which knows every format the schema uses. */ +function schemaValidator(source: Record, name: string): Ajv2020 { + const ajv = new Ajv2020({ strict: false, allErrors: true }); + for (const [format, definition] of Object.entries(fullFormats)) { + ajv.addFormat(format, definition); + } + for (const [format, max] of [ + ["uint16", 2 ** 16 - 1], + ["uint32", 2 ** 32 - 1], + ["uint64", 2 ** 64 - 1], + ] as const) { + ajv.addFormat(format, { + type: "number", + validate: (value: number) => Number.isInteger(value) && value >= 0 && value <= max, + }); + } + // A format without a definition passes every value, so each one must be known. + for (const format of schemaFormats(source)) { + if (!ajv.formats[format]) throw new Error(`No definition of the ACP schema format ${format}`); + } + ajv.addSchema(source, name); + return ajv; } -// A format without a definition passes every value, so each one must be known. -for (const format of schemaFormats(schema)) { - if (!ajv.formats[format]) throw new Error(`No definition of the ACP schema format ${format}`); -} -ajv.addSchema(schema, "acp"); + +const ajv = schemaValidator(schema, "acp"); +const ajvV2 = schemaValidator(v2Schema, "acp-v2"); /** Every `format` value in a JSON schema. */ function schemaFormats(node: unknown, found = new Set()): Set { @@ -112,3 +125,62 @@ export function validateRecorded(record: Recorded, extensions: ReadonlySet `${record.kind} ${error.instancePath || "/"} ${error.message ?? "is invalid"}`, ); } + +/** + * The v2 definition of a message that the agent sends, found from the method + * and side that each definition of the v2 schema names: a request or + * notification that the client handles, or the response to a request that + * the agent handles. + */ +function v2Definition(message: WireRecorded): string { + const side = message.kind === "response" ? "agent" : "client"; + const suffix = + message.kind === "response" + ? "Response" + : message.kind === "request" + ? "Request" + : "Notification"; + const names = Object.entries(v2Schema.$defs) + .filter( + ([name, definition]) => + definition["x-method"] === message.method && + definition["x-side"] === side && + name.endsWith(suffix), + ) + .map(([name]) => name); + if (names.length !== 1) { + throw new Error( + `The ACP v2 schema has ${names.length} definitions of ${message.kind} ${message.method}`, + ); + } + return names[0]; +} + +const v2Validators = new Map(); + +/** + * Returns a description of each way a message that the agent sent to a v2 + * client breaks the draft v2 schema, or an empty list. Extension methods only + * need their `_` prefix, and an error response a code and a message. + */ +export function validateV2Message(message: WireRecorded): string[] { + const label = `${message.kind} ${message.method}`; + if (message.method.startsWith("_")) return []; + if (message.kind === "error") { + const { code, message: text } = message.payload as { code?: unknown; message?: unknown }; + return Number.isInteger(code) && typeof text === "string" + ? [] + : [`${label} is not a JSON-RPC error`]; + } + const definition = v2Definition(message); + let validate = v2Validators.get(definition); + if (!validate) { + validate = ajvV2.getSchema(`acp-v2#/$defs/${definition}`); + if (!validate) throw new Error(`The ACP v2 schema has no ${definition}`); + v2Validators.set(definition, validate); + } + if (validate(message.payload)) return []; + return (validate.errors ?? []).map( + (error) => `${label} ${error.instancePath || "/"} ${error.message ?? "is invalid"}`, + ); +}