mirror of
https://github.com/agentclientprotocol/claude-agent-acp.git
synced 2026-10-27 04:30:59 +00:00
test: run the ACP scenarios on the experimental v2 surface (#1230)
Last part of step 3 of opt-in ACP v2 support (#1222, #1226): the scenario suite now also runs on the v2 surface. It's test-only. ## What `src/tests/acp-scenarios-v2.test.ts` runs all 40 scenarios of the ACP scenario suite the way `serveAcp` serves v2: through the protocol router, driven by the SDK's own v2 client. Per scenario: - `initialize`, then `session/new`, or `session/resume` with `replayFrom` for a transcript; - one `session/prompt` per turn, waiting for the turn's `idle`. The agent's JSON-RPC messages are recorded off the wire, so a message the client would reject is still recorded and checked. Three checks: 1. **Status table.** A scenario passes when every turn ends with a stop reason other than `_error`. Otherwise it records why it stopped: the rejected request's error, or the failed turn's notice. | Status today | Scenarios | |---|---| | passes | 7: `session-setup`, `text-and-thinking`, `goal`, `rate-limit-and-origin`, the three async-task scenarios | | `tool_call` not translated yet (step 4) | 30 | | `plan` not translated yet (step 5) | 2 | | replay not supported yet (step 4) | 1 | Each step that translates more updates must flip its entries to `passes`, so the table is that step's checklist. 2. **Schema.** Every recorded message, including those of scenarios that stop, validates against the SDK's draft v2 JSON schema. The definition is looked up from the method and side the schema itself names (`x-method`/`x-side`). One test checks that the validator rejects a chunk without `messageId`, so the check can't pass vacuously. 3. **Golden files** for the passing scenarios, in `src/tests/acp-scenarios/__snapshots__/v2/`. For example, `text-and-thinking.jsonl` shows the prompt lifecycle from #1226: `user_message`, `running`, the `session/prompt` answer, the chunks with their `messageId`, and `idle end_turn` with usage. `harness.ts` gains `runScenarioV2` next to `runScenario`, sharing the scripted Claude Code, the id normalization and the elicitation answer. `schema.ts` gains `validateV2Message` next to `validateRecorded`. ## Testing - Dropping `messageId` from the v2 chunk translation fails both the schema check and the golden files of the scenarios with chunks. - The v2 file is stable across repeated runs and takes about 1.3 s; the full suite stays at about 4.4 s. `npm run build`, `npm run check`, `npm run test:run` (2293 passed).
This commit is contained in:
194
src/tests/acp-scenarios-v2.test.ts
Normal file
194
src/tests/acp-scenarios-v2.test.ts
Normal file
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* Every scenario of `acp-scenarios/scenarios.ts` on the experimental ACP v2
|
||||
* surface: through the protocol router, with the SDK's v2 client.
|
||||
*
|
||||
* - Status: each scenario either passes, every prompt turn ending with a stop
|
||||
* reason, or stops for the reason in {@link V2_STATUS}. A step of the v2
|
||||
* support that translates more of the agent's messages turns scenarios from
|
||||
* stopped into passing, and updates the table.
|
||||
* - Schema: every message the agent sends, also in a scenario that stops, is
|
||||
* valid against the draft v2 schema of the SDK.
|
||||
* - Golden files: `acp-scenarios/__snapshots__/v2/<scenario>.jsonl` for each
|
||||
* passing scenario. Run `npx vitest run src/tests/acp-scenarios-v2.test.ts -u`
|
||||
* to update them.
|
||||
*/
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import type { ClaudeAcpAgent } from "../acp-agent.js";
|
||||
import {
|
||||
resetIds,
|
||||
runScenarioV2,
|
||||
type ScenarioRunV2,
|
||||
type WireRecorded,
|
||||
} from "./acp-scenarios/harness.js";
|
||||
import { SCENARIOS } from "./acp-scenarios/scenarios.js";
|
||||
import { canonical } from "./acp-scenarios/compare.js";
|
||||
import { validateV2Message } from "./acp-scenarios/schema.js";
|
||||
|
||||
vi.mock("@anthropic-ai/claude-agent-sdk", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("@anthropic-ai/claude-agent-sdk")>();
|
||||
const harness = await import("./acp-scenarios/harness.js");
|
||||
return {
|
||||
...actual,
|
||||
query: harness.mockedQuery,
|
||||
getSessionMessages: harness.mockedSessionMessages,
|
||||
};
|
||||
});
|
||||
|
||||
// The recordings replace only the ids that the run generated, so the harness
|
||||
// learns each id that `randomUUID` returns.
|
||||
vi.mock("node:crypto", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("node:crypto")>();
|
||||
const harness = await import("./acp-scenarios/harness.js");
|
||||
return {
|
||||
...actual,
|
||||
randomUUID: (...args: Parameters<typeof actual.randomUUID>) => {
|
||||
const id = actual.randomUUID(...args);
|
||||
harness.noteGeneratedId(id);
|
||||
return id;
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const PASSES = "passes";
|
||||
/** Step 4 of `docs/acp-v2.md`. */
|
||||
const TOOL_CALLS = "The ACP v2 surface does not translate tool_call session updates yet";
|
||||
const REPLAY = "Invalid params: The ACP v2 surface does not replay session history yet";
|
||||
/** Step 5 of `docs/acp-v2.md`. */
|
||||
const PLANS = "The ACP v2 surface does not translate plan session updates yet";
|
||||
|
||||
/**
|
||||
* How each scenario runs on the v2 surface: it passes, or stops for a reason
|
||||
* that names the step of `docs/acp-v2.md` that removes it.
|
||||
*/
|
||||
const V2_STATUS: Record<string, string> = {
|
||||
"session-setup": PASSES,
|
||||
"text-and-thinking": PASSES,
|
||||
"bash-foreground": TOOL_CALLS,
|
||||
"bash-error": TOOL_CALLS,
|
||||
"bash-background": TOOL_CALLS,
|
||||
"async-task-held-until-tool-id": PASSES,
|
||||
"async-task-ends-before-tool-id": PASSES,
|
||||
"async-task-released-at-turn-end": PASSES,
|
||||
read: TOOL_CALLS,
|
||||
"write-new": TOOL_CALLS,
|
||||
"write-existing": TOOL_CALLS,
|
||||
"edit-with-permission": TOOL_CALLS,
|
||||
"edit-rejected": TOOL_CALLS,
|
||||
"notebook-edit": TOOL_CALLS,
|
||||
"grep-and-glob": TOOL_CALLS,
|
||||
"web-fetch-and-search": TOOL_CALLS,
|
||||
"subagent-task-legacy": TOOL_CALLS,
|
||||
"subagent-agent-async": TOOL_CALLS,
|
||||
"subagent-native-sessions": TOOL_CALLS,
|
||||
"subagent-nested": TOOL_CALLS,
|
||||
"subagent-late-child-update": TOOL_CALLS,
|
||||
"subagent-transcript-extension": TOOL_CALLS,
|
||||
"todo-write": PLANS,
|
||||
"task-create-update": PLANS,
|
||||
goal: PASSES,
|
||||
"network-permission": TOOL_CALLS,
|
||||
"exit-plan-approve": TOOL_CALLS,
|
||||
"exit-plan-reject": TOOL_CALLS,
|
||||
"ask-user-question": TOOL_CALLS,
|
||||
skill: TOOL_CALLS,
|
||||
"mcp-tool": TOOL_CALLS,
|
||||
"task-output-and-stop": TOOL_CALLS,
|
||||
"memory-recall": TOOL_CALLS,
|
||||
"permission-denied": TOOL_CALLS,
|
||||
"tool-progress": TOOL_CALLS,
|
||||
"rate-limit-and-origin": PASSES,
|
||||
"compaction-legacy": TOOL_CALLS,
|
||||
"compaction-update": TOOL_CALLS,
|
||||
"compaction-failed-legacy": TOOL_CALLS,
|
||||
"session-load-replay": REPLAY,
|
||||
};
|
||||
|
||||
const runs = new Map<string, ScenarioRunV2>();
|
||||
let configDir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
// A run must not depend on the machine: no remote login, no Claude CLI, no
|
||||
// user settings, and the bypass mode also for root.
|
||||
for (const name of ["SSH_CONNECTION", "SSH_CLIENT", "SSH_TTY", "NO_BROWSER"]) {
|
||||
vi.stubEnv(name, "");
|
||||
}
|
||||
vi.stubEnv("CLAUDE_CODE_REMOTE", "");
|
||||
vi.stubEnv("ANTHROPIC_MODEL", "");
|
||||
vi.stubEnv("IS_SANDBOX", "1");
|
||||
vi.stubEnv("CLAUDE_CODE_EXECUTABLE", "/usr/bin/false");
|
||||
configDir = fs.mkdtempSync(path.join(os.tmpdir(), "acp-scenario-config-"));
|
||||
vi.stubEnv("CLAUDE_CONFIG_DIR", configDir);
|
||||
const { acpProtocolRouter } = await import("../serve.js");
|
||||
for (const scenario of SCENARIOS) {
|
||||
resetIds();
|
||||
const agents: ClaudeAcpAgent[] = [];
|
||||
const quiet = { log: () => {}, error: () => {} };
|
||||
runs.set(
|
||||
scenario.name,
|
||||
await runScenarioV2(
|
||||
(stream) => acpProtocolRouter(quiet, (agent) => agents.push(agent)).connect(stream),
|
||||
scenario,
|
||||
),
|
||||
);
|
||||
await Promise.all(agents.map((agent) => agent.dispose()));
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
afterAll(() => {
|
||||
vi.unstubAllEnvs();
|
||||
fs.rmSync(configDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function run(scenario: string): ScenarioRunV2 {
|
||||
const recorded = runs.get(scenario);
|
||||
if (!recorded) throw new Error(`no v2 run of ${scenario}`);
|
||||
return recorded;
|
||||
}
|
||||
|
||||
function toJsonLines(recorded: WireRecorded[]): string {
|
||||
return recorded.map((record) => `${canonical(record)}\n`).join("");
|
||||
}
|
||||
|
||||
describe("ACP v2 scenarios", () => {
|
||||
it("run as the status table says", () => {
|
||||
expect(
|
||||
Object.fromEntries(SCENARIOS.map((scenario) => [scenario.name, run(scenario.name).status])),
|
||||
).toEqual(V2_STATUS);
|
||||
});
|
||||
|
||||
describe("schema check", () => {
|
||||
const chunk = (update: Record<string, unknown>): WireRecorded => ({
|
||||
kind: "notification",
|
||||
method: "session/update",
|
||||
payload: { sessionId: "s", update: { sessionUpdate: "agent_message_chunk", ...update } },
|
||||
});
|
||||
|
||||
it("rejects a message that breaks the v2 schema", () => {
|
||||
const content = { type: "text", text: "hi" };
|
||||
expect(validateV2Message(chunk({ messageId: "m", content }))).toEqual([]);
|
||||
// v2 requires the id of the message a chunk belongs to.
|
||||
expect(validateV2Message(chunk({ content }))).not.toEqual([]);
|
||||
});
|
||||
|
||||
it.each(SCENARIOS.map((scenario) => scenario.name))("%s", (scenario) => {
|
||||
expect(run(scenario).raw.flatMap((message) => validateV2Message(message))).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("v2 golden files", () => {
|
||||
const passing = SCENARIOS.map((scenario) => scenario.name).filter(
|
||||
(scenario) => V2_STATUS[scenario] === PASSES,
|
||||
);
|
||||
it.each(passing)("%s", async (scenario) => {
|
||||
await expect(toJsonLines(run(scenario).normalized)).toMatchFileSnapshot(
|
||||
path.join(here, "acp-scenarios", "__snapshots__", "v2", `${scenario}.jsonl`),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
7
src/tests/acp-scenarios/__snapshots__/v2/async-task-ends-before-tool-id.jsonl
generated
Normal file
7
src/tests/acp-scenarios/__snapshots__/v2/async-task-ends-before-tool-id.jsonl
generated
Normal file
@@ -0,0 +1,7 @@
|
||||
{"kind":"response","method":"initialize","payload":{"_meta":{"steering":{"supported":true}},"authMethods":[],"capabilities":{"_meta":{"authStatus":{},"claudeCode":{"promptQueueing":true}}},"info":{"name":"@agentclientprotocol/claude-agent-acp","title":"Claude Agent","version":"<version>"},"protocolVersion":2}}
|
||||
{"kind":"response","method":"session/new","payload":{"configOptions":[{"category":"mode","configId":"mode","currentValue":"default","description":"Session permission mode","name":"Mode","options":[{"description":"Always ask before making changes","name":"Manual","value":"default"},{"description":"Automatically accept all file edits","name":"Accept edits","value":"acceptEdits"},{"description":"Create a plan before making changes","name":"Plan","value":"plan"},{"description":"Claude handles permission decisions","name":"Auto","value":"auto"},{"description":"Accepts all permissions","name":"Bypass permissions","value":"bypassPermissions"}],"type":"select"},{"category":"model","configId":"model","currentValue":"claude-sonnet-4-6","description":"AI model to use","name":"Model","options":[{"description":"Fast","name":"Claude Sonnet 4.6","value":"claude-sonnet-4-6"}],"type":"select"}],"sessionId":"<session>"}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"availableCommands":[{"description":"Show the MCP servers and their status, or reconnect, enable, or disable a server","input":{"hint":"[reconnect|enable|disable [<server>|all]]","type":"text"},"name":"mcp"}],"sessionUpdate":"available_commands_update"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"content":[{"text":"prompt 1","type":"text"}],"messageId":"<id-1>","sessionUpdate":"user_message"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"sessionUpdate":"state_update","state":"running"}}}
|
||||
{"kind":"response","method":"session/prompt","payload":{"messageId":"<id-1>"}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"_meta":{"quota":{"model_usage":[],"token_count":{"cachedInputTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"reasoningOutputTokens":0,"totalTokens":0}}},"sessionUpdate":"state_update","state":"idle","stopReason":"end_turn","usage":{"cachedReadTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"totalTokens":0}}}}
|
||||
7
src/tests/acp-scenarios/__snapshots__/v2/async-task-held-until-tool-id.jsonl
generated
Normal file
7
src/tests/acp-scenarios/__snapshots__/v2/async-task-held-until-tool-id.jsonl
generated
Normal file
@@ -0,0 +1,7 @@
|
||||
{"kind":"response","method":"initialize","payload":{"_meta":{"steering":{"supported":true}},"authMethods":[],"capabilities":{"_meta":{"authStatus":{},"claudeCode":{"promptQueueing":true}}},"info":{"name":"@agentclientprotocol/claude-agent-acp","title":"Claude Agent","version":"<version>"},"protocolVersion":2}}
|
||||
{"kind":"response","method":"session/new","payload":{"configOptions":[{"category":"mode","configId":"mode","currentValue":"default","description":"Session permission mode","name":"Mode","options":[{"description":"Always ask before making changes","name":"Manual","value":"default"},{"description":"Automatically accept all file edits","name":"Accept edits","value":"acceptEdits"},{"description":"Create a plan before making changes","name":"Plan","value":"plan"},{"description":"Claude handles permission decisions","name":"Auto","value":"auto"},{"description":"Accepts all permissions","name":"Bypass permissions","value":"bypassPermissions"}],"type":"select"},{"category":"model","configId":"model","currentValue":"claude-sonnet-4-6","description":"AI model to use","name":"Model","options":[{"description":"Fast","name":"Claude Sonnet 4.6","value":"claude-sonnet-4-6"}],"type":"select"}],"sessionId":"<session>"}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"availableCommands":[{"description":"Show the MCP servers and their status, or reconnect, enable, or disable a server","input":{"hint":"[reconnect|enable|disable [<server>|all]]","type":"text"},"name":"mcp"}],"sessionUpdate":"available_commands_update"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"content":[{"text":"prompt 1","type":"text"}],"messageId":"<id-1>","sessionUpdate":"user_message"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"sessionUpdate":"state_update","state":"running"}}}
|
||||
{"kind":"response","method":"session/prompt","payload":{"messageId":"<id-1>"}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"_meta":{"quota":{"model_usage":[],"token_count":{"cachedInputTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"reasoningOutputTokens":0,"totalTokens":0}}},"sessionUpdate":"state_update","state":"idle","stopReason":"end_turn","usage":{"cachedReadTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"totalTokens":0}}}}
|
||||
7
src/tests/acp-scenarios/__snapshots__/v2/async-task-released-at-turn-end.jsonl
generated
Normal file
7
src/tests/acp-scenarios/__snapshots__/v2/async-task-released-at-turn-end.jsonl
generated
Normal file
@@ -0,0 +1,7 @@
|
||||
{"kind":"response","method":"initialize","payload":{"_meta":{"steering":{"supported":true}},"authMethods":[],"capabilities":{"_meta":{"authStatus":{},"claudeCode":{"promptQueueing":true}}},"info":{"name":"@agentclientprotocol/claude-agent-acp","title":"Claude Agent","version":"<version>"},"protocolVersion":2}}
|
||||
{"kind":"response","method":"session/new","payload":{"configOptions":[{"category":"mode","configId":"mode","currentValue":"default","description":"Session permission mode","name":"Mode","options":[{"description":"Always ask before making changes","name":"Manual","value":"default"},{"description":"Automatically accept all file edits","name":"Accept edits","value":"acceptEdits"},{"description":"Create a plan before making changes","name":"Plan","value":"plan"},{"description":"Claude handles permission decisions","name":"Auto","value":"auto"},{"description":"Accepts all permissions","name":"Bypass permissions","value":"bypassPermissions"}],"type":"select"},{"category":"model","configId":"model","currentValue":"claude-sonnet-4-6","description":"AI model to use","name":"Model","options":[{"description":"Fast","name":"Claude Sonnet 4.6","value":"claude-sonnet-4-6"}],"type":"select"}],"sessionId":"<session>"}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"availableCommands":[{"description":"Show the MCP servers and their status, or reconnect, enable, or disable a server","input":{"hint":"[reconnect|enable|disable [<server>|all]]","type":"text"},"name":"mcp"}],"sessionUpdate":"available_commands_update"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"content":[{"text":"prompt 1","type":"text"}],"messageId":"<id-1>","sessionUpdate":"user_message"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"sessionUpdate":"state_update","state":"running"}}}
|
||||
{"kind":"response","method":"session/prompt","payload":{"messageId":"<id-1>"}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"_meta":{"quota":{"model_usage":[],"token_count":{"cachedInputTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"reasoningOutputTokens":0,"totalTokens":0}}},"sessionUpdate":"state_update","state":"idle","stopReason":"end_turn","usage":{"cachedReadTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"totalTokens":0}}}}
|
||||
7
src/tests/acp-scenarios/__snapshots__/v2/goal.jsonl
generated
Normal file
7
src/tests/acp-scenarios/__snapshots__/v2/goal.jsonl
generated
Normal file
@@ -0,0 +1,7 @@
|
||||
{"kind":"response","method":"initialize","payload":{"_meta":{"steering":{"supported":true}},"authMethods":[],"capabilities":{"_meta":{"authStatus":{},"claudeCode":{"promptQueueing":true}}},"info":{"name":"@agentclientprotocol/claude-agent-acp","title":"Claude Agent","version":"<version>"},"protocolVersion":2}}
|
||||
{"kind":"response","method":"session/new","payload":{"configOptions":[{"category":"mode","configId":"mode","currentValue":"default","description":"Session permission mode","name":"Mode","options":[{"description":"Always ask before making changes","name":"Manual","value":"default"},{"description":"Automatically accept all file edits","name":"Accept edits","value":"acceptEdits"},{"description":"Create a plan before making changes","name":"Plan","value":"plan"},{"description":"Claude handles permission decisions","name":"Auto","value":"auto"},{"description":"Accepts all permissions","name":"Bypass permissions","value":"bypassPermissions"}],"type":"select"},{"category":"model","configId":"model","currentValue":"claude-sonnet-4-6","description":"AI model to use","name":"Model","options":[{"description":"Fast","name":"Claude Sonnet 4.6","value":"claude-sonnet-4-6"}],"type":"select"}],"sessionId":"<session>"}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"availableCommands":[{"description":"Show the MCP servers and their status, or reconnect, enable, or disable a server","input":{"hint":"[reconnect|enable|disable [<server>|all]]","type":"text"},"name":"mcp"}],"sessionUpdate":"available_commands_update"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"content":[{"text":"/goal Ship the feature","type":"text"}],"messageId":"<id-1>","sessionUpdate":"user_message"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"sessionUpdate":"state_update","state":"running"}}}
|
||||
{"kind":"response","method":"session/prompt","payload":{"messageId":"<id-1>"}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"_meta":{"quota":{"model_usage":[],"token_count":{"cachedInputTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"reasoningOutputTokens":0,"totalTokens":0}}},"sessionUpdate":"state_update","state":"idle","stopReason":"end_turn","usage":{"cachedReadTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"totalTokens":0}}}}
|
||||
13
src/tests/acp-scenarios/__snapshots__/v2/rate-limit-and-origin.jsonl
generated
Normal file
13
src/tests/acp-scenarios/__snapshots__/v2/rate-limit-and-origin.jsonl
generated
Normal file
@@ -0,0 +1,13 @@
|
||||
{"kind":"response","method":"initialize","payload":{"_meta":{"steering":{"supported":true}},"authMethods":[],"capabilities":{"_meta":{"authStatus":{},"claudeCode":{"promptQueueing":true}}},"info":{"name":"@agentclientprotocol/claude-agent-acp","title":"Claude Agent","version":"<version>"},"protocolVersion":2}}
|
||||
{"kind":"response","method":"session/new","payload":{"configOptions":[{"category":"mode","configId":"mode","currentValue":"default","description":"Session permission mode","name":"Mode","options":[{"description":"Always ask before making changes","name":"Manual","value":"default"},{"description":"Automatically accept all file edits","name":"Accept edits","value":"acceptEdits"},{"description":"Create a plan before making changes","name":"Plan","value":"plan"},{"description":"Claude handles permission decisions","name":"Auto","value":"auto"},{"description":"Accepts all permissions","name":"Bypass permissions","value":"bypassPermissions"}],"type":"select"},{"category":"model","configId":"model","currentValue":"claude-sonnet-4-6","description":"AI model to use","name":"Model","options":[{"description":"Fast","name":"Claude Sonnet 4.6","value":"claude-sonnet-4-6"}],"type":"select"}],"sessionId":"<session>"}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"availableCommands":[{"description":"Show the MCP servers and their status, or reconnect, enable, or disable a server","input":{"hint":"[reconnect|enable|disable [<server>|all]]","type":"text"},"name":"mcp"}],"sessionUpdate":"available_commands_update"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"content":[{"text":"prompt 1","type":"text"}],"messageId":"<id-1>","sessionUpdate":"user_message"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"sessionUpdate":"state_update","state":"running"}}}
|
||||
{"kind":"response","method":"session/prompt","payload":{"messageId":"<id-1>"}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"_meta":{"_claude/model":"claude-sonnet-4-6"},"sessionUpdate":"usage_update","size":200000,"used":2}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"content":{"text":"Work","type":"text"},"messageId":"msg_rate","sessionUpdate":"agent_message_chunk"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"content":{"text":"ing.","type":"text"},"messageId":"msg_rate","sessionUpdate":"agent_message_chunk"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"_meta":{"_claude/model":"claude-sonnet-4-6"},"sessionUpdate":"usage_update","size":200000,"used":3}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"_meta":{"_claude/model":"claude-sonnet-4-6","_claude/rateLimit":{"resetsAt":1700000000,"status":"allowed_warning","utilization":0.9}},"sessionUpdate":"usage_update","size":200000,"used":2}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"_meta":{"_claude/model":"claude-sonnet-4-6","_claude/origin":{"kind":"human"}},"cost":{"amount":0,"currency":"USD"},"sessionUpdate":"usage_update","size":200000,"used":2}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"_meta":{"quota":{"model_usage":[],"token_count":{"cachedInputTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"reasoningOutputTokens":0,"totalTokens":0}}},"sessionUpdate":"state_update","state":"idle","stopReason":"end_turn","usage":{"cachedReadTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"totalTokens":0}}}}
|
||||
3
src/tests/acp-scenarios/__snapshots__/v2/session-setup.jsonl
generated
Normal file
3
src/tests/acp-scenarios/__snapshots__/v2/session-setup.jsonl
generated
Normal file
@@ -0,0 +1,3 @@
|
||||
{"kind":"response","method":"initialize","payload":{"_meta":{"steering":{"supported":true}},"authMethods":[],"capabilities":{"_meta":{"authStatus":{},"claudeCode":{"promptQueueing":true}}},"info":{"name":"@agentclientprotocol/claude-agent-acp","title":"Claude Agent","version":"<version>"},"protocolVersion":2}}
|
||||
{"kind":"response","method":"session/new","payload":{"configOptions":[{"category":"mode","configId":"mode","currentValue":"default","description":"Session permission mode","name":"Mode","options":[{"description":"Always ask before making changes","name":"Manual","value":"default"},{"description":"Automatically accept all file edits","name":"Accept edits","value":"acceptEdits"},{"description":"Create a plan before making changes","name":"Plan","value":"plan"},{"description":"Claude handles permission decisions","name":"Auto","value":"auto"},{"description":"Accepts all permissions","name":"Bypass permissions","value":"bypassPermissions"}],"type":"select"},{"category":"model","configId":"model","currentValue":"claude-sonnet-4-6","description":"AI model to use","name":"Model","options":[{"description":"Fast","name":"Claude Sonnet 4.6","value":"claude-sonnet-4-6"}],"type":"select"}],"sessionId":"<session>"}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"availableCommands":[{"description":"Show the MCP servers and their status, or reconnect, enable, or disable a server","input":{"hint":"[reconnect|enable|disable [<server>|all]]","type":"text"},"name":"mcp"}],"sessionUpdate":"available_commands_update"}}}
|
||||
14
src/tests/acp-scenarios/__snapshots__/v2/text-and-thinking.jsonl
generated
Normal file
14
src/tests/acp-scenarios/__snapshots__/v2/text-and-thinking.jsonl
generated
Normal file
@@ -0,0 +1,14 @@
|
||||
{"kind":"response","method":"initialize","payload":{"_meta":{"steering":{"supported":true}},"authMethods":[],"capabilities":{"_meta":{"authStatus":{},"claudeCode":{"promptQueueing":true}}},"info":{"name":"@agentclientprotocol/claude-agent-acp","title":"Claude Agent","version":"<version>"},"protocolVersion":2}}
|
||||
{"kind":"response","method":"session/new","payload":{"configOptions":[{"category":"mode","configId":"mode","currentValue":"default","description":"Session permission mode","name":"Mode","options":[{"description":"Always ask before making changes","name":"Manual","value":"default"},{"description":"Automatically accept all file edits","name":"Accept edits","value":"acceptEdits"},{"description":"Create a plan before making changes","name":"Plan","value":"plan"},{"description":"Claude handles permission decisions","name":"Auto","value":"auto"},{"description":"Accepts all permissions","name":"Bypass permissions","value":"bypassPermissions"}],"type":"select"},{"category":"model","configId":"model","currentValue":"claude-sonnet-4-6","description":"AI model to use","name":"Model","options":[{"description":"Fast","name":"Claude Sonnet 4.6","value":"claude-sonnet-4-6"}],"type":"select"}],"sessionId":"<session>"}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"availableCommands":[{"description":"Show the MCP servers and their status, or reconnect, enable, or disable a server","input":{"hint":"[reconnect|enable|disable [<server>|all]]","type":"text"},"name":"mcp"}],"sessionUpdate":"available_commands_update"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"content":[{"text":"prompt 1","type":"text"}],"messageId":"<id-1>","sessionUpdate":"user_message"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"sessionUpdate":"state_update","state":"running"}}}
|
||||
{"kind":"response","method":"session/prompt","payload":{"messageId":"<id-1>"}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"_meta":{"_claude/model":"claude-sonnet-4-6"},"sessionUpdate":"usage_update","size":200000,"used":2}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"content":{"text":"Let me thin","type":"text"},"messageId":"msg_text","sessionUpdate":"agent_thought_chunk"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"content":{"text":"k about it.","type":"text"},"messageId":"msg_text","sessionUpdate":"agent_thought_chunk"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"content":{"text":"Here is t","type":"text"},"messageId":"msg_text","sessionUpdate":"agent_message_chunk"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"content":{"text":"he answer.","type":"text"},"messageId":"msg_text","sessionUpdate":"agent_message_chunk"}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"_meta":{"_claude/model":"claude-sonnet-4-6"},"sessionUpdate":"usage_update","size":200000,"used":3}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"_meta":{"_claude/model":"claude-sonnet-4-6"},"cost":{"amount":0,"currency":"USD"},"sessionUpdate":"usage_update","size":200000,"used":2}}}
|
||||
{"kind":"notification","method":"session/update","payload":{"sessionId":"<session>","update":{"_meta":{"quota":{"model_usage":[],"token_count":{"cachedInputTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"reasoningOutputTokens":0,"totalTokens":0}}},"sessionUpdate":"state_update","state":"idle","stopReason":"end_turn","usage":{"cachedReadTokens":0,"cachedWriteTokens":0,"inputTokens":0,"outputTokens":0,"totalTokens":0}}}}
|
||||
@@ -8,6 +8,10 @@
|
||||
* order that Claude Code uses. The harness records every message that the
|
||||
* agent sends to the client.
|
||||
*
|
||||
* {@link runScenarioV2} runs the same scenarios on the experimental ACP v2
|
||||
* surface, through the protocol router and the SDK's v2 client, and records
|
||||
* the agent's JSON-RPC messages off the wire.
|
||||
*
|
||||
* The harness is vitest-free. The test file mocks `query` and
|
||||
* `getSessionMessages` of the SDK with {@link mockedQuery} and
|
||||
* {@link mockedSessionMessages}.
|
||||
@@ -18,6 +22,7 @@ import type {
|
||||
RequestPermissionRequest,
|
||||
SessionNotification,
|
||||
} from "@agentclientprotocol/sdk";
|
||||
import * as v2 from "@agentclientprotocol/sdk/experimental/v2";
|
||||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
@@ -265,18 +270,39 @@ export interface ScenarioRun {
|
||||
normalized: Recorded[];
|
||||
}
|
||||
|
||||
/** Runs one scenario for one profile and records every outbound message. */
|
||||
export async function runScenario(
|
||||
Agent: AgentClass,
|
||||
profile: Profile,
|
||||
scenario: Scenario,
|
||||
): Promise<ScenarioRun> {
|
||||
/**
|
||||
* Prepares a run of `scenario`: its working directory with the scenario's
|
||||
* files, and the script that {@link mockedQuery} runs. Returns the directory.
|
||||
*/
|
||||
function startRun(scenario: Scenario): string {
|
||||
const cwd = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "acp-scenario-")));
|
||||
for (const [file, text] of Object.entries(scenario.files ?? {})) {
|
||||
const target = path.join(cwd, file);
|
||||
fs.mkdirSync(path.dirname(target), { recursive: true });
|
||||
fs.writeFileSync(target, text);
|
||||
}
|
||||
generatedIds.clear();
|
||||
activeScript = {
|
||||
scenario,
|
||||
cwd,
|
||||
transcript: scenario.transcript?.({ cwd, sessionId: SESSION_ID }) ?? [],
|
||||
};
|
||||
return cwd;
|
||||
}
|
||||
|
||||
function endRun(cwd: string): void {
|
||||
activeScript = undefined;
|
||||
sdkSessionId = undefined;
|
||||
fs.rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
/** Runs one scenario for one profile and records every outbound message. */
|
||||
export async function runScenario(
|
||||
Agent: AgentClass,
|
||||
profile: Profile,
|
||||
scenario: Scenario,
|
||||
): Promise<ScenarioRun> {
|
||||
const cwd = startRun(scenario);
|
||||
const recorded: Recorded[] = [];
|
||||
const record = (kind: Recorded["kind"], payload: unknown) =>
|
||||
recorded.push({ kind, payload: structuredClone(payload) });
|
||||
@@ -295,14 +321,7 @@ export async function runScenario(
|
||||
},
|
||||
async createElicitation(request: CreateElicitationRequest) {
|
||||
record("createElicitation", request);
|
||||
const schema = (request as any).requestedSchema as
|
||||
{ properties?: Record<string, any> } | undefined;
|
||||
const content: Record<string, unknown> = {};
|
||||
for (const [key, property] of Object.entries(schema?.properties ?? {})) {
|
||||
const values = property.enum ?? property.oneOf?.map((o: any) => o.const);
|
||||
if (Array.isArray(values) && values.length > 0) content[key] = values[0];
|
||||
}
|
||||
return { action: "accept", content };
|
||||
return elicitationAnswer(request);
|
||||
},
|
||||
async completeElicitation(params: unknown) {
|
||||
record("completeElicitation", params);
|
||||
@@ -325,12 +344,6 @@ export async function runScenario(
|
||||
? { _meta: { ...profile.capabilities._meta, ...scenario.capabilities?._meta } }
|
||||
: {}),
|
||||
};
|
||||
generatedIds.clear();
|
||||
activeScript = {
|
||||
scenario,
|
||||
cwd,
|
||||
transcript: scenario.transcript?.({ cwd, sessionId: SESSION_ID }) ?? [],
|
||||
};
|
||||
const logger = { log: () => {}, error: () => {}, warn: () => {}, debug: () => {} };
|
||||
const agent = new Agent(client, logger);
|
||||
let sessionId: string | undefined;
|
||||
@@ -364,17 +377,220 @@ export async function runScenario(
|
||||
normalized: normalize(raw, cwd, acpSessionId, generatedIds) as Recorded[],
|
||||
};
|
||||
} finally {
|
||||
activeScript = undefined;
|
||||
sdkSessionId = undefined;
|
||||
try {
|
||||
if (sessionId) await agent.unstable_closeSession?.({ sessionId });
|
||||
} catch {
|
||||
// The session is already gone.
|
||||
}
|
||||
fs.rmSync(cwd, { recursive: true, force: true });
|
||||
endRun(cwd);
|
||||
}
|
||||
}
|
||||
|
||||
/** The client's answer to an elicitation: it accepts, with the first choice of each field. */
|
||||
function elicitationAnswer(request: unknown): {
|
||||
action: "accept";
|
||||
content: Record<string, string | number | boolean | string[]>;
|
||||
} {
|
||||
const schema = (request as { requestedSchema?: { properties?: Record<string, any> } })
|
||||
.requestedSchema;
|
||||
const content: Record<string, string | number | boolean | string[]> = {};
|
||||
for (const [key, property] of Object.entries(schema?.properties ?? {})) {
|
||||
const values = property.enum ?? property.oneOf?.map((o: any) => o.const);
|
||||
if (Array.isArray(values) && values.length > 0) content[key] = values[0];
|
||||
}
|
||||
return { action: "accept", content };
|
||||
}
|
||||
|
||||
/** One JSON-RPC message that the agent sent to a v2 client. */
|
||||
export interface WireRecorded {
|
||||
/** A request or notification to the client, or the answer to one of its requests. */
|
||||
kind: "request" | "notification" | "response" | "error";
|
||||
/** The method of the request or notification; for an answer, of the client's request. */
|
||||
method: string;
|
||||
/** The params, the result, or the error. */
|
||||
payload: unknown;
|
||||
}
|
||||
|
||||
/** How a scenario ran on the ACP v2 surface. */
|
||||
export interface ScenarioRunV2 {
|
||||
/** The messages as the agent sent them. */
|
||||
raw: WireRecorded[];
|
||||
/** The messages with the run-specific values replaced (see {@link normalize}). */
|
||||
normalized: WireRecorded[];
|
||||
/**
|
||||
* `passes` when every prompt turn ended with a stop reason other than
|
||||
* `_error`, or else why the scenario stopped: the error of a rejected
|
||||
* request, or the notice of a turn that failed.
|
||||
*/
|
||||
status: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The v2 counterpart of the plain profile: v2 has no client file system or
|
||||
* terminals, so a v2 client declares only elicitation.
|
||||
*/
|
||||
const V2_CAPABILITIES = { elicitation: { form: {} } };
|
||||
|
||||
/**
|
||||
* Runs one scenario on the ACP v2 surface. `connect` serves the agent side
|
||||
* of the stream, as `serveAcp` does. The SDK's v2 client drives `initialize`,
|
||||
* `session/new` (or `session/resume` with `replayFrom` for a transcript), and
|
||||
* one `session/prompt` per turn, and waits for the turn's `idle`. Every
|
||||
* message that the agent sends is recorded off the wire, so a message that the
|
||||
* client rejects is still recorded.
|
||||
*/
|
||||
export async function runScenarioV2(
|
||||
connect: (stream: v2.Stream) => void,
|
||||
scenario: Scenario,
|
||||
): Promise<ScenarioRunV2> {
|
||||
const cwd = startRun(scenario);
|
||||
const raw: WireRecorded[] = [];
|
||||
const methodsOfClientRequests = new Map<string | number, string>();
|
||||
let recording = true;
|
||||
let sessionId: string | undefined;
|
||||
// The stop reasons of the session's idle state updates, and its notices.
|
||||
const idles: unknown[] = [];
|
||||
const notices: string[] = [];
|
||||
|
||||
const each = (message: v2.AnyWireMessage) =>
|
||||
(Array.isArray(message) ? message : [message]) as Record<string, any>[];
|
||||
const toAgent = new TransformStream<v2.AnyWireMessage, v2.AnyWireMessage>({
|
||||
transform(message, controller) {
|
||||
for (const m of each(message)) {
|
||||
if ("method" in m && "id" in m) methodsOfClientRequests.set(m.id, m.method);
|
||||
}
|
||||
controller.enqueue(message);
|
||||
},
|
||||
});
|
||||
const toClient = new TransformStream<v2.AnyWireMessage, v2.AnyWireMessage>({
|
||||
transform(message, controller) {
|
||||
for (const m of each(message)) {
|
||||
if (recording) raw.push(wireRecord(m, methodsOfClientRequests));
|
||||
}
|
||||
controller.enqueue(message);
|
||||
},
|
||||
});
|
||||
connect({ readable: toAgent.readable, writable: toClient.writable });
|
||||
|
||||
let status = "passes";
|
||||
const stop = (reason: string) => {
|
||||
if (status === "passes") status = reason;
|
||||
};
|
||||
const client = v2
|
||||
.client({ name: "v2-scenario-client" })
|
||||
.onNotification(v2.methods.client.session.update, ({ params }) => {
|
||||
if (params.sessionId !== sessionId) return;
|
||||
const update = params.update as {
|
||||
sessionUpdate: string;
|
||||
state?: string;
|
||||
stopReason?: unknown;
|
||||
title?: string;
|
||||
};
|
||||
if (update.sessionUpdate === "state_update" && update.state === "idle") {
|
||||
idles.push(update.stopReason);
|
||||
}
|
||||
if (update.sessionUpdate === "notice" && update.title) notices.push(update.title);
|
||||
})
|
||||
.onRequest(v2.methods.client.elicitation.create, ({ params }) => elicitationAnswer(params))
|
||||
.onNotification(v2.methods.client.elicitation.complete, () => {})
|
||||
.onNotification(
|
||||
"_auth/status_update",
|
||||
(params) => params,
|
||||
() => {},
|
||||
);
|
||||
|
||||
try {
|
||||
await client.connectWith(
|
||||
{ readable: toClient.readable, writable: toAgent.writable },
|
||||
async (agent) => {
|
||||
await agent.request(v2.methods.agent.initialize, {
|
||||
protocolVersion: v2.PROTOCOL_VERSION,
|
||||
info: { name: "v2-scenario-client", version: "1.0.0" },
|
||||
capabilities: {
|
||||
...V2_CAPABILITIES,
|
||||
...(scenario.capabilities?._meta ? { _meta: scenario.capabilities._meta } : {}),
|
||||
},
|
||||
});
|
||||
try {
|
||||
if (scenario.transcript) {
|
||||
sessionId = SESSION_ID;
|
||||
await agent.request(v2.methods.agent.session.resume, {
|
||||
sessionId: SESSION_ID,
|
||||
cwd,
|
||||
replayFrom: { type: "start" },
|
||||
});
|
||||
} else {
|
||||
sessionId = (await agent.request(v2.methods.agent.session.new, { cwd })).sessionId;
|
||||
}
|
||||
} catch (error) {
|
||||
stop(errorText(error));
|
||||
return;
|
||||
}
|
||||
await settle();
|
||||
for (let turn = 0; turn < scenario.turns.length; turn++) {
|
||||
const idlesBefore = idles.length;
|
||||
try {
|
||||
await agent.request(v2.methods.agent.session.prompt, {
|
||||
sessionId,
|
||||
prompt: [{ type: "text", text: scenario.prompts?.[turn] ?? `prompt ${turn + 1}` }],
|
||||
});
|
||||
} catch (error) {
|
||||
stop(errorText(error));
|
||||
return;
|
||||
}
|
||||
if (!(await until(() => idles.length > idlesBefore))) {
|
||||
stop("the turn reported no idle state");
|
||||
return;
|
||||
}
|
||||
if (idles[idlesBefore] === "_error") {
|
||||
stop(notices.at(-1) ?? "the turn failed");
|
||||
return;
|
||||
}
|
||||
await settle();
|
||||
}
|
||||
recording = false;
|
||||
await agent.request(v2.methods.agent.session.close, { sessionId }).catch(() => {});
|
||||
},
|
||||
);
|
||||
return {
|
||||
raw,
|
||||
normalized: normalize(raw, cwd, sessionId ?? "", generatedIds) as WireRecorded[],
|
||||
status,
|
||||
};
|
||||
} finally {
|
||||
endRun(cwd);
|
||||
}
|
||||
}
|
||||
|
||||
function wireRecord(
|
||||
message: Record<string, any>,
|
||||
methodsOfClientRequests: ReadonlyMap<string | number, string>,
|
||||
): WireRecorded {
|
||||
if ("method" in message) {
|
||||
return {
|
||||
kind: "id" in message ? "request" : "notification",
|
||||
method: message.method,
|
||||
payload: structuredClone(message.params),
|
||||
};
|
||||
}
|
||||
const method = methodsOfClientRequests.get(message.id) ?? "<unknown request>";
|
||||
return "error" in message
|
||||
? { kind: "error", method, payload: structuredClone(message.error) }
|
||||
: { kind: "response", method, payload: structuredClone(message.result) };
|
||||
}
|
||||
|
||||
function errorText(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
/** Waits until `condition` holds, for at most two seconds. */
|
||||
async function until(condition: () => boolean): Promise<boolean> {
|
||||
for (let i = 0; i < 1000 && !condition(); i++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 2));
|
||||
}
|
||||
return condition();
|
||||
}
|
||||
|
||||
async function settle() {
|
||||
for (let i = 0; i < 5; i++) await new Promise((resolve) => setTimeout(resolve, 2));
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
/**
|
||||
* Validates recorded outbound messages against the ACP JSON schema that
|
||||
* `@agentclientprotocol/sdk` ships.
|
||||
* Validates recorded outbound messages against the ACP JSON schemas that
|
||||
* `@agentclientprotocol/sdk` ships: v1 ({@link validateRecorded}) and the
|
||||
* draft v2 ({@link validateV2Message}).
|
||||
*
|
||||
* The session updates of the AIR extensions (`async_task_*`, `subagent_*`)
|
||||
* are not in the ACP schema. {@link validateRecorded} accepts them only for a
|
||||
@@ -15,10 +16,13 @@ import { createRequire } from "node:module";
|
||||
// exports of these CommonJS modules are not constructable or callable there.
|
||||
import { Ajv2020, type ValidateFunction } from "ajv/dist/2020.js";
|
||||
import { fullFormats } from "ajv-formats/dist/formats.js";
|
||||
import type { Recorded } from "./harness.js";
|
||||
import type { Recorded, WireRecorded } from "./harness.js";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const schema = require("@agentclientprotocol/sdk/schema/schema.json") as Record<string, unknown>;
|
||||
const v2Schema = require("@agentclientprotocol/sdk/schema/v2/schema.unstable.json") as {
|
||||
$defs: Record<string, { "x-method"?: string; "x-side"?: string }>;
|
||||
};
|
||||
|
||||
/** The session update kinds of the AIR extensions. They are not in the ACP schema. */
|
||||
export const EXTENSION_SESSION_UPDATES = new Set([
|
||||
@@ -29,23 +33,32 @@ export const EXTENSION_SESSION_UPDATES = new Set([
|
||||
"subagent_state_update",
|
||||
]);
|
||||
|
||||
const ajv = new Ajv2020({ strict: false, allErrors: true });
|
||||
for (const [name, format] of Object.entries(fullFormats)) ajv.addFormat(name, format);
|
||||
for (const [name, max] of [
|
||||
["uint16", 2 ** 16 - 1],
|
||||
["uint32", 2 ** 32 - 1],
|
||||
["uint64", 2 ** 64 - 1],
|
||||
] as const) {
|
||||
ajv.addFormat(name, {
|
||||
type: "number",
|
||||
validate: (value: number) => Number.isInteger(value) && value >= 0 && value <= max,
|
||||
});
|
||||
/** A validator of one ACP schema, which knows every format the schema uses. */
|
||||
function schemaValidator(source: Record<string, unknown>, name: string): Ajv2020 {
|
||||
const ajv = new Ajv2020({ strict: false, allErrors: true });
|
||||
for (const [format, definition] of Object.entries(fullFormats)) {
|
||||
ajv.addFormat(format, definition);
|
||||
}
|
||||
for (const [format, max] of [
|
||||
["uint16", 2 ** 16 - 1],
|
||||
["uint32", 2 ** 32 - 1],
|
||||
["uint64", 2 ** 64 - 1],
|
||||
] as const) {
|
||||
ajv.addFormat(format, {
|
||||
type: "number",
|
||||
validate: (value: number) => Number.isInteger(value) && value >= 0 && value <= max,
|
||||
});
|
||||
}
|
||||
// A format without a definition passes every value, so each one must be known.
|
||||
for (const format of schemaFormats(source)) {
|
||||
if (!ajv.formats[format]) throw new Error(`No definition of the ACP schema format ${format}`);
|
||||
}
|
||||
ajv.addSchema(source, name);
|
||||
return ajv;
|
||||
}
|
||||
// A format without a definition passes every value, so each one must be known.
|
||||
for (const format of schemaFormats(schema)) {
|
||||
if (!ajv.formats[format]) throw new Error(`No definition of the ACP schema format ${format}`);
|
||||
}
|
||||
ajv.addSchema(schema, "acp");
|
||||
|
||||
const ajv = schemaValidator(schema, "acp");
|
||||
const ajvV2 = schemaValidator(v2Schema, "acp-v2");
|
||||
|
||||
/** Every `format` value in a JSON schema. */
|
||||
function schemaFormats(node: unknown, found = new Set<string>()): Set<string> {
|
||||
@@ -112,3 +125,62 @@ export function validateRecorded(record: Recorded, extensions: ReadonlySet<strin
|
||||
(error) => `${record.kind} ${error.instancePath || "/"} ${error.message ?? "is invalid"}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The v2 definition of a message that the agent sends, found from the method
|
||||
* and side that each definition of the v2 schema names: a request or
|
||||
* notification that the client handles, or the response to a request that
|
||||
* the agent handles.
|
||||
*/
|
||||
function v2Definition(message: WireRecorded): string {
|
||||
const side = message.kind === "response" ? "agent" : "client";
|
||||
const suffix =
|
||||
message.kind === "response"
|
||||
? "Response"
|
||||
: message.kind === "request"
|
||||
? "Request"
|
||||
: "Notification";
|
||||
const names = Object.entries(v2Schema.$defs)
|
||||
.filter(
|
||||
([name, definition]) =>
|
||||
definition["x-method"] === message.method &&
|
||||
definition["x-side"] === side &&
|
||||
name.endsWith(suffix),
|
||||
)
|
||||
.map(([name]) => name);
|
||||
if (names.length !== 1) {
|
||||
throw new Error(
|
||||
`The ACP v2 schema has ${names.length} definitions of ${message.kind} ${message.method}`,
|
||||
);
|
||||
}
|
||||
return names[0];
|
||||
}
|
||||
|
||||
const v2Validators = new Map<string, ValidateFunction>();
|
||||
|
||||
/**
|
||||
* Returns a description of each way a message that the agent sent to a v2
|
||||
* client breaks the draft v2 schema, or an empty list. Extension methods only
|
||||
* need their `_` prefix, and an error response a code and a message.
|
||||
*/
|
||||
export function validateV2Message(message: WireRecorded): string[] {
|
||||
const label = `${message.kind} ${message.method}`;
|
||||
if (message.method.startsWith("_")) return [];
|
||||
if (message.kind === "error") {
|
||||
const { code, message: text } = message.payload as { code?: unknown; message?: unknown };
|
||||
return Number.isInteger(code) && typeof text === "string"
|
||||
? []
|
||||
: [`${label} is not a JSON-RPC error`];
|
||||
}
|
||||
const definition = v2Definition(message);
|
||||
let validate = v2Validators.get(definition);
|
||||
if (!validate) {
|
||||
validate = ajvV2.getSchema(`acp-v2#/$defs/${definition}`);
|
||||
if (!validate) throw new Error(`The ACP v2 schema has no ${definition}`);
|
||||
v2Validators.set(definition, validate);
|
||||
}
|
||||
if (validate(message.payload)) return [];
|
||||
return (validate.errors ?? []).map(
|
||||
(error) => `${label} ${error.instancePath || "/"} ${error.message ?? "is invalid"}`,
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user