claude-agent-acp/scripts/smoke-binary.mjs

#!/usr/bin/env node
// Smoke-tests a built agent over ACP. Defaults to the executable from
// scripts/build-binary.mjs, with `claude` next to it. Pass another command
// after `--` to compare, e.g.
// `node scripts/smoke-binary.mjs --full -- node "$PWD/dist/index.js"` (the full
// run starts the agent in a scratch directory, so paths must be absolute).
//
//   node scripts/smoke-binary.mjs            # no login needed
//   node scripts/smoke-binary.mjs --full     # talks to the model; needs a logged-in Claude
//
// SMOKE_DEBUG=1 prints every tool call the full run saw.
//
// The basic run checks `--version`, `--cli --version`, `initialize` (v1 and
// v2) and the `terminal-auth` login command. The
// full run also has Claude use Bash, Read (text and image), Edit and a
// subagent in a scratch directory, then cancels a turn, lists the session,
// loads it in a fresh agent process, and forks and deletes it.
//
// Claude Code's native builds have no Grep or Glob tool: its Bash shell wraps
// `grep` and `find` in functions that run the ugrep and bfs built into the
// Claude binary at `$CLAUDE_CODE_EXECPATH`. The full run checks that those
// run, since they are the helpers a packaged build could fail to reach.

import { spawn, spawnSync } from "node:child_process";
import {
  copyFileSync,
  mkdirSync,
  mkdtempSync,
  readFileSync,
  rmSync,
  symlinkSync,
  writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import readline from "node:readline";
import { fileURLToPath } from "node:url";
import { parseArgs } from "node:util";
import { deflateSync } from "node:zlib";

const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const { values, positionals } = parseArgs({
  options: { full: { type: "boolean", default: false } },
  allowPositionals: true,
});
const exe = process.platform === "win32" ? ".exe" : "";
const [command, ...commandArgs] =
  positionals.length > 0 ? positionals : [join(repoRoot, "dist-bin", `claude-agent-acp${exe}`)];

let failures = 0;
function check(ok, label, detail) {
  console.log(
    `${ok ? "ok  " : "FAIL"} ${label}${!ok && detail !== undefined ? `: ${detail}` : ""}`,
  );
  if (!ok) failures++;
  return ok;
}

// ---------------------------------------------------------------- basic

function runSync(args, [cmd, ...cmdArgs] = [command, ...commandArgs], cwd) {
  const result = spawnSync(cmd, [...cmdArgs, ...args], {
    cwd,
    encoding: "utf8",
    timeout: 60_000,
  });
  return { stdout: (result.stdout ?? "").trim(), error: result.error ?? result.stderr };
}

const version = JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf8")).version;
const own = runSync(["--version"]);
check(own.stdout === version, `--version prints ${version}`, own.stdout || own.error);
const cli = runSync(["--cli", "--version"]);
check(
  /\(Claude Code\)$/.test(cli.stdout),
  "--cli --version runs the bundled Claude CLI",
  cli.stdout || cli.error,
);

// The release archive's layout: the executable finds `claude` in the directory
// of its real path. Only for the default executable; a `--` command has no
// such layout.
if (positionals.length === 0) {
  const scratch = mkdtempSync(join(tmpdir(), "acp-layout-"));
  try {
    if (process.platform !== "win32") {
      const link = join(scratch, "claude-agent-acp");
      symlinkSync(command, link);
      const linked = runSync(["--cli", "--version"], [link]);
      check(
        /\(Claude Code\)$/.test(linked.stdout),
        "a symlink to the executable finds the bundled CLI",
        linked.stdout || linked.error,
      );
      rmSync(link);
    }
    const alone = join(scratch, `claude-agent-acp${exe}`);
    copyFileSync(command, alone);
    const missing = runSync(["--cli", "--version"], [alone]);
    check(
      String(missing.error).includes("The Claude CLI is missing"),
      "without claude next to it, says the CLI is missing",
      missing.stdout || missing.error,
    );

    // Started in a project directory, the executable must not read the
    // project's dotenv files or bunfig.toml. Node stands in for `claude`, to
    // print the environment the CLI is spawned with.
    copyFileSync(process.execPath, join(scratch, `claude${exe}`));
    const project = join(scratch, "project");
    mkdirSync(project);
    writeFileSync(join(project, ".env"), "ACP_SMOKE_DOTENV=leaked\n");
    writeFileSync(join(project, ".env.local"), "ACP_SMOKE_DOTENV_LOCAL=leaked\n");
    writeFileSync(join(project, "bunfig.toml"), "= not toml\n");
    const probe = runSync(
      ["--cli", "-p", "[process.env.ACP_SMOKE_DOTENV, process.env.ACP_SMOKE_DOTENV_LOCAL].join()"],
      [alone],
      project,
    );
    check(
      probe.stdout === ",",
      "ignores .env, .env.local and bunfig.toml in the working directory",
      probe.stdout || probe.error,
    );
  } finally {
    rmSync(scratch, { recursive: true, force: true });
  }
}

/** Starts the agent and speaks newline-delimited JSON-RPC to it. */
function startAgent(cwd, onNotification) {
  const child = spawn(command, commandArgs, { cwd, stdio: ["pipe", "pipe", "pipe"] });
  const stderr = [];
  child.stderr.on("data", (chunk) => stderr.push(chunk));
  let nextId = 0;
  const pending = new Map();
  const write = (message) =>
    child.stdin.write(JSON.stringify({ jsonrpc: "2.0", ...message }) + "\n");
  readline.createInterface({ input: child.stdout }).on("line", (line) => {
    let message;
    try {
      message = JSON.parse(line);
    } catch {
      check(false, "stdout carries only JSON-RPC", line.slice(0, 200));
      return;
    }
    if (message.method === undefined && pending.has(message.id)) {
      pending.get(message.id)(message);
      pending.delete(message.id);
    } else if (message.method === "session/request_permission") {
      const allow =
        message.params.options.find((o) => o.kind === "allow_once") ?? message.params.options[0];
      write({
        id: message.id,
        result: { outcome: { outcome: "selected", optionId: allow.optionId } },
      });
    } else if (message.id !== undefined && message.method !== undefined) {
      write({
        id: message.id,
        error: { code: -32601, message: `smoke client does not handle ${message.method}` },
      });
    } else {
      onNotification?.(message);
    }
  });
  const exited = new Promise((resolve) => child.on("exit", resolve));
  // Answer anything still pending if the agent dies, instead of hanging.
  let exitError;
  exited.then((code) => {
    exitError = { message: `agent exited with ${code}: ${Buffer.concat(stderr)}` };
    for (const resolve of pending.values()) resolve({ error: exitError });
    pending.clear();
  });
  return {
    request(method, params) {
      const id = ++nextId;
      return new Promise((resolve) => {
        if (exitError) return resolve({ error: exitError });
        pending.set(id, resolve);
        write({ id, method, params });
      });
    },
    notify: (method, params) => write({ method, params }),
    async stop() {
      child.stdin.end();
      const code = await Promise.race([
        exited,
        new Promise((r) => setTimeout(() => r("timeout"), 10_000)),
      ]);
      if (code === "timeout") child.kill();
      return { code, stderr: Buffer.concat(stderr).toString() };
    },
  };
}

const initializeParams = { protocolVersion: 1, clientCapabilities: {} };
{
  const agent = startAgent(process.cwd());
  const init = await agent.request("initialize", initializeParams);
  check(
    init.result?.agentInfo?.version === version,
    "initialize answers",
    JSON.stringify(init.error),
  );
  const { code } = await agent.stop();
  check(code === 0, "exits 0 when stdin closes", code);
}
{
  // The `terminal-auth` login command reruns the agent; run it with
  // `--version` in place of `auth login --console` instead of logging in.
  // `--version` wins over a stray argument, so also check that no path inside
  // a compiled executable leaked into the arguments.
  const agent = startAgent(process.cwd());
  const init = await agent.request("initialize", {
    protocolVersion: 1,
    clientCapabilities: { _meta: { "terminal-auth": true } },
  });
  await agent.stop();
  const login = init.result?.authMethods?.find((m) => m.id === "console-login")?._meta?.[
    "terminal-auth"
  ];
  const cliArgs = login?.args?.slice(0, login.args.indexOf("--cli") + 1);
  const rerun = login
    ? runSync([...cliArgs, "--version"], [login.command])
    : { stdout: "", error: "no terminal-auth console-login method" };
  check(
    /\(Claude Code\)$/.test(rerun.stdout) &&
      !login.args.some((arg) => arg.includes("$bunfs") || arg.includes("~BUN")),
    "the terminal-auth login command reruns the agent",
    `${JSON.stringify(login)} -> ${rerun.stdout || rerun.error}`,
  );
}
{
  const agent = startAgent(process.cwd());
  const init = await agent.request("initialize", {
    protocolVersion: 2,
    info: { name: "smoke-binary", version },
    capabilities: {},
  });
  check(
    init.result?.protocolVersion === 2,
    "initialize answers ACP v2",
    JSON.stringify(init.error),
  );
  await agent.stop();
}

// ---------------------------------------------------------------- full

if (values.full) {
  const work = mkdtempSync(join(tmpdir(), "acp-smoke-"));
  try {
    await fullSmoke(work);
  } finally {
    rmSync(work, { recursive: true, force: true });
  }
}

/** A solid red 16x16 PNG, so the model has something to name. */
function redPng() {
  const crcTable = Array.from({ length: 256 }, (_, n) => {
    let c = n;
    for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
    return c >>> 0;
  });
  const crc = (buf) => {
    let c = 0xffffffff;
    for (const byte of buf) c = crcTable[(c ^ byte) & 0xff] ^ (c >>> 8);
    return (c ^ 0xffffffff) >>> 0;
  };
  const chunk = (type, data) => {
    const body = Buffer.concat([Buffer.from(type), data]);
    const out = Buffer.alloc(body.length + 8);
    out.writeUInt32BE(data.length, 0);
    body.copy(out, 4);
    out.writeUInt32BE(crc(body), body.length + 4);
    return out;
  };
  const size = 16;
  const header = Buffer.alloc(13);
  header.writeUInt32BE(size, 0);
  header.writeUInt32BE(size, 4);
  header[8] = 8; // bit depth
  header[9] = 2; // RGB
  const row = Buffer.concat([Buffer.from([0]), Buffer.alloc(size * 3, Buffer.from([255, 0, 0]))]);
  return Buffer.concat([
    Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]),
    chunk("IHDR", header),
    chunk("IDAT", deflateSync(Buffer.concat(Array(size).fill(row)))),
    chunk("IEND", Buffer.alloc(0)),
  ]);
}

/** Writes its results to files so the checks don't depend on the model's reply. */
function searchCommand(token) {
  return [
    `grep -rl "${token}" . > grep.out`,
    'find . -name "*.md" > find.out',
    '{ echo "$CLAUDE_CODE_EXECPATH"; grep --version | head -1; find --version | head -1; } > tools.out',
  ].join("; ");
}

function searchResults(work) {
  const read = (name) => {
    try {
      return readFileSync(join(work, name), "utf8");
    } catch (error) {
      return String(error);
    }
  };
  const [execPath = "", grepVersion = "", findVersion = ""] = read("tools.out").split("\n");
  return { grep: read("grep.out"), find: read("find.out"), execPath, grepVersion, findVersion };
}

async function fullSmoke(work) {
  const token = `SMOKE_TOKEN_${process.pid}`;
  mkdirSync(join(work, "notes"));
  mkdirSync(join(work, "src"));
  writeFileSync(join(work, "notes", "alpha.md"), `# Alpha\n\nThe token is ${token}.\n`);
  writeFileSync(join(work, "notes", "beta.md"), "# Beta\n\nNothing here.\n");
  writeFileSync(join(work, "src", "gamma.ts"), `export const value = "${token}";\n`);
  writeFileSync(join(work, "pixel.png"), redPng());

  const toolCalls = new Map();
  let text = "";
  const onNotification = (message) => {
    if (message.method !== "session/update") return;
    const update = message.params.update;
    if (update.sessionUpdate === "agent_message_chunk" && update.content.type === "text") {
      text += update.content.text;
    } else if (
      update.sessionUpdate === "tool_call" ||
      update.sessionUpdate === "tool_call_update"
    ) {
      const call = toolCalls.get(update.toolCallId) ?? {};
      toolCalls.set(update.toolCallId, {
        ...call,
        name: update._meta?.claudeCode?.toolName ?? call.name,
        status: update.status ?? call.status,
        title: update.title ?? call.title,
      });
    }
  };

  const agent = startAgent(work, onNotification);
  await agent.request("initialize", initializeParams);
  const created = await agent.request("session/new", { cwd: work, mcpServers: [] });
  if (!check(created.result?.sessionId, "session/new", JSON.stringify(created.error))) {
    console.error((await agent.stop()).stderr);
    return;
  }
  const sessionId = created.result.sessionId;

  const started = Date.now();
  const prompt = await agent.request("session/prompt", {
    sessionId,
    prompt: [
      {
        type: "text",
        text: [
          "This is an automated smoke test of your tools. Do each step with the named tool, in order, one tool call per step:",
          `1. Bash: run exactly \`${searchCommand(token)}\``,
          "2. Read: read notes/alpha.md.",
          "3. Read: read pixel.png and note its color.",
          `4. Edit: in notes/alpha.md replace "${token}" with "EDITED".`,
          "5. Bash: run `cat notes/alpha.md`.",
          '6. Agent: launch a subagent (subagent_type "Explore") that counts the .ts files under src/ and reports the number.',
          "Then reply with one line: SMOKE-DONE color=<the color of pixel.png> tsfiles=<the subagent's count>.",
        ].join("\n"),
      },
    ],
  });
  check(
    prompt.result?.stopReason === "end_turn",
    "prompt ends with end_turn",
    JSON.stringify(prompt.result ?? prompt.error),
  );
  console.log(`     (turn took ${Math.round((Date.now() - started) / 1000)}s)`);

  const calls = [...toolCalls.values()];
  if (process.env.SMOKE_DEBUG) console.log(JSON.stringify(calls, null, 1));
  const searched = searchResults(work);
  check(
    searched.grep.includes("alpha.md") && searched.grep.includes("gamma.ts"),
    "grep found the token",
    searched.grep,
  );
  check(
    searched.find.includes("alpha.md") && searched.find.includes("beta.md"),
    "find listed the .md files",
    searched.find,
  );
  check(
    /^ugrep /.test(searched.grepVersion),
    "grep ran Claude's embedded ugrep",
    searched.grepVersion,
  );
  check(/^bfs /.test(searched.findVersion), "find ran Claude's embedded bfs", searched.findVersion);
  console.log(`     (CLAUDE_CODE_EXECPATH=${searched.execPath})`);
  for (const name of ["Read", "Edit", "Bash"]) {
    const used = calls.filter((c) => c.name === name);
    check(
      used.length > 0 && used.every((c) => c.status === "completed"),
      `${name} ran and completed`,
      JSON.stringify(used),
    );
  }
  const subagent = calls.filter((c) => c.name === "Agent" || c.name === "Task");
  check(
    subagent.some((c) => c.status === "completed"),
    "subagent ran and completed",
    JSON.stringify(subagent),
  );
  const failed = calls.filter((c) => c.status === "failed");
  check(failed.length === 0, "no tool call failed", JSON.stringify(failed));
  check(
    readFileSync(join(work, "notes", "alpha.md"), "utf8").includes("EDITED"),
    "Edit changed the file on disk",
  );
  check(
    /SMOKE-DONE/.test(text) && /color=\W*red/i.test(text),
    "image read as red",
    text.slice(-300),
  );
  check(/tsfiles=\W*1\b/.test(text), "subagent counted 1 .ts file", text.slice(-300));

  // A cancel interrupts a running Bash command and ends the turn as cancelled.
  const cancelled = agent.request("session/prompt", {
    sessionId,
    prompt: [
      { type: "text", text: "Use the Bash tool to run `sleep 60`. Do not run anything else." },
    ],
  });
  const deadline = Date.now() + 60_000;
  const bashRunning = () =>
    [...toolCalls.values()].some((c) => c.name === "Bash" && c.status !== "completed");
  while (!bashRunning() && Date.now() < deadline) {
    await new Promise((r) => setTimeout(r, 200));
  }
  agent.notify("session/cancel", { sessionId });
  const cancelResult = await cancelled;
  check(
    cancelResult.result?.stopReason === "cancelled",
    "session/cancel ends the turn as cancelled",
    JSON.stringify(cancelResult.result ?? cancelResult.error),
  );

  const listed = await agent.request("session/list", { cwd: work });
  check(
    listed.result?.sessions?.some((s) => s.sessionId === sessionId),
    "session/list finds the session",
    JSON.stringify(listed.result ?? listed.error),
  );
  const { code, stderr } = await agent.stop();
  check(code === 0, "exits 0 when stdin closes", code);
  if (failures > 0) console.error(stderr);

  // A fresh process, so the replay comes from the transcript on disk.
  let replayed = "";
  const reloaded = startAgent(work, (message) => {
    const update = message.params?.update;
    if (update?.sessionUpdate === "user_message_chunk" && update.content.type === "text")
      replayed += update.content.text;
  });
  await reloaded.request("initialize", initializeParams);
  const loaded = await reloaded.request("session/load", { sessionId, cwd: work, mcpServers: [] });
  check(!loaded.error, "session/load in a new process", JSON.stringify(loaded.error));
  check(replayed.includes(token), "session/load replays the first prompt", replayed.slice(0, 200));

  // Fork and delete go through the SDK's transcript helpers, not the CLI.
  const forked = await reloaded.request("session/fork", { sessionId, cwd: work, mcpServers: [] });
  const forkId = forked.result?.sessionId;
  check(
    forkId && forkId !== sessionId,
    "session/fork",
    JSON.stringify(forked.result ?? forked.error),
  );
  if (forkId) {
    const deleted = await reloaded.request("session/delete", { sessionId: forkId });
    const after = await reloaded.request("session/list", { cwd: work });
    check(
      !deleted.error && !after.result?.sessions?.some((s) => s.sessionId === forkId),
      "session/delete removes the fork",
      JSON.stringify(deleted.error ?? after.result),
    );
  }
  await reloaded.stop();
}

console.log(failures === 0 ? "\nsmoke test passed" : `\n${failures} check(s) failed`);
process.exit(failures === 0 ? 0 : 1);