test(e2e): parse bootstrap registrations

Signed-off-by: loganj <loganj@squareup.com>

desktop/package.json

@@ -23,7 +23,8 @@
     "test:e2e:release-smoke": "pnpm build:e2e && playwright test --config=playwright.release-smoke.config.ts",
     "test:e2e:report": "playwright show-report",
     "tauri:build": "tauri build",
-    "check:e2e-bootstrap": "node ./scripts/check-e2e-bootstrap.mjs"
+    "check:e2e-bootstrap": "pnpm test:check-e2e-bootstrap && node ./scripts/check-e2e-bootstrap.mjs",
+    "test:check-e2e-bootstrap": "node --test ./scripts/check-e2e-bootstrap.test.mjs"
   },
   "dependencies": {
     "@dnd-kit/core": "^6.3.1",

desktop/scripts/check-e2e-bootstrap.mjs

@@ -1,24 +1,173 @@
 import fs from "node:fs";
 import path from "node:path";
+import ts from "typescript";
 
-const projectRoot = path.resolve(import.meta.dirname, "..");
-const specs = fs
-  .readdirSync(path.join(projectRoot, "tests/e2e"))
-  .filter((name) => /\.(spec|perf)\.ts$/.test(name));
+const SPEC_PATTERN = /\.(spec|perf)\.ts$/;
+const CANONICAL_TEST_HELPER = "../helpers/test";
 const exempt = new Set(["agents-everywhere.live.spec.ts"]);
-const violations = [];
-for (const name of specs) {
-  if (exempt.has(name)) continue;
-  const source = fs.readFileSync(
-    path.join(projectRoot, "tests/e2e", name),
-    "utf8",
+
+function namedImport(declaration, name) {
+  return declaration.importClause?.namedBindings?.elements.some(
+    (element) =>
+      element.name.text === name && element.propertyName === undefined,
   );
-  if (!source.includes('from "../helpers/test"')) continue;
-  if (!/await\s+bootstrapE2ePage\(/.test(source)) violations.push(name);
 }
-if (violations.length) {
-  console.error(
-    `E2E specs must register bootstrapE2ePage after their setup:\n${violations.join("\n")}`,
+
+function callbackOf(call) {
+  const candidate = call.arguments.at(-1);
+  return candidate &&
+    (ts.isArrowFunction(candidate) || ts.isFunctionExpression(candidate))
+    ? candidate
+    : undefined;
+}
+
+function testMember(call) {
+  return ts.isPropertyAccessExpression(call.expression) &&
+    ts.isIdentifier(call.expression.expression) &&
+    call.expression.expression.text === "test"
+    ? call.expression.name.text
+    : undefined;
+}
+
+function isTestRegistration(call) {
+  if (ts.isIdentifier(call.expression)) return call.expression.text === "test";
+  const member = testMember(call);
+  return (
+    member !== undefined &&
+    ![
+      "beforeEach",
+      "afterEach",
+      "beforeAll",
+      "afterAll",
+      "describe",
+      "use",
+      "setTimeout",
+      "slow",
+      "step",
+      "skip",
+      "fixme",
+      "fail",
+    ].includes(member)
   );
-  process.exit(1);
+}
+
+function collectFunctionDeclarations(file) {
+  const functions = new Map();
+  const visit = (node) => {
+    if (ts.isFunctionDeclaration(node) && node.name)
+      functions.set(node.name.text, node);
+    if (
+      ts.isVariableDeclaration(node) &&
+      ts.isIdentifier(node.name) &&
+      node.initializer &&
+      (ts.isArrowFunction(node.initializer) ||
+        ts.isFunctionExpression(node.initializer))
+    ) {
+      functions.set(node.name.text, node.initializer);
+    }
+    ts.forEachChild(node, visit);
+  };
+  visit(file);
+  return functions;
+}
+
+function callbackBootstraps(callback, functions) {
+  const visitedFunctions = new Set();
+  const inspect = (node) => {
+    let found = false;
+    const visit = (child) => {
+      if (
+        ts.isAwaitExpression(child) &&
+        ts.isCallExpression(child.expression) &&
+        ts.isIdentifier(child.expression.expression) &&
+        child.expression.expression.text === "bootstrapE2ePage"
+      ) {
+        found = true;
+        return;
+      }
+      if (ts.isCallExpression(child) && ts.isIdentifier(child.expression)) {
+        const declaration = functions.get(child.expression.text);
+        if (declaration && !visitedFunctions.has(child.expression.text)) {
+          visitedFunctions.add(child.expression.text);
+          if (inspect(declaration.body)) found = true;
+        }
+      }
+      if (!found) ts.forEachChild(child, visit);
+    };
+    ts.forEachChild(node, visit);
+    return found;
+  };
+  return inspect(callback.body);
+}
+
+function scanFile(file, functions) {
+  const hooks = [];
+  const tests = [];
+  const visit = (node) => {
+    if (ts.isCallExpression(node)) {
+      const callback = callbackOf(node);
+      const member = testMember(node);
+      if (member === "beforeEach" && callback)
+        hooks.push(callbackBootstraps(callback, functions));
+      else if (isTestRegistration(node) && callback)
+        tests.push(callbackBootstraps(callback, functions));
+    }
+    ts.forEachChild(node, visit);
+  };
+  visit(file);
+  return { hooks, tests };
+}
+
+export function checkSource(source, filename = "fixture.spec.ts") {
+  const file = ts.createSourceFile(
+    filename,
+    source,
+    ts.ScriptTarget.Latest,
+    true,
+  );
+  const helperImports = file.statements.filter(
+    (statement) =>
+      ts.isImportDeclaration(statement) &&
+      ts.isStringLiteral(statement.moduleSpecifier) &&
+      statement.moduleSpecifier.text === CANONICAL_TEST_HELPER,
+  );
+  if (
+    !helperImports.some((declaration) => namedImport(declaration, "test")) ||
+    !helperImports.some((declaration) =>
+      namedImport(declaration, "bootstrapE2ePage"),
+    )
+  ) {
+    return "must directly import unaliased test and bootstrapE2ePage from ../helpers/test";
+  }
+
+  const { hooks, tests } = scanFile(file, collectFunctionDeclarations(file));
+  if (!tests.length) return "does not register a browser test";
+  if (!hooks.some(Boolean) && !tests.some(Boolean)) {
+    return "must await bootstrapE2ePage on every test path, directly or through a registered test.beforeEach";
+  }
+}
+
+export function runCheck(projectRoot) {
+  const violations = [];
+  for (const name of fs
+    .readdirSync(path.join(projectRoot, "tests/e2e"))
+    .filter((name) => SPEC_PATTERN.test(name))) {
+    if (exempt.has(name)) continue;
+    const violation = checkSource(
+      fs.readFileSync(path.join(projectRoot, "tests/e2e", name), "utf8"),
+      name,
+    );
+    if (violation) violations.push(`${name}: ${violation}`);
+  }
+  return violations;
+}
+
+if (process.argv[1] === import.meta.filename) {
+  const violations = runCheck(path.resolve(import.meta.dirname, ".."));
+  if (violations.length) {
+    console.error(
+      `E2E specs must register bootstrapE2ePage after their setup:\n${violations.join("\n")}`,
+    );
+    process.exit(1);
+  }
 }

desktop/scripts/check-e2e-bootstrap.test.mjs

@@ -0,0 +1,67 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+const projects = [
+  ["desktop", "../scripts/check-e2e-bootstrap.mjs"],
+  ["web", "../scripts/check-e2e-bootstrap.mjs"],
+];
+
+const validDirect = `
+  import { test, bootstrapE2ePage } from "../helpers/test";
+  test("uses bootstrap", async ({ page }) => { await bootstrapE2ePage(page); });
+`;
+const validBeforeEach = `
+  import { test, bootstrapE2ePage } from "../helpers/test";
+  test.beforeEach(async ({ page }) => { await bootstrapE2ePage(page); });
+  test("uses registered hook", async ({ page }) => { await page.title(); });
+`;
+const validBeforeNavigate = `
+  import { test, bootstrapE2ePage } from "../helpers/test";
+  test("draft seed", async ({ page }) => {
+    await bootstrapE2ePage(page, { beforeNavigate: async () => page.addInitScript(() => {}) });
+  });
+`;
+
+for (const [project, checkerPath] of projects) {
+  const { checkSource } = await import(checkerPath);
+  test(`${project}: accepts direct, hook, and beforeNavigate bootstrap patterns`, () => {
+    for (const source of [validDirect, validBeforeEach, validBeforeNavigate]) {
+      assert.equal(checkSource(source), undefined);
+    }
+  });
+
+  for (const [name, source] of [
+    [
+      ".ts helper suffix",
+      `import { test, bootstrapE2ePage } from "../helpers/test.ts"; test("x", async () => {});`,
+    ],
+    [
+      "import alias",
+      `import { test as e2eTest, bootstrapE2ePage } from "../helpers/test"; e2eTest("x", async () => { await bootstrapE2ePage(); });`,
+    ],
+    [
+      "re-export",
+      `import { test, bootstrapE2ePage } from "./reexport"; test("x", async () => { await bootstrapE2ePage(); });`,
+    ],
+    [
+      "dynamic import",
+      `const { test, bootstrapE2ePage } = await import("../helpers/test"); test("x", async () => { await bootstrapE2ePage(); });`,
+    ],
+    [
+      "dead-code call",
+      `import { test, bootstrapE2ePage } from "../helpers/test"; async function never(page) { await bootstrapE2ePage(page); } test("x", async () => {});`,
+    ],
+    [
+      "comment",
+      `import { test, bootstrapE2ePage } from "../helpers/test"; // await bootstrapE2ePage(page)\ntest("x", async () => {});`,
+    ],
+    [
+      "string",
+      `import { test, bootstrapE2ePage } from "../helpers/test"; const hint = "await bootstrapE2ePage(page)"; test("x", async () => {});`,
+    ],
+  ]) {
+    test(`${project}: rejects ${name}`, () => {
+      assert.notEqual(checkSource(source), undefined);
+    });
+  }
+}

desktop/tests/e2e/drafts-all-fix-screenshots.spec.ts

@@ -1,7 +1,7 @@
-import { expect, test } from "../helpers/test";
+import { expect, test, bootstrapE2ePage } from "../helpers/test";
 
 import { waitForAnimations } from "../helpers/animations";
-import { E2E_APP_ORIGIN, bootstrapE2ePage } from "../helpers/bootstrap";
+import { E2E_APP_ORIGIN } from "../helpers/bootstrap";
 import { TEST_IDENTITIES, installMockBridge } from "../helpers/bridge";
 
 const SHOTS = "test-results/drafts-all-fix";

web/package.json

@@ -15,7 +15,8 @@
     "preview": "vite preview",
     "test:e2e": "pnpm build && playwright test",
     "test:e2e:smoke": "pnpm build && playwright test --project=smoke",
-    "check:e2e-bootstrap": "node ./scripts/check-e2e-bootstrap.mjs"
+    "check:e2e-bootstrap": "pnpm test:check-e2e-bootstrap && node ./scripts/check-e2e-bootstrap.mjs",
+    "test:check-e2e-bootstrap": "node --test ./scripts/check-e2e-bootstrap.test.mjs"
   },
   "dependencies": {
     "@fontsource-variable/inter": "^5.2.8",

web/scripts/check-e2e-bootstrap.mjs

@@ -1,24 +1,173 @@
 import fs from "node:fs";
 import path from "node:path";
+import ts from "typescript";
 
-const projectRoot = path.resolve(import.meta.dirname, "..");
-const specs = fs
-  .readdirSync(path.join(projectRoot, "tests/e2e"))
-  .filter((name) => /\.(spec|perf)\.ts$/.test(name));
+const SPEC_PATTERN = /\.(spec|perf)\.ts$/;
+const CANONICAL_TEST_HELPER = "../helpers/test";
 const exempt = new Set(["agents-everywhere.live.spec.ts"]);
-const violations = [];
-for (const name of specs) {
-  if (exempt.has(name)) continue;
-  const source = fs.readFileSync(
-    path.join(projectRoot, "tests/e2e", name),
-    "utf8",
+
+function namedImport(declaration, name) {
+  return declaration.importClause?.namedBindings?.elements.some(
+    (element) =>
+      element.name.text === name && element.propertyName === undefined,
   );
-  if (!source.includes('from "../helpers/test"')) continue;
-  if (!/await\s+bootstrapE2ePage\(/.test(source)) violations.push(name);
 }
-if (violations.length) {
-  console.error(
-    `E2E specs must register bootstrapE2ePage after their setup:\n${violations.join("\n")}`,
+
+function callbackOf(call) {
+  const candidate = call.arguments.at(-1);
+  return candidate &&
+    (ts.isArrowFunction(candidate) || ts.isFunctionExpression(candidate))
+    ? candidate
+    : undefined;
+}
+
+function testMember(call) {
+  return ts.isPropertyAccessExpression(call.expression) &&
+    ts.isIdentifier(call.expression.expression) &&
+    call.expression.expression.text === "test"
+    ? call.expression.name.text
+    : undefined;
+}
+
+function isTestRegistration(call) {
+  if (ts.isIdentifier(call.expression)) return call.expression.text === "test";
+  const member = testMember(call);
+  return (
+    member !== undefined &&
+    ![
+      "beforeEach",
+      "afterEach",
+      "beforeAll",
+      "afterAll",
+      "describe",
+      "use",
+      "setTimeout",
+      "slow",
+      "step",
+      "skip",
+      "fixme",
+      "fail",
+    ].includes(member)
   );
-  process.exit(1);
+}
+
+function collectFunctionDeclarations(file) {
+  const functions = new Map();
+  const visit = (node) => {
+    if (ts.isFunctionDeclaration(node) && node.name)
+      functions.set(node.name.text, node);
+    if (
+      ts.isVariableDeclaration(node) &&
+      ts.isIdentifier(node.name) &&
+      node.initializer &&
+      (ts.isArrowFunction(node.initializer) ||
+        ts.isFunctionExpression(node.initializer))
+    ) {
+      functions.set(node.name.text, node.initializer);
+    }
+    ts.forEachChild(node, visit);
+  };
+  visit(file);
+  return functions;
+}
+
+function callbackBootstraps(callback, functions) {
+  const visitedFunctions = new Set();
+  const inspect = (node) => {
+    let found = false;
+    const visit = (child) => {
+      if (
+        ts.isAwaitExpression(child) &&
+        ts.isCallExpression(child.expression) &&
+        ts.isIdentifier(child.expression.expression) &&
+        child.expression.expression.text === "bootstrapE2ePage"
+      ) {
+        found = true;
+        return;
+      }
+      if (ts.isCallExpression(child) && ts.isIdentifier(child.expression)) {
+        const declaration = functions.get(child.expression.text);
+        if (declaration && !visitedFunctions.has(child.expression.text)) {
+          visitedFunctions.add(child.expression.text);
+          if (inspect(declaration.body)) found = true;
+        }
+      }
+      if (!found) ts.forEachChild(child, visit);
+    };
+    ts.forEachChild(node, visit);
+    return found;
+  };
+  return inspect(callback.body);
+}
+
+function scanFile(file, functions) {
+  const hooks = [];
+  const tests = [];
+  const visit = (node) => {
+    if (ts.isCallExpression(node)) {
+      const callback = callbackOf(node);
+      const member = testMember(node);
+      if (member === "beforeEach" && callback)
+        hooks.push(callbackBootstraps(callback, functions));
+      else if (isTestRegistration(node) && callback)
+        tests.push(callbackBootstraps(callback, functions));
+    }
+    ts.forEachChild(node, visit);
+  };
+  visit(file);
+  return { hooks, tests };
+}
+
+export function checkSource(source, filename = "fixture.spec.ts") {
+  const file = ts.createSourceFile(
+    filename,
+    source,
+    ts.ScriptTarget.Latest,
+    true,
+  );
+  const helperImports = file.statements.filter(
+    (statement) =>
+      ts.isImportDeclaration(statement) &&
+      ts.isStringLiteral(statement.moduleSpecifier) &&
+      statement.moduleSpecifier.text === CANONICAL_TEST_HELPER,
+  );
+  if (
+    !helperImports.some((declaration) => namedImport(declaration, "test")) ||
+    !helperImports.some((declaration) =>
+      namedImport(declaration, "bootstrapE2ePage"),
+    )
+  ) {
+    return "must directly import unaliased test and bootstrapE2ePage from ../helpers/test";
+  }
+
+  const { hooks, tests } = scanFile(file, collectFunctionDeclarations(file));
+  if (!tests.length) return "does not register a browser test";
+  if (!hooks.some(Boolean) && !tests.some(Boolean)) {
+    return "must await bootstrapE2ePage on every test path, directly or through a registered test.beforeEach";
+  }
+}
+
+export function runCheck(projectRoot) {
+  const violations = [];
+  for (const name of fs
+    .readdirSync(path.join(projectRoot, "tests/e2e"))
+    .filter((name) => SPEC_PATTERN.test(name))) {
+    if (exempt.has(name)) continue;
+    const violation = checkSource(
+      fs.readFileSync(path.join(projectRoot, "tests/e2e", name), "utf8"),
+      name,
+    );
+    if (violation) violations.push(`${name}: ${violation}`);
+  }
+  return violations;
+}
+
+if (process.argv[1] === import.meta.filename) {
+  const violations = runCheck(path.resolve(import.meta.dirname, ".."));
+  if (violations.length) {
+    console.error(
+      `E2E specs must register bootstrapE2ePage after their setup:\n${violations.join("\n")}`,
+    );
+    process.exit(1);
+  }
 }

web/scripts/check-e2e-bootstrap.test.mjs

@@ -0,0 +1,67 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+const projects = [
+  ["desktop", "../scripts/check-e2e-bootstrap.mjs"],
+  ["web", "../scripts/check-e2e-bootstrap.mjs"],
+];
+
+const validDirect = `
+  import { test, bootstrapE2ePage } from "../helpers/test";
+  test("uses bootstrap", async ({ page }) => { await bootstrapE2ePage(page); });
+`;
+const validBeforeEach = `
+  import { test, bootstrapE2ePage } from "../helpers/test";
+  test.beforeEach(async ({ page }) => { await bootstrapE2ePage(page); });
+  test("uses registered hook", async ({ page }) => { await page.title(); });
+`;
+const validBeforeNavigate = `
+  import { test, bootstrapE2ePage } from "../helpers/test";
+  test("draft seed", async ({ page }) => {
+    await bootstrapE2ePage(page, { beforeNavigate: async () => page.addInitScript(() => {}) });
+  });
+`;
+
+for (const [project, checkerPath] of projects) {
+  const { checkSource } = await import(checkerPath);
+  test(`${project}: accepts direct, hook, and beforeNavigate bootstrap patterns`, () => {
+    for (const source of [validDirect, validBeforeEach, validBeforeNavigate]) {
+      assert.equal(checkSource(source), undefined);
+    }
+  });
+
+  for (const [name, source] of [
+    [
+      ".ts helper suffix",
+      `import { test, bootstrapE2ePage } from "../helpers/test.ts"; test("x", async () => {});`,
+    ],
+    [
+      "import alias",
+      `import { test as e2eTest, bootstrapE2ePage } from "../helpers/test"; e2eTest("x", async () => { await bootstrapE2ePage(); });`,
+    ],
+    [
+      "re-export",
+      `import { test, bootstrapE2ePage } from "./reexport"; test("x", async () => { await bootstrapE2ePage(); });`,
+    ],
+    [
+      "dynamic import",
+      `const { test, bootstrapE2ePage } = await import("../helpers/test"); test("x", async () => { await bootstrapE2ePage(); });`,
+    ],
+    [
+      "dead-code call",
+      `import { test, bootstrapE2ePage } from "../helpers/test"; async function never(page) { await bootstrapE2ePage(page); } test("x", async () => {});`,
+    ],
+    [
+      "comment",
+      `import { test, bootstrapE2ePage } from "../helpers/test"; // await bootstrapE2ePage(page)\ntest("x", async () => {});`,
+    ],
+    [
+      "string",
+      `import { test, bootstrapE2ePage } from "../helpers/test"; const hint = "await bootstrapE2ePage(page)"; test("x", async () => {});`,
+    ],
+  ]) {
+    test(`${project}: rejects ${name}`, () => {
+      assert.notEqual(checkSource(source), undefined);
+    });
+  }
+}