diff --git a/desktop/playwright.config.ts b/desktop/playwright.config.ts index be9d73ed0a..0469497d8d 100644 --- a/desktop/playwright.config.ts +++ b/desktop/playwright.config.ts @@ -137,6 +137,7 @@ export default defineConfig({ "**/persona-sync.spec.ts", "**/team-snapshot.spec.ts", "**/agents-everywhere.live.spec.ts", + "**/relay-restart.live.spec.ts", "**/parity-ancestor-island.spec.ts", ], use: { diff --git a/desktop/tests/e2e/helpers/twoRelayHarness.ts b/desktop/tests/e2e/helpers/twoRelayHarness.ts index 2eded2d9bc..98acd86398 100644 --- a/desktop/tests/e2e/helpers/twoRelayHarness.ts +++ b/desktop/tests/e2e/helpers/twoRelayHarness.ts @@ -17,7 +17,7 @@ type OwnedProcess = { name: string; child: ChildProcess; logPath: string }; export class TwoRelayHarness { readonly root: string; - readonly relays: readonly [RelaySpec, RelaySpec]; + readonly relays: readonly RelaySpec[]; private readonly processes: OwnedProcess[] = []; get ownedPids(): number[] { @@ -26,12 +26,12 @@ export class TwoRelayHarness { ); } - private constructor(root: string, relays: readonly [RelaySpec, RelaySpec]) { + private constructor(root: string, relays: readonly RelaySpec[]) { this.root = root; this.relays = relays; } - static async create(relays: readonly [RelaySpec, RelaySpec]) { + static async create(relays: readonly RelaySpec[]) { return new TwoRelayHarness( await mkdtemp(join(tmpdir(), "buzz-ae-e2e-")), relays, @@ -46,6 +46,45 @@ export class TwoRelayHarness { ); } + /** + * SIGTERM one relay by name and wait for it to exit on its own. Unlike + * `stop()`, this never escalates to SIGKILL: the relay's graceful drain + * (readiness 503 → 5s grace → 1012 close broadcast → listener drain) is + * exactly what restart tests exercise, so a forced kill would invalidate + * the run. Throws if the process does not exit within `timeoutMs`. + */ + async terminateRelayGracefully(name: string, timeoutMs = 45_000) { + const owned = [...this.processes] + .reverse() + .find( + (candidate) => + candidate.name === name && + candidate.child.exitCode === null && + candidate.child.signalCode === null, + ); + if (!owned) throw new Error(`no live relay process named ${name}`); + const exited = new Promise((resolveExit) => + owned.child.once("exit", () => resolveExit()), + ); + this.signal(owned.child, "SIGTERM"); + if ( + !(await Promise.race([exited.then(() => true), delay(timeoutMs, false)])) + ) { + throw new Error( + `${name} did not exit within ${timeoutMs}ms of SIGTERM — graceful drain is stuck`, + ); + } + } + + /** Start a fresh process for a relay spec on its original ports. */ + async restartRelay(name: string, binary = process.env.BUZZ_E2E_RELAY_BIN) { + if (!binary) + throw new Error("BUZZ_E2E_RELAY_BIN is required for the live gate"); + const relay = this.relays.find((candidate) => candidate.name === name); + if (!relay) throw new Error(`no relay spec named ${name}`); + await this.startRelay(binary, relay); + } + async startAcp( name: string, relayWsUrl: string, diff --git a/desktop/tests/e2e/relay-restart.live.spec.ts b/desktop/tests/e2e/relay-restart.live.spec.ts new file mode 100644 index 0000000000..057ef46f52 --- /dev/null +++ b/desktop/tests/e2e/relay-restart.live.spec.ts @@ -0,0 +1,96 @@ +import { expect, test, type Page } from "@playwright/test"; + +import { installBridge } from "../helpers/bridge"; +import { TwoRelayHarness, type RelaySpec } from "./helpers/twoRelayHarness"; + +// Live gate: boots a REAL buzz-relay process, points the app at it, SIGTERMs +// the relay mid-session, restarts it on the same port, and asserts the client +// converges back to "connected". This proves the full restart story end to +// end: the relay's graceful-drain 1012 close broadcast (server side) and the +// client's dial-failure retry + 1012 fast-reconnect (desktop side) — the two +// halves that synthetic mock-websocket specs cannot compose. +// +// Requires: BUZZ_E2E_RELAY_RESTART=1, BUZZ_E2E_RELAY_BIN, and +// BUZZ_E2E_DATABASE_URL (plus reachable Redis and media object store, same +// infra as the agents-everywhere live gate). +const enabled = process.env.BUZZ_E2E_RELAY_RESTART === "1"; + +function required(name: string, value: string | undefined): string { + if (!value) throw new Error(`${name} is required for the live gate`); + return value; +} + +async function connectionState(page: Page): Promise { + return page.evaluate(() => { + const win = window as Window & { + __BUZZ_E2E_GET_RELAY_CONNECTION_STATE__?: () => string; + }; + return win.__BUZZ_E2E_GET_RELAY_CONNECTION_STATE__?.() ?? "uninstalled"; + }); +} + +test.describe("relay restart live gate", () => { + test.skip(!enabled, "set BUZZ_E2E_RELAY_RESTART=1 to run live gate"); + + test("client reconnects after the relay is SIGTERMed and restarted", async ({ + page, + }) => { + test.setTimeout(180_000); + const portBase = 26_000 + (process.pid % 3_000); + const spec: RelaySpec = { + name: "relay-restart", + ports: { + main: portBase, + health: portBase + 3_000, + metrics: portBase + 6_000, + }, + databaseUrl: required( + "BUZZ_E2E_DATABASE_URL", + process.env.BUZZ_E2E_DATABASE_URL, + ), + redisUrl: + process.env.BUZZ_E2E_REDIS_RESTART ?? "redis://127.0.0.1:6379/13", + }; + const harness = await TwoRelayHarness.create([spec]); + try { + await harness.startRelays(); + + const relayHttpUrl = `http://127.0.0.1:${spec.ports.main}`; + await installBridge(page, { + mode: "relay", + user: "tyler", + relayHttpUrl, + relayWsUrl: `ws://127.0.0.1:${spec.ports.main}`, + }); + await page.goto("/"); + + // Baseline: the app converges to a live authenticated session. + await expect + .poll(() => connectionState(page), { timeout: 60_000 }) + .toBe("connected"); + + // Roll the pod. Graceful drain: readiness 503 → 5s grace → 1012 close + // broadcast → process exit. The client must observe the close (not a + // silent stall) and start retrying. + await harness.terminateRelayGracefully(spec.name); + await expect + .poll(() => connectionState(page), { timeout: 30_000 }) + .not.toBe("connected"); + + // Bring the "new pod" up on the same address, exactly like a k8s + // restart behind a stable service endpoint. + await harness.restartRelay(spec.name); + + // The client's retry loop must find the fresh relay and converge back + // to connected without any user interaction. + await expect + .poll(() => connectionState(page), { timeout: 60_000 }) + .toBe("connected"); + } catch (error) { + console.error(await harness.logs()); + throw error; + } finally { + await harness.stop(); + } + }); +});