test(desktop): live relay kill/restart reconnect gate (#2583)

Signed-off-by: Tyler Longwell <tlongwell@block.xyz>
Co-authored-by: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
Co-authored-by: Tyler Longwell <tlongwell@block.xyz>

desktop/playwright.config.ts

@@ -137,6 +137,7 @@ export default defineConfig({
         "**/persona-sync.spec.ts",
         "**/team-snapshot.spec.ts",
         "**/agents-everywhere.live.spec.ts",
+        "**/relay-restart.live.spec.ts",
         "**/parity-ancestor-island.spec.ts",
       ],
       use: {

desktop/tests/e2e/helpers/twoRelayHarness.ts

@@ -17,7 +17,7 @@ type OwnedProcess = { name: string; child: ChildProcess; logPath: string };
 
 export class TwoRelayHarness {
   readonly root: string;
-  readonly relays: readonly [RelaySpec, RelaySpec];
+  readonly relays: readonly RelaySpec[];
   private readonly processes: OwnedProcess[] = [];
 
   get ownedPids(): number[] {
@@ -26,12 +26,12 @@ export class TwoRelayHarness {
     );
   }
 
-  private constructor(root: string, relays: readonly [RelaySpec, RelaySpec]) {
+  private constructor(root: string, relays: readonly RelaySpec[]) {
     this.root = root;
     this.relays = relays;
   }
 
-  static async create(relays: readonly [RelaySpec, RelaySpec]) {
+  static async create(relays: readonly RelaySpec[]) {
     return new TwoRelayHarness(
       await mkdtemp(join(tmpdir(), "buzz-ae-e2e-")),
       relays,
@@ -46,6 +46,45 @@ export class TwoRelayHarness {
     );
   }
 
+  /**
+   * SIGTERM one relay by name and wait for it to exit on its own. Unlike
+   * `stop()`, this never escalates to SIGKILL: the relay's graceful drain
+   * (readiness 503 → 5s grace → 1012 close broadcast → listener drain) is
+   * exactly what restart tests exercise, so a forced kill would invalidate
+   * the run. Throws if the process does not exit within `timeoutMs`.
+   */
+  async terminateRelayGracefully(name: string, timeoutMs = 45_000) {
+    const owned = [...this.processes]
+      .reverse()
+      .find(
+        (candidate) =>
+          candidate.name === name &&
+          candidate.child.exitCode === null &&
+          candidate.child.signalCode === null,
+      );
+    if (!owned) throw new Error(`no live relay process named ${name}`);
+    const exited = new Promise<void>((resolveExit) =>
+      owned.child.once("exit", () => resolveExit()),
+    );
+    this.signal(owned.child, "SIGTERM");
+    if (
+      !(await Promise.race([exited.then(() => true), delay(timeoutMs, false)]))
+    ) {
+      throw new Error(
+        `${name} did not exit within ${timeoutMs}ms of SIGTERM — graceful drain is stuck`,
+      );
+    }
+  }
+
+  /** Start a fresh process for a relay spec on its original ports. */
+  async restartRelay(name: string, binary = process.env.BUZZ_E2E_RELAY_BIN) {
+    if (!binary)
+      throw new Error("BUZZ_E2E_RELAY_BIN is required for the live gate");
+    const relay = this.relays.find((candidate) => candidate.name === name);
+    if (!relay) throw new Error(`no relay spec named ${name}`);
+    await this.startRelay(binary, relay);
+  }
+
   async startAcp(
     name: string,
     relayWsUrl: string,

desktop/tests/e2e/relay-restart.live.spec.ts

@@ -0,0 +1,96 @@
+import { expect, test, type Page } from "@playwright/test";
+
+import { installBridge } from "../helpers/bridge";
+import { TwoRelayHarness, type RelaySpec } from "./helpers/twoRelayHarness";
+
+// Live gate: boots a REAL buzz-relay process, points the app at it, SIGTERMs
+// the relay mid-session, restarts it on the same port, and asserts the client
+// converges back to "connected". This proves the full restart story end to
+// end: the relay's graceful-drain 1012 close broadcast (server side) and the
+// client's dial-failure retry + 1012 fast-reconnect (desktop side) — the two
+// halves that synthetic mock-websocket specs cannot compose.
+//
+// Requires: BUZZ_E2E_RELAY_RESTART=1, BUZZ_E2E_RELAY_BIN, and
+// BUZZ_E2E_DATABASE_URL (plus reachable Redis and media object store, same
+// infra as the agents-everywhere live gate).
+const enabled = process.env.BUZZ_E2E_RELAY_RESTART === "1";
+
+function required(name: string, value: string | undefined): string {
+  if (!value) throw new Error(`${name} is required for the live gate`);
+  return value;
+}
+
+async function connectionState(page: Page): Promise<string> {
+  return page.evaluate(() => {
+    const win = window as Window & {
+      __BUZZ_E2E_GET_RELAY_CONNECTION_STATE__?: () => string;
+    };
+    return win.__BUZZ_E2E_GET_RELAY_CONNECTION_STATE__?.() ?? "uninstalled";
+  });
+}
+
+test.describe("relay restart live gate", () => {
+  test.skip(!enabled, "set BUZZ_E2E_RELAY_RESTART=1 to run live gate");
+
+  test("client reconnects after the relay is SIGTERMed and restarted", async ({
+    page,
+  }) => {
+    test.setTimeout(180_000);
+    const portBase = 26_000 + (process.pid % 3_000);
+    const spec: RelaySpec = {
+      name: "relay-restart",
+      ports: {
+        main: portBase,
+        health: portBase + 3_000,
+        metrics: portBase + 6_000,
+      },
+      databaseUrl: required(
+        "BUZZ_E2E_DATABASE_URL",
+        process.env.BUZZ_E2E_DATABASE_URL,
+      ),
+      redisUrl:
+        process.env.BUZZ_E2E_REDIS_RESTART ?? "redis://127.0.0.1:6379/13",
+    };
+    const harness = await TwoRelayHarness.create([spec]);
+    try {
+      await harness.startRelays();
+
+      const relayHttpUrl = `http://127.0.0.1:${spec.ports.main}`;
+      await installBridge(page, {
+        mode: "relay",
+        user: "tyler",
+        relayHttpUrl,
+        relayWsUrl: `ws://127.0.0.1:${spec.ports.main}`,
+      });
+      await page.goto("/");
+
+      // Baseline: the app converges to a live authenticated session.
+      await expect
+        .poll(() => connectionState(page), { timeout: 60_000 })
+        .toBe("connected");
+
+      // Roll the pod. Graceful drain: readiness 503 → 5s grace → 1012 close
+      // broadcast → process exit. The client must observe the close (not a
+      // silent stall) and start retrying.
+      await harness.terminateRelayGracefully(spec.name);
+      await expect
+        .poll(() => connectionState(page), { timeout: 30_000 })
+        .not.toBe("connected");
+
+      // Bring the "new pod" up on the same address, exactly like a k8s
+      // restart behind a stable service endpoint.
+      await harness.restartRelay(spec.name);
+
+      // The client's retry loop must find the fresh relay and converge back
+      // to connected without any user interaction.
+      await expect
+        .poll(() => connectionState(page), { timeout: 60_000 })
+        .toBe("connected");
+    } catch (error) {
+      console.error(await harness.logs());
+      throw error;
+    } finally {
+      await harness.stop();
+    }
+  });
+});