fix(gateway): per-run deadline + queue bound on Mutex - 08-12 freeze fix

One chain call that never settled, run under the per-key Mutex with no
deadline, convoyed every subsequent request on that key silently while
/models and /account stayed healthy (2026-08-12 incident). The per-fetch
30s aborts in the Ogmios providers predate the freeze and did not prevent
it; the unbounded thing was the composition under the mutex.

Mutex.run() now races each fn against a deadline (default 180s): on
expiry the run rejects with MutexTimeoutError and the chain advances, so
queued work proceeds instead of freezing. A zombie that settles later can
at worst lose a UTxO-contention race with the run that replaced it -
strictly better than a permanent convoy. New work is refused with
MutexQueueFullError past a queue bound (default 64) instead of queueing
without limit. Serialization and rejection-tolerance semantics are
unchanged (covered by tests). Session locks in transcripts.ts construct
the same class and inherit both protections.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

tests/unit/gateway-mutex.test.ts

@@ -0,0 +1,121 @@
+/**
+ * gateway-mutex.test.ts — Mutex deadline + queue bound (2026-08-12 freeze fix).
+ *
+ * Incident: one chain call that never settled, run under a per-key Mutex with
+ * no deadline, convoyed every subsequent request on that key silently while
+ * /models and /account stayed healthy. The fix gives run() a deadline that
+ * abandons the stuck operation (advancing the chain) and a queue bound that
+ * rejects new work instead of queueing without limit.
+ *
+ * Serialization semantics of the original class must be preserved.
+ */
+
+import { describe, it, expect, vi, afterEach } from "vitest";
+import {
+  Mutex,
+  MutexTimeoutError,
+  MutexQueueFullError,
+} from "../../gateway/src/sdk/registry.js";
+
+afterEach(() => {
+  vi.useRealTimers();
+});
+
+// ─── Preserved semantics ─────────────────────────────────────────────────────
+
+describe("Mutex — serialization (preserved semantics)", () => {
+  it("runs fns strictly one at a time, in order", async () => {
+    const mutex = new Mutex();
+    const order: number[] = [];
+    let firstRunning = false;
+
+    const first = mutex.run(async () => {
+      firstRunning = true;
+      await new Promise((r) => setTimeout(r, 20));
+      order.push(1);
+      firstRunning = false;
+    });
+    const second = mutex.run(async () => {
+      expect(firstRunning).toBe(false);
+      order.push(2);
+    });
+
+    await Promise.all([first, second]);
+    expect(order).toEqual([1, 2]);
+  });
+
+  it("a rejecting fn does not poison the chain", async () => {
+    const mutex = new Mutex();
+
+    await expect(mutex.run(async () => Promise.reject(new Error("boom")))).rejects.toThrow("boom");
+    await expect(mutex.run(async () => "alive")).resolves.toBe("alive");
+  });
+});
+
+// ─── Deadline (the freeze fix) ───────────────────────────────────────────────
+
+describe("Mutex — per-run deadline", () => {
+  it("rejects a stuck run with MutexTimeoutError once the deadline passes", async () => {
+    vi.useFakeTimers();
+    const mutex = new Mutex({ timeoutMs: 1_000 });
+
+    const stuck = mutex.run(() => new Promise<never>(() => {}), "stuck-chain-read");
+    // Attach the rejection expectation BEFORE advancing time so the rejection
+    // is not treated as unhandled.
+    const assertion = expect(stuck).rejects.toBeInstanceOf(MutexTimeoutError);
+
+    await vi.advanceTimersByTimeAsync(1_001);
+    await assertion;
+  });
+
+  it("queued work proceeds after a stuck run is abandoned (no convoy)", async () => {
+    vi.useFakeTimers();
+    const mutex = new Mutex({ timeoutMs: 1_000 });
+
+    const stuck = mutex.run(() => new Promise<never>(() => {}), "stuck");
+    const stuckAssertion = expect(stuck).rejects.toBeInstanceOf(MutexTimeoutError);
+    const queued = mutex.run(async () => "unblocked");
+
+    await vi.advanceTimersByTimeAsync(1_001);
+    await stuckAssertion;
+    await expect(queued).resolves.toBe("unblocked");
+  });
+
+  it("a run that finishes in time is unaffected by the deadline", async () => {
+    vi.useFakeTimers();
+    const mutex = new Mutex({ timeoutMs: 1_000 });
+
+    const quick = mutex.run(async () => "done");
+    await vi.advanceTimersByTimeAsync(1);
+    await expect(quick).resolves.toBe("done");
+  });
+});
+
+// ─── Queue bound ─────────────────────────────────────────────────────────────
+
+describe("Mutex — queue bound", () => {
+  it("rejects immediately with MutexQueueFullError when the queue is full", async () => {
+    const mutex = new Mutex({ maxQueue: 2 });
+    let releaseFirst!: () => void;
+    const gate = new Promise<void>((r) => {
+      releaseFirst = r;
+    });
+
+    const first = mutex.run(() => gate.then(() => "first"));
+    const second = mutex.run(async () => "second");
+    const third = mutex.run(async () => "third");
+
+    await expect(third).rejects.toBeInstanceOf(MutexQueueFullError);
+
+    releaseFirst();
+    await expect(first).resolves.toBe("first");
+    await expect(second).resolves.toBe("second");
+  });
+
+  it("slots free up once runs settle", async () => {
+    const mutex = new Mutex({ maxQueue: 1 });
+
+    await mutex.run(async () => "a");
+    await expect(mutex.run(async () => "b")).resolves.toBe("b");
+  });
+});