feat(deploy): add HF-router Llama-3.3-70B mainnet supplier (hf-llama33)

OpenAI-compatible supplier backed by the HuggingFace Inference Providers router
(meta-llama/Llama-3.3-70B-Instruct, llm.text.generate.v1). Config only; mirrors
the glm one-off template with OPENAI_BASE_URL=https://router.huggingface.co.

- deploy/mainnet/docker-compose.supplier-hf-llama33.yml
- register the supplier wallet in monitor-wallets OPERATOR_SOURCES

The env file (supplier/.env.hf-llama33) holds the wallet key + HF token and is
never committed; wallet-monitor/wallets.json is gitignored runtime data.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

buyer/scripts/monitor-wallets.ts

@@ -41,6 +41,13 @@ interface OperatorSource {
 }
 
 const OPERATOR_SOURCES: OperatorSource[] = [
+  {
+    label: "supplier-hf-llama33@vector-marketplace",
+    host: "root@91.98.147.172",
+    envPath: "/root/agents-marketplace/supplier/.env.hf-llama33",
+    keyVar: "SUPPLIER_PRIV_KEY_HEX",
+    role: "supplier",
+  },
   {
     label: "buyer@vector-marketplace",
     host: "root@91.98.147.172",

deploy/mainnet/docker-compose.supplier-hf-llama33.yml

@@ -0,0 +1,92 @@
+# deploy/mainnet/docker-compose.supplier-hf-llama33.yml — Supplier — MAINNET
+#   (HuggingFace Inference Providers router / meta-llama/Llama-3.3-70B-Instruct
+#    — one-off `llm.text.generate.v1`)
+#
+# HF-router-backed supplier. LLM_BACKEND=openai pointed at the HuggingFace
+# Inference Providers router (OpenAI-compatible at https://router.huggingface.co).
+# No local proxy container — reachable over public HTTPS. Direct analogue of the
+# OpenRouter/Hetzner-backed suppliers; the router is OpenAI-compatible so this is
+# config only (V1 HF integration).
+#
+# Pre-flight:
+#   1. DNS: mp-suppliers-hf-llama33.vector.apexfusion.org → host
+#      (individual DNS-only A record → 91.98.147.172; NO wildcard exists —
+#       letsencrypt HTTP-01 issuance needs the unproxied direct record)
+#   2. supplier/.env.hf-llama33 written on host (chmod 600) with at minimum:
+#        SUPPLIER_PRIV_KEY_HEX, SUPPLIER_ADDRESS, SUPPLIER_PKH,
+#        SUPPLIER_PUB_KEY_HEX, OPENAI_API_KEY (hf_ token), OGMIOS_URL,
+#        NETWORK_ID=1, VECTOR_ZERO_TIME_MS, ESCROW_REF_UTXO, ADVERT_REF_UTXO,
+#        ADVERT_REF, LIVE_CHAIN=1
+#      Advert posted with --capability-id llm.text.generate.v1.
+#      Do NOT set OPENAI_REASONING — the HF router rejects it (HTTP 400) and the
+#      supplier's config guard refuses to boot.
+#   3. Networks already exist (created by the canonical supplier compose):
+#        docker network create marketplace-mainnet-net   # one-shot
+#        # traefik-net is created by the Traefik project
+
+name: marketplace-mainnet-supplier-hf-llama33
+
+services:
+  supplier:
+    build:
+      context: ../..
+      dockerfile: supplier/Dockerfile
+    container_name: marketplace-mainnet-supplier-hf-llama33
+    restart: unless-stopped
+    env_file:
+      # Self-contained env for this HF-router-backed variant. Holds this
+      # supplier's distinct SUPPLIER_PRIV_KEY_HEX (+ derived wallet vars),
+      # ADVERT_REF, the HF OPENAI_API_KEY, and the shared mainnet plumbing
+      # duplicated in full (NETWORK_ID, OGMIOS_URL, VECTOR_ZERO_TIME_MS,
+      # ref-script UTxOs, LIVE_CHAIN). No env_file ordering / override magic.
+      - path: ../../supplier/.env.hf-llama33
+        required: false
+    environment:
+      LLM_BACKEND: openai
+      # No /v1 suffix — supplier/src/openai.ts appends /v1/chat/completions
+      # (HF router serves at https://router.huggingface.co/v1/...).
+      OPENAI_BASE_URL: https://router.huggingface.co
+      # First-token latency across hosted providers can exceed local backends;
+      # kept ≤ the advert's max_processing_ms (300000) per fleet footgun #4.
+      OPENAI_TIMEOUT_MS: "300000"
+      # OPENAI_MAX_TOKENS intentionally unset: no max_tokens forwarded upstream,
+      # the provider grants context − prompt. Advert cap = model context length.
+      PORT: "8080"
+      # Enable LiveOgmiosProvider so submitTx/awaitTx actually broadcast.
+      LIVE_CHAIN: "1"
+      # NETWORK_ID and OGMIOS_URL intentionally not pinned here — they MUST
+      # come from the operator-provided supplier/.env.hf-llama33 on a mainnet host.
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://localhost:8080/healthz >/dev/null || exit 1"]
+      interval: 20s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
+    networks:
+      - marketplace-mainnet-net
+      - traefik-net
+    labels:
+      - "traefik.enable=true"
+      - "traefik.docker.network=traefik-net"
+      - "traefik.http.routers.mp-suppliers-hf-llama33-mainnet.entrypoints=web"
+      - "traefik.http.routers.mp-suppliers-hf-llama33-mainnet.rule=Host(`mp-suppliers-hf-llama33.vector.apexfusion.org`)"
+      - "traefik.http.routers.mp-suppliers-hf-llama33-mainnet.middlewares=mp-suppliers-hf-llama33-mainnet-https"
+      - "traefik.http.middlewares.mp-suppliers-hf-llama33-mainnet-https.redirectscheme.scheme=https"
+      - "traefik.http.middlewares.mp-suppliers-hf-llama33-mainnet-https.redirectscheme.permanent=true"
+      - "traefik.http.routers.mp-suppliers-hf-llama33-mainnet-tls.entrypoints=websecure"
+      - "traefik.http.routers.mp-suppliers-hf-llama33-mainnet-tls.rule=Host(`mp-suppliers-hf-llama33.vector.apexfusion.org`)"
+      - "traefik.http.routers.mp-suppliers-hf-llama33-mainnet-tls.tls.certresolver=letsencrypt"
+      - "traefik.http.services.mp-suppliers-hf-llama33-mainnet.loadbalancer.server.port=8080"
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+
+networks:
+  marketplace-mainnet-net:
+    external: true
+    name: marketplace-mainnet-net
+  traefik-net:
+    external: true
+    name: traefik-net

gateway/src/account/routes.ts

@@ -19,6 +19,26 @@ import { badRequest, forbidden } from "../openai/errors.js";
 
 const ACTIVE_ESCROW_STATES = new Set(["Open", "Claimed", "Submitted"]);
 
+/** Bound on per-request escrow liveness lookups; rows past the cap count as
+ * locked unverified (a key with this many active escrows has bigger problems). */
+const LIVENESS_CHECK_CAP = 32;
+
+/** True when the escrow UTxO behind an indexer row is still unspent. The
+ * indexer never updates rows on terminal spends (Accept/Reclaim/Release
+ * consume the escrow without a new escrow output), so "active" rows can be
+ * long spent. Unparseable refs and failed lookups count as live so errors
+ * inflate the locked figure rather than hide funds. */
+async function escrowUtxoIsLive(deps: GatewayDeps, utxoRef: string): Promise<boolean> {
+  const m = /^([0-9a-fA-F]{64})#(\d+)$/.exec(utxoRef);
+  if (!m) return true;
+  try {
+    const utxo = await deps.chain.queryUtxo({ txHash: m[1], index: Number(m[2]) });
+    return utxo !== null;
+  } catch {
+    return true;
+  }
+}
+
 function ap3x(lovelace: bigint): string {
   return (Number(lovelace) / 1e6).toFixed(2);
 }
@@ -74,8 +94,15 @@ export function makeAccountHandler(deps: GatewayDeps) {
       if (r.ok) {
         const rows = (await r.json()) as Array<Record<string, unknown>>;
         if (Array.isArray(rows)) {
-          for (const row of rows) {
-            if (!ACTIVE_ESCROW_STATES.has(String(row.state))) continue;
+          const active = rows.filter((row) => ACTIVE_ESCROW_STATES.has(String(row.state)));
+          const liveness = await Promise.all(
+            active.map((row, i) =>
+              i < LIVENESS_CHECK_CAP ? escrowUtxoIsLive(deps, String(row.utxo_ref ?? "")) : Promise.resolve(true),
+            ),
+          );
+          for (let i = 0; i < active.length; i++) {
+            if (!liveness[i]) continue;
+            const row = active[i];
             lockedInEscrow +=
               BigInt(String(row.payment_lovelace ?? "0")) +
               BigInt(String(row.buyer_bond_lovelace ?? "0")) +

gateway/src/sdk/registry.ts

@@ -15,19 +15,88 @@ import { open as unseal } from "../crypto/seal.js";
 import { deriveWalletKey } from "../wallet.js";
 import type { ApiKeyRow } from "../db/store.js";
 
-/** Tiny promise-chain mutex: run(fn) executes fns strictly one at a time. */
+/** Default per-run deadline. Chain reads and tx submits settle in seconds;
+ * anything past this is the 2026-08-12 freeze mode (a fetch that never
+ * settles), not a slow success. */
+export const DEFAULT_MUTEX_TIMEOUT_MS = 180_000;
+
+/** Default queue bound per mutex. A healthy key drains its queue in seconds;
+ * a queue this deep means the head is stuck and callers should fail fast. */
+export const DEFAULT_MUTEX_MAX_QUEUE = 64;
+
+/** A run() exceeded its deadline and was abandoned so queued work can proceed. */
+export class MutexTimeoutError extends Error {
+  constructor(label: string, timeoutMs: number) {
+    super(`mutex run '${label}' exceeded ${timeoutMs}ms and was abandoned; queued work proceeds`);
+    this.name = "MutexTimeoutError";
+  }
+}
+
+/** run() was refused because the queue is at its bound (head likely stuck). */
+export class MutexQueueFullError extends Error {
+  constructor(label: string, depth: number) {
+    super(`mutex queue full (${depth} pending) — refusing '${label}'; the head of the queue is likely stuck`);
+    this.name = "MutexQueueFullError";
+  }
+}
+
+function runWithDeadline<T>(fn: () => Promise<T>, timeoutMs: number, label: string): Promise<T> {
+  return new Promise<T>((resolvePromise, rejectPromise) => {
+    const timer = setTimeout(() => rejectPromise(new MutexTimeoutError(label, timeoutMs)), timeoutMs);
+    // Node-only nicety: don't hold the process open for a pending deadline.
+    timer.unref?.();
+    fn().then(
+      (v) => {
+        clearTimeout(timer);
+        resolvePromise(v);
+      },
+      (e) => {
+        clearTimeout(timer);
+        rejectPromise(e);
+      },
+    );
+  });
+}
+
+/**
+ * Tiny promise-chain mutex: run(fn) executes fns strictly one at a time —
+ * now with a per-run deadline and a queue bound (2026-08-12 freeze fix).
+ *
+ * On deadline the stuck fn is NOT cancelled (promises aren't); it is
+ * abandoned: its run() rejects with MutexTimeoutError and the chain advances.
+ * A zombie that settles later can at worst lose a UTxO-contention race with
+ * the run that replaced it — strictly better than the alternative, where one
+ * stuck chain call convoys every subsequent request on the key forever.
+ */
 export class Mutex {
   private tail: Promise<unknown> = Promise.resolve();
+  private depth = 0;
 
-  run<T>(fn: () => Promise<T>): Promise<T> {
-    const result = this.tail.then(fn, fn);
-    // Keep the chain alive even if fn rejects; swallow here so the next
-    // run() isn't poisoned by a prior rejection.
-    this.tail = result.then(
+  constructor(private readonly opts: { timeoutMs?: number; maxQueue?: number } = {}) {}
+
+  run<T>(fn: () => Promise<T>, label = "op"): Promise<T> {
+    const timeoutMs = this.opts.timeoutMs ?? DEFAULT_MUTEX_TIMEOUT_MS;
+    const maxQueue = this.opts.maxQueue ?? DEFAULT_MUTEX_MAX_QUEUE;
+
+    if (this.depth >= maxQueue) {
+      return Promise.reject(new MutexQueueFullError(label, this.depth));
+    }
+    this.depth++;
+
+    // Keep the chain alive even if fn rejects or times out; swallow here so
+    // the next run() isn't poisoned by a prior rejection.
+    const result = this.tail.then(
+      () => runWithDeadline(fn, timeoutMs, label),
+      () => runWithDeadline(fn, timeoutMs, label),
+    );
+    const settled = result.finally(() => {
+      this.depth--;
+    });
+    this.tail = settled.then(
       () => undefined,
       () => undefined,
     );
-    return result;
+    return settled;
   }
 }
 

tests/unit/gateway-account-locked.test.ts

@@ -0,0 +1,132 @@
+/**
+ * gateway-account-locked.test.ts — /account locked_in_escrow_lovelace accuracy.
+ *
+ * The indexer never updates escrow rows on terminal spends (Accept/Reclaim/
+ * Release consume the UTxO without producing a new escrow output), so rows
+ * can sit in an "active" state long after the funds moved. The account
+ * handler must verify each active row's UTxO is still unspent on-chain
+ * before counting it as locked; a spent ref is a stale row, not locked funds.
+ *
+ * Note: supplier_bond stays in the sum on purpose — the buyer fronts
+ * payment + buyer_bond + supplier_bond at PostEscrow (see postEscrow.ts
+ * totalLocked), so all three legs are the key's money while the escrow lives.
+ */
+
+import { describe, it, expect, vi } from "vitest";
+import type { Request, Response } from "express";
+import { makeAccountHandler } from "../../gateway/src/account/routes.js";
+
+const LIVE_REF = "a".repeat(64) + "#0";
+const STALE_REF = "b".repeat(64) + "#1";
+
+function escrowRow(utxoRef: string, state: string) {
+  return {
+    utxo_ref: utxoRef,
+    state,
+    payment_lovelace: "2000000",
+    buyer_bond_lovelace: "1000000",
+    supplier_bond_lovelace: "1000000",
+  };
+}
+
+function fakeKeyRow() {
+  return {
+    id: "key-1",
+    key_hash: "h",
+    key_prefix: "vk_live_abc",
+    label: null,
+    wallet_pkh: "c".repeat(56),
+    deposit_address: "addr1_deposit",
+    enc_priv_nonce: "",
+    enc_priv_ct: "",
+    enc_priv_tag: "",
+    master_key_version: 1,
+    created_at: 0,
+    disabled: 0,
+    demo: 0,
+  };
+}
+
+function makeDeps(rows: Array<Record<string, unknown>>, opts?: { queryUtxo?: (ref: { txHash: string; index: number }) => Promise<unknown> }) {
+  const queryUtxo =
+    opts?.queryUtxo ??
+    (async (ref: { txHash: string; index: number }) => {
+      const canonical = `${ref.txHash}#${ref.index}`;
+      return canonical === LIVE_REF ? { ref, lovelace: 4_000_000n } : null;
+    });
+  return {
+    config: { indexerUrl: "http://indexer.test", networkId: 1, masterKeyHex: "00" },
+    store: {
+      sumCostLovelace: () => 0n,
+      countUsage: () => 0,
+      listUsage: () => [],
+    },
+    chain: {
+      queryUtxosByAddress: async () => [],
+      queryUtxo: vi.fn(queryUtxo),
+    },
+    registry: {},
+    fetchFn: vi.fn(async () => ({
+      ok: true,
+      json: async () => rows,
+    })),
+  } as never;
+}
+
+/**
+ * asyncHandler returns a fire-and-forget (req, res) => void wrapper, so the
+ * test must await the res.json() call itself, not the handler invocation.
+ * Errors also surface through json() (sendError maps them to a JSON body).
+ */
+async function callAccount(deps: never): Promise<Record<string, never>> {
+  const handler = makeAccountHandler(deps);
+  const req = { gatewayKey: fakeKeyRow() } as unknown as Request;
+  const body = await new Promise<unknown>((resolveBody, rejectBody) => {
+    const timer = setTimeout(() => rejectBody(new Error("handler never responded")), 2_000);
+    const res = {
+      status: (_code: number) => res,
+      json: (b: unknown) => {
+        clearTimeout(timer);
+        resolveBody(b);
+        return res;
+      },
+    } as unknown as Response;
+    handler(req, res);
+  });
+  if ((body as { error?: unknown }).error) {
+    throw new Error(`handler returned error body: ${JSON.stringify(body)}`);
+  }
+  return body as Record<string, never>;
+}
+
+describe("GET /account — locked_in_escrow_lovelace", () => {
+  it("counts payment + both bonds for an active escrow whose UTxO is live", async () => {
+    const body = await callAccount(makeDeps([escrowRow(LIVE_REF, "Open")]));
+    expect(body.balance.locked_in_escrow_lovelace).toBe("4000000");
+  });
+
+  it("excludes an active-state row whose UTxO is terminally spent (stale indexer row)", async () => {
+    const body = await callAccount(
+      makeDeps([escrowRow(LIVE_REF, "Open"), escrowRow(STALE_REF, "Submitted")]),
+    );
+    // Only the live escrow counts: 4_000_000, not 8_000_000.
+    expect(body.balance.locked_in_escrow_lovelace).toBe("4000000");
+  });
+
+  it("skips terminal-state rows without a chain lookup", async () => {
+    const deps = makeDeps([escrowRow(LIVE_REF, "Accepted"), escrowRow(STALE_REF, "Reclaimed")]);
+    const body = await callAccount(deps);
+    expect(body.balance.locked_in_escrow_lovelace).toBe("0");
+    expect((deps as { chain: { queryUtxo: ReturnType<typeof vi.fn> } }).chain.queryUtxo).not.toHaveBeenCalled();
+  });
+
+  it("counts a row conservatively when the chain lookup fails (best-effort)", async () => {
+    const deps = makeDeps([escrowRow(LIVE_REF, "Open")], {
+      queryUtxo: async () => {
+        throw new Error("ogmios unreachable");
+      },
+    });
+    const body = await callAccount(deps);
+    expect(body.balance.locked_in_escrow_lovelace).toBe("4000000");
+  });
+});

tests/unit/gateway-mutex.test.ts

@@ -0,0 +1,121 @@
+/**
+ * gateway-mutex.test.ts — Mutex deadline + queue bound (2026-08-12 freeze fix).
+ *
+ * Incident: one chain call that never settled, run under a per-key Mutex with
+ * no deadline, convoyed every subsequent request on that key silently while
+ * /models and /account stayed healthy. The fix gives run() a deadline that
+ * abandons the stuck operation (advancing the chain) and a queue bound that
+ * rejects new work instead of queueing without limit.
+ *
+ * Serialization semantics of the original class must be preserved.
+ */
+
+import { describe, it, expect, vi, afterEach } from "vitest";
+import {
+  Mutex,
+  MutexTimeoutError,
+  MutexQueueFullError,
+} from "../../gateway/src/sdk/registry.js";
+
+afterEach(() => {
+  vi.useRealTimers();
+});
+
+// ─── Preserved semantics ─────────────────────────────────────────────────────
+
+describe("Mutex — serialization (preserved semantics)", () => {
+  it("runs fns strictly one at a time, in order", async () => {
+    const mutex = new Mutex();
+    const order: number[] = [];
+    let firstRunning = false;
+
+    const first = mutex.run(async () => {
+      firstRunning = true;
+      await new Promise((r) => setTimeout(r, 20));
+      order.push(1);
+      firstRunning = false;
+    });
+    const second = mutex.run(async () => {
+      expect(firstRunning).toBe(false);
+      order.push(2);
+    });
+
+    await Promise.all([first, second]);
+    expect(order).toEqual([1, 2]);
+  });
+
+  it("a rejecting fn does not poison the chain", async () => {
+    const mutex = new Mutex();
+
+    await expect(mutex.run(async () => Promise.reject(new Error("boom")))).rejects.toThrow("boom");
+    await expect(mutex.run(async () => "alive")).resolves.toBe("alive");
+  });
+});
+
+// ─── Deadline (the freeze fix) ───────────────────────────────────────────────
+
+describe("Mutex — per-run deadline", () => {
+  it("rejects a stuck run with MutexTimeoutError once the deadline passes", async () => {
+    vi.useFakeTimers();
+    const mutex = new Mutex({ timeoutMs: 1_000 });
+
+    const stuck = mutex.run(() => new Promise<never>(() => {}), "stuck-chain-read");
+    // Attach the rejection expectation BEFORE advancing time so the rejection
+    // is not treated as unhandled.
+    const assertion = expect(stuck).rejects.toBeInstanceOf(MutexTimeoutError);
+
+    await vi.advanceTimersByTimeAsync(1_001);
+    await assertion;
+  });
+
+  it("queued work proceeds after a stuck run is abandoned (no convoy)", async () => {
+    vi.useFakeTimers();
+    const mutex = new Mutex({ timeoutMs: 1_000 });
+
+    const stuck = mutex.run(() => new Promise<never>(() => {}), "stuck");
+    const stuckAssertion = expect(stuck).rejects.toBeInstanceOf(MutexTimeoutError);
+    const queued = mutex.run(async () => "unblocked");
+
+    await vi.advanceTimersByTimeAsync(1_001);
+    await stuckAssertion;
+    await expect(queued).resolves.toBe("unblocked");
+  });
+
+  it("a run that finishes in time is unaffected by the deadline", async () => {
+    vi.useFakeTimers();
+    const mutex = new Mutex({ timeoutMs: 1_000 });
+
+    const quick = mutex.run(async () => "done");
+    await vi.advanceTimersByTimeAsync(1);
+    await expect(quick).resolves.toBe("done");
+  });
+});
+
+// ─── Queue bound ─────────────────────────────────────────────────────────────
+
+describe("Mutex — queue bound", () => {
+  it("rejects immediately with MutexQueueFullError when the queue is full", async () => {
+    const mutex = new Mutex({ maxQueue: 2 });
+    let releaseFirst!: () => void;
+    const gate = new Promise<void>((r) => {
+      releaseFirst = r;
+    });
+
+    const first = mutex.run(() => gate.then(() => "first"));
+    const second = mutex.run(async () => "second");
+    const third = mutex.run(async () => "third");
+
+    await expect(third).rejects.toBeInstanceOf(MutexQueueFullError);
+
+    releaseFirst();
+    await expect(first).resolves.toBe("first");
+    await expect(second).resolves.toBe("second");
+  });
+
+  it("slots free up once runs settle", async () => {
+    const mutex = new Mutex({ maxQueue: 1 });
+
+    await mutex.run(async () => "a");
+    await expect(mutex.run(async () => "b")).resolves.toBe("b");
+  });
+});