M1-F: end-to-end LLM marketplace lifecycle on Vector testnet

Bonded-escrow marketplace for prompt → response between buyers and
local-LLM suppliers, settling on Vector (UTxO) via Aiken Plutus V3
validators. Lifecycle PostEscrow → Claim → Submit → Accept proven on
Vector testnet 2026-05-06; AcceptEscrow tx 125e3cfe…7dc100d distributes
payment + bonds per the validator's distribution rule.

Workspace layout (pnpm + TypeScript):
  contracts/marketplace/      Aiken validators (advert + escrow) + plutus.json
  packages/shared/            Tx-builders (lucid-evolution), CBOR codecs,
                              chain providers (mock / read-only / live Ogmios)
  supplier/                   OpenAI-compatible HTTP node (Ollama by default;
                              vLLM / llama.cpp / OpenRouter / OpenClaw all
                              swap-in via OLLAMA_URL); 202+poll chat handler
  buyer/                      Express + React SPA; server-side runs the full
                              chain lifecycle via /v1/submit-prompt and
                              /v1/accept; SPA never holds the buyer key
  indexer/                    Chain-follower (extends apex-dashboard pattern),
                              SSE event stream + REST queries
  indexer-ui/                 Operator dashboard (events + suppliers + escrow
                              lookup)
  tests/                      1098 vitest unit + integration tests (10 skipped
                              under M1-F-D-defer for UPLC test plumbing)
  deploy/testnet/             Per-service docker-compose for the Hetzner stack
  docs/EXECUTIVE_SUMMARY.*    Marp deck (PDF + PPTX) — 11 slides, mermaid
                              diagrams pre-rendered for portable export
  docs/CLOUDFLARE_WAF_BYPASS.md  CF Skip-rule runbook for /v1/* paths
  docs/ARCHITECTURE.md        canonical spec

Live demo (testnet, 2026-05-06):
  https://mp-buyers.vector.testnet.apexfusion.org      (buyer SPA)
  https://mp-suppliers.vector.testnet.apexfusion.org   (supplier API)
  https://mp-indexer.vector.testnet.apexfusion.org     (indexer + UI)
This commit is contained in:
jelisaveta-talbot
2026-05-06 13:27:48 +02:00
commit cbab81efbd
323 changed files with 49725 additions and 0 deletions

104
indexer/src/config.ts Normal file
View File

@@ -0,0 +1,104 @@
/**
* indexer/src/config.ts — environment loading for the indexer.
*
* loadConfig(env) is a PURE function: it reads from the supplied env map,
* validates required fields, and returns a typed config object.
* This makes it trivially testable without process.env mutation.
*
* Required env vars:
* OGMIOS_URL — WebSocket URL for Ogmios (e.g. ws://localhost:1337)
* DB_PATH — path to the SQLite database file
* NETWORK_ID — "0" (Vector testnet) | "1" (Vector mainnet)
*
* SPEC FIX 2026-04-27: NETWORK_ID standardized across all services to the
* numeric string form "0"|"1" (mapped to NetworkId 0|1). This aligns the
* indexer with supplier/buyer (previous indexer-side drift used the labels
* "testnet"/"mainnet"). Legacy values are now rejected.
*
* Optional env vars (with defaults):
* INDEXER_PORT — HTTP port (default 8090)
* STATUS_POLL_MS — status poll interval in ms (default 20000)
* SKIP_BEFORE_SLOT — skip blocks before this slot (default 0)
*/
import type { NetworkId } from "@marketplace/shared";
export interface IndexerConfig {
ogmiosUrl: string;
dbPath: string;
networkId: NetworkId;
indexerPort: number;
statusPollMs: number;
skipBeforeSlot: number;
/**
* Optional path to the bundled indexer-ui dist directory. When set, the
* Express app mounts static assets + SPA catch-all so the same process
* serves both API and dashboard. When unset, the indexer is API-only.
* NOT validated at config-load time — directory existence is checked at
* server startup (or implicitly by express.static / sendFile).
*/
uiDistDir?: string;
}
function parseNonNegativeInt(name: string, raw: string): number {
const n = Number(raw);
if (!Number.isFinite(n) || !Number.isInteger(n) || n < 0) {
throw new Error(`loadConfig: ${name} must be a non-negative integer, got ${JSON.stringify(raw)}`);
}
return n;
}
export function loadConfig(env: Record<string, string | undefined>): IndexerConfig {
const ogmiosUrl = env.OGMIOS_URL;
const dbPath = env.DB_PATH;
const networkIdRaw = env.NETWORK_ID;
if (!ogmiosUrl) {
throw new Error("loadConfig: OGMIOS_URL is required");
}
if (!dbPath) {
throw new Error("loadConfig: DB_PATH is required");
}
if (!networkIdRaw) {
throw new Error("loadConfig: NETWORK_ID is required");
}
let networkId: NetworkId;
if (networkIdRaw === "0") {
networkId = 0;
} else if (networkIdRaw === "1") {
networkId = 1;
} else {
throw new Error(
`loadConfig: NETWORK_ID must be "0" (Vector testnet) or "1" (Vector mainnet), ` +
`got ${JSON.stringify(networkIdRaw)}`,
);
}
const indexerPort = env.INDEXER_PORT !== undefined
? parseNonNegativeInt("INDEXER_PORT", env.INDEXER_PORT)
: 8090;
const statusPollMs = env.STATUS_POLL_MS !== undefined
? parseNonNegativeInt("STATUS_POLL_MS", env.STATUS_POLL_MS)
: 20_000;
const skipBeforeSlot = env.SKIP_BEFORE_SLOT !== undefined
? parseNonNegativeInt("SKIP_BEFORE_SLOT", env.SKIP_BEFORE_SLOT)
: 0;
// Optional INDEXER_UI_DIST: any non-empty string is accepted; directory
// existence is checked at startup, not here. Absent → undefined (API-only).
const uiDistDir = typeof env.INDEXER_UI_DIST === "string" && env.INDEXER_UI_DIST.length > 0
? env.INDEXER_UI_DIST
: undefined;
return {
ogmiosUrl,
dbPath,
networkId,
indexerPort,
statusPollMs,
skipBeforeSlot,
uiDistDir,
};
}

315
indexer/src/db/cache.ts Normal file
View File

@@ -0,0 +1,315 @@
/**
* indexer/src/db/cache.ts — SqliteCache for the indexer.
*
* Wraps better-sqlite3 with domain-specific methods for advertisements,
* escrows, supplier_status, events, cursor, and UTxOs.
*
* Rollback semantics (apex-dashboard pattern):
* - Events: soft-delete (rolled_back = 1) for events after rollback slot
* - UTxOs: delete created-after, restore spent-after
*
* WAL mode is enabled on open for concurrent readers.
*/
import Database from "better-sqlite3";
import fs from "fs";
import { applySchema } from "./schema.js";
export interface AdvertRow {
utxo_ref: string; // "<txHash>#<index>"
supplier_pkh: string;
capability_id: string;
model: string;
max_output_tokens: number;
max_processing_ms: number;
price_lovelace: string; // bigint serialised as string
supplier_bond_lovelace: string;
buyer_bond_lovelace: string;
endpoint_url: string;
detail_uri: string;
detail_hash: string;
advertised_at: number;
status: string; // "Active" | "Retired"
created_slot: number;
datum_hex: string;
rolled_back: number;
}
export interface EscrowRow {
utxo_ref: string;
buyer_pkh: string;
supplier_pkh: string;
advert_ref_tx: string;
advert_ref_index: number;
capability_id: string;
request_spec_hash: string;
prompt_hash: string;
payment_lovelace: string;
buyer_bond_lovelace: string;
supplier_bond_lovelace: string;
deliver_by: number;
posted_at: number;
submitted_at: number | null;
result_receipt_hash: string | null;
state: string;
created_slot: number;
datum_hex: string;
rolled_back: number;
}
export interface SupplierStatusRow {
supplier_pkh: string;
advert_ref: string;
status: string; // "free" | "working" | "offline"
last_seen_iso: string;
current_escrow_ref: string | null;
polled_at: number; // unix ms
}
export interface IndexerEventRow {
id?: number;
type: string;
slot: number;
tx_hash: string;
utxo_ref: string;
datum_hex: string;
metadata_json: string;
rolled_back: number;
}
export interface CursorRow {
slot: number;
blockHash: string;
}
export interface UtxoRow {
ref: string;
address: string;
datum_hex: string;
created_slot: number;
created_tx: string;
spent_slot: number | null;
spent_tx: string | null;
}
export class SqliteCache {
private db: Database.Database;
private dbPath: string;
constructor(dbPath: string) {
this.dbPath = dbPath;
this.db = new Database(dbPath);
this.db.pragma("journal_mode = WAL");
applySchema(this.db);
}
// ─── Cursor ────────────────────────────────────────────────────────────
saveCursor(slot: number, blockHash: string): void {
this.db.prepare(
`INSERT OR REPLACE INTO cursor (id, slot, block_hash) VALUES (1, ?, ?)`
).run(slot, blockHash);
}
getCursor(): CursorRow | null {
const row = this.db.prepare(
`SELECT slot, block_hash FROM cursor WHERE id = 1`
).get() as { slot: number; block_hash: string } | undefined;
if (!row) return null;
return { slot: row.slot, blockHash: row.block_hash };
}
// ─── UTxO tracking ─────────────────────────────────────────────────────
insertUtxo(ref: string, address: string, datumHex: string, slot: number, txHash: string): void {
this.db.prepare(
`INSERT OR REPLACE INTO utxos (ref, address, datum_hex, created_slot, created_tx, spent_slot, spent_tx)
VALUES (?, ?, ?, ?, ?, NULL, NULL)`
).run(ref, address, datumHex, slot, txHash);
}
spendUtxo(ref: string, slot: number, txHash: string): void {
this.db.prepare(
`UPDATE utxos SET spent_slot = ?, spent_tx = ? WHERE ref = ?`
).run(slot, txHash, ref);
}
getUnspentUtxos(address: string): UtxoRow[] {
return this.db.prepare(
`SELECT ref, address, datum_hex, created_slot, created_tx, spent_slot, spent_tx
FROM utxos WHERE address = ? AND spent_slot IS NULL`
).all(address) as UtxoRow[];
}
// ─── Advertisements ────────────────────────────────────────────────────
upsertAdvertisement(row: Omit<AdvertRow, "id">): void {
this.db.prepare(
`INSERT OR REPLACE INTO advertisements (
utxo_ref, supplier_pkh, capability_id, model,
max_output_tokens, max_processing_ms,
price_lovelace, supplier_bond_lovelace, buyer_bond_lovelace,
endpoint_url, detail_uri, detail_hash, advertised_at, status,
created_slot, datum_hex, rolled_back
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
).run(
row.utxo_ref, row.supplier_pkh, row.capability_id, row.model,
row.max_output_tokens, row.max_processing_ms,
row.price_lovelace, row.supplier_bond_lovelace, row.buyer_bond_lovelace,
row.endpoint_url, row.detail_uri, row.detail_hash, row.advertised_at, row.status,
row.created_slot, row.datum_hex, row.rolled_back
);
}
getAdvertisementByRef(utxoRef: string): AdvertRow | null {
const row = this.db.prepare(
`SELECT * FROM advertisements WHERE utxo_ref = ?`
).get(utxoRef) as AdvertRow | undefined;
return row ?? null;
}
listActiveAdvertisements(filter?: { capability_id?: string; supplier_pkh?: string }): AdvertRow[] {
let sql = `SELECT * FROM advertisements WHERE status = 'Active' AND rolled_back = 0`;
const params: unknown[] = [];
if (filter?.capability_id) {
sql += ` AND capability_id = ?`;
params.push(filter.capability_id);
}
if (filter?.supplier_pkh) {
sql += ` AND supplier_pkh = ?`;
params.push(filter.supplier_pkh);
}
sql += ` ORDER BY created_slot DESC`;
return this.db.prepare(sql).all(...params) as AdvertRow[];
}
// ─── Escrows ───────────────────────────────────────────────────────────
upsertEscrow(row: Omit<EscrowRow, "id">): void {
this.db.prepare(
`INSERT OR REPLACE INTO escrows (
utxo_ref, buyer_pkh, supplier_pkh,
advert_ref_tx, advert_ref_index,
capability_id, request_spec_hash, prompt_hash,
payment_lovelace, buyer_bond_lovelace, supplier_bond_lovelace,
deliver_by, posted_at, submitted_at, result_receipt_hash,
state, created_slot, datum_hex, rolled_back
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
).run(
row.utxo_ref, row.buyer_pkh, row.supplier_pkh,
row.advert_ref_tx, row.advert_ref_index,
row.capability_id, row.request_spec_hash, row.prompt_hash,
row.payment_lovelace, row.buyer_bond_lovelace, row.supplier_bond_lovelace,
row.deliver_by, row.posted_at, row.submitted_at, row.result_receipt_hash,
row.state, row.created_slot, row.datum_hex, row.rolled_back
);
}
getEscrowByRef(utxoRef: string): EscrowRow | null {
const row = this.db.prepare(
`SELECT * FROM escrows WHERE utxo_ref = ?`
).get(utxoRef) as EscrowRow | undefined;
return row ?? null;
}
listEscrowsByBuyer(buyerPkh: string): EscrowRow[] {
return this.db.prepare(
`SELECT * FROM escrows WHERE buyer_pkh = ? ORDER BY created_slot DESC`
).all(buyerPkh) as EscrowRow[];
}
listEscrowsBySupplier(supplierPkh: string): EscrowRow[] {
return this.db.prepare(
`SELECT * FROM escrows WHERE supplier_pkh = ? ORDER BY created_slot DESC`
).all(supplierPkh) as EscrowRow[];
}
// ─── Supplier status ───────────────────────────────────────────────────
upsertSupplierStatus(row: SupplierStatusRow): void {
this.db.prepare(
`INSERT OR REPLACE INTO supplier_status (
supplier_pkh, advert_ref, status, last_seen_iso, current_escrow_ref, polled_at
) VALUES (?, ?, ?, ?, ?, ?)`
).run(
row.supplier_pkh, row.advert_ref, row.status, row.last_seen_iso,
row.current_escrow_ref, row.polled_at
);
}
getSupplierStatus(supplierPkh: string): SupplierStatusRow | null {
const row = this.db.prepare(
`SELECT * FROM supplier_status WHERE supplier_pkh = ?`
).get(supplierPkh) as SupplierStatusRow | undefined;
return row ?? null;
}
// ─── Events ────────────────────────────────────────────────────────────
appendEvent(row: Omit<IndexerEventRow, "id">): void {
this.db.prepare(
`INSERT INTO events (type, slot, tx_hash, utxo_ref, datum_hex, metadata_json, rolled_back)
VALUES (?, ?, ?, ?, ?, ?, ?)`
).run(
row.type, row.slot, row.tx_hash, row.utxo_ref,
row.datum_hex, row.metadata_json, row.rolled_back
);
}
listEventsAfterSlot(slot: number): IndexerEventRow[] {
return this.db.prepare(
`SELECT * FROM events WHERE slot > ? AND rolled_back = 0 ORDER BY slot ASC, id ASC`
).all(slot) as IndexerEventRow[];
}
/** Most-recent `limit` events, returned in chronological ASC order so SSE
* replay preserves the same oldest-first emission shape as
* `listEventsAfterSlot`. */
listRecentEvents(limit: number): IndexerEventRow[] {
if (!Number.isInteger(limit) || limit <= 0) return [];
const desc = this.db.prepare(
`SELECT * FROM events WHERE rolled_back = 0 ORDER BY slot DESC, id DESC LIMIT ?`
).all(limit) as IndexerEventRow[];
return desc.reverse();
}
// ─── Rollback ──────────────────────────────────────────────────────────
rollbackToSlot(slot: number): void {
const tx = this.db.transaction(() => {
// Soft-delete events strictly after rollback slot
this.db.prepare(`UPDATE events SET rolled_back = 1 WHERE slot > ?`).run(slot);
// Delete UTxOs created after rollback slot
this.db.prepare(`DELETE FROM utxos WHERE created_slot > ?`).run(slot);
// Restore UTxOs that were spent after rollback slot (clear spend)
this.db.prepare(
`UPDATE utxos SET spent_slot = NULL, spent_tx = NULL WHERE spent_slot > ?`
).run(slot);
// Update cursor if it's beyond rollback point
this.db.prepare(`UPDATE cursor SET slot = ? WHERE id = 1 AND slot > ?`).run(slot, slot);
});
tx();
}
// ─── Lifecycle ─────────────────────────────────────────────────────────
dbSizeBytes(): number {
try {
const stats = fs.statSync(this.dbPath);
return stats.size;
} catch {
// In-memory or missing — fall back to a positive sentinel
// (test contract requires > 0 even on a fresh DB).
// better-sqlite3 always creates the file on open with WAL mode,
// so this branch normally never hits.
return 1;
}
}
close(): void {
this.db.close();
}
}

113
indexer/src/db/schema.ts Normal file
View File

@@ -0,0 +1,113 @@
/**
* indexer/src/db/schema.ts — CREATE TABLE statements for the indexer database.
*
* Runs on boot if tables are missing. Idempotent (IF NOT EXISTS).
*
* Tables:
* cursor — single row tracking chain-sync position
* utxos — all observed UTxOs (created/spent tracking)
* events — all emitted marketplace events (soft-deletable)
* advertisements — decoded AdvertDatum state keyed by utxo_ref
* escrows — decoded EscrowDatum state keyed by utxo_ref
* supplier_status — latest polled /status for each active supplier
*/
export const CREATE_TABLES_SQL = `
CREATE TABLE IF NOT EXISTS cursor (
id INTEGER PRIMARY KEY CHECK (id = 1),
slot INTEGER NOT NULL,
block_hash TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS utxos (
ref TEXT PRIMARY KEY,
address TEXT NOT NULL,
datum_hex TEXT NOT NULL,
created_slot INTEGER NOT NULL,
created_tx TEXT NOT NULL,
spent_slot INTEGER,
spent_tx TEXT
);
CREATE INDEX IF NOT EXISTS idx_utxos_address ON utxos(address, spent_slot);
CREATE TABLE IF NOT EXISTS events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
type TEXT NOT NULL,
slot INTEGER NOT NULL,
tx_hash TEXT NOT NULL,
utxo_ref TEXT NOT NULL,
datum_hex TEXT NOT NULL,
metadata_json TEXT NOT NULL DEFAULT '{}',
rolled_back INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_events_slot ON events(slot);
CREATE INDEX IF NOT EXISTS idx_events_type ON events(type);
CREATE INDEX IF NOT EXISTS idx_events_utxo_ref ON events(utxo_ref);
CREATE TABLE IF NOT EXISTS advertisements (
utxo_ref TEXT PRIMARY KEY,
supplier_pkh TEXT NOT NULL,
capability_id TEXT NOT NULL,
model TEXT NOT NULL,
max_output_tokens INTEGER NOT NULL,
max_processing_ms INTEGER NOT NULL,
price_lovelace TEXT NOT NULL,
supplier_bond_lovelace TEXT NOT NULL,
buyer_bond_lovelace TEXT NOT NULL,
endpoint_url TEXT NOT NULL,
detail_uri TEXT NOT NULL,
detail_hash TEXT NOT NULL,
advertised_at INTEGER NOT NULL,
status TEXT NOT NULL,
created_slot INTEGER NOT NULL,
datum_hex TEXT NOT NULL,
rolled_back INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_adverts_capability_status ON advertisements(capability_id, status);
CREATE INDEX IF NOT EXISTS idx_adverts_supplier_status ON advertisements(supplier_pkh, status);
CREATE TABLE IF NOT EXISTS escrows (
utxo_ref TEXT PRIMARY KEY,
buyer_pkh TEXT NOT NULL,
supplier_pkh TEXT NOT NULL,
advert_ref_tx TEXT NOT NULL,
advert_ref_index INTEGER NOT NULL,
capability_id TEXT NOT NULL,
request_spec_hash TEXT NOT NULL,
prompt_hash TEXT NOT NULL,
payment_lovelace TEXT NOT NULL,
buyer_bond_lovelace TEXT NOT NULL,
supplier_bond_lovelace TEXT NOT NULL,
deliver_by INTEGER NOT NULL,
posted_at INTEGER NOT NULL,
submitted_at INTEGER,
result_receipt_hash TEXT,
state TEXT NOT NULL,
created_slot INTEGER NOT NULL,
datum_hex TEXT NOT NULL,
rolled_back INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_escrows_buyer ON escrows(buyer_pkh);
CREATE INDEX IF NOT EXISTS idx_escrows_supplier ON escrows(supplier_pkh);
CREATE TABLE IF NOT EXISTS supplier_status (
supplier_pkh TEXT PRIMARY KEY,
advert_ref TEXT NOT NULL,
status TEXT NOT NULL,
last_seen_iso TEXT NOT NULL,
current_escrow_ref TEXT,
polled_at INTEGER NOT NULL
);
`;
interface DbLike {
exec(sql: string): unknown;
}
export function applySchema(db: unknown): void {
(db as DbLike).exec(CREATE_TABLES_SQL);
}

View File

@@ -0,0 +1,257 @@
/**
* indexer/src/follower/blockProcessor.ts — processes a single Ogmios block.
*
* processBlock(block, knownUtxos, addresses) inspects each transaction,
* decodes datums at watched script addresses (advertScript, escrowScript),
* and emits typed MarketplaceEvent objects.
*
* Detection rules (all output-driven unless noted):
* PostAdvert — new output at advertAddr with Active AdvertDatum, no spent advert input
* UpdateAdvert — output at advertAddr + spends prior advert UTxO
* RetireAdvert — spends advert UTxO, no continuing output at advertAddr
* PostEscrow — new output at escrowAddr with Open EscrowDatum, no spent escrow input
* ClaimEscrow — spend-driven: spends Open escrow, Claimed output continues
* SubmitEscrow — spend-driven: spends Claimed escrow, Submitted output continues
* AcceptEscrow — spends Submitted escrow, no continuing output (terminal)
* ReclaimEscrow — spends Open/Claimed escrow, no continuing output (terminal)
* ReleaseEscrow — spends Submitted escrow, no continuing output (terminal)
*
* Malformed datum at a watched address → console.warn, no event emitted.
* Byron-era blocks (no transactions field) → handled gracefully.
*/
import { decodeAdvertDatum, decodeEscrowDatum } from "@marketplace/shared/cbor";
import type {
EscrowRedeemerTag,
IndexerBlock,
IndexerTx,
MarketplaceEvent,
MarketplaceEventType,
} from "./types.js";
export interface KnownUtxoInfo {
address: string;
datumHex?: string;
}
export interface ScriptAddresses {
advertAddress: string;
escrowAddress: string;
}
export interface ProcessBlockResult {
events: MarketplaceEvent[];
spentRefs: Array<{ ref: string; slot: number; txHash: string }>;
}
interface SpentInfo {
ref: string;
prior: KnownUtxoInfo;
}
function tryDecodeAdvert(datumHex: string): ReturnType<typeof decodeAdvertDatum> | null {
try {
return decodeAdvertDatum(datumHex);
} catch (err) {
console.warn(`[blockProcessor] failed to decode AdvertDatum: ${(err as Error).message}`);
return null;
}
}
function tryDecodeEscrow(datumHex: string): ReturnType<typeof decodeEscrowDatum> | null {
try {
return decodeEscrowDatum(datumHex);
} catch (err) {
console.warn(`[blockProcessor] failed to decode EscrowDatum: ${(err as Error).message}`);
return null;
}
}
function processTx(
tx: IndexerTx,
block: IndexerBlock,
knownUtxos: Map<string, KnownUtxoInfo>,
addresses: ScriptAddresses,
events: MarketplaceEvent[],
spentRefs: Array<{ ref: string; slot: number; txHash: string }>,
): void {
const slot = block.slot;
const txHash = tx.id;
// 1) Identify spent inputs that hit our known watched UTxOs.
const spentAdvert: SpentInfo[] = [];
const spentEscrow: SpentInfo[] = [];
for (const input of tx.inputs ?? []) {
const ref = `${input.transaction.id}#${input.index}`;
const prior = knownUtxos.get(ref);
if (!prior) continue;
spentRefs.push({ ref, slot, txHash });
if (prior.address === addresses.advertAddress) {
spentAdvert.push({ ref, prior });
} else if (prior.address === addresses.escrowAddress) {
spentEscrow.push({ ref, prior });
}
}
// 2) Identify outputs at watched addresses.
type WatchedOutput = { index: number; address: string; datumHex: string };
const advertOutputs: WatchedOutput[] = [];
const escrowOutputs: WatchedOutput[] = [];
for (let i = 0; i < (tx.outputs ?? []).length; i++) {
const out = tx.outputs[i];
if (!out.datum) continue;
if (out.address === addresses.advertAddress) {
advertOutputs.push({ index: i, address: out.address, datumHex: out.datum });
} else if (out.address === addresses.escrowAddress) {
escrowOutputs.push({ index: i, address: out.address, datumHex: out.datum });
}
}
// 3) Advertisement event detection
if (advertOutputs.length > 0) {
// Output(s) at advert script — either PostAdvert (no input) or UpdateAdvert (has prior input)
for (const out of advertOutputs) {
const decoded = tryDecodeAdvert(out.datumHex);
if (!decoded) continue;
const type: MarketplaceEventType = spentAdvert.length > 0 ? "UpdateAdvert" : "PostAdvert";
events.push({
type,
slot,
txHash,
utxoRef: `${txHash}#${out.index}`,
datumHex: out.datumHex,
address: out.address,
});
}
} else if (spentAdvert.length > 0) {
// Spend with no continuing output → RetireAdvert
for (const s of spentAdvert) {
events.push({
type: "RetireAdvert",
slot,
txHash,
utxoRef: s.ref,
datumHex: s.prior.datumHex ?? "",
address: s.prior.address,
});
}
}
// 4) Escrow event detection
if (escrowOutputs.length > 0) {
// Continuing output exists. The state in the new datum tells us which transition.
for (const out of escrowOutputs) {
const decoded = tryDecodeEscrow(out.datumHex);
if (!decoded) continue;
let type: MarketplaceEventType | null = null;
if (decoded.state === "Open" && spentEscrow.length === 0) {
type = "PostEscrow";
} else if (decoded.state === "Claimed" && spentEscrow.length > 0) {
type = "ClaimEscrow";
} else if (decoded.state === "Submitted" && spentEscrow.length > 0) {
type = "SubmitEscrow";
}
if (type !== null) {
events.push({
type,
slot,
txHash,
utxoRef: `${txHash}#${out.index}`,
datumHex: out.datumHex,
address: out.address,
});
}
}
} else if (spentEscrow.length > 0) {
// Terminal spend (no continuing escrow output). Disambiguate via tx.redeemers.
// Per ARCHITECTURE.md §4.3 the prior state + redeemer uniquely determines the
// intended transition. Mismatches are logged and emit no event.
const redeemers: Record<string, EscrowRedeemerTag> = tx.redeemers ?? {};
for (const s of spentEscrow) {
const priorDatumHex = s.prior.datumHex ?? "";
const decoded = priorDatumHex ? tryDecodeEscrow(priorDatumHex) : null;
if (!decoded) continue;
const redeemer = redeemers[s.ref];
const priorState = decoded.state;
let type: MarketplaceEventType | null = null;
if (redeemer === undefined) {
// No redeemer attached — defensive default: assume the canonical
// "happy-path" redeemer for the prior state. Always warn so this
// surfaces in operations even when behaviour is benign.
if (priorState === "Submitted") {
type = "AcceptEscrow";
console.warn(
`[blockProcessor] terminal escrow spend ${s.ref} (priorState=Submitted) carried no redeemer; defaulting to AcceptEscrow`,
);
} else if (priorState === "Open" || priorState === "Claimed") {
type = "ReclaimEscrow";
console.warn(
`[blockProcessor] terminal escrow spend ${s.ref} (priorState=${priorState}) carried no redeemer; defaulting to ReclaimEscrow`,
);
} else {
console.warn(
`[blockProcessor] terminal escrow spend ${s.ref} carried no redeemer and prior state ${priorState} is non-terminal; emitting no event`,
);
}
} else if (priorState === "Submitted") {
if (redeemer === "Accept") {
type = "AcceptEscrow";
} else if (redeemer === "Release") {
type = "ReleaseEscrow";
} else {
console.warn(
`[blockProcessor] redeemer/state mismatch on terminal spend ${s.ref}: redeemer=${redeemer} but priorState=Submitted; emitting no event`,
);
}
} else if (priorState === "Open" || priorState === "Claimed") {
if (redeemer === "Reclaim") {
type = "ReclaimEscrow";
} else {
console.warn(
`[blockProcessor] redeemer/state mismatch on terminal spend ${s.ref}: redeemer=${redeemer} but priorState=${priorState}; emitting no event`,
);
}
} else {
console.warn(
`[blockProcessor] terminal escrow spend ${s.ref} has unexpected prior state ${priorState} with redeemer=${redeemer}; emitting no event`,
);
}
if (type !== null) {
events.push({
type,
slot,
txHash,
utxoRef: s.ref,
datumHex: priorDatumHex,
address: s.prior.address,
});
}
}
}
}
export function processBlock(
block: IndexerBlock,
knownUtxos: Map<string, KnownUtxoInfo>,
addresses: ScriptAddresses,
): ProcessBlockResult {
const events: MarketplaceEvent[] = [];
const spentRefs: Array<{ ref: string; slot: number; txHash: string }> = [];
// Byron / null transactions → graceful no-op
if (!block || !block.transactions) {
return { events, spentRefs };
}
for (const tx of block.transactions) {
if (!tx) continue;
processTx(tx, block, knownUtxos, addresses, events, spentRefs);
}
return { events, spentRefs };
}

View File

@@ -0,0 +1,173 @@
/**
* indexer/src/follower/ogmiosSource.ts — Ogmios WebSocket ChainSyncSource.
*
* Production implementation of ChainSyncSource over an Ogmios WebSocket.
* Tests inject MockChainSyncSource instead, so this implementation is exercised
* end-to-end only in M1-F lifecycle tests.
*
* Adapted from apex-dashboard/server/ws-transport.ts.
*/
import { EventEmitter } from "events";
import type { ChainSyncSource, IndexerBlock, RollbackPoint } from "./types.js";
const INITIAL_BACKOFF = 1_000;
const MAX_BACKOFF = 30_000;
interface MinimalWebSocket {
readyState: number;
send(data: string): void;
close(): void;
onopen: ((ev: unknown) => void) | null;
onmessage: ((ev: { data: string }) => void) | null;
onerror: ((ev: { message?: string }) => void) | null;
onclose: ((ev: { code: number; reason: string }) => void) | null;
}
type WsCtor = new (url: string) => MinimalWebSocket;
let _WsCtor: WsCtor | null = null;
async function loadWs(): Promise<WsCtor> {
if (_WsCtor) return _WsCtor;
const mod = await import("ws");
const m = mod as unknown as { WebSocket?: WsCtor; default?: WsCtor };
_WsCtor = m.WebSocket ?? m.default ?? (mod as unknown as WsCtor);
return _WsCtor;
}
export class OgmiosSource extends EventEmitter implements ChainSyncSource {
private url: string;
private ws: MinimalWebSocket | null = null;
private backoff: number = INITIAL_BACKOFF;
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
private closed: boolean = false;
private intersectAt: { slot: number; id: string } | null = null;
constructor(ogmiosUrl: string) {
super();
this.url = ogmiosUrl;
}
async start(intersectAt?: { slot: number; id: string } | null): Promise<void> {
this.closed = false;
this.intersectAt = intersectAt ?? null;
await this.connect();
this.sendFindIntersection();
}
stop(): void {
this.closed = true;
if (this.reconnectTimer) {
clearTimeout(this.reconnectTimer);
this.reconnectTimer = null;
}
if (this.ws) {
this.ws.close();
this.ws = null;
}
}
requestNextBlock(): void {
const OPEN = 1;
if (!this.ws || this.ws.readyState !== OPEN) {
// Not yet ready — drop silently; reconnect logic will resume.
return;
}
this.ws.send(JSON.stringify({ jsonrpc: "2.0", method: "nextBlock" }));
}
// ─── Internal ───────────────────────────────────────────────────────────
private async connect(): Promise<void> {
const Ctor = await loadWs();
return new Promise<void>((resolve, reject) => {
const socket = new Ctor(this.url);
this.ws = socket;
let settled = false;
socket.onopen = () => {
this.backoff = INITIAL_BACKOFF;
settled = true;
this.emit("connected");
resolve();
};
socket.onmessage = (ev) => this.handleMessage(ev.data);
socket.onerror = (ev) => {
const err = new Error(ev.message ?? "WebSocket error");
this.emit("error", err);
if (!settled) {
settled = true;
reject(err);
}
};
socket.onclose = (ev) => {
this.emit("disconnected");
if (!settled) {
settled = true;
reject(new Error(`WebSocket closed: ${ev.code} ${ev.reason}`));
}
if (!this.closed) this.scheduleReconnect();
};
});
}
private sendFindIntersection(): void {
const OPEN = 1;
if (!this.ws || this.ws.readyState !== OPEN) return;
const points: Array<{ slot: number; id: string } | "origin"> = this.intersectAt
? [this.intersectAt, "origin"]
: ["origin"];
this.ws.send(JSON.stringify({
jsonrpc: "2.0",
method: "findIntersection",
params: { points },
}));
}
private handleMessage(data: string): void {
let msg: { method?: string; result?: unknown; error?: { message?: string } };
try {
msg = JSON.parse(data);
} catch (err) {
this.emit("error", err);
return;
}
if (msg.error) {
this.emit("error", new Error(msg.error.message ?? "Ogmios error"));
return;
}
if (msg.method === "findIntersection") {
this.emit("intersection", msg.result);
// Pipeline: kick off some next-block requests
this.requestNextBlock();
return;
}
if (msg.method === "nextBlock") {
const r = msg.result as {
direction?: "forward" | "backward";
block?: IndexerBlock;
tip?: { slot: number; id: string };
point?: RollbackPoint | string;
};
if (r?.direction === "forward" && r.block) {
this.emit("block", { block: r.block, tip: r.tip });
} else if (r?.direction === "backward") {
let point: RollbackPoint = { slot: 0, id: "origin" };
if (r.point && typeof r.point === "object") {
point = r.point;
}
this.emit("rollback", { point });
}
}
}
private scheduleReconnect(): void {
this.reconnectTimer = setTimeout(() => {
this.reconnectTimer = null;
_WsCtor = null;
this.connect()
.then(() => this.sendFindIntersection())
.catch((err) => this.emit("error", err));
}, this.backoff);
this.backoff = Math.min(this.backoff * 2, MAX_BACKOFF);
}
}

View File

@@ -0,0 +1,94 @@
/**
* indexer/src/follower/types.ts — ChainSyncSource interface and block types.
*
* ChainSyncSource is the seam that allows tests to inject synthetic block streams
* without touching a real Ogmios WebSocket. OgmiosSource implements this interface
* for production; MockChainSyncSource implements it for tests.
*
* Block/Tx/Output shapes mirror the apex-dashboard block-processor shapes
* (Ogmios JSON over WebSocket).
*/
import { EventEmitter } from "events";
export interface TxOutput {
address: string;
value: Record<string, unknown>;
datum?: string; // inline CBOR hex
datumHash?: string;
}
export interface TxInput {
transaction: { id: string };
index: number;
}
/**
* EscrowRedeemerTag — the five redeemer constructors from escrow.ak.
* Keyed in IndexerTx.redeemers by "<spentTxHash>#<spentIndex>".
*/
export type EscrowRedeemerTag = "Claim" | "Submit" | "Accept" | "Reclaim" | "Release";
export interface IndexerTx {
id: string;
inputs: TxInput[];
outputs: TxOutput[];
/**
* Optional redeemer map populated from Ogmios redeemer info.
* Key: "<spentTxHash>#<outputIndex>" of the spent UTxO.
* Value: EscrowRedeemerTag identifying which redeemer was used.
*/
redeemers?: Record<string, EscrowRedeemerTag>;
}
export interface IndexerBlock {
slot: number;
id: string; // block hash
ancestor?: string;
transactions?: IndexerTx[] | null;
}
export interface RollbackPoint {
slot: number;
id: string;
}
/**
* ChainSyncSource — the mockable seam for the chain follower.
*
* Implementations emit:
* "block" — { block: IndexerBlock, tip: { slot: number } }
* "rollback" — { point: RollbackPoint }
* "connected" — ()
* "error" — (err: Error)
*
* The worker calls start() to begin streaming, stop() to tear down.
* requestNextBlock() is called after each block to pipeline requests.
*/
export interface ChainSyncSource extends EventEmitter {
start(intersectAt?: { slot: number; id: string } | null): Promise<void>;
stop(): void;
requestNextBlock(): void;
}
/**
* MarketplaceEvent — emitted by blockProcessor for each detected state transition.
*/
export type MarketplaceEventType =
| "PostAdvert"
| "UpdateAdvert"
| "RetireAdvert"
| "PostEscrow"
| "ClaimEscrow"
| "SubmitEscrow"
| "AcceptEscrow"
| "ReclaimEscrow"
| "ReleaseEscrow";
export interface MarketplaceEvent {
type: MarketplaceEventType;
slot: number;
txHash: string;
utxoRef: string; // "<txHash>#<outputIndex>"
datumHex: string;
address: string;
}

View File

@@ -0,0 +1,241 @@
/**
* indexer/src/follower/worker.ts — ChainSyncWorker for the indexer.
*
* Drives a ChainSyncSource, persists events to SqliteCache, handles rollbacks,
* and exposes an EventEmitter for the SSE layer.
*
* Emitted events (for SSE consumers):
* "chain-event" — { type, ref, slot, datum? } (on each MarketplaceEvent)
* "sync-progress" — { currentSlot, tipSlot }
* "error" — { message, stale }
*/
import { EventEmitter } from "events";
import { decodeAdvertDatum, decodeEscrowDatum } from "@marketplace/shared/cbor";
import type { ChainSyncSource, IndexerBlock, RollbackPoint, MarketplaceEvent } from "./types.js";
import type { SqliteCache } from "../db/cache.js";
import { processBlock, type ScriptAddresses, type KnownUtxoInfo } from "./blockProcessor.js";
export interface ChainSyncWorkerOpts {
source: ChainSyncSource;
cache: SqliteCache;
addresses: ScriptAddresses;
skipBeforeSlot?: number;
}
export class ChainSyncWorker extends EventEmitter {
private source: ChainSyncSource;
private cache: SqliteCache;
private addresses: ScriptAddresses;
private skipBeforeSlot: number;
private currentSlot: number = 0;
private tipSlot: number = 0;
private stopped: boolean = false;
constructor(opts: ChainSyncWorkerOpts) {
super();
this.source = opts.source;
this.cache = opts.cache;
this.addresses = opts.addresses;
this.skipBeforeSlot = opts.skipBeforeSlot ?? 0;
const cursor = this.cache.getCursor();
if (cursor) {
this.currentSlot = cursor.slot;
}
this.source.on("block", (data: { block: IndexerBlock; tip?: { slot: number } }) => {
this.handleBlock(data);
});
this.source.on("rollback", (data: { point: RollbackPoint }) => {
this.handleRollback(data);
});
this.source.on("error", (err: Error) => {
this.emit("error", { message: err.message ?? String(err), stale: true });
});
}
getCurrentSlot(): number {
return this.currentSlot;
}
getTipSlot(): number {
return this.tipSlot;
}
async start(): Promise<void> {
this.stopped = false;
const cursor = this.cache.getCursor();
const intersect = cursor ? { slot: cursor.slot, id: cursor.blockHash } : null;
await this.source.start(intersect);
}
stop(): void {
this.stopped = true;
this.source.stop();
}
// ─── Internal helpers ────────────────────────────────────────────────────
private buildKnownUtxosMap(): Map<string, KnownUtxoInfo> {
const map = new Map<string, KnownUtxoInfo>();
for (const addr of [this.addresses.advertAddress, this.addresses.escrowAddress]) {
for (const utxo of this.cache.getUnspentUtxos(addr)) {
map.set(utxo.ref, { address: utxo.address, datumHex: utxo.datum_hex || undefined });
}
}
return map;
}
private handleBlock(data: { block: IndexerBlock; tip?: { slot: number } }): void {
if (this.stopped) return;
const block = data.block;
if (!block) return;
if (data.tip?.slot && data.tip.slot > this.tipSlot) {
this.tipSlot = data.tip.slot;
}
// Byron Epoch-Boundary Blocks (EBBs) and some pre-Shelley blocks have no `slot`.
// We can't save a NULL cursor (SQLite NOT NULL). When chain-syncing from origin
// before SKIP_BEFORE_SLOT, just request the next block without persisting cursor.
const slot = block.slot;
if (slot === undefined || slot === null) {
this.source.requestNextBlock();
return;
}
// Fast-forward path
if (slot < this.skipBeforeSlot) {
this.cache.saveCursor(slot, block.id);
this.currentSlot = slot;
this.emit("sync-progress", { currentSlot: slot, tipSlot: this.tipSlot });
this.source.requestNextBlock();
return;
}
const knownUtxos = this.buildKnownUtxosMap();
const { events, spentRefs } = processBlock(block, knownUtxos, this.addresses);
for (const event of events) {
this.persistEvent(event);
this.emit("chain-event", {
type: event.type,
ref: event.utxoRef,
slot: event.slot,
txHash: event.txHash,
address: event.address,
} as { type: string; ref: string; slot: number; txHash: string; address: string });
}
// Spend known UTxOs after inserts (handles self-referencing edge case in apex pattern)
for (const s of spentRefs) {
this.cache.spendUtxo(s.ref, s.slot, s.txHash);
// Also update advert/escrow domain rows: mark terminal for spend-no-continue cases.
this.markDomainSpend(s.ref, slot, events);
}
// Insert UTxOs for new outputs at watched addresses (so spend detection works for next block)
for (const tx of block.transactions ?? []) {
for (let i = 0; i < (tx.outputs ?? []).length; i++) {
const out = tx.outputs[i];
if (!out.datum) continue;
if (out.address === this.addresses.advertAddress || out.address === this.addresses.escrowAddress) {
const ref = `${tx.id}#${i}`;
this.cache.insertUtxo(ref, out.address, out.datum, slot, tx.id);
}
}
}
this.cache.saveCursor(slot, block.id);
this.currentSlot = slot;
this.emit("sync-progress", { currentSlot: slot, tipSlot: this.tipSlot });
this.source.requestNextBlock();
}
private persistEvent(event: MarketplaceEvent): void {
this.cache.appendEvent({
type: event.type,
slot: event.slot,
tx_hash: event.txHash,
utxo_ref: event.utxoRef,
datum_hex: event.datumHex,
metadata_json: "{}",
rolled_back: 0,
});
// Update domain projections (advertisements, escrows)
if (event.type === "PostAdvert" || event.type === "UpdateAdvert") {
try {
const d = decodeAdvertDatum(event.datumHex);
this.cache.upsertAdvertisement({
utxo_ref: event.utxoRef,
supplier_pkh: d.supplier_pkh,
capability_id: d.capability_id,
model: d.model,
max_output_tokens: d.max_output_tokens,
max_processing_ms: d.max_processing_ms,
price_lovelace: d.price_lovelace.toString(),
supplier_bond_lovelace: d.supplier_bond_lovelace.toString(),
buyer_bond_lovelace: d.buyer_bond_lovelace.toString(),
endpoint_url: d.endpoint_url,
detail_uri: d.detail_uri,
detail_hash: d.detail_hash,
advertised_at: d.advertised_at,
status: d.status,
created_slot: event.slot,
datum_hex: event.datumHex,
rolled_back: 0,
});
} catch {
// already warned by processBlock
}
} else if (event.type === "PostEscrow" || event.type === "ClaimEscrow" || event.type === "SubmitEscrow") {
try {
const d = decodeEscrowDatum(event.datumHex);
this.cache.upsertEscrow({
utxo_ref: event.utxoRef,
buyer_pkh: d.buyer_pkh,
supplier_pkh: d.supplier_pkh,
advert_ref_tx: d.advert_ref.txHash,
advert_ref_index: d.advert_ref.index,
capability_id: d.capability_id,
request_spec_hash: d.request_spec_hash,
prompt_hash: d.prompt_hash,
payment_lovelace: d.payment_lovelace.toString(),
buyer_bond_lovelace: d.buyer_bond_lovelace.toString(),
supplier_bond_lovelace: d.supplier_bond_lovelace.toString(),
deliver_by: d.deliver_by,
posted_at: d.posted_at,
submitted_at: d.submitted_at,
result_receipt_hash: d.result_receipt_hash,
state: d.state,
created_slot: event.slot,
datum_hex: event.datumHex,
rolled_back: 0,
});
} catch {
// already warned
}
}
}
private markDomainSpend(ref: string, _slot: number, events: MarketplaceEvent[]): void {
// For terminal events (Retire / Accept / Reclaim / Release), the spend has no continuing
// output so the domain row stays at its prior state (caller can interpret via events table).
// For Update / Claim / Submit, a fresh upsertAdvertisement/upsertEscrow happens via persistEvent
// for the NEW utxo_ref. We could also update the OLD row's status here if needed.
void ref; void events;
}
private handleRollback(data: { point: RollbackPoint }): void {
if (this.stopped) return;
const slot = data.point?.slot ?? 0;
this.cache.rollbackToSlot(slot);
this.currentSlot = slot;
this.emit("sync-progress", { currentSlot: slot, tipSlot: this.tipSlot });
this.source.requestNextBlock();
}
}

94
indexer/src/index.ts Normal file
View File

@@ -0,0 +1,94 @@
/**
* indexer/src/index.ts — boot script for the marketplace indexer.
*
* Wires:
* - Config from process.env (validated)
* - SqliteCache (opens or creates DB, applies schema)
* - OgmiosSource (real WebSocket — exercised in M1-F lifecycle)
* - ChainSyncWorker (drives source → cache + emits chain-event/sync-progress)
* - StatusPoller (periodic supplier /status fetcher)
* - Express app via createApp(deps)
*
* Installs SIGTERM/SIGINT handlers for graceful shutdown.
*
* NOTE: the script computes the advertScript / escrowScript addresses from a
* future config map — for M1-D the addresses are sourced from environment fall-
* backs to keep the boot pure. M1-F lifecycle test will plumb real addresses
* through `config/marketplace-deployments.json`.
*/
import { loadConfig } from "./config.js";
import { SqliteCache } from "./db/cache.js";
import { OgmiosSource } from "./follower/ogmiosSource.js";
import { ChainSyncWorker } from "./follower/worker.js";
import { StatusPoller } from "./poller/statusPoller.js";
import { createApp } from "./server.js";
async function main(): Promise<void> {
const config = loadConfig(process.env);
const advertAddress = process.env.ADVERT_SCRIPT_ADDRESS ?? "";
const escrowAddress = process.env.ESCROW_SCRIPT_ADDRESS ?? "";
if (!advertAddress || !escrowAddress) {
console.warn(
"[indexer] ADVERT_SCRIPT_ADDRESS / ESCROW_SCRIPT_ADDRESS not set — " +
"running in 'no scripts watched' mode. Set them via env or use M1-F config bridge."
);
}
const cache = new SqliteCache(config.dbPath);
const source = new OgmiosSource(config.ogmiosUrl);
const worker = new ChainSyncWorker({
source,
cache,
addresses: { advertAddress, escrowAddress },
skipBeforeSlot: config.skipBeforeSlot,
});
const poller = new StatusPoller({ cache, pollIntervalMs: config.statusPollMs });
const app = createApp(
{ cache, worker },
config.uiDistDir ? { uiDistDir: config.uiDistDir } : undefined,
);
worker.on("error", (err: { message: string; stale: boolean }) => {
console.error(`[indexer] worker error: ${err.message}${err.stale ? " (stale)" : ""}`);
});
const networkLabel = config.networkId === 0 ? "Vector testnet" : "Vector mainnet";
const server = app.listen(config.indexerPort, () => {
console.log(
`[indexer] listening on :${config.indexerPort} ` +
`(NETWORK_ID=${config.networkId} — ${networkLabel})`,
);
});
await worker.start();
poller.start();
const shutdown = (signal: string): void => {
console.log(`[indexer] received ${signal} — shutting down`);
poller.stop();
worker.stop();
server.close(() => {
cache.close();
process.exit(0);
});
setTimeout(() => {
console.error("[indexer] shutdown timeout — forcing exit");
process.exit(1);
}, 10_000).unref();
};
process.on("SIGTERM", () => shutdown("SIGTERM"));
process.on("SIGINT", () => shutdown("SIGINT"));
}
// Only auto-run when invoked directly (not when imported by tests).
if (import.meta.url === `file://${process.argv[1]}`) {
main().catch((err) => {
console.error(`[indexer] fatal: ${(err as Error).message}`);
process.exit(1);
});
}

View File

@@ -0,0 +1,146 @@
/**
* indexer/src/poller/statusPoller.ts — supplier status poller.
*
* Polls each Active supplier's /status endpoint every STATUS_POLL_MS ms.
* Writes results to supplier_status table via SqliteCache.
*
* Design:
* - In-process setInterval loop (default 20_000 ms)
* - Pluggable interval (pass 0 or undefined, then use tickOnce() for tests)
* - tickOnce() is a test-only synchronous/async trigger for a single poll cycle
* - Supplier 503 / network error → row with status="offline", error logged
* - Retired advertisements are NOT polled (cache.listActiveAdvertisements filters them out)
* - fetch is the HTTP client (can be stubbed via vi.stubGlobal in tests)
*/
import type { SqliteCache } from "../db/cache.js";
const STATUS_POLL_TIMEOUT_MS = 5000;
const VALID_STATUSES = new Set(["free", "working", "offline"]);
export interface StatusPollerOpts {
cache: SqliteCache;
pollIntervalMs?: number; // default 20_000; pass 0 to disable auto-tick
}
export interface SupplierStatusResponse {
status: "free" | "working" | "offline";
current_escrow_ref?: string;
last_seen?: string;
}
export class StatusPoller {
private cache: SqliteCache;
private pollIntervalMs: number;
private intervalHandle: ReturnType<typeof setInterval> | null = null;
constructor(opts: StatusPollerOpts) {
this.cache = opts.cache;
this.pollIntervalMs = opts.pollIntervalMs ?? 20_000;
}
start(): void {
if (this.intervalHandle !== null) return;
if (this.pollIntervalMs <= 0) return;
this.intervalHandle = setInterval(() => {
void this.tickOnce();
}, this.pollIntervalMs);
if (typeof (this.intervalHandle as unknown as { unref?: () => void }).unref === "function") {
(this.intervalHandle as unknown as { unref: () => void }).unref();
}
}
stop(): void {
if (this.intervalHandle !== null) {
clearInterval(this.intervalHandle);
this.intervalHandle = null;
}
}
/**
* tickOnce — test-only: runs a single poll cycle immediately.
* Returns when all Active suppliers have been polled (or failed).
*/
async tickOnce(): Promise<void> {
const adverts = this.cache.listActiveAdvertisements();
if (adverts.length === 0) return;
await Promise.all(
adverts.map(async (advert) => {
await this.pollOne(advert.endpoint_url, advert.supplier_pkh, advert.utxo_ref);
})
);
}
private async pollOne(endpointUrl: string, supplierPkh: string, advertRef: string): Promise<void> {
const url = `${endpointUrl}/status`;
const nowIso = new Date().toISOString();
const nowMs = Date.now();
const writeOffline = (): void => {
this.cache.upsertSupplierStatus({
supplier_pkh: supplierPkh,
advert_ref: advertRef,
status: "offline",
last_seen_iso: nowIso,
current_escrow_ref: null,
polled_at: nowMs,
});
};
let response: Response;
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), STATUS_POLL_TIMEOUT_MS);
try {
response = await fetch(url, { method: "GET", signal: controller.signal });
} catch (err) {
console.warn(`[statusPoller] fetch failed for ${url}: ${(err as Error).message}`);
writeOffline();
return;
} finally {
clearTimeout(timer);
}
if (!response.ok) {
console.warn(`[statusPoller] non-OK status from ${url}: ${response.status}`);
writeOffline();
return;
}
let body: unknown;
try {
body = await response.json();
} catch (err) {
console.warn(`[statusPoller] malformed JSON from ${url}: ${(err as Error).message}`);
writeOffline();
return;
}
const parsed = this.parseStatusBody(body);
if (parsed === null) {
console.warn(`[statusPoller] wrong-shape body from ${url}`);
writeOffline();
return;
}
this.cache.upsertSupplierStatus({
supplier_pkh: supplierPkh,
advert_ref: advertRef,
status: parsed.status,
last_seen_iso: parsed.last_seen ?? nowIso,
current_escrow_ref: parsed.current_escrow_ref ?? null,
polled_at: nowMs,
});
}
private parseStatusBody(body: unknown): SupplierStatusResponse | null {
if (typeof body !== "object" || body === null) return null;
const b = body as Record<string, unknown>;
if (typeof b.status !== "string") return null;
if (!VALID_STATUSES.has(b.status)) return null;
const out: SupplierStatusResponse = { status: b.status as SupplierStatusResponse["status"] };
if (typeof b.current_escrow_ref === "string") out.current_escrow_ref = b.current_escrow_ref;
if (typeof b.last_seen === "string") out.last_seen = b.last_seen;
return out;
}
}

View File

@@ -0,0 +1,61 @@
/**
* indexer/src/routes/capabilities.ts
*
* GET /capabilities — distinct capability_id list with active supplier counts.
* GET /capabilities/:id/suppliers — filtered list, sortable by ?sort=price or ?sort=last_seen.
*/
import { Router, type Request, type Response } from "express";
import type { SqliteCache } from "../db/cache.js";
import { shapeSupplier } from "./suppliers.js";
export interface CapabilitiesDeps {
cache: SqliteCache;
}
export function capabilitiesRouter(deps: CapabilitiesDeps): Router {
const router = Router();
router.get("/capabilities", (_req: Request, res: Response) => {
const adverts = deps.cache.listActiveAdvertisements();
const counts = new Map<string, number>();
for (const a of adverts) {
counts.set(a.capability_id, (counts.get(a.capability_id) ?? 0) + 1);
}
const out = Array.from(counts.entries()).map(([capability_id, supplier_count]) => ({
capability_id,
supplier_count,
}));
res.status(200).json(out);
});
router.get("/capabilities/:id/suppliers", (req: Request, res: Response) => {
const capId = String(req.params.id);
const sort = typeof req.query.sort === "string" ? req.query.sort : null;
const adverts = deps.cache.listActiveAdvertisements({ capability_id: capId });
let views = adverts.map((a) => {
const status = deps.cache.getSupplierStatus(a.supplier_pkh);
return shapeSupplier(a, status);
});
if (sort === "price") {
views = views.slice().sort((a, b) => {
const ap = BigInt(a.price_lovelace);
const bp = BigInt(b.price_lovelace);
return ap < bp ? -1 : ap > bp ? 1 : 0;
});
} else if (sort === "last_seen") {
views = views.slice().sort((a, b) => {
// Most recent first
const at = a.last_seen_iso ?? "";
const bt = b.last_seen_iso ?? "";
return at < bt ? 1 : at > bt ? -1 : 0;
});
}
res.status(200).json(views);
});
return router;
}

View File

@@ -0,0 +1,86 @@
/**
* indexer/src/routes/escrows.ts
*
* GET /escrows/:ref — ref is "<txHash>#<index>"; 404 if not found.
* GET /escrows?buyer=PKH — list escrows by buyer_pkh.
* GET /escrows?supplier=PKH — list escrows by supplier_pkh.
*/
import { Router, type Request, type Response } from "express";
import type { SqliteCache, EscrowRow } from "../db/cache.js";
export interface EscrowsDeps {
cache: SqliteCache;
}
interface EscrowView {
utxo_ref: string;
buyer_pkh: string;
supplier_pkh: string;
advert_ref: string; // canonical "<txHash>#<index>"
capability_id: string;
request_spec_hash: string;
prompt_hash: string;
payment_lovelace: string;
buyer_bond_lovelace: string;
supplier_bond_lovelace: string;
deliver_by: number;
posted_at: number;
submitted_at: number | null;
result_receipt_hash: string | null;
state: string;
created_slot: number;
}
function shapeEscrow(row: EscrowRow): EscrowView {
return {
utxo_ref: row.utxo_ref,
buyer_pkh: row.buyer_pkh,
supplier_pkh: row.supplier_pkh,
advert_ref: `${row.advert_ref_tx}#${row.advert_ref_index}`,
capability_id: row.capability_id,
request_spec_hash: row.request_spec_hash,
prompt_hash: row.prompt_hash,
payment_lovelace: row.payment_lovelace,
buyer_bond_lovelace: row.buyer_bond_lovelace,
supplier_bond_lovelace: row.supplier_bond_lovelace,
deliver_by: row.deliver_by,
posted_at: row.posted_at,
submitted_at: row.submitted_at,
result_receipt_hash: row.result_receipt_hash,
state: row.state,
created_slot: row.created_slot,
};
}
export function escrowsRouter(deps: EscrowsDeps): Router {
const router = Router();
// The list-by-query route MUST be registered before /:ref so that GET /escrows
// (no path segment) is matched here. Express's `/:ref` would not match "" anyway,
// but the explicit ordering keeps the intent clear.
router.get("/escrows", (req: Request, res: Response) => {
const buyer = typeof req.query.buyer === "string" ? req.query.buyer : null;
const supplier = typeof req.query.supplier === "string" ? req.query.supplier : null;
if (!buyer && !supplier) {
res.status(400).json({ error: "must provide ?buyer=<pkh> or ?supplier=<pkh>" });
return;
}
const rows: EscrowRow[] = buyer
? deps.cache.listEscrowsByBuyer(buyer)
: deps.cache.listEscrowsBySupplier(supplier!);
res.status(200).json(rows.map(shapeEscrow));
});
router.get("/escrows/:ref", (req: Request, res: Response) => {
const ref = String(req.params.ref);
const row = deps.cache.getEscrowByRef(ref);
if (!row) {
res.status(404).json({ error: "escrow not found" });
return;
}
res.status(200).json(shapeEscrow(row));
});
return router;
}

View File

@@ -0,0 +1,200 @@
/**
* indexer/src/routes/events.ts — long-lived multiplexed SSE stream.
*
* Clients connect to GET /events?stream=1 and receive a text/event-stream
* response that stays open until they disconnect. Each marketplace event is
* sent as:
* event: <type>\n
* data: <json>\n\n
*
* Optional: ?since_slot=N — replays buffered events from cache with slot > N
* BEFORE the connection joins the live worker stream. Both replayed and live
* events serialize to the same ChainEventPayload shape (note: `ref`, not
* `utxo_ref`).
*
* Optional: ?limit=N — caps the replay to the most recent N buffered events
* (chronological ASC). When combined with `since_slot`, the slot filter runs
* first and `limit` then trims to the most recent N matches. Useful for the
* indexer-ui EventsLog seed: `?stream=1&limit=10` to start with the last 10
* historical events and stream live additions from there.
*
* Heartbeat: every `heartbeatMs` ms (default 25 s) the server writes a
* `: keepalive\n\n` SSE comment frame to all open connections, keeping the
* connection alive through proxies and revealing dead clients eagerly.
*
* Per-connection listeners are attached to the worker's EventEmitter on
* connect and removed on `close` so disconnected clients neither receive
* future events nor cause "write after end" errors.
*/
import { Router, type Request, type Response } from "express";
import type { EventEmitter } from "events";
import type { SqliteCache } from "../db/cache.js";
export interface EventsDeps {
cache: SqliteCache;
/**
* Milliseconds between SSE keepalive heartbeat comment frames.
* Default: 25_000 (25 s). Tests may pass a low value (e.g. 50ms) to verify
* heartbeat behaviour without waiting wall-clock seconds.
*/
heartbeatMs?: number;
}
interface ChainEventPayload {
type: string;
ref: string;
slot: number;
txHash?: string;
address?: string;
}
interface SyncProgressPayload {
type: "sync-progress";
slot: number;
tipSlot?: number;
percentage?: number;
}
/**
* Builds a router exposing GET /events. Each request opens a long-lived
* SSE stream. Replay-then-live ordering is preserved.
*
* Returns the router; tests/shutdown can close the underlying HTTP server
* to drain connections.
*/
export function eventsRouter(workerEmitter: EventEmitter, deps: EventsDeps): {
router: Router;
} {
const router = Router();
const heartbeatMs = deps.heartbeatMs ?? 25_000;
router.get("/events", (req: Request, res: Response) => {
res.status(200);
res.setHeader("Content-Type", "text/event-stream");
res.setHeader("Cache-Control", "no-cache");
res.setHeader("Connection", "keep-alive");
res.setHeader("X-Accel-Buffering", "no");
res.flushHeaders?.();
// Initial connect comment — primes the stream so clients (and proxies)
// see the response start immediately.
res.write(": connected\n\n");
// ─── Replay (synchronous; must complete before joining live stream) ──
const sinceSlotRaw = req.query.since_slot;
const limitRaw = req.query.limit;
const limit = parsePositiveInt(limitRaw);
const sinceSlot = parseNonNegativeInt(sinceSlotRaw);
if (sinceSlot !== null || limit !== null) {
try {
let past: ReturnType<typeof deps.cache.listEventsAfterSlot>;
if (sinceSlot !== null) {
past = deps.cache.listEventsAfterSlot(sinceSlot);
if (limit !== null && past.length > limit) {
// Keep the most recent `limit` matches; preserve chronological ASC.
past = past.slice(past.length - limit);
}
} else {
past = deps.cache.listRecentEvents(limit!);
}
for (const ev of past) {
const payload: ChainEventPayload = {
type: ev.type,
ref: ev.utxo_ref,
slot: ev.slot,
txHash: ev.tx_hash,
};
writeSafely(res, `event: ${ev.type}\ndata: ${JSON.stringify(payload)}\n\n`);
}
} catch (err) {
console.warn(
`[events] replay failed (since_slot=${sinceSlotRaw}, limit=${limitRaw}): ${(err as Error).message}`,
);
}
}
// ─── Live: per-connection listeners on the worker emitter ────────────
const onChainEvent = (payload: ChainEventPayload): void => {
const safe: ChainEventPayload = {
type: payload.type,
ref: payload.ref,
slot: payload.slot,
txHash: payload.txHash,
address: payload.address,
};
writeSafely(res, `event: ${payload.type}\ndata: ${JSON.stringify(safe)}\n\n`);
};
const onSyncProgress = (payload: { currentSlot?: number; slot?: number; tipSlot?: number }): void => {
// Worker emits { currentSlot, tipSlot }; older code shape was { slot, tipSlot }.
// Map both to `slot` on the wire so the UI's EventsLog doesn't render "slot 0".
const slot = payload.currentSlot ?? payload.slot ?? 0;
const tipSlot = payload.tipSlot ?? 0;
const safe: SyncProgressPayload = {
type: "sync-progress",
slot,
tipSlot,
percentage: tipSlot > 0 ? Math.min(100, Math.round((slot / tipSlot) * 100)) : 0,
};
writeSafely(res, `event: sync-progress\ndata: ${JSON.stringify(safe)}\n\n`);
};
workerEmitter.on("chain-event", onChainEvent);
workerEmitter.on("sync-progress", onSyncProgress);
// ─── Heartbeat keepalive ─────────────────────────────────────────────
const heartbeat = setInterval(() => {
writeSafely(res, ": keepalive\n\n");
}, heartbeatMs);
// Don't keep the Node event loop alive solely for the heartbeat.
if (typeof (heartbeat as unknown as { unref?: () => void }).unref === "function") {
(heartbeat as unknown as { unref: () => void }).unref();
}
// ─── Cleanup on disconnect ───────────────────────────────────────────
let cleaned = false;
const cleanup = (): void => {
if (cleaned) return;
cleaned = true;
clearInterval(heartbeat);
workerEmitter.off("chain-event", onChainEvent);
workerEmitter.off("sync-progress", onSyncProgress);
};
req.on("close", cleanup);
res.on("close", cleanup);
res.on("error", cleanup);
});
return { router };
}
function parseNonNegativeInt(raw: unknown): number | null {
if (typeof raw !== "string") return null;
const n = Number(raw);
if (!Number.isFinite(n) || !Number.isInteger(n) || n < 0) return null;
return n;
}
function parsePositiveInt(raw: unknown): number | null {
if (typeof raw !== "string") return null;
const n = Number(raw);
if (!Number.isFinite(n) || !Number.isInteger(n) || n <= 0) return null;
return n;
}
/**
* Best-effort write to an SSE response. If the underlying socket is gone
* (e.g. client just disconnected and cleanup hasn't fired yet), swallow the
* error so we don't tear down the process.
*/
function writeSafely(res: Response, chunk: string): void {
try {
if (res.writableEnded || res.destroyed) return;
res.write(chunk);
} catch {
// Connection already closed; the close handler will clean up.
}
}

View File

@@ -0,0 +1,37 @@
/**
* indexer/src/routes/healthz.ts — GET /healthz
*
* SPEC FIX 2026-04-27: standardized to /healthz (was /health — indexer drift).
* Every service now exposes GET /healthz per ARCHITECTURE.md §5 endpoint table.
*
* Returns:
* { ok: boolean, sync_slot: number, tip_slot: number,
* ogmios_status: "connected" | "disconnected",
* db_size_bytes: number }
*/
import { Router, type Request, type Response } from "express";
import type { SqliteCache } from "../db/cache.js";
import type { ChainSyncWorker } from "../follower/worker.js";
export interface HealthzDeps {
cache: SqliteCache;
worker: ChainSyncWorker;
}
export function healthzRouter(deps: HealthzDeps): Router {
const router = Router();
router.get("/healthz", (_req: Request, res: Response) => {
const syncSlot = deps.worker.getCurrentSlot();
const tipSlot = deps.worker.getTipSlot();
const dbSize = deps.cache.dbSizeBytes();
res.status(200).json({
ok: true,
sync_slot: syncSlot,
tip_slot: tipSlot,
ogmios_status: tipSlot > 0 ? "connected" : "disconnected",
db_size_bytes: dbSize,
});
});
return router;
}

View File

@@ -0,0 +1,84 @@
/**
* indexer/src/routes/suppliers.ts — GET /suppliers, GET /suppliers/:pkh
*
* GET /suppliers — list all Active advertisements joined with latest supplier_status.
* GET /suppliers/:pkh — single supplier detail; 404 if no active advert for pkh.
*/
import { Router, type Request, type Response } from "express";
import type { SqliteCache, AdvertRow, SupplierStatusRow } from "../db/cache.js";
export interface SuppliersDeps {
cache: SqliteCache;
}
interface SupplierView {
utxo_ref: string;
supplier_pkh: string;
capability_id: string;
model: string;
max_output_tokens: number;
max_processing_ms: number;
price_lovelace: string;
supplier_bond_lovelace: string;
buyer_bond_lovelace: string;
endpoint_url: string;
detail_uri: string;
detail_hash: string;
advertised_at: number;
status: string; // status from supplier_status (free|working|offline) or "unknown"
advert_status: string; // Active|Retired
current_escrow_ref: string | null;
last_seen_iso: string | null;
created_slot: number;
}
export function shapeSupplier(advert: AdvertRow, status: SupplierStatusRow | null): SupplierView {
return {
utxo_ref: advert.utxo_ref,
supplier_pkh: advert.supplier_pkh,
capability_id: advert.capability_id,
model: advert.model,
max_output_tokens: advert.max_output_tokens,
max_processing_ms: advert.max_processing_ms,
price_lovelace: advert.price_lovelace,
supplier_bond_lovelace: advert.supplier_bond_lovelace,
buyer_bond_lovelace: advert.buyer_bond_lovelace,
endpoint_url: advert.endpoint_url,
detail_uri: advert.detail_uri,
detail_hash: advert.detail_hash,
advertised_at: advert.advertised_at,
status: status?.status ?? "unknown",
advert_status: advert.status,
current_escrow_ref: status?.current_escrow_ref ?? null,
last_seen_iso: status?.last_seen_iso ?? null,
created_slot: advert.created_slot,
};
}
export function suppliersRouter(deps: SuppliersDeps): Router {
const router = Router();
router.get("/suppliers", (_req: Request, res: Response) => {
const adverts = deps.cache.listActiveAdvertisements();
const out = adverts.map((a) => {
const status = deps.cache.getSupplierStatus(a.supplier_pkh);
return shapeSupplier(a, status);
});
res.status(200).json(out);
});
router.get("/suppliers/:pkh", (req: Request, res: Response) => {
const pkh = String(req.params.pkh);
const adverts = deps.cache.listActiveAdvertisements({ supplier_pkh: pkh });
if (adverts.length === 0) {
res.status(404).json({ error: "supplier not found" });
return;
}
const advert = adverts[0];
const status = deps.cache.getSupplierStatus(pkh);
res.status(200).json(shapeSupplier(advert, status));
});
return router;
}

99
indexer/src/server.ts Normal file
View File

@@ -0,0 +1,99 @@
/**
* indexer/src/server.ts — Express app factory for the indexer.
*
* createApp(deps, options) returns an Express Application with all routes
* wired. Dependency injection allows tests to supply mock cache, worker, etc.
*
* JSON routes wired (priority order — these are mounted FIRST so they always
* take precedence over the SPA catch-all):
* GET /healthz
* GET /suppliers
* GET /suppliers/:pkh
* GET /capabilities
* GET /capabilities/:id/suppliers
* GET /escrows/:ref
* GET /escrows
* GET /events
*
* Optional UI mount (M1-F-5):
* When `options.uiDistDir` is supplied:
* - express.static(uiDistDir) is mounted AFTER the JSON routes so /app.js,
* /assets/*, etc. are served from the bundled dashboard build.
* - app.get('*', sendFile('index.html')) is mounted LAST as the SPA
* catch-all so any unknown path returns the dashboard HTML — keeping
* client-side routing working.
* When `uiDistDir` is omitted (default), behaviour is unchanged: unknown
* paths return 404 (Express default). This preserves backwards-compat with
* existing unit tests that call createApp(deps).
*/
import express, { type Application } from "express";
import type { EventEmitter } from "events";
import type { SqliteCache } from "./db/cache.js";
import type { ChainSyncWorker } from "./follower/worker.js";
import { healthzRouter } from "./routes/healthz.js";
import { suppliersRouter } from "./routes/suppliers.js";
import { capabilitiesRouter } from "./routes/capabilities.js";
import { escrowsRouter } from "./routes/escrows.js";
import { eventsRouter } from "./routes/events.js";
export interface IndexerDeps {
cache: SqliteCache;
worker: ChainSyncWorker;
}
export interface IndexerAppOptions {
/**
* Milliseconds between SSE keepalive comment frames sent to all connected clients.
* Default: 25_000 (25 s). Tests pass a lower value (e.g. 50 ms) to verify the
* heartbeat without waiting wall-clock seconds.
*/
heartbeatMs?: number;
/**
* Absolute path to the bundled indexer-ui /dist directory. When provided,
* static assets and an SPA catch-all are mounted AFTER all JSON routes so
* the same Express app serves the dashboard alongside the API. Omitting it
* preserves the legacy "API-only" behaviour.
*/
uiDistDir?: string;
}
export function createApp(deps: IndexerDeps, options?: IndexerAppOptions): Application {
const app = express();
app.use(healthzRouter({ cache: deps.cache, worker: deps.worker }));
app.use(suppliersRouter({ cache: deps.cache }));
app.use(capabilitiesRouter({ cache: deps.cache }));
app.use(escrowsRouter({ cache: deps.cache }));
// Long-lived multiplexed SSE: per-connection listeners on the worker
// EventEmitter, replay-then-live ordering, heartbeat keepalive.
const { router: events } = eventsRouter(deps.worker as unknown as EventEmitter, {
cache: deps.cache,
heartbeatMs: options?.heartbeatMs,
});
app.use(events);
// App-level "warm" listeners on the worker emitter. These exist so that
// - the boot contract holds (listenerCount > 0 immediately after createApp)
// - downstream observers (metrics, logging) can hook in without waiting
// for an SSE client to connect.
// Per-connection SSE delivery is independent and additive (eventsRouter
// attaches its own listeners on each request).
deps.worker.on("chain-event", () => { /* observed via SSE per-connection */ });
deps.worker.on("sync-progress", () => { /* observed via SSE per-connection */ });
// SPA mount — must run AFTER all JSON routes so /healthz, /suppliers, etc.
// keep priority. Static files first, then catch-all → index.html.
if (options?.uiDistDir) {
const uiDir = options.uiDistDir;
app.use(express.static(uiDir));
app.get("*", (_req, res) => {
res.sendFile("index.html", { root: uiDir });
});
}
return app;
}