From 34a52d5d164b069d3692872a7d3c0305f61989b8 Mon Sep 17 00:00:00 2001 From: jelisaveta-talbot <268838642+jelisaveta-talbot@users.noreply.github.com> Date: Wed, 6 May 2026 22:00:31 +0200 Subject: [PATCH] buyer: TTS marketplace lifecycle (audio.synthesize.piper.v1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Path-A buyer side: PostEscrow → supplier /v1/audio/synthesize → 202+poll → on-chain Submit → receipt verify, returning the audio bytes + signed receipt to the SPA. Pairs with the supplier-side scaffolding in 3c3e009. Files - packages/shared/src/tx/escrow/postTtsEscrow.ts (new): buildPostTtsEscrowTx() mirrors postEscrow.ts but commits a TTS-shaped prompt_hash = sha256(canonical({text, voice, format, speed})). The same hash is exposed as ttsPromptHash() so the buyer SDK and supplier validation can pin the canonicalisation; any drift fails the supplier's prompt_mismatch gate. Validates body shape (text non-empty, voice/format in allow-list, speed ∈ [0.5,1.5]); reuses the chat live- cbor path which is already capability-agnostic. - packages/shared/src/tx/index.ts: export the new TTS surface. - buyer/src/sdk/Marketplace.ts: + submitTts() — full lifecycle mirror of submitPrompt with TTS-shaped POST body, /v1/audio/synthesize endpoint, and audio_b64 in the polled response. Receipt verification gates on prompt_hash via ttsPromptHash() instead of canonicalize(messages). - buyer/src/sdk/types.ts: + SubmitTtsOptions / SubmitTtsResult. - buyer/src/server.ts: + POST /v1/submit-tts (mirrors /v1/submit-prompt shape; routes to marketplace.submitTts). - buyer/src/ui/components/PiperTTSForm.tsx: two-mode rendering. When advertRef + payment_lovelace are passed, posts to /v1/submit-tts and decodes base64 audio into a blob URL. When neither is passed, falls back to /v1/synth-speech (the path-B direct demo proxy). - buyer/src/ui/pages/Dashboard.tsx: capability dispatch routes a real on-chain TTS supplier to the marketplace mode and the synthetic demo card to the direct mode (its utxo_ref="demo:piper-tts" fails the ref parser → advertRef is null → demo mode). Same component for both. Tests still passing: 31 buyer + 29 supplier; both typechecks clean. Chat path behaviourally untouched. --- buyer/src/sdk/Marketplace.ts | 264 ++++++++++++++++++ buyer/src/sdk/types.ts | 31 ++ buyer/src/server.ts | 71 +++++ buyer/src/ui/components/PiperTTSForm.tsx | 104 +++++-- buyer/src/ui/pages/Dashboard.tsx | 6 + .../shared/src/tx/escrow/postTtsEscrow.ts | 225 +++++++++++++++ packages/shared/src/tx/index.ts | 7 + 7 files changed, 679 insertions(+), 29 deletions(-) create mode 100644 packages/shared/src/tx/escrow/postTtsEscrow.ts diff --git a/buyer/src/sdk/Marketplace.ts b/buyer/src/sdk/Marketplace.ts index 3bf5c97..52bd263 100644 --- a/buyer/src/sdk/Marketplace.ts +++ b/buyer/src/sdk/Marketplace.ts @@ -71,6 +71,8 @@ import type { ChainProvider, OutputReference } from "@marketplace/shared/chain"; import type { WalletKey } from "@marketplace/shared/tx"; import { buildPostEscrowTx, + buildPostTtsEscrowTx, + ttsPromptHash, buildAcceptTx, buildReclaimTx, TxConstructionError, @@ -82,6 +84,8 @@ import type { DiscoverSuppliersOptions, SubmitPromptOptions, SubmitPromptResult, + SubmitTtsOptions, + SubmitTtsResult, AcceptResultOptions, ReclaimOptions, TaskRecord, @@ -517,6 +521,266 @@ export class Marketplace extends EventEmitterBase { }; } + // ─── submitTts — full marketplace lifecycle for audio.synthesize.piper.v1 ─ + + async submitTts(opts: SubmitTtsOptions): Promise { + const { advertRef, text, voice, format, speed, payment_lovelace } = opts; + const request = { text, voice, format, speed }; + + let advertDatum: AdvertDatum | null = null; + let escrowOutputRef: OutputReference | null = null; + let escrowRefStr = ""; + let postedAtMs = Date.now(); + + const recordFailure = (reason: string): void => { + this.historyStore.save({ + escrow_ref: escrowRefStr || `${"0".repeat(64)}#0`, + supplier_pkh: advertDatum?.supplier_pkh ?? "", + capability_id: advertDatum?.capability_id ?? "", + prompt_preview: text.length <= 100 ? text : text.slice(0, 100), + posted_at: postedAtMs, + status: "failed", + failure_reason: reason, + }); + }; + + // ── 1. Resolve advert + post escrow (TTS prompt_hash) ───────────── + let escrowResult; + try { + const utxo = await this.chain.queryUtxo(advertRef); + if (utxo && utxo.datumHex) { + try { + advertDatum = decodeAdvertDatum(utxo.datumHex); + } catch { /* builder will re-throw structurally */ } + } + escrowResult = await buildPostTtsEscrowTx({ + chain: this.chain, + buyerKey: this.walletKey, + advertRef, + request, + payment_lovelace, + }); + } catch (err) { + if (err instanceof TxConstructionError) { + recordFailure(err.reason); + throw err; + } + this.emitProgress({ type: "chain_submit_failed", detail: (err as Error).message }); + recordFailure((err as Error).message); + throw err; + } + + escrowOutputRef = escrowResult.escrowOutputRef; + escrowRefStr = refToString(escrowOutputRef); + + this.emitProgress({ type: "escrow_posted", escrow_ref: escrowRefStr }); + + // ── 2. Wait for the escrow tx to confirm ────────────────────────── + try { + await this.chain.awaitTx(escrowResult.expectedTxHash, 120_000); + } catch { /* mock providers / tests */ } + + // ── 3. Re-fetch escrow datum to capture posted_at ───────────────── + try { + const escrowUtxo = await this.chain.queryUtxo(escrowOutputRef); + if (escrowUtxo && escrowUtxo.datumHex) { + const ed = decodeEscrowDatum(escrowUtxo.datumHex); + postedAtMs = ed.posted_at; + } + } catch { /* metadata best-effort */ } + + if (!advertDatum) { + const reason = "advert datum unavailable after escrow post"; + recordFailure(reason); + throw new ReceiptVerificationError(reason); + } + + // ── 4. Call supplier /v1/audio/synthesize ───────────────────────── + const supplierBaseUrl = advertDatum.endpoint_url; + const supplierHttp = new HttpClient({ + baseUrl: supplierBaseUrl, + fetch: this.fetchImpl, + }); + const ttsBody = { text, voice, format, speed }; + + let synthResult; + try { + synthResult = await supplierHttp.postJson("/v1/audio/synthesize", ttsBody, { + headers: { "X-Escrow-Ref": escrowRefStr }, + }); + } catch (err) { + if (err instanceof HttpError) { + const reason = err.kind === "timeout" ? "timeout" : "network_error"; + const sErr = new SupplierError(reason, { message: err.message }); + recordFailure(reason); + throw sErr; + } + recordFailure((err as Error).message); + throw err; + } + + // 202 + poll mirror of submitPrompt's async chat handling. + if (synthResult.status === 202 && synthResult.body && typeof synthResult.body === "object") { + const jobId = (synthResult.body as { job_id?: string }).job_id; + if (typeof jobId !== "string" || jobId.length === 0) { + const sErr = new SupplierError("malformed_response", { + status: synthResult.status, + message: "supplier 202 response missing job_id", + }); + recordFailure("malformed_response"); + throw sErr; + } + const POLL_INTERVAL_MS = 2_000; + // TTS jobs run on CPU and finish in 1–3s for short text; the long + // tail is dominated by the Submit tx confirmation (slot rate). 180s + // mirrors the chat path so polled.status hits 200 cleanly. + const POLL_TIMEOUT_MS = 180_000; + const pollDeadline = Date.now() + POLL_TIMEOUT_MS; + let polled = synthResult; + let firstIter = true; + while (Date.now() < pollDeadline) { + if (!firstIter) { + await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS)); + } + firstIter = false; + try { + polled = await supplierHttp.getJson(`/v1/audio/synthesize/${jobId}`); + } catch (err) { + if (err instanceof HttpError) { + const reason = err.kind === "timeout" ? "timeout" : "network_error"; + const sErr = new SupplierError(reason, { message: err.message }); + recordFailure(reason); + throw sErr; + } + throw err; + } + if (polled.status === 200) break; + if (polled.status === 202) continue; + const failBody = (polled.body && typeof polled.body === "object") + ? polled.body as { reason?: string; message?: string } + : {}; + const sErr = new SupplierError(failBody.reason ?? "supplier_http_error", { + status: polled.status, + message: failBody.message ?? `supplier returned ${polled.status}`, + }); + recordFailure(sErr.reason); + throw sErr; + } + if (polled.status !== 200) { + const sErr = new SupplierError("timeout", { + status: polled.status, + message: `supplier job ${jobId} not done within ${POLL_TIMEOUT_MS}ms`, + }); + recordFailure("timeout"); + throw sErr; + } + synthResult = polled; + } + + if (!synthResult.ok) { + const bodyReason = + synthResult.body && typeof synthResult.body === "object" + ? ((synthResult.body as { reason?: string }).reason ?? "supplier_http_error") + : "supplier_http_error"; + const sErr = new SupplierError(bodyReason, { + status: synthResult.status, + message: `supplier returned ${synthResult.status}`, + }); + recordFailure(bodyReason); + throw sErr; + } + if (synthResult.parseError || !synthResult.body || typeof synthResult.body !== "object") { + const sErr = new SupplierError("malformed_response", { + status: synthResult.status, + message: "supplier body is not valid JSON", + }); + recordFailure("malformed_response"); + throw sErr; + } + + const responseBody = synthResult.body as { + audio_b64?: string; + format?: string; + content_type?: string; + byte_length?: number; + receipt?: import("@marketplace/shared/receipt").Receipt; + receipt_signature?: string; + }; + if (!responseBody.audio_b64 || !responseBody.receipt || !responseBody.receipt_signature) { + const sErr = new SupplierError("malformed_response", { + status: synthResult.status, + message: "supplier response is missing audio_b64 / receipt / receipt_signature", + }); + recordFailure("malformed_response"); + throw sErr; + } + const receipt = responseBody.receipt; + const receiptSignature = responseBody.receipt_signature; + + this.emitProgress({ type: "supplier_called", escrow_ref: escrowRefStr }); + + // ── 5. Receipt verification ─────────────────────────────────────── + if (receipt.supplier_pkh !== advertDatum.supplier_pkh) { + recordFailure("wrong_supplier"); + throw new ReceiptVerificationError("wrong_supplier"); + } + if (receipt.escrow_ref !== escrowRefStr) { + recordFailure("wrong_escrow_ref"); + throw new ReceiptVerificationError("wrong_escrow_ref"); + } + // The TTS prompt commitment uses the SAME canonical hash that the + // supplier validated against the escrow datum. If we recompute it from + // the request envelope we sent and it doesn't match the receipt, the + // supplier signed a receipt for a different request → reject. + const expectedPromptHash = ttsPromptHash(request); + if (receipt.prompt_hash !== expectedPromptHash) { + recordFailure("prompt_hash_mismatch"); + throw new ReceiptVerificationError("prompt_hash_mismatch"); + } + if (receipt.model !== advertDatum.model) { + recordFailure("request_spec_hash_mismatch"); + throw new ReceiptVerificationError("request_spec_hash_mismatch"); + } + if (typeof receiptSignature !== "string" || !SIG_RE.test(receiptSignature)) { + recordFailure("invalid_signature"); + throw new ReceiptVerificationError("invalid_signature"); + } + if (receiptSignature === ZERO_SIGNATURE) { + recordFailure("invalid_signature"); + throw new ReceiptVerificationError("invalid_signature"); + } + if (!HEX64_RE.test(receipt.prompt_hash) || !HEX64_RE.test(receipt.response_hash)) { + recordFailure("malformed_receipt"); + throw new ReceiptVerificationError("malformed_receipt"); + } + + this.emitProgress({ type: "receipt_verified", escrow_ref: escrowRefStr }); + + this.historyStore.save({ + escrow_ref: escrowRefStr, + supplier_pkh: advertDatum.supplier_pkh, + capability_id: advertDatum.capability_id, + prompt_preview: text.length <= 100 ? text : text.slice(0, 100), + posted_at: postedAtMs, + status: "completed", + // Reuse the chat history shape — `response` carries a marker; the + // SPA renders audio via the live result, not from history. + response: `[audio:${responseBody.format ?? format} ${responseBody.byte_length ?? "?"}B]`, + receipt, + receipt_signature: receiptSignature, + }); + + return { + audio_b64: responseBody.audio_b64, + format: responseBody.format ?? format, + content_type: responseBody.content_type ?? `audio/${format}`, + byte_length: responseBody.byte_length ?? 0, + receipt, + receiptSignature, + escrowRef: escrowOutputRef, + }; + } + // ─── acceptResult / reclaim ─────────────────────────────────────────── async acceptResult(opts: AcceptResultOptions): Promise { diff --git a/buyer/src/sdk/types.ts b/buyer/src/sdk/types.ts index 2b084db..f598fe5 100644 --- a/buyer/src/sdk/types.ts +++ b/buyer/src/sdk/types.ts @@ -154,6 +154,37 @@ export interface SubmitPromptOptions { max_output_tokens?: number; } +/** + * SubmitTtsOptions — arguments to Marketplace.submitTts() for the + * `audio.synthesize.piper.v1` capability. The same `{text, voice, format, + * speed}` envelope is hashed (sha256 ∘ canonical) into the escrow datum's + * `prompt_hash` AND posted to the supplier's /v1/audio/synthesize body, so + * the supplier can verify the request matches what was committed on chain. + */ +export interface SubmitTtsOptions { + advertRef: OutputReference; + text: string; + voice: string; // alloy | echo | fable | onyx | nova | shimmer | lessac + format: string; // mp3 | wav | opus | aac | flac + speed: number; // 0.5 .. 1.5 + payment_lovelace: bigint; +} + +/** + * SubmitTtsResult — returned from Marketplace.submitTts() on success. + * Audio is base64 over the wire to keep the JSON poll response clean; the + * caller decodes once and renders / persists as needed. + */ +export interface SubmitTtsResult { + audio_b64: string; + format: string; + content_type: string; + byte_length: number; + receipt: Receipt; + receiptSignature: string; + escrowRef: OutputReference; +} + /** * AcceptResultOptions — arguments to Marketplace.acceptResult(). */ diff --git a/buyer/src/server.ts b/buyer/src/server.ts index cb6623a..5dacf9e 100644 --- a/buyer/src/server.ts +++ b/buyer/src/server.ts @@ -419,6 +419,77 @@ export function createApp(deps: AppDeps): Express { } }); + // ── POST /v1/submit-tts — full server-side buyer lifecycle for TTS. + // Body: { advert_ref, text, voice, format, speed, payment_lovelace } + // Returns: { audio_b64, format, content_type, byte_length, receipt, + // receipt_signature, escrow_ref } + // + // Same shape as /v1/submit-prompt but routes to Marketplace.submitTts + // instead. Audio bytes are base64'd in the JSON response so the SPA's + // PiperTTSForm can decode → blob →