buyer/pdf: durable job store + past-work list + resume-on-return

The summarization runs server-side, but the SPA held the job only in
component state and jobs lived only in memory — so navigating away from
the Book Summarizer reset the UI (losing the handle to a still-running
job) and past summaries vanished on restart.

- PdfJobDb: SQLite store (its own file under ARCHIVE_DIR) persisting
  every job on each progress update.
- JobStore: createdAt, persist-on-update, viewOf()/list() that serve
  live jobs first then fall back to the durable record; non-live jobs
  whose worker died surface as a new "interrupted" status.
- routes: GET /v1/pdf-jobs (newest-first list); GET /:id + summary.md
  serve past jobs via viewOf; /:id/events sends a single terminal frame
  for a finished/past job (no hang) and live frames otherwise.
- SPA: RecentJobs list (open any job → reconnect to live progress or
  show result), localStorage resume of the active job on return, and a
  "back to list (keeps running)" affordance.
- mainnet buyer compose: persistent named volume + ARCHIVE_DIR so the
  archive + pdf-jobs.db survive container recreation/redeploys.

Tests: persistence-across-restart, interrupted detection, list, and
past-job serving (incl. terminal SSE). Full buyer suite green (185).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
david
2026-06-03 11:59:20 +02:00
parent c552c4caa3
commit cf77811331
10 changed files with 674 additions and 50 deletions

View File

@@ -5,7 +5,11 @@
*/
import { describe, it, expect } from "vitest";
import { mkdtempSync, rmSync } from "fs";
import { tmpdir } from "os";
import { join } from "path";
import { JobStore, type RunCallFn } from "../../buyer/src/pdf/summarize-job.js";
import { PdfJobDb } from "../../buyer/src/pdf/job-db.js";
import { loadPdfCaps } from "../../buyer/src/pdf/caps.js";
import type { Chunk } from "../../buyer/src/pdf/types.js";
import type { Marketplace } from "../../buyer/src/sdk/Marketplace.js";
@@ -166,6 +170,66 @@ describe("JobStore orchestrator", () => {
expect(typeof job.finalSummary).toBe("string");
});
it("persists a completed job durably across a JobStore 'restart'", async () => {
const dir = mkdtempSync(join(tmpdir(), "pdfjobs-"));
try {
const canned: RunCallFn = async (sup) => ({
response: "S",
escrowRef: `${"f".repeat(64)}#${Math.floor(Math.random() * 1e9)}`,
supplierPkh: sup.supplierPkh,
model: sup.model,
receipt: {},
receiptSignature: "sig",
});
const db1 = new PdfJobDb(dir);
const store1 = new JobStore({
marketplace: MARKETPLACE, chain: CHAIN, walletKey: WALLET, indexerUrl: "http://i",
caps: loadPdfCaps({}), db: db1, runCall: canned, walletBalance: async () => 10_000_000_000n,
});
const job = store1.createJob("persist.pdf", 3, chunks(3));
await waitForDone(store1, job);
expect(job.status).toBe("completed");
db1.close();
// Fresh process: new db connection + new store (empty in-memory map).
const db2 = new PdfJobDb(dir);
const store2 = new JobStore({
marketplace: MARKETPLACE, chain: CHAIN, walletKey: WALLET, indexerUrl: "http://i",
caps: loadPdfCaps({}), db: db2,
});
const v = store2.viewOf(job.jobId);
expect(v).not.toBeNull();
expect(v?.status).toBe("completed");
expect(typeof v?.final_summary_md).toBe("string");
expect((v?.escrow_refs.length ?? 0)).toBeGreaterThan(0);
expect(store2.list().find((j) => j.job_id === job.jobId)?.has_summary).toBe(true);
db2.close();
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("reports a persisted-but-not-live running job as interrupted", async () => {
const dir = mkdtempSync(join(tmpdir(), "pdfjobs-"));
try {
const db = new PdfJobDb(dir);
db.upsert({
job_id: "job_crashed", filename: "x.pdf", status: "running", page_count: 1, chunk_count: 2,
coverage_done: 1, coverage_total: 2, running_cost_lovelace: "200000", final_summary_md: null,
chunk_results_json: "[]", escrow_refs_json: "[]", created_at: 1, updated_at: 2,
});
const store = new JobStore({
marketplace: MARKETPLACE, chain: CHAIN, walletKey: WALLET, indexerUrl: "http://i",
caps: loadPdfCaps({}), db,
});
expect(store.viewOf("job_crashed")?.status).toBe("interrupted");
expect(store.list().find((j) => j.job_id === "job_crashed")?.status).toBe("interrupted");
db.close();
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("fails cleanly when no capable suppliers exist", async () => {
const emptyMarket = { discoverSuppliers: async () => [] } as unknown as Marketplace;
const store = new JobStore({

View File

@@ -6,8 +6,12 @@
*/
import { describe, it, expect } from "vitest";
import { mkdtempSync, rmSync } from "fs";
import { tmpdir } from "os";
import { join } from "path";
import { createApp } from "../../buyer/src/server.js";
import { JobStore, type RunCallFn } from "../../buyer/src/pdf/summarize-job.js";
import { PdfJobDb } from "../../buyer/src/pdf/job-db.js";
import { loadPdfCaps } from "../../buyer/src/pdf/caps.js";
import type { Chunk } from "../../buyer/src/pdf/types.js";
import type { Marketplace } from "../../buyer/src/sdk/Marketplace.js";
@@ -44,7 +48,7 @@ const MARKETPLACE = {
const WALLET = { address: "addr_test", pubKeyHash: "pkh", pubKeyHex: "", privateKeyHex: "0".repeat(64) } as WalletKey;
const CHAIN = {} as ChainProvider;
function makeStore(opts?: { balance?: bigint; caps?: Record<string, string> }): JobStore {
function makeStore(opts?: { balance?: bigint; caps?: Record<string, string>; db?: PdfJobDb }): JobStore {
let n = 0;
const runCall: RunCallFn = async (sup) => {
n += 1;
@@ -65,6 +69,7 @@ function makeStore(opts?: { balance?: bigint; caps?: Record<string, string> }):
caps: loadPdfCaps(opts?.caps ?? {}),
runCall,
walletBalance: async () => opts?.balance ?? 10_000_000_000n,
db: opts?.db,
});
}
@@ -100,6 +105,65 @@ describe("/v1/pdf-* routes", () => {
expect(["pdf_parse_failed", "scanned_or_image_pdf", "no_text"]).toContain(res.body.error);
});
it("lists jobs newest-first via GET /v1/pdf-jobs", async () => {
const store = makeStore();
const a = store.createJob("First.pdf", 1, chunks(1));
const b = store.createJob("Second.pdf", 2, chunks(2));
const app = createApp({ jobStore: store, pdfCaps: loadPdfCaps({}) });
const { default: request } = await import("supertest");
const res = await request(app).get("/v1/pdf-jobs");
expect(res.status).toBe(200);
expect(Array.isArray(res.body.jobs)).toBe(true);
const ids = res.body.jobs.map((j: { job_id: string }) => j.job_id);
expect(ids).toContain(a.jobId);
expect(ids).toContain(b.jobId);
const found = res.body.jobs.find((j: { job_id: string }) => j.job_id === b.jobId);
expect(found.filename).toBe("Second.pdf");
expect(found.status).toBe("estimated");
});
it("serves a past (persisted, non-live) job's view, summary.md, and terminal SSE", async () => {
const dir = mkdtempSync(join(tmpdir(), "pdfroutes-"));
try {
const db1 = new PdfJobDb(dir);
const store1 = makeStore({ db: db1 });
const seeded = store1.createJob("Past.pdf", 2, chunks(2));
const app1 = createApp({ jobStore: store1, pdfCaps: loadPdfCaps({}) });
const { default: request } = await import("supertest");
await request(app1).post(`/v1/pdf-jobs/${seeded.jobId}/start`).send({ confirm: true });
let view = seeded.view();
for (let i = 0; i < 100 && view.status === "running"; i++) {
await new Promise((r) => setTimeout(r, 10));
view = (await request(app1).get(`/v1/pdf-jobs/${seeded.jobId}`)).body;
}
expect(view.status).toBe("completed");
db1.close();
// Fresh store (no live job) backed by the same db = "after restart".
const db2 = new PdfJobDb(dir);
const store2 = makeStore({ db: db2 });
const app2 = createApp({ jobStore: store2, pdfCaps: loadPdfCaps({}) });
const v = await request(app2).get(`/v1/pdf-jobs/${seeded.jobId}`);
expect(v.status).toBe(200);
expect(v.body.status).toBe("completed");
expect(v.body.final_summary_md.length).toBeGreaterThan(0);
const md = await request(app2).get(`/v1/pdf-jobs/${seeded.jobId}/summary.md`);
expect(md.status).toBe(200);
// SSE on a non-live job returns a single terminal frame and ENDS (no hang).
const ev = await request(app2).get(`/v1/pdf-jobs/${seeded.jobId}/events`).buffer(true);
expect(ev.text).toContain("event: done");
const list = await request(app2).get("/v1/pdf-jobs");
expect(list.body.jobs.find((j: { job_id: string }) => j.job_id === seeded.jobId)?.has_summary).toBe(true);
db2.close();
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("404s estimate for an unknown job", async () => {
const app = createApp({ jobStore: makeStore(), pdfCaps: loadPdfCaps({}) });
const { default: request } = await import("supertest");