buyer/pdf: durable job store + past-work list + resume-on-return
The summarization runs server-side, but the SPA held the job only in component state and jobs lived only in memory — so navigating away from the Book Summarizer reset the UI (losing the handle to a still-running job) and past summaries vanished on restart. - PdfJobDb: SQLite store (its own file under ARCHIVE_DIR) persisting every job on each progress update. - JobStore: createdAt, persist-on-update, viewOf()/list() that serve live jobs first then fall back to the durable record; non-live jobs whose worker died surface as a new "interrupted" status. - routes: GET /v1/pdf-jobs (newest-first list); GET /:id + summary.md serve past jobs via viewOf; /:id/events sends a single terminal frame for a finished/past job (no hang) and live frames otherwise. - SPA: RecentJobs list (open any job → reconnect to live progress or show result), localStorage resume of the active job on return, and a "back to list (keeps running)" affordance. - mainnet buyer compose: persistent named volume + ARCHIVE_DIR so the archive + pdf-jobs.db survive container recreation/redeploys. Tests: persistence-across-restart, interrupted detection, list, and past-job serving (incl. terminal SSE). Full buyer suite green (185). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -5,7 +5,11 @@
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { mkdtempSync, rmSync } from "fs";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import { JobStore, type RunCallFn } from "../../buyer/src/pdf/summarize-job.js";
|
||||
import { PdfJobDb } from "../../buyer/src/pdf/job-db.js";
|
||||
import { loadPdfCaps } from "../../buyer/src/pdf/caps.js";
|
||||
import type { Chunk } from "../../buyer/src/pdf/types.js";
|
||||
import type { Marketplace } from "../../buyer/src/sdk/Marketplace.js";
|
||||
@@ -166,6 +170,66 @@ describe("JobStore orchestrator", () => {
|
||||
expect(typeof job.finalSummary).toBe("string");
|
||||
});
|
||||
|
||||
it("persists a completed job durably across a JobStore 'restart'", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "pdfjobs-"));
|
||||
try {
|
||||
const canned: RunCallFn = async (sup) => ({
|
||||
response: "S",
|
||||
escrowRef: `${"f".repeat(64)}#${Math.floor(Math.random() * 1e9)}`,
|
||||
supplierPkh: sup.supplierPkh,
|
||||
model: sup.model,
|
||||
receipt: {},
|
||||
receiptSignature: "sig",
|
||||
});
|
||||
const db1 = new PdfJobDb(dir);
|
||||
const store1 = new JobStore({
|
||||
marketplace: MARKETPLACE, chain: CHAIN, walletKey: WALLET, indexerUrl: "http://i",
|
||||
caps: loadPdfCaps({}), db: db1, runCall: canned, walletBalance: async () => 10_000_000_000n,
|
||||
});
|
||||
const job = store1.createJob("persist.pdf", 3, chunks(3));
|
||||
await waitForDone(store1, job);
|
||||
expect(job.status).toBe("completed");
|
||||
db1.close();
|
||||
|
||||
// Fresh process: new db connection + new store (empty in-memory map).
|
||||
const db2 = new PdfJobDb(dir);
|
||||
const store2 = new JobStore({
|
||||
marketplace: MARKETPLACE, chain: CHAIN, walletKey: WALLET, indexerUrl: "http://i",
|
||||
caps: loadPdfCaps({}), db: db2,
|
||||
});
|
||||
const v = store2.viewOf(job.jobId);
|
||||
expect(v).not.toBeNull();
|
||||
expect(v?.status).toBe("completed");
|
||||
expect(typeof v?.final_summary_md).toBe("string");
|
||||
expect((v?.escrow_refs.length ?? 0)).toBeGreaterThan(0);
|
||||
expect(store2.list().find((j) => j.job_id === job.jobId)?.has_summary).toBe(true);
|
||||
db2.close();
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("reports a persisted-but-not-live running job as interrupted", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "pdfjobs-"));
|
||||
try {
|
||||
const db = new PdfJobDb(dir);
|
||||
db.upsert({
|
||||
job_id: "job_crashed", filename: "x.pdf", status: "running", page_count: 1, chunk_count: 2,
|
||||
coverage_done: 1, coverage_total: 2, running_cost_lovelace: "200000", final_summary_md: null,
|
||||
chunk_results_json: "[]", escrow_refs_json: "[]", created_at: 1, updated_at: 2,
|
||||
});
|
||||
const store = new JobStore({
|
||||
marketplace: MARKETPLACE, chain: CHAIN, walletKey: WALLET, indexerUrl: "http://i",
|
||||
caps: loadPdfCaps({}), db,
|
||||
});
|
||||
expect(store.viewOf("job_crashed")?.status).toBe("interrupted");
|
||||
expect(store.list().find((j) => j.job_id === "job_crashed")?.status).toBe("interrupted");
|
||||
db.close();
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("fails cleanly when no capable suppliers exist", async () => {
|
||||
const emptyMarket = { discoverSuppliers: async () => [] } as unknown as Marketplace;
|
||||
const store = new JobStore({
|
||||
|
||||
@@ -6,8 +6,12 @@
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { mkdtempSync, rmSync } from "fs";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import { createApp } from "../../buyer/src/server.js";
|
||||
import { JobStore, type RunCallFn } from "../../buyer/src/pdf/summarize-job.js";
|
||||
import { PdfJobDb } from "../../buyer/src/pdf/job-db.js";
|
||||
import { loadPdfCaps } from "../../buyer/src/pdf/caps.js";
|
||||
import type { Chunk } from "../../buyer/src/pdf/types.js";
|
||||
import type { Marketplace } from "../../buyer/src/sdk/Marketplace.js";
|
||||
@@ -44,7 +48,7 @@ const MARKETPLACE = {
|
||||
const WALLET = { address: "addr_test", pubKeyHash: "pkh", pubKeyHex: "", privateKeyHex: "0".repeat(64) } as WalletKey;
|
||||
const CHAIN = {} as ChainProvider;
|
||||
|
||||
function makeStore(opts?: { balance?: bigint; caps?: Record<string, string> }): JobStore {
|
||||
function makeStore(opts?: { balance?: bigint; caps?: Record<string, string>; db?: PdfJobDb }): JobStore {
|
||||
let n = 0;
|
||||
const runCall: RunCallFn = async (sup) => {
|
||||
n += 1;
|
||||
@@ -65,6 +69,7 @@ function makeStore(opts?: { balance?: bigint; caps?: Record<string, string> }):
|
||||
caps: loadPdfCaps(opts?.caps ?? {}),
|
||||
runCall,
|
||||
walletBalance: async () => opts?.balance ?? 10_000_000_000n,
|
||||
db: opts?.db,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -100,6 +105,65 @@ describe("/v1/pdf-* routes", () => {
|
||||
expect(["pdf_parse_failed", "scanned_or_image_pdf", "no_text"]).toContain(res.body.error);
|
||||
});
|
||||
|
||||
it("lists jobs newest-first via GET /v1/pdf-jobs", async () => {
|
||||
const store = makeStore();
|
||||
const a = store.createJob("First.pdf", 1, chunks(1));
|
||||
const b = store.createJob("Second.pdf", 2, chunks(2));
|
||||
const app = createApp({ jobStore: store, pdfCaps: loadPdfCaps({}) });
|
||||
const { default: request } = await import("supertest");
|
||||
const res = await request(app).get("/v1/pdf-jobs");
|
||||
expect(res.status).toBe(200);
|
||||
expect(Array.isArray(res.body.jobs)).toBe(true);
|
||||
const ids = res.body.jobs.map((j: { job_id: string }) => j.job_id);
|
||||
expect(ids).toContain(a.jobId);
|
||||
expect(ids).toContain(b.jobId);
|
||||
const found = res.body.jobs.find((j: { job_id: string }) => j.job_id === b.jobId);
|
||||
expect(found.filename).toBe("Second.pdf");
|
||||
expect(found.status).toBe("estimated");
|
||||
});
|
||||
|
||||
it("serves a past (persisted, non-live) job's view, summary.md, and terminal SSE", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "pdfroutes-"));
|
||||
try {
|
||||
const db1 = new PdfJobDb(dir);
|
||||
const store1 = makeStore({ db: db1 });
|
||||
const seeded = store1.createJob("Past.pdf", 2, chunks(2));
|
||||
const app1 = createApp({ jobStore: store1, pdfCaps: loadPdfCaps({}) });
|
||||
const { default: request } = await import("supertest");
|
||||
await request(app1).post(`/v1/pdf-jobs/${seeded.jobId}/start`).send({ confirm: true });
|
||||
let view = seeded.view();
|
||||
for (let i = 0; i < 100 && view.status === "running"; i++) {
|
||||
await new Promise((r) => setTimeout(r, 10));
|
||||
view = (await request(app1).get(`/v1/pdf-jobs/${seeded.jobId}`)).body;
|
||||
}
|
||||
expect(view.status).toBe("completed");
|
||||
db1.close();
|
||||
|
||||
// Fresh store (no live job) backed by the same db = "after restart".
|
||||
const db2 = new PdfJobDb(dir);
|
||||
const store2 = makeStore({ db: db2 });
|
||||
const app2 = createApp({ jobStore: store2, pdfCaps: loadPdfCaps({}) });
|
||||
|
||||
const v = await request(app2).get(`/v1/pdf-jobs/${seeded.jobId}`);
|
||||
expect(v.status).toBe(200);
|
||||
expect(v.body.status).toBe("completed");
|
||||
expect(v.body.final_summary_md.length).toBeGreaterThan(0);
|
||||
|
||||
const md = await request(app2).get(`/v1/pdf-jobs/${seeded.jobId}/summary.md`);
|
||||
expect(md.status).toBe(200);
|
||||
|
||||
// SSE on a non-live job returns a single terminal frame and ENDS (no hang).
|
||||
const ev = await request(app2).get(`/v1/pdf-jobs/${seeded.jobId}/events`).buffer(true);
|
||||
expect(ev.text).toContain("event: done");
|
||||
|
||||
const list = await request(app2).get("/v1/pdf-jobs");
|
||||
expect(list.body.jobs.find((j: { job_id: string }) => j.job_id === seeded.jobId)?.has_summary).toBe(true);
|
||||
db2.close();
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("404s estimate for an unknown job", async () => {
|
||||
const app = createApp({ jobStore: makeStore(), pdfCaps: loadPdfCaps({}) });
|
||||
const { default: request } = await import("supertest");
|
||||
|
||||
Reference in New Issue
Block a user