c552c4caa3
Adds an authenticated upload flow to the buyer web app: an operator uploads a PDF, the server extracts + paragraph-aligned-chunks it (~2.2k tokens), then runs a hierarchical map-reduce summarization by paying marketplace suppliers per chunk — one on-chain escrow per map call and per reduce node. One book upload fans out into dozens of real mainnet escrows (the intended marketplace traffic) while producing a genuine full-book summary. Server module (buyer/src/pdf/): PDF extraction (text-only, rejects scanned), token-aware chunker, supplier discovery/round-robin filtered to capable models (kimi/deepseek/gpt; excludes qwen2.5:0.5b), pre-spend cost estimate + wallet-floor guard, and a JobStore orchestrator that reuses submitPrompt → findSubmittedEscrowRef → runAccept per call. v1 serializes PostEscrow under a mutex (the shared tx builder can't pin wallet inputs, so parallel escrows would double-spend); retries failed calls on a different supplier then continues with gaps. 6 /v1/pdf-* routes (multipart upload, estimate, confirm-gated start, SSE progress, result, .md download) behind the existing session auth + page/chunk/ balance caps. SPA adds a 4-step BookSummarizer wizard. Tests: chunk/estimate/supplier-pool/extract unit tests + orchestrator (happy/gap/failover) + routes integration. Full buyer suite green (181 tests); SPA build + typecheck clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
65 lines
2.5 KiB
TypeScript
65 lines
2.5 KiB
TypeScript
/**
|
|
* buyer-pdf-chunk.test.ts — the paragraph/sentence-aligned chunker must never
|
|
* cut a sentence in half, must respect the target size, and must be
|
|
* deterministic.
|
|
*/
|
|
|
|
import { describe, it, expect } from "vitest";
|
|
import { chunkText } from "../../buyer/src/pdf/chunk.js";
|
|
|
|
describe("chunkText", () => {
|
|
const sentences = Array.from(
|
|
{ length: 50 },
|
|
(_, i) => `This is sentence number ${i + 1} with some filler words added to bump the token count up.`,
|
|
);
|
|
const prose = sentences.join(" ");
|
|
|
|
it("never splits mid-sentence — every chunk ends on sentence punctuation", () => {
|
|
const chunks = chunkText(prose, { targetTokens: 60, maxTokens: 200 });
|
|
expect(chunks.length).toBeGreaterThan(1);
|
|
for (const c of chunks) {
|
|
expect(/[.!?]$/.test(c.text.trim())).toBe(true);
|
|
}
|
|
});
|
|
|
|
it("preserves every sentence verbatim (no loss, no breakage)", () => {
|
|
const chunks = chunkText(prose, { targetTokens: 60, maxTokens: 200 });
|
|
for (const s of sentences) {
|
|
expect(chunks.some((c) => c.text.includes(s))).toBe(true);
|
|
}
|
|
});
|
|
|
|
it("assigns sequential 0-based indices", () => {
|
|
const chunks = chunkText(prose, { targetTokens: 60, maxTokens: 200 });
|
|
chunks.forEach((c, i) => expect(c.index).toBe(i));
|
|
});
|
|
|
|
it("is deterministic", () => {
|
|
const a = chunkText(prose, { targetTokens: 60, maxTokens: 200 });
|
|
const b = chunkText(prose, { targetTokens: 60, maxTokens: 200 });
|
|
expect(b.map((c) => c.text)).toEqual(a.map((c) => c.text));
|
|
});
|
|
|
|
it("keeps each chunk within a bounded size of the target", () => {
|
|
const chunks = chunkText(prose, { targetTokens: 60, maxTokens: 200 });
|
|
for (const c of chunks) {
|
|
// packing flushes before exceeding target, so a chunk is at most
|
|
// target + one unit (a single sentence here, ~20 tokens) ≤ maxTokens.
|
|
expect(c.tokenEstimate).toBeLessThanOrEqual(200);
|
|
}
|
|
});
|
|
|
|
it("hard-splits a single oversize sentence (no punctuation) without losing words", () => {
|
|
const giant = Array.from({ length: 400 }, (_, i) => `word${i}`).join(" ");
|
|
const chunks = chunkText(giant, { targetTokens: 50, maxTokens: 80 });
|
|
expect(chunks.length).toBeGreaterThan(1);
|
|
const reassembled = chunks.map((c) => c.text).join(" ").split(/\s+/).sort();
|
|
expect(reassembled).toEqual(giant.split(/\s+/).sort());
|
|
});
|
|
|
|
it("returns no chunks for empty/whitespace input", () => {
|
|
expect(chunkText("", { targetTokens: 60, maxTokens: 200 })).toEqual([]);
|
|
expect(chunkText(" \n\n ", { targetTokens: 60, maxTokens: 200 })).toEqual([]);
|
|
});
|
|
});
|