9a4f9f0699
The demo endpoint lost its session on nearly every second message: affinity
required clients to re-echo the assistant reply byte-identically, so clients
that strip <think> blocks or reshape whitespace (Open WebUI, litellm) missed
every turn, opened a fresh escrow session on the next free single-slot
supplier, and left the old one pinned. Open WebUI's background title/tag-gen
requests each pinned another supplier — one message could exhaust all three
kimi replicas ("no available chat supplier").
- affinity: lenient assistant-content matching (think-strip + whitespace
collapse); user/system/tool messages and tool_calls stay byte-strict;
strict prefixes outrank lenient ones. The mirror is never rewritten with
client text, so receipt hashes stay valid.
- demoChat: trim trailing assistant/system messages (regenerate semantics)
instead of 400 "the last message must have role user or tool".
- supplier: roll the transcript back when the LLM call fails so retries
don't duplicate the delta (ChatSessionStore.truncateTranscript).
- demoChat: LRU-evict a demo session when no supplier is available (supplier
/v1/chat/end frees the slot, sweeper Accepts the escrow) and retry the
open with selectCandidates ignoreStatusFor to bypass the ~20s-stale
indexer status poll. Max 2 evictions per request.
- sweeper: idle janitor closes demo sessions unused for DEMO_SESSION_IDLE_MS
(default 3 min, below the supplier's 5 min CHAT_IDLE_TIMEOUT_MS) so
abandoned one-off sessions release their suppliers quickly.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
89 lines
3.3 KiB
TypeScript
89 lines
3.3 KiB
TypeScript
/**
|
|
* tx/chatMessage.ts — canonical ChatMessage normalization + equivalence.
|
|
*
|
|
* The supplier's session transcript and the gateway's in-memory mirror must be
|
|
* hash-identical (receipt response_hash = sha256(canonicalize(transcript))),
|
|
* so both sides build messages through normalizeChatMessage: optional fields
|
|
* are truly absent when empty, content is never null, and vendor extras are
|
|
* stripped. chatMessagesEquivalent is the tolerant comparison used by the
|
|
* demo-endpoint transcript-prefix affinity matcher.
|
|
*/
|
|
|
|
import type { ChatMessage, ToolCall } from "./types.js";
|
|
|
|
const ROLES = new Set(["system", "user", "assistant", "tool"]);
|
|
|
|
function normalizeToolCall(raw: unknown): ToolCall | null {
|
|
if (typeof raw !== "object" || raw === null) return null;
|
|
const tc = raw as Record<string, unknown>;
|
|
const fn = tc.function as Record<string, unknown> | undefined;
|
|
if (typeof tc.id !== "string" || tc.id === "") return null;
|
|
if (tc.type !== "function") return null;
|
|
if (typeof fn !== "object" || fn === null) return null;
|
|
if (typeof fn.name !== "string" || fn.name === "") return null;
|
|
if (typeof fn.arguments !== "string") return null;
|
|
return { id: tc.id, type: "function", function: { name: fn.name, arguments: fn.arguments } };
|
|
}
|
|
|
|
/**
|
|
* Validate and canonicalize an untrusted message-shaped value.
|
|
* Returns null when the value is not a usable ChatMessage.
|
|
*/
|
|
export function normalizeChatMessage(raw: unknown): ChatMessage | null {
|
|
if (typeof raw !== "object" || raw === null) return null;
|
|
const m = raw as Record<string, unknown>;
|
|
if (typeof m.role !== "string" || !ROLES.has(m.role)) return null;
|
|
const role = m.role as ChatMessage["role"];
|
|
|
|
let content: string;
|
|
if (typeof m.content === "string") content = m.content;
|
|
else if (m.content === null || m.content === undefined) content = "";
|
|
else return null;
|
|
|
|
const msg: ChatMessage = { role, content };
|
|
|
|
if (m.tool_calls !== undefined && m.tool_calls !== null) {
|
|
if (role !== "assistant" || !Array.isArray(m.tool_calls)) return null;
|
|
const calls: ToolCall[] = [];
|
|
for (const raw of m.tool_calls) {
|
|
const tc = normalizeToolCall(raw);
|
|
if (!tc) return null;
|
|
calls.push(tc);
|
|
}
|
|
if (calls.length > 0) msg.tool_calls = calls;
|
|
}
|
|
|
|
if (m.tool_call_id !== undefined && m.tool_call_id !== null) {
|
|
if (role !== "tool" || typeof m.tool_call_id !== "string" || m.tool_call_id === "") return null;
|
|
msg.tool_call_id = m.tool_call_id;
|
|
}
|
|
if (role === "tool" && msg.tool_call_id === undefined) return null;
|
|
|
|
return msg;
|
|
}
|
|
|
|
export function toolCallsEqual(a?: ToolCall[], b?: ToolCall[]): boolean {
|
|
const la = a ?? [];
|
|
const lb = b ?? [];
|
|
if (la.length !== lb.length) return false;
|
|
return la.every((tc, i) =>
|
|
tc.id === lb[i].id &&
|
|
tc.function.name === lb[i].function.name &&
|
|
tc.function.arguments === lb[i].function.arguments,
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Tolerant equality for affinity prefix matching. Both sides are expected to
|
|
* already be normalized; comparison ignores incidental key order and treats
|
|
* missing tool fields as empty.
|
|
*/
|
|
export function chatMessagesEquivalent(a: ChatMessage, b: ChatMessage): boolean {
|
|
return (
|
|
a.role === b.role &&
|
|
a.content === b.content &&
|
|
(a.tool_call_id ?? "") === (b.tool_call_id ?? "") &&
|
|
toolCallsEqual(a.tool_calls, b.tool_calls)
|
|
);
|
|
}
|