From b92bff56a7395b68a57848e9d1b1ef3724de37a6 Mon Sep 17 00:00:00 2001 From: david Date: Thu, 2 Jul 2026 16:47:30 +0200 Subject: [PATCH] feat(supplier): env-gated OpenRouter reasoning-off + max_tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The supplier sent a bare {model,messages,stream} to the OpenAI/OpenRouter endpoint — no max_tokens and no reasoning control — so reasoning ("thinking") models like kimi-k2.6 emitted reasoning tokens that could consume the completion budget and finish with reason=length (truncated/empty answer = failed job). deepseek-v4-flash has no thinking; kimi-k2.6 does by default. Add two optional, env-gated request params (defaults preserve current behavior, so ChatMock/DeepSeek-direct suppliers are unaffected): - OPENAI_REASONING=off -> send reasoning:{enabled:false} (disable thinking: faster, cheaper, straight answers; no reasoning-starved length truncation) - OPENAI_MAX_TOKENS=N -> forward max_tokens:N. Must stay below the model context (kimi context = 262144), so the advert's 262144 cap cannot be sent verbatim (provider 400s when prompt+max_tokens > context). openai.ts builds the body conditionally; config.ts parses the env; wired through jobRunner.ts (text) and server.ts (chat-session stream). Deployed on the 4 kimi + 3 deepseek-flash suppliers with OPENAI_REASONING=off and OPENAI_MAX_TOKENS=131072 (env files are host-only/gitignored). Co-Authored-By: Claude Opus 4.8 --- supplier/src/config.ts | 40 +++++++++++++++++++++++++++++++++++++++ supplier/src/jobRunner.ts | 2 ++ supplier/src/openai.ts | 30 ++++++++++++++++++++++++----- supplier/src/server.ts | 2 ++ 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/supplier/src/config.ts b/supplier/src/config.ts index e11b7b6..9436147 100644 --- a/supplier/src/config.ts +++ b/supplier/src/config.ts @@ -79,6 +79,19 @@ export interface SupplierConfig { * proxies). Required for hosted OpenAI-compatible APIs (DeepSeek, OpenAI, etc.). */ openaiApiKey: string; + /** + * When true, upstream OpenAI/OpenRouter calls send `reasoning:{enabled:false}` + * to disable "thinking" tokens (faster/cheaper straight answers; avoids + * reasoning starving the completion into a length-truncated empty answer). + * Set via OPENAI_REASONING=off. Default false. OpenRouter-only. + */ + openaiReasoningDisabled: boolean; + /** + * Optional `max_tokens` ceiling forwarded on upstream calls. 0 = omit (the + * provider then grants the max available = context − prompt). Must stay below + * the model context or the provider 400s (e.g. kimi context = 262144). + */ + openaiMaxTokens: number; /** Empty string when capabilityKind="chat". */ piperUrl: string; advertRef: AdvertRef; @@ -230,6 +243,31 @@ export function loadConfig(env: Record): SupplierCon const openaiApiKey = env.OPENAI_API_KEY ?? ""; + // OPENAI_REASONING — "off"/"false"/"0"/"no" disables OpenRouter reasoning + // ("thinking") by sending reasoning:{enabled:false} on every upstream call. + // Unset/any other value leaves the model's own default untouched (no param + // sent). Only valid for OpenRouter-backed suppliers — do NOT set for + // ChatMock/DeepSeek-direct, which reject the param. + const reasoningRaw = (env.OPENAI_REASONING ?? "").trim().toLowerCase(); + const openaiReasoningDisabled = + reasoningRaw === "off" || + reasoningRaw === "false" || + reasoningRaw === "0" || + reasoningRaw === "no"; + + // OPENAI_MAX_TOKENS — optional hard ceiling forwarded as `max_tokens`. Unset + // or non-positive omits it (provider grants max available = context − prompt). + // Keep it below the model context (kimi 262144) or the provider 400s when + // prompt+max_tokens exceeds context. + const maxTokStr = env.OPENAI_MAX_TOKENS; + let openaiMaxTokens = 0; + if (maxTokStr !== undefined && maxTokStr !== "") { + if (!POS_INT_RE.test(maxTokStr)) { + throw new Error("loadConfig: OPENAI_MAX_TOKENS must be a positive integer"); + } + openaiMaxTokens = Number(maxTokStr); + } + // CHAT_IDLE_TIMEOUT_MS — idle auto-end window for chat sessions (default 5 min). const chatIdleStr = env.CHAT_IDLE_TIMEOUT_MS; let chatIdleTimeoutMs = 300_000; @@ -270,6 +308,8 @@ export function loadConfig(env: Record): SupplierCon ollamaUrl, openaiBaseUrl, openaiApiKey, + openaiReasoningDisabled, + openaiMaxTokens, piperUrl, advertRef, networkId, diff --git a/supplier/src/jobRunner.ts b/supplier/src/jobRunner.ts index 60f7c46..677fa62 100644 --- a/supplier/src/jobRunner.ts +++ b/supplier/src/jobRunner.ts @@ -92,6 +92,8 @@ export async function runChatJob(params: RunChatJobParams): Promise { messages: requestBody.messages, timeoutMs: deps.config.openaiTimeoutMs, apiKey: deps.config.openaiApiKey, + maxTokens: deps.config.openaiMaxTokens, + disableReasoning: deps.config.openaiReasoningDisabled, }); } else { inference = await ollama.callOllama({ diff --git a/supplier/src/openai.ts b/supplier/src/openai.ts index e8dbdb1..6f333be 100644 --- a/supplier/src/openai.ts +++ b/supplier/src/openai.ts @@ -39,6 +39,20 @@ export interface CallOpenAiParams { * e.g. the ChatMock localhost proxy. */ apiKey?: string; + /** + * Forwarded as `max_tokens` when a positive number; omitted otherwise. The + * caller is responsible for keeping prompt+max_tokens within the model's + * context window (some providers 400 when it exceeds context). + */ + maxTokens?: number; + /** + * When true, sends `reasoning: { enabled: false }` to disable OpenRouter + * "thinking" tokens — faster/cheaper straight answers, and avoids reasoning + * starving the completion budget into a length-truncated (empty) answer. + * Only valid for OpenRouter endpoints; leave false for ChatMock / direct + * DeepSeek, which don't accept the param. + */ + disableReasoning?: boolean; } export interface OpenAiResult { @@ -68,9 +82,12 @@ function isAbortError(err: unknown): boolean { } export async function callOpenAi(params: CallOpenAiParams): Promise { - const { baseUrl, model, messages, timeoutMs, apiKey } = params; + const { baseUrl, model, messages, timeoutMs, apiKey, maxTokens, disableReasoning } = params; const url = `${baseUrl}/v1/chat/completions`; - const body = JSON.stringify({ model, messages, stream: false }); + const payload: Record = { model, messages, stream: false }; + if (typeof maxTokens === "number" && maxTokens > 0) payload.max_tokens = maxTokens; + if (disableReasoning) payload.reasoning = { enabled: false }; + const body = JSON.stringify(payload); const headers: Record = { "content-type": "application/json" }; if (apiKey) { @@ -187,14 +204,17 @@ export async function callOpenAiStream( params: CallOpenAiParams, onToken: (delta: string) => void, ): Promise { - const { baseUrl, model, messages, timeoutMs, apiKey } = params; + const { baseUrl, model, messages, timeoutMs, apiKey, maxTokens, disableReasoning } = params; const url = `${baseUrl}/v1/chat/completions`; - const body = JSON.stringify({ + const payload: Record = { model, messages, stream: true, stream_options: { include_usage: true }, - }); + }; + if (typeof maxTokens === "number" && maxTokens > 0) payload.max_tokens = maxTokens; + if (disableReasoning) payload.reasoning = { enabled: false }; + const body = JSON.stringify(payload); const headers: Record = { "content-type": "application/json", diff --git a/supplier/src/server.ts b/supplier/src/server.ts index 836adf1..d0823e6 100644 --- a/supplier/src/server.ts +++ b/supplier/src/server.ts @@ -856,6 +856,8 @@ function makeChatSessionHandlers(deps: ResolvedDeps) { messages: record.transcript, timeoutMs: deps.config.openaiTimeoutMs, apiKey: deps.config.openaiApiKey, + maxTokens: deps.config.openaiMaxTokens, + disableReasoning: deps.config.openaiReasoningDisabled, }, (delta) => sse({ type: "token", value: delta }), );