diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 629a17c98d..84425f276f 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -71,8 +71,13 @@ import { type ProviderCandidate, type ScoringWeights, } from "./autoCombo/scoring.ts"; -import { getResolvedModelCapabilities, supportsToolCalling } from "./modelCapabilities.ts"; +import { + getResolvedModelCapabilities, + supportsReasoning, + supportsToolCalling, +} from "./modelCapabilities.ts"; import { estimateTokens } from "./contextManager.ts"; +import { getReasoningTokens } from "../../src/lib/usage/tokenAccounting.ts"; import { getSessionConnection } from "./sessionManager.ts"; import { orderTargetsByEvalScores } from "./evalRouting.ts"; import { generateRoutingHints } from "./manifestAdapter"; @@ -99,7 +104,10 @@ import { recordProviderCooldown, recordProviderSuccess, } from "./providerCooldownTracker.ts"; -import { resolveResilienceSettings, type ResilienceSettings } from "../../src/lib/resilience/settings"; +import { + resolveResilienceSettings, + type ResilienceSettings, +} from "../../src/lib/resilience/settings"; // Status codes that should mark round-robin target semaphores as cooling down. const TRANSIENT_FOR_SEMAPHORE = [429, 502, 503, 504]; @@ -492,6 +500,28 @@ export async function validateResponseQuality( return { valid: false, reason: "empty content and no tool_calls in response" }; } + // Issue #3587: Reasoning models (deepseek-v4-flash, nemotron, etc.) may consume + // ALL max_tokens for reasoning_tokens, leaving content empty. When content is + // empty but reasoning_content exists, and usage shows reasoning consumed nearly + // all completion tokens, treat as invalid so the combo loop retries with more + // tokens or falls back to a non-reasoning model. + const contentIsEmpty = content === null || content === undefined || content === ""; + if (contentIsEmpty && hasReasoningContent && !hasToolCalls) { + const usage = json?.usage as Record | undefined; + if (usage) { + const completionTokens = Number(usage.completion_tokens) || 0; + const reasoningTokens = getReasoningTokens(usage); + // If reasoning consumed 90%+ of completion tokens, the model ran out of + // budget before producing any content output. + if (completionTokens > 0 && reasoningTokens >= completionTokens * 0.9) { + return { + valid: false, + reason: `reasoning consumed ${reasoningTokens}/${completionTokens} tokens — no content output`, + }; + } + } + } + return { valid: true, clonedResponse: new Response(text, { @@ -3359,10 +3389,7 @@ export async function handleComboChat({ Boolean(provider && provider !== "unknown") && isProviderInCooldown(provider, target.connectionId ?? undefined, resilienceSettings) ) { - log.info( - "COMBO", - `Skipping ${modelStr} — provider ${provider} in global cooldown` - ); + log.info("COMBO", `Skipping ${modelStr} — provider ${provider} in global cooldown`); if (i > 0) fallbackCount++; return null; } @@ -3547,6 +3574,26 @@ export async function handleComboChat({ ); } } + + // Issue #3587: Reasoning models (deepseek-v4-flash, nemotron, etc.) consume + // ALL max_tokens for reasoning_tokens, leaving content empty. Add a buffer + // to max_tokens so the model has enough tokens for both reasoning and content. + if (supportsReasoning(modelStr)) { + const bodyRecord = attemptBody as Record; + const currentMaxTokens = Number(bodyRecord.max_tokens) || 0; + if (currentMaxTokens > 0) { + // Add 50% buffer + 1000 floor to ensure reasoning + content both fit + const bufferedMaxTokens = Math.max( + currentMaxTokens + 1000, + Math.ceil(currentMaxTokens * 1.5) + ); + bodyRecord.max_tokens = bufferedMaxTokens; + log.info( + "COMBO", + `Reasoning model ${modelStr}: buffered max_tokens ${currentMaxTokens} -> ${bufferedMaxTokens}` + ); + } + } const result = await handleSingleModelWithTimeout(attemptBody, modelStr, { ...targetForAttempt, failoverBeforeRetry: config.failoverBeforeRetry, @@ -4208,10 +4255,7 @@ async function handleRoundRobinCombo({ Boolean(provider && provider !== "unknown") && isProviderInCooldown(provider, target.connectionId as string | undefined, resilienceSettings) ) { - log.info( - "COMBO-RR", - `Skipping ${modelStr} — provider ${provider} in global cooldown` - ); + log.info("COMBO-RR", `Skipping ${modelStr} — provider ${provider} in global cooldown`); if (offset > 0) fallbackCount++; continue; } @@ -4271,7 +4315,32 @@ async function handleRoundRobinCombo({ `[RR #${counter}] → ${modelStr}${offset > 0 ? ` (fallback +${offset})` : ""}${retry > 0 ? ` (retry ${retry})` : ""}` ); - const result = await handleSingleModel(body, modelStr, { + // Issue #3587: Reasoning models consume ALL max_tokens for reasoning_tokens. + // Add buffer to ensure reasoning + content both fit. Apply the buffer to a + // per-attempt COPY — never mutate the shared `body` — so it does not compound + // across round-robin iterations/retries (otherwise 4096 -> 6144 -> 9216 -> ... + // as each reasoning model re-reads an already-buffered value and overshoots the + // model's real limit, triggering 400s). + let attemptBody = body; + if (supportsReasoning(modelStr)) { + const currentMaxTokens = Number((body as Record).max_tokens) || 0; + if (currentMaxTokens > 0) { + const bufferedMaxTokens = Math.max( + currentMaxTokens + 1000, + Math.ceil(currentMaxTokens * 1.5) + ); + attemptBody = { + ...(body as Record), + max_tokens: bufferedMaxTokens, + } as typeof body; + log.info( + "COMBO-RR", + `Reasoning model ${modelStr}: buffered max_tokens ${currentMaxTokens} -> ${bufferedMaxTokens}` + ); + } + } + + const result = await handleSingleModel(attemptBody, modelStr, { ...targetForAttempt, failoverBeforeRetry: config.failoverBeforeRetry, }); diff --git a/tests/unit/combo-quality-validator-reasoning.test.ts b/tests/unit/combo-quality-validator-reasoning.test.ts index f0948cde56..0650acd1d8 100644 --- a/tests/unit/combo-quality-validator-reasoning.test.ts +++ b/tests/unit/combo-quality-validator-reasoning.test.ts @@ -105,3 +105,115 @@ test("#2341 reasoning_content as non-string is ignored (defensive)", async () => // Non-string reasoning_content shouldn't count as content; still rejected. assert.equal(out.valid, false); }); + +test("#3587 reasoning consumed 90%+ of tokens → invalid (token exhaustion)", async () => { + const res = makeResponse({ + choices: [ + { + message: { + content: null, + reasoning_content: "Deep reasoning about the problem...", + }, + }, + ], + usage: { + completion_tokens: 4096, + reasoning_tokens: 3800, + }, + }); + const out = await validateResponseQuality(res, false, silentLog); + assert.equal(out.valid, false, "should be invalid: reasoning exhausted tokens"); + assert.match(out.reason ?? "", /reasoning consumed/i); +}); + +test("#3587 reasoning consumed < 90% of tokens → valid (normal reasoning)", async () => { + const res = makeResponse({ + choices: [ + { + message: { + content: null, + reasoning_content: "Some reasoning", + }, + }, + ], + usage: { + completion_tokens: 4096, + reasoning_tokens: 500, + }, + }); + const out = await validateResponseQuality(res, false, silentLog); + assert.equal(out.valid, true, "should be valid: reasoning has room"); +}); + +test("#3587 reasoning with no usage data → valid (can't determine)", async () => { + const res = makeResponse({ + choices: [ + { + message: { + content: null, + reasoning_content: "Some reasoning", + }, + }, + ], + }); + const out = await validateResponseQuality(res, false, silentLog); + assert.equal(out.valid, true, "should be valid: no usage to check"); +}); + +test("#3587 content present + reasoning + tokens exhausted → valid (has content)", async () => { + const res = makeResponse({ + choices: [ + { + message: { + content: "Final answer here", + reasoning_content: "Deep reasoning", + }, + }, + ], + usage: { + completion_tokens: 4096, + reasoning_tokens: 3800, + }, + }); + const out = await validateResponseQuality(res, false, silentLog); + assert.equal(out.valid, true, "should be valid: content is present"); +}); + +test("#3587 reasoning via completion_tokens_details.reasoning_tokens → invalid", async () => { + const res = makeResponse({ + choices: [ + { + message: { + content: null, + reasoning_content: "Step-by-step analysis...", + }, + }, + ], + usage: { + completion_tokens: 10000, + completion_tokens_details: { reasoning_tokens: 9500 }, + }, + }); + const out = await validateResponseQuality(res, false, silentLog); + assert.equal(out.valid, false, "should be invalid: reasoning exhausted via details"); + assert.match(out.reason ?? "", /reasoning consumed/i); +}); + +test("#3587 edge: completion_tokens=0 → safe (no division by zero)", async () => { + const res = makeResponse({ + choices: [ + { + message: { + content: null, + reasoning_content: "Tiny reasoning", + }, + }, + ], + usage: { + completion_tokens: 0, + reasoning_tokens: 0, + }, + }); + const out = await validateResponseQuality(res, false, silentLog); + assert.equal(out.valid, true, "should be valid: can't divide by zero"); +}); diff --git a/tests/unit/combo-routing-engine.test.ts b/tests/unit/combo-routing-engine.test.ts index 5e828f17cc..3322d54f47 100644 --- a/tests/unit/combo-routing-engine.test.ts +++ b/tests/unit/combo-routing-engine.test.ts @@ -2870,3 +2870,125 @@ test("handleComboChat aborts combo when 503 response does NOT contain the unavai result.status === 503 ); }); + +test("#3587 reasoning model gets max_tokens buffer applied", async () => { + saveModelsDevCapabilities({ + openai: { + "gpt-4o-reasoning": capabilityEntry(4096, { reasoning: true }), + }, + }); + + const bodies: Array> = []; + const result = await handleComboChat({ + body: { max_tokens: 4096 }, + combo: { + name: "reasoning-buffer", + models: ["openai/gpt-4o-reasoning"], + }, + handleSingleModel: async (body: any) => { + bodies.push(body); + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + relayOptions: null as any, + allCombos: null, + }); + + assert.equal(result.ok, true); + assert.equal(bodies.length, 1, "should have called handleSingleModel once"); + // 4096 * 1.5 = 6144; max(4096+1000, 6144) = 6144 + assert.equal(bodies[0].max_tokens, 6144, "max_tokens should be buffered for reasoning model"); +}); + +test("#3587 non-reasoning model does not get max_tokens buffer", async () => { + saveModelsDevCapabilities({ + openai: { + "gpt-4o-plain": capabilityEntry(4096, { reasoning: false }), + }, + }); + + const bodies: Array> = []; + const result = await handleComboChat({ + body: { max_tokens: 4096 }, + combo: { + name: "no-reasoning-buffer", + models: ["openai/gpt-4o-plain"], + }, + handleSingleModel: async (body: any) => { + bodies.push(body); + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + relayOptions: null as any, + allCombos: null, + }); + + assert.equal(result.ok, true); + assert.equal(bodies.length, 1, "should have called handleSingleModel once"); + // Non-reasoning model: max_tokens should NOT be buffered + assert.equal( + bodies[0].max_tokens, + 4096, + "max_tokens should remain unchanged for non-reasoning model" + ); +}); + +test("#3587 round-robin buffer does NOT compound across reasoning models", async () => { + // Two reasoning models in a round-robin combo. The first fails (400) so the + // loop falls through to the second. The buffer must be computed from the + // ORIGINAL max_tokens for each attempt — never from an already-buffered value — + // so both attempts see 6144 (4096 * 1.5), not [6144, 9216, ...]. Regression for + // the shared-`body` mutation that compounded the buffer on every RR iteration. + saveModelsDevCapabilities({ + openai: { + "rr-reasoning-a": capabilityEntry(4096, { reasoning: true }), + "rr-reasoning-b": capabilityEntry(4096, { reasoning: true }), + }, + }); + + const seen: Array<{ model: string; maxTokens: unknown }> = []; + const result = await handleComboChat({ + body: { max_tokens: 4096 }, + combo: { + name: "rr-reasoning-no-compound", + strategy: "round-robin", + models: ["openai/rr-reasoning-a", "openai/rr-reasoning-b"], + }, + handleSingleModel: async (body: any, modelStr: any) => { + seen.push({ model: modelStr, maxTokens: body.max_tokens }); + if (modelStr === "openai/rr-reasoning-a") { + return new Response(JSON.stringify({ error: { message: "transient" } }), { + status: 400, + headers: { "content-type": "application/json" }, + }); + } + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: { + comboDefaults: { + concurrencyPerModel: 1, + queueTimeoutMs: 5, + maxRetries: 0, + retryDelayMs: 1, + }, + }, + relayOptions: null as any, + allCombos: null, + }); + + assert.equal(result.status, 200); + assert.equal(seen.length, 2, "both reasoning models should have been attempted"); + // Each attempt buffers from the original 4096 → 6144. No compounding. + assert.equal(seen[0].maxTokens, 6144, "first reasoning model buffered from original"); + assert.equal( + seen[1].maxTokens, + 6144, + "second reasoning model must ALSO buffer from original 4096, not 6144" + ); +});