fix: add reasoning token buffer for combo routing (fixes #3587) (#3588)

Integrated into release/v3.8.21

open-sse/services/combo.ts

@@ -71,8 +71,13 @@ import {
   type ProviderCandidate,
   type ScoringWeights,
 } from "./autoCombo/scoring.ts";
-import { getResolvedModelCapabilities, supportsToolCalling } from "./modelCapabilities.ts";
+import {
+  getResolvedModelCapabilities,
+  supportsReasoning,
+  supportsToolCalling,
+} from "./modelCapabilities.ts";
 import { estimateTokens } from "./contextManager.ts";
+import { getReasoningTokens } from "../../src/lib/usage/tokenAccounting.ts";
 import { getSessionConnection } from "./sessionManager.ts";
 import { orderTargetsByEvalScores } from "./evalRouting.ts";
 import { generateRoutingHints } from "./manifestAdapter";
@@ -99,7 +104,10 @@ import {
   recordProviderCooldown,
   recordProviderSuccess,
 } from "./providerCooldownTracker.ts";
-import { resolveResilienceSettings, type ResilienceSettings } from "../../src/lib/resilience/settings";
+import {
+  resolveResilienceSettings,
+  type ResilienceSettings,
+} from "../../src/lib/resilience/settings";
 
 // Status codes that should mark round-robin target semaphores as cooling down.
 const TRANSIENT_FOR_SEMAPHORE = [429, 502, 503, 504];
@@ -492,6 +500,28 @@ export async function validateResponseQuality(
     return { valid: false, reason: "empty content and no tool_calls in response" };
   }
 
+  // Issue #3587: Reasoning models (deepseek-v4-flash, nemotron, etc.) may consume
+  // ALL max_tokens for reasoning_tokens, leaving content empty. When content is
+  // empty but reasoning_content exists, and usage shows reasoning consumed nearly
+  // all completion tokens, treat as invalid so the combo loop retries with more
+  // tokens or falls back to a non-reasoning model.
+  const contentIsEmpty = content === null || content === undefined || content === "";
+  if (contentIsEmpty && hasReasoningContent && !hasToolCalls) {
+    const usage = json?.usage as Record<string, unknown> | undefined;
+    if (usage) {
+      const completionTokens = Number(usage.completion_tokens) || 0;
+      const reasoningTokens = getReasoningTokens(usage);
+      // If reasoning consumed 90%+ of completion tokens, the model ran out of
+      // budget before producing any content output.
+      if (completionTokens > 0 && reasoningTokens >= completionTokens * 0.9) {
+        return {
+          valid: false,
+          reason: `reasoning consumed ${reasoningTokens}/${completionTokens} tokens — no content output`,
+        };
+      }
+    }
+  }
+
   return {
     valid: true,
     clonedResponse: new Response(text, {
@@ -3359,10 +3389,7 @@ export async function handleComboChat({
           Boolean(provider && provider !== "unknown") &&
           isProviderInCooldown(provider, target.connectionId ?? undefined, resilienceSettings)
         ) {
-          log.info(
-            "COMBO",
-            `Skipping ${modelStr} — provider ${provider} in global cooldown`
-          );
+          log.info("COMBO", `Skipping ${modelStr} — provider ${provider} in global cooldown`);
           if (i > 0) fallbackCount++;
           return null;
         }
@@ -3547,6 +3574,26 @@ export async function handleComboChat({
               );
             }
           }
+
+          // Issue #3587: Reasoning models (deepseek-v4-flash, nemotron, etc.) consume
+          // ALL max_tokens for reasoning_tokens, leaving content empty. Add a buffer
+          // to max_tokens so the model has enough tokens for both reasoning and content.
+          if (supportsReasoning(modelStr)) {
+            const bodyRecord = attemptBody as Record<string, unknown>;
+            const currentMaxTokens = Number(bodyRecord.max_tokens) || 0;
+            if (currentMaxTokens > 0) {
+              // Add 50% buffer + 1000 floor to ensure reasoning + content both fit
+              const bufferedMaxTokens = Math.max(
+                currentMaxTokens + 1000,
+                Math.ceil(currentMaxTokens * 1.5)
+              );
+              bodyRecord.max_tokens = bufferedMaxTokens;
+              log.info(
+                "COMBO",
+                `Reasoning model ${modelStr}: buffered max_tokens ${currentMaxTokens} -> ${bufferedMaxTokens}`
+              );
+            }
+          }
           const result = await handleSingleModelWithTimeout(attemptBody, modelStr, {
             ...targetForAttempt,
             failoverBeforeRetry: config.failoverBeforeRetry,
@@ -4208,10 +4255,7 @@ async function handleRoundRobinCombo({
       Boolean(provider && provider !== "unknown") &&
       isProviderInCooldown(provider, target.connectionId as string | undefined, resilienceSettings)
     ) {
-      log.info(
-        "COMBO-RR",
-        `Skipping ${modelStr} — provider ${provider} in global cooldown`
-      );
+      log.info("COMBO-RR", `Skipping ${modelStr} — provider ${provider} in global cooldown`);
       if (offset > 0) fallbackCount++;
       continue;
     }
@@ -4271,7 +4315,32 @@ async function handleRoundRobinCombo({
           `[RR #${counter}] → ${modelStr}${offset > 0 ? ` (fallback +${offset})` : ""}${retry > 0 ? ` (retry ${retry})` : ""}`
         );
 
-        const result = await handleSingleModel(body, modelStr, {
+        // Issue #3587: Reasoning models consume ALL max_tokens for reasoning_tokens.
+        // Add buffer to ensure reasoning + content both fit. Apply the buffer to a
+        // per-attempt COPY — never mutate the shared `body` — so it does not compound
+        // across round-robin iterations/retries (otherwise 4096 -> 6144 -> 9216 -> ...
+        // as each reasoning model re-reads an already-buffered value and overshoots the
+        // model's real limit, triggering 400s).
+        let attemptBody = body;
+        if (supportsReasoning(modelStr)) {
+          const currentMaxTokens = Number((body as Record<string, unknown>).max_tokens) || 0;
+          if (currentMaxTokens > 0) {
+            const bufferedMaxTokens = Math.max(
+              currentMaxTokens + 1000,
+              Math.ceil(currentMaxTokens * 1.5)
+            );
+            attemptBody = {
+              ...(body as Record<string, unknown>),
+              max_tokens: bufferedMaxTokens,
+            } as typeof body;
+            log.info(
+              "COMBO-RR",
+              `Reasoning model ${modelStr}: buffered max_tokens ${currentMaxTokens} -> ${bufferedMaxTokens}`
+            );
+          }
+        }
+
+        const result = await handleSingleModel(attemptBody, modelStr, {
           ...targetForAttempt,
           failoverBeforeRetry: config.failoverBeforeRetry,
         });

tests/unit/combo-quality-validator-reasoning.test.ts

@@ -105,3 +105,115 @@ test("#2341 reasoning_content as non-string is ignored (defensive)", async () =>
   // Non-string reasoning_content shouldn't count as content; still rejected.
   assert.equal(out.valid, false);
 });
+
+test("#3587 reasoning consumed 90%+ of tokens → invalid (token exhaustion)", async () => {
+  const res = makeResponse({
+    choices: [
+      {
+        message: {
+          content: null,
+          reasoning_content: "Deep reasoning about the problem...",
+        },
+      },
+    ],
+    usage: {
+      completion_tokens: 4096,
+      reasoning_tokens: 3800,
+    },
+  });
+  const out = await validateResponseQuality(res, false, silentLog);
+  assert.equal(out.valid, false, "should be invalid: reasoning exhausted tokens");
+  assert.match(out.reason ?? "", /reasoning consumed/i);
+});
+
+test("#3587 reasoning consumed < 90% of tokens → valid (normal reasoning)", async () => {
+  const res = makeResponse({
+    choices: [
+      {
+        message: {
+          content: null,
+          reasoning_content: "Some reasoning",
+        },
+      },
+    ],
+    usage: {
+      completion_tokens: 4096,
+      reasoning_tokens: 500,
+    },
+  });
+  const out = await validateResponseQuality(res, false, silentLog);
+  assert.equal(out.valid, true, "should be valid: reasoning has room");
+});
+
+test("#3587 reasoning with no usage data → valid (can't determine)", async () => {
+  const res = makeResponse({
+    choices: [
+      {
+        message: {
+          content: null,
+          reasoning_content: "Some reasoning",
+        },
+      },
+    ],
+  });
+  const out = await validateResponseQuality(res, false, silentLog);
+  assert.equal(out.valid, true, "should be valid: no usage to check");
+});
+
+test("#3587 content present + reasoning + tokens exhausted → valid (has content)", async () => {
+  const res = makeResponse({
+    choices: [
+      {
+        message: {
+          content: "Final answer here",
+          reasoning_content: "Deep reasoning",
+        },
+      },
+    ],
+    usage: {
+      completion_tokens: 4096,
+      reasoning_tokens: 3800,
+    },
+  });
+  const out = await validateResponseQuality(res, false, silentLog);
+  assert.equal(out.valid, true, "should be valid: content is present");
+});
+
+test("#3587 reasoning via completion_tokens_details.reasoning_tokens → invalid", async () => {
+  const res = makeResponse({
+    choices: [
+      {
+        message: {
+          content: null,
+          reasoning_content: "Step-by-step analysis...",
+        },
+      },
+    ],
+    usage: {
+      completion_tokens: 10000,
+      completion_tokens_details: { reasoning_tokens: 9500 },
+    },
+  });
+  const out = await validateResponseQuality(res, false, silentLog);
+  assert.equal(out.valid, false, "should be invalid: reasoning exhausted via details");
+  assert.match(out.reason ?? "", /reasoning consumed/i);
+});
+
+test("#3587 edge: completion_tokens=0 → safe (no division by zero)", async () => {
+  const res = makeResponse({
+    choices: [
+      {
+        message: {
+          content: null,
+          reasoning_content: "Tiny reasoning",
+        },
+      },
+    ],
+    usage: {
+      completion_tokens: 0,
+      reasoning_tokens: 0,
+    },
+  });
+  const out = await validateResponseQuality(res, false, silentLog);
+  assert.equal(out.valid, true, "should be valid: can't divide by zero");
+});

tests/unit/combo-routing-engine.test.ts

@@ -2870,3 +2870,125 @@ test("handleComboChat aborts combo when 503 response does NOT contain the unavai
       result.status === 503
   );
 });
+
+test("#3587 reasoning model gets max_tokens buffer applied", async () => {
+  saveModelsDevCapabilities({
+    openai: {
+      "gpt-4o-reasoning": capabilityEntry(4096, { reasoning: true }),
+    },
+  });
+
+  const bodies: Array<Record<string, unknown>> = [];
+  const result = await handleComboChat({
+    body: { max_tokens: 4096 },
+    combo: {
+      name: "reasoning-buffer",
+      models: ["openai/gpt-4o-reasoning"],
+    },
+    handleSingleModel: async (body: any) => {
+      bodies.push(body);
+      return okResponse();
+    },
+    isModelAvailable: async () => true,
+    log: createLog(),
+    settings: null,
+    relayOptions: null as any,
+    allCombos: null,
+  });
+
+  assert.equal(result.ok, true);
+  assert.equal(bodies.length, 1, "should have called handleSingleModel once");
+  // 4096 * 1.5 = 6144; max(4096+1000, 6144) = 6144
+  assert.equal(bodies[0].max_tokens, 6144, "max_tokens should be buffered for reasoning model");
+});
+
+test("#3587 non-reasoning model does not get max_tokens buffer", async () => {
+  saveModelsDevCapabilities({
+    openai: {
+      "gpt-4o-plain": capabilityEntry(4096, { reasoning: false }),
+    },
+  });
+
+  const bodies: Array<Record<string, unknown>> = [];
+  const result = await handleComboChat({
+    body: { max_tokens: 4096 },
+    combo: {
+      name: "no-reasoning-buffer",
+      models: ["openai/gpt-4o-plain"],
+    },
+    handleSingleModel: async (body: any) => {
+      bodies.push(body);
+      return okResponse();
+    },
+    isModelAvailable: async () => true,
+    log: createLog(),
+    settings: null,
+    relayOptions: null as any,
+    allCombos: null,
+  });
+
+  assert.equal(result.ok, true);
+  assert.equal(bodies.length, 1, "should have called handleSingleModel once");
+  // Non-reasoning model: max_tokens should NOT be buffered
+  assert.equal(
+    bodies[0].max_tokens,
+    4096,
+    "max_tokens should remain unchanged for non-reasoning model"
+  );
+});
+
+test("#3587 round-robin buffer does NOT compound across reasoning models", async () => {
+  // Two reasoning models in a round-robin combo. The first fails (400) so the
+  // loop falls through to the second. The buffer must be computed from the
+  // ORIGINAL max_tokens for each attempt — never from an already-buffered value —
+  // so both attempts see 6144 (4096 * 1.5), not [6144, 9216, ...]. Regression for
+  // the shared-`body` mutation that compounded the buffer on every RR iteration.
+  saveModelsDevCapabilities({
+    openai: {
+      "rr-reasoning-a": capabilityEntry(4096, { reasoning: true }),
+      "rr-reasoning-b": capabilityEntry(4096, { reasoning: true }),
+    },
+  });
+
+  const seen: Array<{ model: string; maxTokens: unknown }> = [];
+  const result = await handleComboChat({
+    body: { max_tokens: 4096 },
+    combo: {
+      name: "rr-reasoning-no-compound",
+      strategy: "round-robin",
+      models: ["openai/rr-reasoning-a", "openai/rr-reasoning-b"],
+    },
+    handleSingleModel: async (body: any, modelStr: any) => {
+      seen.push({ model: modelStr, maxTokens: body.max_tokens });
+      if (modelStr === "openai/rr-reasoning-a") {
+        return new Response(JSON.stringify({ error: { message: "transient" } }), {
+          status: 400,
+          headers: { "content-type": "application/json" },
+        });
+      }
+      return okResponse();
+    },
+    isModelAvailable: async () => true,
+    log: createLog(),
+    settings: {
+      comboDefaults: {
+        concurrencyPerModel: 1,
+        queueTimeoutMs: 5,
+        maxRetries: 0,
+        retryDelayMs: 1,
+      },
+    },
+    relayOptions: null as any,
+    allCombos: null,
+  });
+
+  assert.equal(result.status, 200);
+  assert.equal(seen.length, 2, "both reasoning models should have been attempted");
+  // Each attempt buffers from the original 4096 → 6144. No compounding.
+  assert.equal(seen[0].maxTokens, 6144, "first reasoning model buffered from original");
+  assert.equal(
+    seen[1].maxTokens,
+    6144,
+    "second reasoning model must ALSO buffer from original 4096, not 6144"
+  );
+});