From 5b2c2d6be45f59c65228c16098c8f49054e3a3f4 Mon Sep 17 00:00:00 2001 From: Randi <55005611+rdself@users.noreply.github.com> Date: Fri, 12 Jun 2026 07:28:17 -0400 Subject: [PATCH] fix(combo): gate reasoning token buffer (#3700) Integrated into release/v3.8.23. Makes the #3588 reasoning token buffer safe and configurable: only inflates max_tokens when the model has a known, non-default output cap and the buffered value fits inside it; otherwise preserves/clamps the client limit. Adds the reasoningTokenBufferEnabled kill switch (default ON). Validated: combo-routing-engine 81/81, combo-config 25/25, combo-quality-validator-reasoning 12/12, phase1f 10/10, typecheck:core clean. --- docs/guides/USER_GUIDE.md | 6 + open-sse/services/combo.ts | 74 +++--- open-sse/services/comboConfig.ts | 1 + open-sse/services/reasoningTokenBuffer.ts | 40 +++ .../providers/[id]/__tests__/phase1f.test.tsx | 42 ++-- .../settings/components/ComboDefaultsTab.tsx | 24 ++ src/app/api/settings/combo-defaults/route.ts | 1 + src/lib/modelCapabilities.ts | 47 +++- src/shared/validation/schemas.ts | 1 + src/types/settings.ts | 1 + tests/integration/integration-wiring.test.ts | 8 +- tests/integration/resilience-http-e2e.test.ts | 1 + tests/unit/combo-config.test.ts | 38 +++ tests/unit/combo-routing-engine.test.ts | 236 +++++++++++++++++- 14 files changed, 451 insertions(+), 69 deletions(-) create mode 100644 open-sse/services/reasoningTokenBuffer.ts diff --git a/docs/guides/USER_GUIDE.md b/docs/guides/USER_GUIDE.md index 6256824de5..dcc10ce3f9 100644 --- a/docs/guides/USER_GUIDE.md +++ b/docs/guides/USER_GUIDE.md @@ -989,6 +989,12 @@ history, or compressing fallback requests; enabling it allows configured hedging skips, and proactive fallback compression to trade routing/request fidelity for lower tail latency. +Disable **Reasoning token buffer** when upstream providers require strict +`max_tokens` / `maxOutputTokens` limits. When enabled, combo routing only adds reasoning-model +headroom for models with a known output cap and leaves the client token limit unchanged when the +safe buffered value would exceed that cap. If the client limit is already above a known cap, +OmniRoute clamps it down to that cap before sending the upstream request. + --- ### Health Dashboard diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 92b8df3738..3d030ee4b1 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -71,11 +71,7 @@ import { type ProviderCandidate, type ScoringWeights, } from "./autoCombo/scoring.ts"; -import { - getResolvedModelCapabilities, - supportsReasoning, - supportsToolCalling, -} from "./modelCapabilities.ts"; +import { getResolvedModelCapabilities, supportsToolCalling } from "./modelCapabilities.ts"; import { estimateTokens } from "./contextManager.ts"; import { getReasoningTokens } from "../../src/lib/usage/tokenAccounting.ts"; import { getSessionConnection } from "./sessionManager.ts"; @@ -108,6 +104,7 @@ import { resolveResilienceSettings, type ResilienceSettings, } from "../../src/lib/resilience/settings"; +import { resolveReasoningBufferedMaxTokens, toPositiveInteger } from "./reasoningTokenBuffer.ts"; // Status codes that should mark round-robin target semaphores as cooling down. const TRANSIENT_FOR_SEMAPHORE = [429, 502, 503, 504]; @@ -3019,6 +3016,7 @@ export async function handleComboChat({ ? resolveComboConfig(combo, settings) : { ...getDefaultComboConfig(), ...(combo.config || {}) }; const comboTargetTimeoutMs = resolveComboTargetTimeoutMs(config, FETCH_TIMEOUT_MS); + const reasoningTokenBufferEnabled = config.reasoningTokenBufferEnabled !== false; // ── Per-model timeout wrapper ──────────────────────────────────────────── // Combo target timeouts inherit FETCH_TIMEOUT_MS by default. Operators can @@ -3807,23 +3805,25 @@ export async function handleComboChat({ } } - // Issue #3587: Reasoning models (deepseek-v4-flash, nemotron, etc.) consume - // ALL max_tokens for reasoning_tokens, leaving content empty. Add a buffer - // to max_tokens so the model has enough tokens for both reasoning and content. - if (supportsReasoning(modelStr)) { + // Issue #3587: Reasoning models can spend the whole output budget on + // reasoning. Only add headroom when the complete buffer fits inside the + // model's known output cap; otherwise preserve the client's explicit limit. + { const bodyRecord = attemptBody as Record; - const currentMaxTokens = Number(bodyRecord.max_tokens) || 0; - if (currentMaxTokens > 0) { - // Add 50% buffer + 1000 floor to ensure reasoning + content both fit - const bufferedMaxTokens = Math.max( - currentMaxTokens + 1000, - Math.ceil(currentMaxTokens * 1.5) - ); + const currentMaxTokens = toPositiveInteger(bodyRecord.max_tokens); + const bufferedMaxTokens = resolveReasoningBufferedMaxTokens( + modelStr, + bodyRecord.max_tokens, + { enabled: reasoningTokenBufferEnabled } + ); + if (currentMaxTokens !== null && bufferedMaxTokens !== null) { bodyRecord.max_tokens = bufferedMaxTokens; - log.info( - "COMBO", - `Reasoning model ${modelStr}: buffered max_tokens ${currentMaxTokens} -> ${bufferedMaxTokens}` - ); + if (bufferedMaxTokens !== currentMaxTokens) { + log.info( + "COMBO", + `Reasoning model ${modelStr}: adjusted max_tokens ${currentMaxTokens} -> ${bufferedMaxTokens}` + ); + } } } const result = await handleSingleModelWithTimeout(attemptBody, modelStr, { @@ -4426,6 +4426,7 @@ async function handleRoundRobinCombo({ const maxRetries = config.maxRetries ?? 1; const retryDelayMs = resolveDelayMs(config.retryDelayMs, 2000); const fallbackDelayMs = resolveDelayMs(config.fallbackDelayMs, 0); + const reasoningTokenBufferEnabled = config.reasoningTokenBufferEnabled !== false; const resilienceSettings: ResilienceSettings = settings ? resolveResilienceSettings(settings) @@ -4585,27 +4586,30 @@ async function handleRoundRobinCombo({ `[RR #${counter}] → ${modelStr}${offset > 0 ? ` (fallback +${offset})` : ""}${retry > 0 ? ` (retry ${retry})` : ""}` ); - // Issue #3587: Reasoning models consume ALL max_tokens for reasoning_tokens. - // Add buffer to ensure reasoning + content both fit. Apply the buffer to a - // per-attempt COPY — never mutate the shared `body` — so it does not compound - // across round-robin iterations/retries (otherwise 4096 -> 6144 -> 9216 -> ... - // as each reasoning model re-reads an already-buffered value and overshoots the - // model's real limit, triggering 400s). + // Issue #3587: Reasoning models can spend the whole output budget on + // reasoning. Apply any safe buffer to a per-attempt copy so round-robin + // retries never compound across models. let attemptBody = body; - if (supportsReasoning(modelStr)) { - const currentMaxTokens = Number((body as Record).max_tokens) || 0; - if (currentMaxTokens > 0) { - const bufferedMaxTokens = Math.max( - currentMaxTokens + 1000, - Math.ceil(currentMaxTokens * 1.5) - ); + { + const bodyRecord = body as Record; + const currentMaxTokens = toPositiveInteger(bodyRecord.max_tokens); + const bufferedMaxTokens = resolveReasoningBufferedMaxTokens( + modelStr, + bodyRecord.max_tokens, + { enabled: reasoningTokenBufferEnabled } + ); + if ( + currentMaxTokens !== null && + bufferedMaxTokens !== null && + bufferedMaxTokens !== currentMaxTokens + ) { attemptBody = { - ...(body as Record), + ...bodyRecord, max_tokens: bufferedMaxTokens, } as typeof body; log.info( "COMBO-RR", - `Reasoning model ${modelStr}: buffered max_tokens ${currentMaxTokens} -> ${bufferedMaxTokens}` + `Reasoning model ${modelStr}: adjusted max_tokens ${currentMaxTokens} -> ${bufferedMaxTokens}` ); } } diff --git a/open-sse/services/comboConfig.ts b/open-sse/services/comboConfig.ts index e8c9c33a52..5afe81f6aa 100644 --- a/open-sse/services/comboConfig.ts +++ b/open-sse/services/comboConfig.ts @@ -26,6 +26,7 @@ const DEFAULT_COMBO_CONFIG = { maxMessagesForSummary: 30, maxComboDepth: 3, trackMetrics: true, + reasoningTokenBufferEnabled: true, manifestRouting: false, resetAwareSessionWeight: 0.35, resetAwareWeeklyWeight: 0.65, diff --git a/open-sse/services/reasoningTokenBuffer.ts b/open-sse/services/reasoningTokenBuffer.ts new file mode 100644 index 0000000000..23290de713 --- /dev/null +++ b/open-sse/services/reasoningTokenBuffer.ts @@ -0,0 +1,40 @@ +import { getResolvedModelCapabilities } from "../../src/lib/modelCapabilities.ts"; +import { MODEL_SPECS } from "../../src/shared/constants/modelSpecs.ts"; + +const DEFAULT_MAX_OUTPUT_TOKENS = MODEL_SPECS.__default__.maxOutputTokens; + +export function toPositiveInteger(value: unknown): number | null { + const numericValue = + typeof value === "number" + ? value + : typeof value === "string" && value.trim() !== "" + ? Number(value) + : null; + if (numericValue === null || !Number.isFinite(numericValue)) return null; + const normalized = Math.floor(numericValue); + return normalized > 0 ? normalized : null; +} + +export function resolveReasoningBufferedMaxTokens( + modelStr: string, + currentMaxTokens: unknown, + options: { enabled?: boolean } = {} +): number | null { + if (options.enabled === false) return null; + + const current = toPositiveInteger(currentMaxTokens); + if (current === null) return null; + + const capabilities = getResolvedModelCapabilities(modelStr); + if (capabilities.supportsThinking !== true) return null; + + const maxOutputTokens = toPositiveInteger(capabilities.maxOutputTokens); + if (maxOutputTokens === null || maxOutputTokens === DEFAULT_MAX_OUTPUT_TOKENS) return null; + if (current > maxOutputTokens) return maxOutputTokens; + if (current === maxOutputTokens) return current; + + const buffered = Math.max(current + 1000, Math.ceil(current * 1.5)); + if (buffered > maxOutputTokens) return current; + + return buffered; +} diff --git a/src/app/(dashboard)/dashboard/providers/[id]/__tests__/phase1f.test.tsx b/src/app/(dashboard)/dashboard/providers/[id]/__tests__/phase1f.test.tsx index 9ee6137d20..7b38d0e8bd 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/__tests__/phase1f.test.tsx +++ b/src/app/(dashboard)/dashboard/providers/[id]/__tests__/phase1f.test.tsx @@ -10,7 +10,7 @@ // Uses createRoot + act to mount each hook inside a minimal wrapper component // so we test real React hook semantics without a full Next.js server context. -import React, { act } from "react"; +import React, { act, useEffect } from "react"; import { createRoot } from "react-dom/client"; import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; @@ -27,10 +27,7 @@ vi.mock("next/navigation", () => ({ vi.mock("next-intl", () => ({ useTranslations: () => (key: string, values?: Record) => { if (values) { - return Object.entries(values).reduce( - (acc, [k, v]) => acc.replace(`{${k}}`, String(v)), - key - ); + return Object.entries(values).reduce((acc, [k, v]) => acc.replace(`{${k}}`, String(v)), key); } return key; }, @@ -83,10 +80,13 @@ describe("useProviderConnections — initial state", () => { let result: HookResult | null = null; function TestWrapper() { - result = useProviderConnections("openai", true, false); + const hookResult = useProviderConnections("openai", true, false); + useEffect(() => { + result = hookResult; + }, [hookResult]); return ( - {String(result.connections.length)}|{String(result.batchTesting)} + {String(hookResult.connections.length)}|{String(hookResult.batchTesting)} ); } @@ -112,7 +112,10 @@ describe("useProviderConnections — initial state", () => { let result: HookResult | null = null; function TestWrapper() { - result = useProviderConnections("openai", true, false); + const hookResult = useProviderConnections("openai", true, false); + useEffect(() => { + result = hookResult; + }, [hookResult]); return ; } @@ -181,7 +184,10 @@ describe("useProviderSettings — initial state", () => { let result: HookResult | null = null; function TestWrapper() { - result = useProviderSettings("openai"); + const hookResult = useProviderSettings("openai"); + useEffect(() => { + result = hookResult; + }, [hookResult]); return ; } @@ -207,7 +213,10 @@ describe("useProviderSettings — initial state", () => { let result: HookResult | null = null; function TestWrapper() { - result = useProviderSettings("codex"); + const hookResult = useProviderSettings("codex"); + useEffect(() => { + result = hookResult; + }, [hookResult]); return ; } @@ -253,7 +262,10 @@ describe("useProviderModels — initial state", () => { let result: HookResult | null = null; function TestWrapper() { - result = useProviderModels("openai", false); + const hookResult = useProviderModels("openai", false); + useEffect(() => { + result = hookResult; + }, [hookResult]); return ; } @@ -275,7 +287,10 @@ describe("useProviderModels — initial state", () => { let result: HookResult | null = null; function TestWrapper() { - result = useProviderModels("openai", false); + const hookResult = useProviderModels("openai", false); + useEffect(() => { + result = hookResult; + }, [hookResult]); return ; } @@ -316,8 +331,7 @@ describe("useProviderModels — initial state", () => { // Cycle-safety: hooks must NOT import from ProviderDetailPageClient // --------------------------------------------------------------------------- -const HOOKS_DIR = - "/home/diegosouzapw/dev/proxys/OmniRoute/.worktrees/fix-3501-phase1f/src/app/(dashboard)/dashboard/providers/[id]/hooks"; +const HOOKS_DIR = `${process.cwd()}/src/app/(dashboard)/dashboard/providers/[id]/hooks`; describe("Cycle-safety — hooks do not import ProviderDetailPageClient", () => { // We allow the name in JSDoc comments; what we forbid is an actual ES import statement. diff --git a/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx b/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx index 7e8c1537fd..6b254addcf 100644 --- a/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx @@ -91,6 +91,7 @@ export default function ComboDefaultsTab() { retryDelayMs: 2000, maxComboDepth: 3, trackMetrics: true, + reasoningTokenBufferEnabled: true, handoffThreshold: 0.85, handoffModel: "", maxMessagesForSummary: 30, @@ -556,6 +557,29 @@ export default function ComboDefaultsTab() { } /> +
+
+

+ {translateOrFallback(t, "reasoningTokenBuffer", "Reasoning token buffer")} +

+

+ {translateOrFallback( + t, + "reasoningTokenBufferDesc", + "Allow combo routing to add max_tokens headroom only for known reasoning models when the full buffer fits inside a known output cap." + )} +

+
+ + setComboDefaults((prev) => ({ + ...prev, + reasoningTokenBufferEnabled: prev.reasoningTokenBufferEnabled === false, + })) + } + /> +

diff --git a/src/app/api/settings/combo-defaults/route.ts b/src/app/api/settings/combo-defaults/route.ts index 79a2812fe5..9e6ca358e4 100644 --- a/src/app/api/settings/combo-defaults/route.ts +++ b/src/app/api/settings/combo-defaults/route.ts @@ -55,6 +55,7 @@ export async function GET(request: Request) { maxMessagesForSummary: 30, maxComboDepth: 3, trackMetrics: true, + reasoningTokenBufferEnabled: true, zeroLatencyOptimizationsEnabled: false, }, providerOverrides, diff --git a/src/lib/modelCapabilities.ts b/src/lib/modelCapabilities.ts index 3a2bd1d1b0..379f5a535d 100644 --- a/src/lib/modelCapabilities.ts +++ b/src/lib/modelCapabilities.ts @@ -176,6 +176,40 @@ function getStaticSpec(modelId: string | null, rawModel: string | null): ModelSp return undefined; } +function getStaticSpecCanonicalModelId(modelId: string | null, rawModel: string | null) { + const candidates = [modelId, rawModel].filter( + (candidate): candidate is string => typeof candidate === "string" && candidate.length > 0 + ); + for (const candidate of candidates) { + const lower = candidate.toLowerCase(); + for (const [canonical, spec] of Object.entries(MODEL_SPECS)) { + if (canonical === "__default__") continue; + if (canonical.toLowerCase() === lower) return canonical; + if (spec.aliases?.some((alias) => alias.toLowerCase() === lower)) return canonical; + } + } + return null; +} + +function getSyncedCapabilityForResolved( + provider: string | null, + model: string | null, + rawModel: string | null +): SyncedCapabilities { + if (!provider || !model) return null; + + const direct = getSyncedCapability(provider, model); + if (direct) return direct; + + if (rawModel && rawModel !== model) { + const raw = getSyncedCapability(provider, rawModel); + if (raw) return raw; + } + + const canonical = getStaticSpecCanonicalModelId(model, rawModel); + return canonical && canonical !== model ? getSyncedCapability(provider, canonical) : null; +} + function resolveVisionCapability( spec: ModelSpec | undefined, registryModel: { supportsVision?: boolean } | null, @@ -209,10 +243,11 @@ export function getResolvedModelCapabilities(input: CapabilityInput): ResolvedMo const resolved = resolveCapabilityInput(input); const spec = getStaticSpec(resolved.model, resolved.rawModel); const registryModel = getRegistryModel(resolved.provider, resolved.model); - const synced = - resolved.provider && resolved.model - ? getSyncedCapability(resolved.provider, resolved.model) - : null; + const synced = getSyncedCapabilityForResolved( + resolved.provider, + resolved.model, + resolved.rawModel + ); const modalitiesInput = parseModalities(synced?.modalities_input); const modalitiesOutput = parseModalities(synced?.modalities_output); @@ -283,9 +318,7 @@ export function getResolvedModelCapabilities(input: CapabilityInput): ResolvedMo modalitiesOutput, interleavedField: synced?.interleaved_field ?? - (typeof registryModel?.interleavedField === "string" - ? registryModel.interleavedField - : null), + (typeof registryModel?.interleavedField === "string" ? registryModel.interleavedField : null), }; } diff --git a/src/shared/validation/schemas.ts b/src/shared/validation/schemas.ts index 97858f1df0..e855296431 100644 --- a/src/shared/validation/schemas.ts +++ b/src/shared/validation/schemas.ts @@ -643,6 +643,7 @@ const comboRuntimeConfigSchema = z maxMessagesForSummary: z.coerce.number().int().min(5).max(100).optional(), maxComboDepth: z.coerce.number().int().min(1).max(10).optional(), trackMetrics: z.boolean().optional(), + reasoningTokenBufferEnabled: z.boolean().optional(), compressionMode: compressionModeSchema.optional(), failoverBeforeRetry: z.boolean().optional(), maxSetRetries: z.coerce.number().int().min(0).max(10).optional(), diff --git a/src/types/settings.ts b/src/types/settings.ts index b46b68cfc7..d1bd0a4bd2 100644 --- a/src/types/settings.ts +++ b/src/types/settings.ts @@ -56,6 +56,7 @@ export interface ComboDefaults { fallbackDelayMs?: number; maxComboDepth: number; trackMetrics: boolean; + reasoningTokenBufferEnabled?: boolean; concurrencyPerModel?: number; queueTimeoutMs?: number; handoffThreshold?: number; diff --git a/tests/integration/integration-wiring.test.ts b/tests/integration/integration-wiring.test.ts index a4af359c7a..6597b16616 100644 --- a/tests/integration/integration-wiring.test.ts +++ b/tests/integration/integration-wiring.test.ts @@ -526,7 +526,7 @@ describe("Page Integration — combos page empty state", () => { describe("Page Integration — provider test results privacy", () => { const providersSrc = readProjectFile("src/app/(dashboard)/dashboard/providers/page.tsx"); const providerDetailSrc = readProjectFile( - "src/app/(dashboard)/dashboard/providers/[id]/page.tsx" + "src/app/(dashboard)/dashboard/providers/[id]/ProviderDetailPageClient.tsx" ); it("should mask provider test batch names with the global email privacy toggle", () => { @@ -541,7 +541,7 @@ describe("Page Integration — provider test results privacy", () => { it("should mask provider detail test result names with the global email privacy toggle", () => { assert.ok( providerDetailSrc, - "src/app/(dashboard)/dashboard/providers/[id]/page.tsx should exist" + "src/app/(dashboard)/dashboard/providers/[id]/ProviderDetailPageClient.tsx should exist" ); assert.match(providerDetailSrc, /const emailsVisible = useEmailPrivacyStore/); assert.match( @@ -553,7 +553,7 @@ describe("Page Integration — provider test results privacy", () => { it("should resolve provider detail metadata through the shared dashboard catalog", () => { assert.ok( providerDetailSrc, - "src/app/(dashboard)/dashboard/providers/[id]/page.tsx should exist" + "src/app/(dashboard)/dashboard/providers/[id]/ProviderDetailPageClient.tsx should exist" ); assert.match(providerDetailSrc, /resolveDashboardProviderInfo/); }); @@ -561,7 +561,7 @@ describe("Page Integration — provider test results privacy", () => { it("should treat upstream proxy entries as a dedicated management surface", () => { assert.ok( providerDetailSrc, - "src/app/(dashboard)/dashboard/providers/[id]/page.tsx should exist" + "src/app/(dashboard)/dashboard/providers/[id]/ProviderDetailPageClient.tsx should exist" ); assert.match(providerDetailSrc, /isUpstreamProxyProvider/); assert.match(providerDetailSrc, /Managed via Upstream Proxy Settings/); diff --git a/tests/integration/resilience-http-e2e.test.ts b/tests/integration/resilience-http-e2e.test.ts index 7dc7135100..dccee521fa 100644 --- a/tests/integration/resilience-http-e2e.test.ts +++ b/tests/integration/resilience-http-e2e.test.ts @@ -558,6 +558,7 @@ test("resilience API only exposes configuration, not runtime breaker state", asy "connectionCooldown", "legacy", "providerBreaker", + "providerCooldown", "requestQueue", "waitForCooldown", ]); diff --git a/tests/unit/combo-config.test.ts b/tests/unit/combo-config.test.ts index 69eac65af2..783d2106c0 100644 --- a/tests/unit/combo-config.test.ts +++ b/tests/unit/combo-config.test.ts @@ -24,6 +24,7 @@ test("getDefaultComboConfig returns a fresh copy of the defaults", () => { assert.equal(first.failoverBeforeRetry, true); assert.equal(first.maxSetRetries, 0); assert.equal(first.setRetryDelayMs, 2000); + assert.equal(first.reasoningTokenBufferEnabled, true); assert.equal(first.zeroLatencyOptimizationsEnabled, false); assert.equal(first.hedging, false); assert.equal(first.fallbackCompressionMode, "lite"); @@ -70,6 +71,39 @@ test("resolveComboConfig applies the full cascade from defaults to combo overrid assert.ok(!("healthCheckEnabled" in result)); }); +test("resolveComboConfig cascades reasoning token buffer feature flag", () => { + const providerDisabled = resolveComboConfig( + {}, + { + comboDefaults: { + reasoningTokenBufferEnabled: true, + }, + providerOverrides: { + openai: { + reasoningTokenBufferEnabled: false, + }, + }, + }, + "openai" + ); + + const comboEnabled = resolveComboConfig( + { + config: { + reasoningTokenBufferEnabled: true, + }, + }, + { + comboDefaults: { + reasoningTokenBufferEnabled: false, + }, + } + ); + + assert.equal(providerDisabled.reasoningTokenBufferEnabled, false); + assert.equal(comboEnabled.reasoningTokenBufferEnabled, true); +}); + test("resolveComboConfig preserves nested routing defaults for partial overrides", () => { const result = resolveComboConfig( { @@ -132,19 +166,23 @@ test("updateComboDefaultsSchema accepts arbitrarily large timeout defaults and p comboDefaults: { timeoutMs: 3600000, targetTimeoutMs: 30000, + reasoningTokenBufferEnabled: false, }, providerOverrides: { anthropic: { timeoutMs: 5400000, targetTimeoutMs: 45000, + reasoningTokenBufferEnabled: false, }, }, }); assert.equal(parsed.comboDefaults.timeoutMs, 3600000); assert.equal(parsed.comboDefaults.targetTimeoutMs, 30000); + assert.equal(parsed.comboDefaults.reasoningTokenBufferEnabled, false); assert.equal(parsed.providerOverrides.anthropic.timeoutMs, 5400000); assert.equal(parsed.providerOverrides.anthropic.targetTimeoutMs, 45000); + assert.equal(parsed.providerOverrides.anthropic.reasoningTokenBufferEnabled, false); }); test("combo config schema accepts explicit zero-latency opt-in controls", () => { diff --git a/tests/unit/combo-routing-engine.test.ts b/tests/unit/combo-routing-engine.test.ts index 3322d54f47..2f3972f785 100644 --- a/tests/unit/combo-routing-engine.test.ts +++ b/tests/unit/combo-routing-engine.test.ts @@ -15,6 +15,8 @@ const { resolveNestedComboModels, handleComboChat, } = await import("../../open-sse/services/combo.ts"); +const { resolveReasoningBufferedMaxTokens } = + await import("../../open-sse/services/reasoningTokenBuffer.ts"); const { normalizeComboStep } = await import("../../src/lib/combos/steps.ts"); const { registerStrategy } = await import("../../open-sse/services/autoCombo/routerStrategy.ts"); const { touchSession, clearSessions } = await import("../../open-sse/services/sessionManager.ts"); @@ -2874,7 +2876,7 @@ test("handleComboChat aborts combo when 503 response does NOT contain the unavai test("#3587 reasoning model gets max_tokens buffer applied", async () => { saveModelsDevCapabilities({ openai: { - "gpt-4o-reasoning": capabilityEntry(4096, { reasoning: true }), + "gpt-4o-reasoning": capabilityEntry(12000, { reasoning: true, limit_output: 12000 }), }, }); @@ -2885,14 +2887,14 @@ test("#3587 reasoning model gets max_tokens buffer applied", async () => { name: "reasoning-buffer", models: ["openai/gpt-4o-reasoning"], }, - handleSingleModel: async (body: any) => { + handleSingleModel: async (body: Record) => { bodies.push(body); return okResponse(); }, isModelAvailable: async () => true, log: createLog(), settings: null, - relayOptions: null as any, + relayOptions: null, allCombos: null, }); @@ -2902,6 +2904,129 @@ test("#3587 reasoning model gets max_tokens buffer applied", async () => { assert.equal(bodies[0].max_tokens, 6144, "max_tokens should be buffered for reasoning model"); }); +test("#3587 reasoning buffer preserves max_tokens when the full buffer exceeds model cap", async () => { + saveModelsDevCapabilities({ + openai: { + "gemini-high-cap": capabilityEntry(65536, { reasoning: true, limit_output: 65536 }), + }, + }); + + assert.equal( + resolveReasoningBufferedMaxTokens("openai/gemini-high-cap", 64000), + 64000, + "near-cap requests should not be inflated beyond the model's accepted range" + ); + assert.equal( + resolveReasoningBufferedMaxTokens("openai/gemini-high-cap", "4096"), + 6144, + "numeric string max_tokens should be normalized before applying a safe buffer" + ); + assert.equal( + resolveReasoningBufferedMaxTokens("openai/gemini-high-cap", "not-a-number"), + null, + "non-numeric string max_tokens should not be changed" + ); + assert.equal( + resolveReasoningBufferedMaxTokens("openai/gemini-high-cap", 70000), + 65536, + "already over-cap max_tokens should be clamped to a known explicit cap" + ); + + const bodies: Array> = []; + const result = await handleComboChat({ + body: { max_tokens: 64000 }, + combo: { + name: "reasoning-buffer-near-cap", + models: ["openai/gemini-high-cap"], + }, + handleSingleModel: async (body: Record) => { + bodies.push(body); + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + relayOptions: null, + allCombos: null, + }); + + assert.equal(result.ok, true); + assert.equal(bodies.length, 1, "should have called handleSingleModel once"); + assert.equal(bodies[0].max_tokens, 64000, "max_tokens should remain within the cap"); +}); + +test("#3587 reasoning buffer is disabled without explicit model capability data", async () => { + assert.equal( + resolveReasoningBufferedMaxTokens("missing-provider/unknown-reasoning-model", 100), + null, + "unknown models must not receive heuristic token inflation" + ); + + saveModelsDevCapabilities({ + openai: { + "capless-reasoning": capabilityEntry(8192, { + reasoning: true, + limit_output: null, + }), + "default-cap-reasoning": capabilityEntry(8192, { + reasoning: true, + limit_output: 8192, + }), + }, + }); + + assert.equal( + resolveReasoningBufferedMaxTokens("openai/capless-reasoning", 100), + null, + "reasoning metadata without an explicit output cap is not safe enough to inflate" + ); + assert.equal( + resolveReasoningBufferedMaxTokens("openai/default-cap-reasoning", 100), + null, + "default-sized caps are treated as unknown because registry fallbacks use the same value" + ); +}); + +test("#3588 reasoning token buffer feature flag preserves client max_tokens", async () => { + saveModelsDevCapabilities({ + openai: { + "flagged-reasoning": capabilityEntry(12000, { reasoning: true, limit_output: 12000 }), + }, + }); + + assert.equal( + resolveReasoningBufferedMaxTokens("openai/flagged-reasoning", 4096, { enabled: false }), + null, + "disabled feature flag should skip reasoning token inflation" + ); + + const bodies: Array> = []; + const result = await handleComboChat({ + body: { max_tokens: 4096 }, + combo: { + name: "reasoning-buffer-disabled", + models: ["openai/flagged-reasoning"], + }, + handleSingleModel: async (body: Record) => { + bodies.push(body); + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: { + comboDefaults: { + reasoningTokenBufferEnabled: false, + }, + }, + relayOptions: null, + allCombos: null, + }); + + assert.equal(result.ok, true); + assert.equal(bodies.length, 1, "should have called handleSingleModel once"); + assert.equal(bodies[0].max_tokens, 4096, "feature flag should preserve client max_tokens"); +}); + test("#3587 non-reasoning model does not get max_tokens buffer", async () => { saveModelsDevCapabilities({ openai: { @@ -2916,14 +3041,14 @@ test("#3587 non-reasoning model does not get max_tokens buffer", async () => { name: "no-reasoning-buffer", models: ["openai/gpt-4o-plain"], }, - handleSingleModel: async (body: any) => { + handleSingleModel: async (body: Record) => { bodies.push(body); return okResponse(); }, isModelAvailable: async () => true, log: createLog(), settings: null, - relayOptions: null as any, + relayOptions: null, allCombos: null, }); @@ -2945,8 +3070,8 @@ test("#3587 round-robin buffer does NOT compound across reasoning models", async // the shared-`body` mutation that compounded the buffer on every RR iteration. saveModelsDevCapabilities({ openai: { - "rr-reasoning-a": capabilityEntry(4096, { reasoning: true }), - "rr-reasoning-b": capabilityEntry(4096, { reasoning: true }), + "rr-reasoning-a": capabilityEntry(12000, { reasoning: true, limit_output: 12000 }), + "rr-reasoning-b": capabilityEntry(12000, { reasoning: true, limit_output: 12000 }), }, }); @@ -2958,7 +3083,7 @@ test("#3587 round-robin buffer does NOT compound across reasoning models", async strategy: "round-robin", models: ["openai/rr-reasoning-a", "openai/rr-reasoning-b"], }, - handleSingleModel: async (body: any, modelStr: any) => { + handleSingleModel: async (body: Record, modelStr: string) => { seen.push({ model: modelStr, maxTokens: body.max_tokens }); if (modelStr === "openai/rr-reasoning-a") { return new Response(JSON.stringify({ error: { message: "transient" } }), { @@ -2978,7 +3103,7 @@ test("#3587 round-robin buffer does NOT compound across reasoning models", async retryDelayMs: 1, }, }, - relayOptions: null as any, + relayOptions: null, allCombos: null, }); @@ -2992,3 +3117,96 @@ test("#3587 round-robin buffer does NOT compound across reasoning models", async "second reasoning model must ALSO buffer from original 4096, not 6144" ); }); + +test("#3588 round-robin honors disabled reasoning token buffer feature flag", async () => { + saveModelsDevCapabilities({ + openai: { + "rr-flagged-a": capabilityEntry(12000, { reasoning: true, limit_output: 12000 }), + "rr-flagged-b": capabilityEntry(12000, { reasoning: true, limit_output: 12000 }), + }, + }); + + const seen: Array<{ model: string; maxTokens: unknown }> = []; + const result = await handleComboChat({ + body: { max_tokens: 4096 }, + combo: { + name: "rr-reasoning-buffer-disabled", + strategy: "round-robin", + models: ["openai/rr-flagged-a", "openai/rr-flagged-b"], + }, + handleSingleModel: async (body: Record, modelStr: string) => { + seen.push({ model: modelStr, maxTokens: body.max_tokens }); + if (modelStr === "openai/rr-flagged-a") { + return new Response(JSON.stringify({ error: { message: "transient" } }), { + status: 400, + headers: { "content-type": "application/json" }, + }); + } + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: { + comboDefaults: { + concurrencyPerModel: 1, + queueTimeoutMs: 5, + maxRetries: 0, + retryDelayMs: 1, + reasoningTokenBufferEnabled: false, + }, + }, + relayOptions: null, + allCombos: null, + }); + + assert.equal(result.status, 200); + assert.equal(seen.length, 2, "both reasoning models should have been attempted"); + assert.equal(seen[0].maxTokens, 4096, "first model should preserve client max_tokens"); + assert.equal(seen[1].maxTokens, 4096, "second model should preserve client max_tokens"); +}); + +test("#3587 round-robin keeps near-cap reasoning max_tokens unchanged", async () => { + saveModelsDevCapabilities({ + openai: { + "rr-near-cap-a": capabilityEntry(65536, { reasoning: true, limit_output: 65536 }), + "rr-near-cap-b": capabilityEntry(65536, { reasoning: true, limit_output: 65536 }), + }, + }); + + const seen: Array<{ model: string; maxTokens: unknown }> = []; + const result = await handleComboChat({ + body: { max_tokens: 64000 }, + combo: { + name: "rr-reasoning-near-cap", + strategy: "round-robin", + models: ["openai/rr-near-cap-a", "openai/rr-near-cap-b"], + }, + handleSingleModel: async (body: Record, modelStr: string) => { + seen.push({ model: modelStr, maxTokens: body.max_tokens }); + if (modelStr === "openai/rr-near-cap-a") { + return new Response(JSON.stringify({ error: { message: "transient" } }), { + status: 400, + headers: { "content-type": "application/json" }, + }); + } + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: { + comboDefaults: { + concurrencyPerModel: 1, + queueTimeoutMs: 5, + maxRetries: 0, + retryDelayMs: 1, + }, + }, + relayOptions: null, + allCombos: null, + }); + + assert.equal(result.status, 200); + assert.equal(seen.length, 2, "both reasoning models should have been attempted"); + assert.equal(seen[0].maxTokens, 64000, "first reasoning model should keep max_tokens"); + assert.equal(seen[1].maxTokens, 64000, "second reasoning model should keep max_tokens"); +});