From 74c54828fcc6a3325e6d65291d153636d8ef76b5 Mon Sep 17 00:00:00 2001 From: Xiangzhe <32761048+xz-dev@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:49:12 +0800 Subject: [PATCH] [defer] fix(routing): keep approximate context estimates advisory (#10162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merged — per owner decision on RFC #10141: gateway token estimates (chars/4) become advisory-only for combo routing/context checks, never a hard pre-dispatch 400. Locally validated (31/31 focused tests across capability filtering, target resolution, and the #8841 repro), file-size/changelog gates clean, merges conflict-free against the current release tip. Thanks for the well-scoped fix and for flagging this as an RFC first! --- ...0162-approximate-combo-context-advisory.md | 1 + open-sse/services/combo.ts | 40 +--- open-sse/services/combo/comboStructure.ts | 8 +- .../services/combo/knownContextOverflow.ts | 184 ------------------ .../services/combo/resolveAutoStrategy.ts | 25 +-- open-sse/services/combo/targetResolution.ts | 76 ++------ .../capabilities/capabilityFilter.ts | 38 ++-- ...8488-capability-filter-fail-closed.test.ts | 46 ++--- .../unit/combo-context-window-filter.test.ts | 119 +++-------- .../combo-target-resolution-split.test.ts | 33 ++-- ...pro-8841-context-overflow-opencode.test.ts | 46 +---- 11 files changed, 126 insertions(+), 490 deletions(-) create mode 100644 changelog.d/fixes/10162-approximate-combo-context-advisory.md delete mode 100644 open-sse/services/combo/knownContextOverflow.ts diff --git a/changelog.d/fixes/10162-approximate-combo-context-advisory.md b/changelog.d/fixes/10162-approximate-combo-context-advisory.md new file mode 100644 index 0000000000..3c1bc703a0 --- /dev/null +++ b/changelog.d/fixes/10162-approximate-combo-context-advisory.md @@ -0,0 +1 @@ +- **fix(routing):** keep approximate Combo context estimates advisory so requests reach concrete targets instead of returning a pre-dispatch 400 ([#10162](https://github.com/diegosouzapw/OmniRoute/pull/10162)) — thanks @xz-dev diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 982775f69b..71d1f47384 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -234,7 +234,6 @@ import { resolveComboRuntimeUnits, resolveComboTargets, } from "./combo/comboStructure.ts"; -import { getKnownContextOverflow } from "./combo/knownContextOverflow.ts"; import { createInvocationId, finalizeComboTrace, @@ -284,12 +283,7 @@ export { }; export { resolveShadowTargets, scheduleShadowRouting }; export { preScreenTargets }; -export { - resolveComboRuntimeUnits, - resolveComboTargets, - filterTargetsByRequestCompatibility, - getKnownContextOverflow, -}; +export { resolveComboRuntimeUnits, resolveComboTargets, filterTargetsByRequestCompatibility }; export { getComboFromData, getComboModelsFromData, @@ -835,12 +829,6 @@ async function handleComboChatInner({ handleSingleModelWithTimeout, buildAutoCandidates, hiddenModelsByProvider, - clientManagedResponsesContext, - deferContextOverflowWhenCompressible, - compressionExclusions, - sourceFormat, - endpointPath, - requestHeaders, }); if ("earlyResponse" in targetResolution) return targetResolution.earlyResponse; const { stickyWeightedLimit, getWeightedStepKeyForTarget, preScreenMap } = targetResolution; @@ -2719,32 +2707,6 @@ async function handleRoundRobinCombo({ ); const tagFilteredTargets = await applyRequestTagRouting(orderedTargets, body, log); const evalRankedTargets = orderTargetsByEvalScores(tagFilteredTargets, config.evalRouting, log); - const knownContextOverflow = getKnownContextOverflow(evalRankedTargets, body, { - clientManagedResponsesContext, - deferContextOverflowWhenCompressible, - compressionExclusions, - sourceFormat, - endpointPath, - requestHeaders, - }); - if (knownContextOverflow) { - return errorResponseWithComboDiagnostics( - 400, - `Request requires approximately ${knownContextOverflow.requiredContextTokens} tokens, but the largest known context limit in this combo is ${knownContextOverflow.maxKnownContextTokens} tokens. Reduce or compact the request context.`, - { - poolSize: evalRankedTargets.length, - attempted: 0, - excluded: evalRankedTargets.map((target) => ({ - provider: target.provider, - model: target.modelStr, - reason: "context_window", - })), - attemptOrder: [], - terminalReason: "context_length_exceeded", - }, - { code: "context_length_exceeded", type: "invalid_request_error" } - ); - } // Align with the main/auto paths: combo config OR top-level settings (#8488 / #8494). const rrCompatFailOpen = (config as { compatFilterFailOpen?: unknown }).compatFilterFailOpen === true || diff --git a/open-sse/services/combo/comboStructure.ts b/open-sse/services/combo/comboStructure.ts index c125bc9052..258c92da71 100644 --- a/open-sse/services/combo/comboStructure.ts +++ b/open-sse/services/combo/comboStructure.ts @@ -25,7 +25,6 @@ import { dedupeTargetsByExecutionKey, isRecord } from "./comboData.ts"; import { isComboModelVisible } from "./comboVisibility.ts"; import { getTargetProvider, MAX_COMBO_DEPTH } from "./comboPredicates.ts"; import { evaluateContextLimit } from "./contextOverrideGate.ts"; -import { hasEstimableContent } from "./knownContextOverflow.ts"; import { normalizeModelEntry, orderTargetsForWeightedFallback, @@ -480,6 +479,13 @@ function requestRequiresStructuredOutput(body: Record): boolean return type === "json_object" || type === "json_schema"; } +export function hasEstimableContent(value: unknown): boolean { + if (value === undefined || value === null) return false; + if (Array.isArray(value)) return value.length > 0; + if (typeof value === "object") return Object.keys(value).length > 0; + return true; +} + function estimateRequestInputTokens(body: Record): number { const estimatePayload: Record = {}; for (const key of ["messages", "input", "tools", "functions", "response_format"]) { diff --git a/open-sse/services/combo/knownContextOverflow.ts b/open-sse/services/combo/knownContextOverflow.ts deleted file mode 100644 index 532e1a4be6..0000000000 --- a/open-sse/services/combo/knownContextOverflow.ts +++ /dev/null @@ -1,184 +0,0 @@ -/** - * Known context-overflow rejection, extracted from comboStructure.ts to keep - * that file under the file-size cap (#7177). - * - * Fixes: routing a request to a combo whose targets all have a KNOWN (not - * unknown/fail-open) context window too small for the request used to be - * discovered only after every target was tried and failed upstream — burning - * retries/cooldowns on a request that could never succeed. This lets the - * combo dispatcher reject it up front, before exhausting providers. - * - * getKnownContextLimit/hasEstimableContent also - * live here (moved from comboStructure.ts, same file-size-cap motivation): - * they are the "how big is a target's known context window" primitives, so - * they belong next to the overflow check that is their main consumer. - * comboStructure.ts's own compatibility filter now decides fit via its - * evaluateContextLimit (#7052); only hasEstimableContent is imported back. - */ - -import { getResolvedModelCapabilities } from "../modelCapabilities.ts"; -import { isCompressionExcluded, type CompressionExclusions } from "../compression/exclusions.ts"; -import { shouldUseNativeCodexPassthrough } from "../../handlers/chatCore/passthroughHelpers.ts"; -import { deriveRequestCompatibilityRequirements } from "./comboStructure.ts"; -import type { ResolvedComboTarget } from "./types.ts"; - -export type KnownContextOverflow = { - estimatedInputTokens: number; - requestedOutputTokens: number; - requiredContextTokens: number; - maxKnownContextTokens: number; - targetCount: number; -}; - -export type KnownContextOverflowOptions = { - clientManagedResponsesContext?: boolean; - /** - * When prompt compression is enabled for this request (global compression switch - * AND not API-key opted-out), defer the hard preflight so chatCore's compression - * pipeline runs before the final context gate — instead of a raw-body estimate - * rejecting a compressible request up front. (#10225) - */ - deferContextOverflowWhenCompressible?: boolean; - /** Server-side compression exclusions (#8034) — targets matching one cannot run compression. */ - compressionExclusions?: CompressionExclusions; - /** - * #10503: the exact request-shape facts chatCore.ts uses to decide - * `shouldUseNativeCodexPassthrough` (open-sse/handlers/chatCore/passthroughHelpers.ts) — - * threaded down so the deferral decision below can be target-aware instead of - * relying on the looser `clientManagedResponsesContext` proxy. Reused verbatim - * (not re-derived) so the combo-layer decision can never drift from chatCore's own. - */ - sourceFormat?: string | null; - endpointPath?: string | null; - requestHeaders?: Headers | Record | null; -}; - -// #7177: an empty array/object (e.g. a default `messages: []` some combo entrypoints inject -// when the caller sent none) has no real content — counting it would charge a few phantom -// "structural" tokens (JSON.stringify braces/brackets) toward the estimate, which is enough -// to falsely trip the exact-boundary known-context-overflow check for a request that has no -// actual input at all. -export function hasEstimableContent(value: unknown): boolean { - if (value === undefined || value === null) return false; - if (Array.isArray(value)) return value.length > 0; - if (typeof value === "object") return Object.keys(value).length > 0; - return true; -} - -// #7177: known context limit that accounts for the request's own requested -// output tokens — a target whose input+output would together exceed -// maxInputTokens is exactly as incompatible as one whose contextWindow is too -// small, so both bounds go through the same min() so far the tightest wins. -export function getKnownContextLimit( - capabilities: { - maxInputTokens?: number | null; - contextWindow?: number | null; - }, - requestedOutputTokens = 0 -): number | null { - const limits: number[] = []; - if (capabilities.maxInputTokens != null) { - limits.push(capabilities.maxInputTokens + requestedOutputTokens); - } - if (capabilities.contextWindow != null) { - limits.push(capabilities.contextWindow); - } - return limits.length > 0 ? Math.min(...limits) : null; -} - -/** - * Return a hard context-overflow decision only when every target has a known - * context limit and every one of those limits is too small for the request. - * Unknown metadata deliberately keeps the legacy fail-open behavior. - */ -export function getKnownContextOverflow( - targets: ResolvedComboTarget[], - body: Record, - options: KnownContextOverflowOptions = {} -): KnownContextOverflow | null { - if (targets.length === 0) return null; - // Native Codex Responses clients compact their own item history. Let the concrete - // Codex target enforce its effective context limit (including operator overrides) - // instead of rejecting early against a smaller catalog hint. Keep this scoped to - // pools made exclusively from native Codex-capable targets so other Responses - // clients/providers retain the hard preflight. - if ( - options.clientManagedResponsesContext === true && - targets.every( - (target) => target.provider === "codex" || target.provider === "chatgpt-web-codex" - ) - ) { - return null; - } - // #10225 / #10499-sweep #10503: a conservative raw-body context estimate must not - // be treated as proof that a compression-enabled request cannot fit. When - // compression is available for this request AND at least one target can actually - // run it, defer the hard rejection so handleChatCore runs proactive compression - // (chatCore.ts) and its post-compression enforceOutputTokenBudget becomes the - // final context gate — returning a local `context_length_exceeded` only if the - // compressed body still cannot fit (no upstream dispatch). - // - // Target-awareness is load-bearing here: a target is only a valid reason to defer - // when handleChatCore will ACTUALLY attempt compression for it. Two classes are - // excluded from "can compress" even though `isCompressionExcluded` (operator - // exclusions) says nothing about them: - // - Operator-excluded targets (#8034, existing `isCompressionExcluded` check). - // - Native Codex Responses passthrough targets: chatCore.ts unconditionally sets - // `compressionExcluded = nativeCodexPassthrough || ...` for these, computed via - // `shouldUseNativeCodexPassthrough()` (chatCore/passthroughHelpers.ts) — called - // here with the SAME request-shape facts (sourceFormat/endpointPath/headers) - // chatCore itself uses, reused verbatim rather than re-derived from the looser - // `clientManagedResponsesContext` flag (which always requires a VERIFIED native - // client; chatCore's own gate does NOT for provider==="codex" — see - // shouldUseNativeCodexPassthrough's `provider === "codex" || isVerifiedNativeCodexRequest` - // short-circuit). Deferring on such a target's account would let an oversized - // body sail straight through to `fetch()` uncompressed instead of being caught - // by either preflight — silently defeating the whole point of this feature. - // If NO target can compress, the fast raw-body preflight is kept (unchanged). - if ( - options.deferContextOverflowWhenCompressible === true && - targets.some((target) => { - const isNativeCodexPassthroughTarget = shouldUseNativeCodexPassthrough({ - provider: target.provider, - sourceFormat: options.sourceFormat, - endpointPath: options.endpointPath, - body, - headers: options.requestHeaders, - }); - if (isNativeCodexPassthroughTarget) return false; - return !isCompressionExcluded( - { - provider: target.provider, - model: target.modelStr.includes("/") - ? target.modelStr.split("/").slice(1).join("/") - : target.modelStr, - }, - options.compressionExclusions - ); - }) - ) { - return null; - } - const requirements = deriveRequestCompatibilityRequirements(body); - if (requirements.requiredContextTokens <= 0) return null; - - const limits = targets.map((target) => - getKnownContextLimit( - getResolvedModelCapabilities(target.modelStr), - requirements.requestedOutputTokens - ) - ); - if (limits.some((limit) => limit === null)) return null; - - const knownLimits = limits as number[]; - const maxKnownContextTokens = Math.max(...knownLimits); - if (maxKnownContextTokens >= requirements.requiredContextTokens) return null; - - return { - estimatedInputTokens: requirements.estimatedInputTokens, - requestedOutputTokens: requirements.requestedOutputTokens, - requiredContextTokens: requirements.requiredContextTokens, - maxKnownContextTokens, - targetCount: targets.length, - }; -} diff --git a/open-sse/services/combo/resolveAutoStrategy.ts b/open-sse/services/combo/resolveAutoStrategy.ts index 1d6d2bf156..7d6fe2a682 100644 --- a/open-sse/services/combo/resolveAutoStrategy.ts +++ b/open-sse/services/combo/resolveAutoStrategy.ts @@ -179,32 +179,11 @@ export async function resolveAutoStrategyOrder( `Auto strategy: context-window filter kept ${filteredByContext.length}/${eligibleTargets.length} candidates (est. ${estimatedInputTokens} tokens)` ); eligibleTargets = filteredByContext; - } else if (compatFilterFailOpen) { + } else { log.warn( "COMBO", - `Auto strategy: all candidates filtered by context-window policy (est. ${estimatedInputTokens} tokens), falling back to full pool (compatFilterFailOpen)` + `Auto strategy: all candidates filtered by approximate context-window policy (est. ${estimatedInputTokens} tokens), falling back to full pool` ); - } else { - // #8488: every candidate has a known limit below the estimate — surface - // context_length_exceeded rather than dispatching oversized targets. - return { - earlyResponse: errorResponseWithComboDiagnostics( - 400, - `Request requires approximately ${estimatedInputTokens} tokens, but every auto-strategy candidate in combo ${combo.name} has a smaller known context limit`, - { - poolSize: eligibleTargets.length, - attempted: 0, - excluded: eligibleTargets.map((target) => ({ - provider: target.provider, - model: target.modelStr, - reason: "context_window", - })), - attemptOrder: [], - terminalReason: "context_length_exceeded", - }, - { code: "context_length_exceeded", type: "invalid_request_error" } - ), - }; } eligibleTargets = await expandAutoComboCandidatePool(eligibleTargets, combo); diff --git a/open-sse/services/combo/targetResolution.ts b/open-sse/services/combo/targetResolution.ts index 45aa57013f..ff3b0362f7 100644 --- a/open-sse/services/combo/targetResolution.ts +++ b/open-sse/services/combo/targetResolution.ts @@ -8,21 +8,19 @@ * 1. provider-wildcard expansion of the combo + the combos collection (#2562) * 2. weighted step-group resolution + sticky-weighted eligibility * 3. request-tag routing - * 4. known-context-overflow early return - * 5. smart/pipeline-enabled dispatch (auto strategy) - * 6. auto-strategy candidate build / scoring / ordering, or per-strategy ordering - * 7. prompt-cache strategy affinity, session stickiness, eval scores, + * 4. smart/pipeline-enabled dispatch (auto strategy) + * 5. auto-strategy candidate build / scoring / ordering, or per-strategy ordering + * 6. prompt-cache strategy affinity, session stickiness, eval scores, * request compatibility, context requirements - * 8. task-aware reordering - * 9. prompt-cache affinity application - * 10. the parallel pre-screen (priority strategy only) + * 7. task-aware reordering + * 8. prompt-cache affinity application + * 9. the parallel pre-screen (priority strategy only) * - * Behaviour is byte-identical to the inline block it replaces — the two early exits - * (context overflow, pipeline dispatch, auto-strategy `earlyResponse`) become an - * `{ earlyResponse }` result so the host decides to return them, and the values the - * attempt loop still consumes (`orderedTargets`, `stickyWeightedLimit`, - * `getWeightedStepKeyForTarget`, `sticky`, `preScreenMap`) are returned instead of - * closed over. + * Behaviour is byte-identical to the inline block it replaces — pipeline dispatch and + * auto-strategy `earlyResponse` become an `{ earlyResponse }` result so the host decides + * to return them, and the values the attempt loop still consumes (`orderedTargets`, + * `stickyWeightedLimit`, `getWeightedStepKeyForTarget`, `sticky`, `preScreenMap`) are + * returned instead of closed over. * * See _tasks/quality/2026-06-19-DESIGN-godfiles-decomposition.md §4. */ @@ -53,7 +51,6 @@ import { } from "./comboStructure.ts"; import { applyContextRequirements } from "./contextRequirements.ts"; import { recordComboFailure } from "./failureTracker.ts"; -import { getKnownContextOverflow } from "./knownContextOverflow.ts"; import { buildEmptyComboTargetsPayload, buildRecoveryHint } from "./pinRecovery.ts"; import { applyPromptCacheAffinity, @@ -113,16 +110,6 @@ export interface ResolveComboTargetPipelineDeps { */ buildAutoCandidates: ResolveAutoStrategyDeps["buildAutoCandidates"]; hiddenModelsByProvider?: HiddenModelsByProvider; - /** Native Responses clients (for example Codex CLI/Desktop) manage compaction themselves. */ - clientManagedResponsesContext?: boolean; - /** #10225 — defer the hard context-overflow preflight when compression is enabled for this request. */ - deferContextOverflowWhenCompressible?: boolean; - /** Server-side compression exclusions (#8034) — which targets can run compression. */ - compressionExclusions?: import("../compression/exclusions.ts").CompressionExclusions; - /** #10503 — request-shape facts for the target-aware deferral check (see knownContextOverflow.ts). */ - sourceFormat?: string | null; - endpointPath?: string | null; - requestHeaders?: Headers | Record | null; } export interface ResolvedComboTargetPipeline { @@ -323,35 +310,6 @@ function buildWeightedStepKeyMapper( }; } -/** 400 rejection for a request no target in the pool can physically accept. */ -function buildContextOverflowResponse( - overflow: { requiredContextTokens: number; maxKnownContextTokens: number }, - orderedTargets: ResolvedComboTarget[], - log: ComboLogger -): Response { - const { requiredContextTokens, maxKnownContextTokens } = overflow; - log.warn( - "COMBO", - `Request context exceeds every known target limit (${requiredContextTokens} > ${maxKnownContextTokens} tokens)` - ); - return errorResponseWithComboDiagnostics( - 400, - `Request requires approximately ${requiredContextTokens} tokens, but the largest known context limit in this combo is ${maxKnownContextTokens} tokens. Reduce or compact the request context.`, - { - poolSize: orderedTargets.length, - attempted: 0, - excluded: orderedTargets.map((target) => ({ - provider: target.provider, - model: target.modelStr, - reason: "context_window", - })), - attemptOrder: [], - terminalReason: "context_length_exceeded", - }, - { code: "context_length_exceeded", type: "invalid_request_error" } - ); -} - function logTargetPoolSize( strategy: string, allCombos: ComboCollectionLike, @@ -736,18 +694,6 @@ export async function resolveComboTargetPipeline( orderedTargets = await applyRequestTagRouting(orderedTargets, body, log); - const overflow = getKnownContextOverflow(orderedTargets, body, { - clientManagedResponsesContext: deps.clientManagedResponsesContext, - deferContextOverflowWhenCompressible: deps.deferContextOverflowWhenCompressible, - compressionExclusions: deps.compressionExclusions, - sourceFormat: deps.sourceFormat, - endpointPath: deps.endpointPath, - requestHeaders: deps.requestHeaders, - }); - if (overflow) { - return { earlyResponse: buildContextOverflowResponse(overflow, orderedTargets, log) }; - } - logTargetPoolSize(strategy, allCombos, orderedTargets, stickyWeightedKey, log); const pipelineResponse = await dispatchSmartPipeline( diff --git a/src/shared/constants/capabilities/capabilityFilter.ts b/src/shared/constants/capabilities/capabilityFilter.ts index b0c0de8091..4dd0365fb6 100644 --- a/src/shared/constants/capabilities/capabilityFilter.ts +++ b/src/shared/constants/capabilities/capabilityFilter.ts @@ -17,9 +17,11 @@ import { getResolvedModelCapabilities } from "@/lib/modelCapabilities"; import { evaluateContextLimit } from "@omniroute/open-sse/services/combo/contextOverrideGate"; -import { hasEstimableContent } from "@omniroute/open-sse/services/combo/knownContextOverflow"; import { isRecord } from "@omniroute/open-sse/services/combo/comboData"; -import { providerSupportsEmulatedToolCalling } from "@omniroute/open-sse/services/combo/comboStructure"; +import { + hasEstimableContent, + providerSupportsEmulatedToolCalling, +} from "@omniroute/open-sse/services/combo/comboStructure"; import { estimateTokens } from "@omniroute/open-sse/services/contextManager"; // ── Types ───────────────────────────────────────────────────────────────── @@ -89,10 +91,15 @@ function isContextOverflow( capabilities: { maxInputTokens: number | null; contextWindow: number | null }, requirements: { requiredContextTokens: number } ): boolean { - return evaluateContextLimit( - { maxInputTokens: capabilities.maxInputTokens, contextWindow: capabilities.contextWindow }, - { estimatedInputTokens: requirements.requiredContextTokens, requiredContextTokens: requirements.requiredContextTokens } - ) === false; + return ( + evaluateContextLimit( + { maxInputTokens: capabilities.maxInputTokens, contextWindow: capabilities.contextWindow }, + { + estimatedInputTokens: requirements.requiredContextTokens, + requiredContextTokens: requirements.requiredContextTokens, + } + ) === false + ); } function valueContainsImageType(value: Record): boolean { @@ -140,7 +147,9 @@ export function buildCapabilityMismatchMessage( structured_output: `Provider '${provider}' does not support structured output`, context_window: `Request exceeds the context window for ${provider}/${model}`, }; - return msgs[terminalReason] || `Provider '${provider}' does not support the required capabilities`; + return ( + msgs[terminalReason] || `Provider '${provider}' does not support the required capabilities` + ); } /** @@ -167,8 +176,11 @@ function collectCapabilityFailures( maxOutputTokens: number | null; }; - if (requirements.requiresTools && (caps.supportsTools === false || !caps.toolCalling) - && !providerSupportsEmulatedToolCalling(provider)) { + if ( + requirements.requiresTools && + (caps.supportsTools === false || !caps.toolCalling) && + !providerSupportsEmulatedToolCalling(provider) + ) { failures.push("tools"); } if (requirements.requiresVision && caps.supportsVision !== true) { @@ -203,9 +215,13 @@ export function checkRequestCapabilityFit( requirements: RequestCapabilityRequirements, provider?: string | null ): CapabilityFilterResult { - const failures = collectCapabilityFailures(capabilities as Record, requirements, provider); + const failures = collectCapabilityFailures( + capabilities as Record, + requirements, + provider + ); if (failures.length === 0) { return { compatible: true, failures: [] }; } return { compatible: false, failures, terminalReason: primaryFailure(failures) }; -} \ No newline at end of file +} diff --git a/tests/unit/8488-capability-filter-fail-closed.test.ts b/tests/unit/8488-capability-filter-fail-closed.test.ts index 6e3807ce2b..691353e792 100644 --- a/tests/unit/8488-capability-filter-fail-closed.test.ts +++ b/tests/unit/8488-capability-filter-fail-closed.test.ts @@ -19,6 +19,7 @@ const { } = await import("../../open-sse/services/combo/comboStructure.ts"); const { resolveAutoStrategyOrder } = await import("../../open-sse/services/combo/resolveAutoStrategy.ts"); +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); function capabilityEntry(limit_context: number, overrides: Record = {}) { return { @@ -129,10 +130,7 @@ test("#8488 filter: chatgpt-web emulation providers stay eligible for tools (#52 assert.equal(providerSupportsEmulatedToolCalling("openai"), false); const kept = filterTargetsByRequestCompatibility( - [ - target("chatgpt-web", "chatgpt-web/gpt-5.5"), - target("chatgpt-web", "chatgpt-web/o3"), - ], + [target("chatgpt-web", "chatgpt-web/gpt-5.5"), target("chatgpt-web", "chatgpt-web/o3")], { messages: [{ role: "user", content: "Use a tool." }], tools: [{ type: "function", function: { name: "lookup", parameters: {} } }], @@ -146,10 +144,7 @@ test("#8488 filter: chatgpt-web emulation providers stay eligible for tools (#52 ); const exhaustion = describeCapabilityFilterExhaustion( - [ - target("chatgpt-web", "chatgpt-web/gpt-5.5"), - target("chatgpt-web", "chatgpt-web/o3"), - ], + [target("chatgpt-web", "chatgpt-web/gpt-5.5"), target("chatgpt-web", "chatgpt-web/o3")], { messages: [{ role: "user", content: "Use a tool." }], tools: [{ type: "function", function: { name: "lookup", parameters: {} } }], @@ -175,7 +170,10 @@ test("#8488 auto: chatgpt-web emulation survives tool pre-filter (#5240)", async buildAutoCandidates: (async () => []) as never, }); - assert.ok(!("earlyResponse" in result), "must not 400 capability_mismatch for emulation providers"); + assert.ok( + !("earlyResponse" in result), + "must not 400 capability_mismatch for emulation providers" + ); if ("orderedTargets" in result) { assert.equal(result.orderedTargets.length, 1); assert.equal(result.orderedTargets[0].modelStr, "chatgpt-web/gpt-5.5"); @@ -287,7 +285,7 @@ test("#8488 auto: tool pre-filter fail-open opt-in keeps full pool", async () => } }); -test("#8488 auto: context pre-filter fail closed when all known limits too small", async () => { +test("auto context estimate still dispatches when all known limits look too small", async () => { saveModelsDevCapabilities({ openai: { tiny: capabilityEntry(100, { tool_call: true }), @@ -295,22 +293,24 @@ test("#8488 auto: context pre-filter fail closed when all known limits too small }); const hugePrompt = "x".repeat(4000); // ~1000 tokens at 4 chars/token - const result = await resolveAutoStrategyOrder({ - orderedTargets: [target("openai", "openai/tiny")] as never, + const dispatches: string[] = []; + const result = await handleComboChat({ body: { messages: [{ role: "user", content: hugePrompt }] }, - combo: { id: "c1", name: "auto-ctx", config: {} } as never, + combo: { id: "c1", name: "auto-ctx", strategy: "auto", models: ["openai/tiny"] }, + handleSingleModel: async (_body, modelStr) => { + dispatches.push(modelStr); + return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }, + isModelAvailable: async () => true, + log, settings: null, - config: {}, relayOptions: null, - resilienceSettings: { quotaPreflight: { enabled: false } } as never, - log: log as never, - buildAutoCandidates: (async () => []) as never, + allCombos: null, }); - assert.ok("earlyResponse" in result); - if ("earlyResponse" in result) { - assert.equal(result.earlyResponse.status, 400); - const body = await result.earlyResponse.json(); - assert.equal(body?.error?.code, "context_length_exceeded"); - } + assert.equal(result.status, 200); + assert.deepEqual(dispatches, ["openai/tiny"]); }); diff --git a/tests/unit/combo-context-window-filter.test.ts b/tests/unit/combo-context-window-filter.test.ts index 7759b20fb7..442caf85e0 100644 --- a/tests/unit/combo-context-window-filter.test.ts +++ b/tests/unit/combo-context-window-filter.test.ts @@ -15,7 +15,7 @@ process.env.DATA_DIR = TEST_DATA_DIR; const core = await import("../../src/lib/db/core.ts"); const { saveModelsDevCapabilities, clearModelsDevCapabilities } = await import("../../src/lib/modelsDevSync.ts"); -const { filterTargetsByRequestCompatibility, getKnownContextOverflow, handleComboChat } = +const { filterTargetsByRequestCompatibility, handleComboChat } = await import("../../open-sse/services/combo.ts"); const { setModelContextOverride, removeModelContextOverride } = await import("../../src/lib/db/modelContextOverrides.ts"); @@ -212,62 +212,7 @@ test("output-token limits remain a hard compatibility requirement", () => { ); }); -test("known context overflow reports the largest target limit", () => { - saveModelsDevCapabilities({ - "unit-known-context": { - tiny: capabilityEntry(8_000), - small: capabilityEntry(16_000), - }, - }); - - const overflow = getKnownContextOverflow( - [target("unit-known-context/tiny"), target("unit-known-context/small")], - largeContextBody() - ); - - assert.ok(overflow); - assert.ok(overflow.requiredContextTokens > overflow.maxKnownContextTokens); - assert.equal(overflow.maxKnownContextTokens, 16_000); - assert.equal(overflow.targetCount, 2); -}); - -test("#7177 an empty messages array is not counted as real content at an exact-boundary limit", () => { - // Regression: some combo entrypoints default a caller-omitted `messages` to `[]`. The - // estimator used to JSON.stringify whatever keys were merely *present* on the body, - // so an empty array still contributed a few phantom "structural" tokens (JSON braces/ - // brackets), which was enough to trip a false-positive overflow when max_tokens exactly - // equals the target's context window (a common config where limit_input === limit_output - // === limit_context) even though there is no real input to account for. - saveModelsDevCapabilities({ - "unit-known-context": { - exact: capabilityEntry(4_096), - }, - }); - - const overflow = getKnownContextOverflow([target("unit-known-context/exact")], { - messages: [], - max_tokens: 4_096, - }); - - assert.equal(overflow, null); -}); - -test("unknown context metadata keeps overflow detection fail-open", () => { - saveModelsDevCapabilities({ - "unit-known-context": { - tiny: capabilityEntry(8_000), - }, - }); - - const overflow = getKnownContextOverflow( - [target("unit-known-context/tiny"), target("unit-unknown-context/mystery")], - largeContextBody() - ); - - assert.equal(overflow, null); -}); - -test("combo rejects a known oversized request before upstream dispatch", async () => { +test("combo dispatches requests that only an approximate estimate marks oversized", async () => { saveModelsDevCapabilities({ "unit-known-context": { tiny: capabilityEntry(8_000), @@ -285,48 +230,46 @@ test("combo rejects a known oversized request before upstream dispatch", async ( }, handleSingleModel: async () => { dispatches += 1; - return new Response("unexpected", { status: 200 }); + return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); }, log: noopLog, }); - assert.equal(response.status, 400); - assert.equal(dispatches, 0); - const body = await response.json(); - assert.equal(body.error.code, "context_length_exceeded"); - assert.equal(body.diagnostics.terminalReason, "context_length_exceeded"); - assert.equal(body.diagnostics.attempted, 0); + assert.equal(response.status, 200); + assert.equal(dispatches, 1); }); -test("native Responses context bypasses catalog overflow only for all-Codex pools (#8932)", () => { +test("round-robin dispatches requests that only an approximate estimate marks oversized", async () => { saveModelsDevCapabilities({ - codex: { - large: capabilityEntry(272_000), - }, "unit-known-context": { - large: capabilityEntry(272_000), + tiny: capabilityEntry(8_000), + small: capabilityEntry(16_000), }, }); - const body = bigContextBody(275_000); + let dispatches = 0; - assert.equal( - getKnownContextOverflow([target("codex/large")], body, { - clientManagedResponsesContext: true, - }), - null - ); - assert.equal( - getKnownContextOverflow([target("codex/large"), target("chatgpt-web-codex/large")], body, { - clientManagedResponsesContext: true, - }), - null - ); - assert.ok( - getKnownContextOverflow([target("unit-known-context/large")], body, { - clientManagedResponsesContext: true, - }), - "non-Codex pools must retain the catalog overflow guard" - ); + const response = await handleComboChat({ + body: largeContextBody(), + combo: { + name: "known-context-overflow-round-robin", + strategy: "round-robin", + models: ["unit-known-context/tiny", "unit-known-context/small"], + }, + handleSingleModel: async () => { + dispatches += 1; + return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }, + log: noopLog, + }); + + assert.equal(response.status, 200); + assert.equal(dispatches, 1); }); test("native Responses context reaches an all-Codex target beyond its catalog hint (#8932)", async () => { diff --git a/tests/unit/combo-target-resolution-split.test.ts b/tests/unit/combo-target-resolution-split.test.ts index be2b89c7ee..92d95f570a 100644 --- a/tests/unit/combo-target-resolution-split.test.ts +++ b/tests/unit/combo-target-resolution-split.test.ts @@ -5,14 +5,13 @@ import os from "node:os"; import path from "node:path"; // Split guard for the #3501 god-file decomposition (PR 2): the target-resolution -// stage of handleComboChat (wildcard expansion → weighted step groups → known -// context overflow → strategy ordering → stickiness/eval/compat/context filters → -// task-aware reorder → prompt-cache affinity → pre-screen) was extracted verbatim -// into resolveComboTargetPipeline. These tests pin the leaf's own contract: the -// shape it hands back to the attempt loop, the pass-through ordering for the plain -// `priority` path, and the `earlyResponse` exit for a request that exceeds every -// target's known context window. The strategy-specific branches stay covered -// end-to-end by the combo-* consumer suites through combo.ts. +// stage of handleComboChat (wildcard expansion → weighted step groups → strategy +// ordering → stickiness/eval/compat/context filters → task-aware reorder → +// prompt-cache affinity → pre-screen) was extracted verbatim into +// resolveComboTargetPipeline. These tests pin the leaf's own contract: the shape it +// hands back to the attempt loop and pass-through ordering for the plain `priority` +// path. The strategy-specific branches stay covered end-to-end by the combo-* +// consumer suites through combo.ts. const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-target-resolution-")); const ORIGINAL_DATA_DIR = process.env.DATA_DIR; @@ -115,7 +114,7 @@ test("an empty combo yields an empty target pool (combo.ts turns it into a 404)" assert.deepEqual(result.orderedTargets, []); }); -test("request exceeding every known context window returns a 400 earlyResponse", async () => { +test("request exceeding every approximate context hint keeps the target pool", async () => { saveModelsDevCapabilities({ "unit-target-resolution": { tiny: capabilityEntry(8_000), @@ -135,16 +134,12 @@ test("request exceeding every known context window returns a 400 earlyResponse", }) ); - assert.ok("earlyResponse" in result, "expected a context-overflow early response"); - if (!("earlyResponse" in result)) return; - assert.equal(result.earlyResponse.status, 400); - const body = (await result.earlyResponse.json()) as { - error?: { code?: string }; - diagnostics?: { terminalReason?: string; attempted?: number }; - }; - assert.equal(body.error?.code, "context_length_exceeded"); - assert.equal(body.diagnostics?.terminalReason, "context_length_exceeded"); - assert.equal(body.diagnostics?.attempted, 0); + assert.ok(!("earlyResponse" in result), "approximate context hints must not reject the pool"); + if ("earlyResponse" in result) return; + assert.deepEqual( + result.orderedTargets.map((target) => target.modelStr), + ["unit-target-resolution/tiny", "unit-target-resolution/small"] + ); }); // #8790: maxContextWindow rejects every target whose known context window diff --git a/tests/unit/repro-8841-context-overflow-opencode.test.ts b/tests/unit/repro-8841-context-overflow-opencode.test.ts index 6847f47c6f..0dae913167 100644 --- a/tests/unit/repro-8841-context-overflow-opencode.test.ts +++ b/tests/unit/repro-8841-context-overflow-opencode.test.ts @@ -4,21 +4,13 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -const TEST_DATA_DIR = fs.mkdtempSync( - path.join(os.tmpdir(), "omniroute-repro-8841-") -); +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-repro-8841-")); const ORIGINAL_DATA_DIR = process.env.DATA_DIR; process.env.DATA_DIR = TEST_DATA_DIR; -const { getResolvedModelCapabilities } = await import( - "../../src/lib/modelCapabilities.ts" -); -const { getKnownContextOverflow, handleComboChat } = await import( - "../../open-sse/services/combo.ts" -); -const { getTokenLimit } = await import( - "../../open-sse/services/contextManager.ts" -); +const { getResolvedModelCapabilities } = await import("../../src/lib/modelCapabilities.ts"); +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); +const { getTokenLimit } = await import("../../open-sse/services/contextManager.ts"); const core = await import("../../src/lib/db/core.ts"); test.after(() => { @@ -35,18 +27,6 @@ const noopLog = { debug() {}, }; -const target = (m) => ({ - kind: "model", - stepId: m, - executionKey: m, - modelStr: m, - provider: "opencode-zen", - providerId: null, - connectionId: null, - weight: 1, - label: null, -}); - function largeBody() { return { messages: [{ role: "user", content: "x".repeat(840_000) }], @@ -80,15 +60,8 @@ test("#8841 advertised vs compat-filter limit agree", () => { ); }); -test("#8841 oversized request rejected up front (no dispatch)", async () => { +test("#8841 real upstream context overflow remains a fatal 400 after dispatch", async () => { const body = largeBody(); - const pool = [ - target("opencode/mimo-v2.5-free"), - target("opencode/hy3-free"), - ]; - - assert.ok(getKnownContextOverflow(pool, body), "overflow before dispatch"); - let dispatches = 0; const result = await handleComboChat({ body, @@ -96,8 +69,8 @@ test("#8841 oversized request rejected up front (no dispatch)", async () => { name: "pro-coding-repro-8841", strategy: "priority", models: [ - "opencode/mimo-v2.5-free", - "opencode/hy3-free", + { model: "opencode/north-mini-code-free" }, + { model: "opencode/north-mini-code-free" }, ], }, handleSingleModel: async () => { @@ -109,9 +82,8 @@ test("#8841 oversized request rejected up front (no dispatch)", async () => { allCombos: [], }); - assert.equal(dispatches, 0, `no upstream dispatch (got ${dispatches})`); + assert.equal(dispatches, 1, "real upstream overflow must short-circuit fallback"); assert.equal(result.status, 400); const json = await result.json(); assert.equal(json.error?.code, "context_length_exceeded"); - assert.equal(json.diagnostics?.attempted, 0); -}); \ No newline at end of file +});