diff --git a/changelog.d/fixes/8488-capability-filter-fail-closed.md b/changelog.d/fixes/8488-capability-filter-fail-closed.md new file mode 100644 index 0000000000..f5487c0e13 --- /dev/null +++ b/changelog.d/fixes/8488-capability-filter-fail-closed.md @@ -0,0 +1 @@ +- **fix(backend):** capability filters fail closed when every combo target is incompatible, with opt-in `compatFilterFailOpen` ([#8488](https://github.com/diegosouzapw/OmniRoute/issues/8488)) — thanks @Prudhvivuda diff --git a/config/quality/eslint-suppressions.json b/config/quality/eslint-suppressions.json index dc40c33777..6d9ef3f219 100644 --- a/config/quality/eslint-suppressions.json +++ b/config/quality/eslint-suppressions.json @@ -1152,7 +1152,7 @@ }, "tests/unit/combo-routing-engine.test.ts": { "@typescript-eslint/no-explicit-any": { - "count": 271 + "count": 267 } }, "tests/unit/combo-same-provider-cascade.test.ts": { diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 49fc80daad..50f87e6ba4 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -1,4 +1,5 @@ { + "_rebaseline_2026_07_25_8494_capability_filter_fail_closed": "PR #8494 (fix/capability-filters-fail-closed, #8488) own growth: open-sse/services/combo.ts 3640->3693 (+53) adds a fail-closed guard after filterTargetsByRequestCompatibility() — when every eligible target is excluded by request-capability filtering (vision/tools/etc) instead of quota/health, the combo now returns an explicit `capability_mismatch` 400 (describeCapabilityFilterExhaustion, imported from combo/comboStructure.ts) rather than silently falling through to a generic no-targets error, plus a `compatFilterFailOpen` escape hatch (combo config OR settings) mirrored at both the main/auto and round-robin call sites for symmetry. combo/comboStructure.ts (previously under cap, un-frozen) grows 794->918 (+124) — new home for describeCapabilityFilterExhaustion + providerSupportsEmulatedToolCalling (#5240 emulated tool-calling exemption so fail-closed does not regress prompt-emulation-only combos like all-chatgpt-web). Irreducible orchestration wiring at the existing filter chokepoint (same precedent as #7301's universal-cooldown-retry generalization). Companion test tests/unit/combo-routing-engine.test.ts 3409->3449 (+40, fail-closed/fail-open coverage across both call sites) also rebaselined. Covered by tests/unit/8488-capability-filter-fail-closed.test.ts (new) + 95/95 passing across both files. Structural shrink of combo.ts tracked in #3501.", "_rebaseline_2026_07_25_8499_ts7_result_union_predicates": "PR #8499 (backryun, chore/ts7-types-executor-scattered) own growth: muse-spark-web.ts 1396->1405 (+9, irreducible). Under this workspace's `strictNullChecks: false`, the boolean-literal discriminant on `GraphqlResult` (`{ ok: true } | { ok: false; error: string }`) narrows the positive `.ok===true` branch but leaves `!result.ok` at the full union under TS7, making `.error` unreachable to the checker at the two call sites (warmup, mode-switch). Fixed by adding a single `isGraphqlFailure()` type-predicate helper (doc comment + 3-line body) reused at both call sites instead of duplicating the predicate inline — not extractable to a shared module without splitting a single-file executor's local narrowing helper out of its own file. Covered by the existing muse-spark-web executor test suite (no behavior change, pure narrowing fix).", "_rebaseline_2026_07_22_8131_windowshide_cloudflared_spawn": "PR #8167 (Dingding-leo, fix/windows-hide-child-process, #8131) own growth: src/lib/cloudflaredTunnel.ts 934->935 (+1, irreducible call-site wiring — the single `windowsHide: true` option added to the existing cloudflared spawn() options object so no transient conhost.exe/cmd console window flashes open on Windows). Covered by the pre-merge-fix regression test tests/unit/windows-hide-child-process-spawns-8131.test.ts (added for the two additional spawn() sites the PR missed: ServiceSupervisor.ts, versionManager/processManager.ts) plus the windowsHide assertion added to tests/unit/services/installers/runNpm-shell-5379.test.ts (installers/utils.ts buildNpmExecOptions).", "_rebaseline_2026_07_22_8006_adobe_firefly_media_provider": "PR #8006 (artickc, feat/adobe-firefly-media) own growth: adds Adobe Firefly as a media-only (image + video) provider — unofficial IMS/cookie-session bridge for firefly.adobe.com covering IMS cookie->access_token exchange, discovery-catalog fallback, credits/balance usage, and submit+poll dispatch for both image (nano-banana/gpt-image families) and video (Sora 2/Veo 3.1/Kling 3.0) generation, with 408-under-load retry handling. New leaf open-sse/services/adobeFireflyClient.ts frozen at 1958 (>>cap 800) — a single self-contained upstream client (mirrors the qoderCli.ts precedent for a new provider client that is legitimately large on day one: IMS auth, cookie/JWT normalization, payload builders for 2 media types x multiple model families, SSE-less submit/poll state machine, error sanitization); not extractable without scattering a single upstream integration across artificial module boundaries mid-PR. open-sse/config/imageRegistry.ts (existing, previously under cap) grows 800->821 (+21, the new adobe-firefly IMAGE_PROVIDERS entry + models list, additive registry data at the existing registry chokepoint). src/lib/usage/providerLimits.ts 1000->1003 (+3, adobe-firefly/firefly added to the existing apikey-usage-fetcher allowlist, irreducible call-site wiring mirroring the sibling #7994 PromptQL/HyperAgent entries in the same PR group). Covered by tests/unit/adobe-firefly.test.ts (35/35). Structural shrink tracked in #3501.", @@ -209,7 +210,8 @@ "_rebaseline_2026_06_24_task_aware_routing": "Task-aware routing strategy (port PR #2045, OmniRoute #4945): combo.ts 3190->3225 (+35) = one new `else if (strategy === \"task-aware\")` dispatch branch delegating 100% to selectTaskAwareTarget + its imports/log lines. All scoring/classification logic lives OUT of the god-file in the new leaf open-sse/services/taskAwareRouting.ts (553 LOC 3604 (+56, entirely this PR's diff — no other commit touched this file between the PR's merge-base and the release tip). Fixes combo cooldown-wait state recording so a bogus 503 is no longer crystallized when the cooldown-wait vars reset every setTry, adds an OpenAI-format SSE error frame path for combo-exhausted rejections (capturing request body + attempted models), and gives an abandoned per-target dispatch its own timeout instead of leaking a permanent 'pending' dashboard entry. Irreducible additions at the existing handleComboChat dispatch/retry chokepoint (mirrors the prior quota-share/headroom/task-aware strategy-branch precedents already frozen in this file). Covered by the PR's own combo-config + Gemini TPM-ceiling benchmark test additions.", "_rebaseline_2026_07_25_8476_combo_input_bound_homogeneous_scope": "PR #8476 (herjarsa, fix/8375-8459-combo-image-fixes, #8375) own growth: open-sse/services/combo.ts 3642->3679 (+37 net: +29 the PR's own isInputBoundFailure short-circuit for deterministic context_length_exceeded/context_window_exceeded failures, +8 a /green-prs pre-merge fix scoping that short-circuit to homogeneous remainders only — the shipped code fired unconditionally on ANY target, regressing the intentional heterogeneous-combo fallback #6637/isContextOverflow400 protects, exactly as flagged by this PR's own review evidence but never actually implemented in the branch). The fix compares orderedTargets[i+1..] modelStr against the failing target's modelStr at the existing executeTarget dispatch chokepoint (mirrors the sameProviderNext precedent a few lines below) — irreducible call-site wiring, not extractable without hiding the dispatch boundary. Covered by tests/unit/combo-input-bound-failure-8375.test.ts (homogeneous pool still short-circuits) and the new tests/unit/combo-input-bound-heterogeneous-8375.test.ts (heterogeneous combo now correctly falls through to the larger-context target).", - "open-sse/services/combo.ts": 3679, + "open-sse/services/combo.ts": 3642, + "open-sse/services/combo/comboStructure.ts": 917, "_rebaseline_2026_06_26_fidelity_gate_extraction": "Milestone-B fidelity-gate wiring residual: bodyToText+gateAdvance extracted to fidelityGateStep.ts (889->854, -35), but the StackOptions.fidelityGate field, the `const fidelityGate` reads at the two stacked-loop dispatch chokepoints, and the import of FidelityGateConfig are irreducible wiring that cannot leave strategySelector without an architectural refactor of the pre-existing stacked pipeline. Net: 889->854 (+6 vs the pre-Milestone-B frozen 848). Covered by tests/unit/compression/*.test.ts (940 pass).", "_rebaseline_2026_06_28_5243_risk_gate_prepass": "PR #5243 (compression risk-gate pre-pass) own growth: open-sse/services/compression/strategySelector.ts 854->899 (+45). The three exported entry points (applyCompression/applyStackedCompression/applyStackedCompressionAsync) become thin wrappers over pure-extracted private bodies (runCompression/runStackedCompression/runStackedCompressionAsync) so the risk-gate mask->run->restore wrapper sits strictly OUTSIDE the per-step loop — a single universal integration point. The wrapper logic itself (resolveRiskGate/withRiskGate) lives in the new riskGate/strategyWrap.ts (960 (+61 = the opt-in result-memoization branches in applyCompression/applyCompressionAsync — principal+determinism gate, makeMemoKey lookup/store with model+supportsVision folded into the key, recompute-with-memo-off). Default off (memoizeCompressionResults), so zero behavior change. The memo helpers live in the leaf resultMemo.ts ( 0) { + const exhaustion = describeCapabilityFilterExhaustion(preCompatTargets, body, combo.name); + if (exhaustion) { + recordComboFailure(effectiveSessionId, combo.name); + return errorResponseWithComboDiagnostics( + 400, + exhaustion.message, + { + poolSize: preCompatTargets.length, + attempted: 0, + excluded: exhaustion.excluded, + attemptOrder: [], + terminalReason: exhaustion.terminalReason, + recovery: buildRecoveryHint("no_executable_targets"), + }, + { code: "capability_mismatch", type: "invalid_request_error" } + ); + } + } orderedTargets = applyContextRequirements(orderedTargets, config.contextRequirements, log); // Task-aware reordering: only active for strategies ["smart","task","task-aware","task_aware","auto"]. @@ -2859,11 +2886,17 @@ async function handleRoundRobinCombo({ { code: "context_length_exceeded", type: "invalid_request_error" } ); } + // Align with the main/auto paths: combo config OR top-level settings. + const rrCompatFailOpen = + (config as { compatFilterFailOpen?: unknown }).compatFilterFailOpen === true || + (settings as { compatFilterFailOpen?: unknown } | null | undefined)?.compatFilterFailOpen === + true; let filteredTargets = filterTargetsByRequestCompatibility( evalRankedTargets, body, log, - "Context-aware round-robin fallback" + "Context-aware round-robin fallback", + { failOpen: rrCompatFailOpen } ); // #6238: keep the targets the compat pre-filter rejected so they can serve as a // last-resort fallback tier. The pre-filter drops request-incompatible targets @@ -2877,6 +2910,25 @@ async function handleRoundRobinCombo({ ); let modelCount = filteredTargets.length; if (modelCount === 0) { + const exhaustion = describeCapabilityFilterExhaustion( + evalRankedTargets, + body, + rrExpandedCombo?.name || combo?.name + ); + if (exhaustion) { + return errorResponseWithComboDiagnostics( + 400, + exhaustion.message, + { + poolSize: evalRankedTargets.length, + attempted: 0, + excluded: exhaustion.excluded, + attemptOrder: [], + terminalReason: exhaustion.terminalReason, + }, + { code: "capability_mismatch", type: "invalid_request_error" } + ); + } return comboModelNotFoundResponse("Round-robin combo has no executable targets"); } diff --git a/open-sse/services/combo/comboStructure.ts b/open-sse/services/combo/comboStructure.ts index a53550f2de..4bb3aabc33 100644 --- a/open-sse/services/combo/comboStructure.ts +++ b/open-sse/services/combo/comboStructure.ts @@ -13,6 +13,10 @@ import { getModelContextLimit } from "../../../src/lib/modelCapabilities"; import { getComboModelString, normalizeComboStep } from "../../../src/lib/combos/steps.ts"; +import { + getProviderByAlias, + getProviderById, +} from "../../../src/shared/constants/providers.ts"; import { estimateTokens } from "../contextManager.ts"; import { getResolvedModelCapabilities } from "../modelCapabilities.ts"; import { parseModel } from "../model.ts"; @@ -36,6 +40,22 @@ import type { ResolvedComboUnit, } from "./types.ts"; +/** + * #8488 / #5240: web-cookie (and similar) providers honestly advertise + * registry toolCalling:false but still run the prompt-emulated tool shim. + * Combo tools filters must keep those targets eligible so fail-closed does + * not regress emulation-only combos (e.g. all chatgpt-web). + */ +export function providerSupportsEmulatedToolCalling( + providerIdOrAlias: string | null | undefined +): boolean { + if (!providerIdOrAlias) return false; + const provider = + getProviderById(providerIdOrAlias) || getProviderByAlias(providerIdOrAlias) || null; + if (!provider || typeof provider !== "object") return false; + return (provider as { toolCalling?: unknown }).toolCalling === "emulated"; +} + function toTrimmedString(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } @@ -542,7 +562,9 @@ function getTargetCompatibilityFailures( if ( requirements.requiresTools && - (capabilities.supportsTools === false || !capabilities.toolCalling) + (capabilities.supportsTools === false || !capabilities.toolCalling) && + // #5240: prompt-emulated tool providers remain eligible under fail-closed. + !providerSupportsEmulatedToolCalling(target.provider) ) { failures.push("tools"); } @@ -551,8 +573,8 @@ function getTargetCompatibilityFailures( // support is *confirmed* (`=== true`). Treat `false` AND `null` (unknown) as // incompatible: an unknown-capability model receiving the image is exactly how // a text-only model (e.g. ministral) ended up answering "image not provided". - // The caller keeps all targets when none qualify, so combos with no - // confirmed-vision member still behave as before. + // #8488: when none qualify the filter fails closed (empty list) unless the + // operator opts into compatFilterFailOpen. if (requirements.requiresVision && capabilities.supportsVision !== true) { failures.push("vision"); } @@ -573,11 +595,80 @@ function getTargetCompatibilityFailures( return failures; } +export type CompatFilterOptions = { + /** + * Opt-in legacy behavior (#8488): when every target fails a hard capability + * requirement (tools / vision / structured_output), restore the full pool + * instead of returning an empty list. Default is fail-closed. + */ + failOpen?: boolean; +}; + +const HARD_COMPAT_REASONS = new Set(["tools", "vision", "structured_output"]); + +function hasHardCapabilityFailure(reasons: string[]): boolean { + return reasons.some((reason) => HARD_COMPAT_REASONS.has(reason)); +} + +/** + * Summarize a capability-filter exhaustion for a 400-class combo error (#8488). + * Returns null when the empty pool is not attributable to hard requirements. + */ +export function describeCapabilityFilterExhaustion( + targets: ResolvedComboTarget[], + body: Record, + comboName?: string +): { + unmet: string[]; + excluded: Array<{ provider: string; model: string; reason: string }>; + message: string; + terminalReason: string; +} | null { + if (targets.length === 0) return null; + const requirements = deriveRequestCompatibilityRequirements(body); + const rejected = targets.map((target) => ({ + target, + reasons: getTargetCompatibilityFailures(target, requirements), + })); + const unmet = Array.from( + new Set(rejected.flatMap((entry) => entry.reasons.filter((r) => HARD_COMPAT_REASONS.has(r)))) + ); + if (unmet.length === 0) return null; + + const primary = unmet.includes("tools") + ? "tools" + : unmet.includes("vision") + ? "vision" + : unmet[0]; + const toolCount = Array.isArray(body.tools) ? body.tools.length : 0; + const name = comboName && comboName.trim().length > 0 ? comboName : "this combo"; + let message: string; + if (primary === "tools") { + message = `No target in combo ${name} supports tool calling; request carried ${toolCount} tools`; + } else if (primary === "vision") { + message = `No target in combo ${name} has confirmed vision support for this image request`; + } else { + message = `No target in combo ${name} supports structured output for this request`; + } + + return { + unmet, + excluded: rejected.map((entry) => ({ + provider: entry.target.provider, + model: entry.target.modelStr, + reason: entry.reasons.join("+") || primary, + })), + message, + terminalReason: "capability_mismatch", + }; +} + export function filterTargetsByRequestCompatibility( targets: ResolvedComboTarget[], body: Record, log: ComboLogger, - label = "Context-aware fallback" + label = "Context-aware fallback", + options?: CompatFilterOptions ): ResolvedComboTarget[] { if (targets.length === 0) return targets; const requirements = deriveRequestCompatibilityRequirements(body); @@ -655,22 +746,55 @@ export function filterTargetsByRequestCompatibility( if (compatible.length === targets.length) return targets; if (compatible.length === 0) { - log.warn( - "COMBO", - `${label}: all ${targets.length} targets were filtered by request requirements; preserving strategy order` - ); + const hardRejected = rejected.some((entry) => hasHardCapabilityFailure(entry.reasons)); + const failOpen = options?.failOpen === true; + log.debug?.( "COMBO", `${label}: rejected targets ${rejected .map((entry) => `${entry.target.modelStr}(${entry.reasons.join("+")})`) .join(", ")}` ); - // #8332: vision is never safe to guess — this safety net must not resurrect a - // vision-rejected target, even if nothing else remains. + + // #8332: vision is never safe to guess — never resurrect vision-rejected targets. if (requirements.requiresVision) { - return targets.filter((target) => !isVisionIncompatibleTarget(target, requirements)); + const visionSafe = targets.filter( + (target) => !isVisionIncompatibleTarget(target, requirements) + ); + if (visionSafe.length === 0) { + log.warn( + "COMBO", + `${label}: all ${targets.length} targets lack confirmed vision; failing closed (#8488)` + ); + return []; + } + if (failOpen) { + log.warn( + "COMBO", + `${label}: all targets filtered; compatFilterFailOpen restoring non-vision-rejected pool` + ); + return visionSafe; + } + return visionSafe; } - return targets; + + // Context/output-only exhaustion keeps the legacy fail-open path — the honest + // answer is the existing context_length_exceeded gate, not a capability error. + if (!hardRejected || failOpen) { + log.warn( + "COMBO", + `${label}: all ${targets.length} targets were filtered by request requirements; preserving strategy order` + ); + return targets; + } + + // #8488: tools / structured_output — fail closed instead of dispatching to a + // target already known to be incompatible. + log.warn( + "COMBO", + `${label}: all ${targets.length} targets were filtered by hard request requirements; failing closed` + ); + return []; } log.info( diff --git a/open-sse/services/combo/resolveAutoStrategy.ts b/open-sse/services/combo/resolveAutoStrategy.ts index c0d05cf650..6e0ecad032 100644 --- a/open-sse/services/combo/resolveAutoStrategy.ts +++ b/open-sse/services/combo/resolveAutoStrategy.ts @@ -1,4 +1,4 @@ -import { errorResponse, unavailableResponse } from "../../utils/error.ts"; +import { errorResponse, unavailableResponse, errorResponseWithComboDiagnostics } from "../../utils/error.ts"; import { BudgetExceededError, selectProvider as selectAutoProvider } from "../autoCombo/engine.ts"; import { resolveRequestModePack, @@ -17,7 +17,10 @@ import { supportsToolCalling } from "../modelCapabilities.ts"; import type { ResilienceSettings } from "../../../src/lib/resilience/settings"; import { parseAutoConfig } from "./autoConfig.ts"; import { dedupeTargetsByExecutionKey } from "./comboData.ts"; -import { getModelContextLimitForModelString } from "./comboStructure.ts"; +import { + getModelContextLimitForModelString, + providerSupportsEmulatedToolCalling, +} from "./comboStructure.ts"; import { calculatePromptCacheAffinityScores, promptCacheTargetIdentity, @@ -56,7 +59,7 @@ export interface ResolveAutoStrategyDeps { body: Record; combo: ComboLike; settings: Record | null | undefined; - config: { complexityAwareRouting?: boolean }; + config: { complexityAwareRouting?: boolean; compatFilterFailOpen?: boolean }; relayOptions?: { bypassProviderQuotaPolicy?: boolean; sessionId?: string | null; @@ -105,16 +108,48 @@ export async function resolveAutoStrategyOrder( const requestHasTools = Array.isArray(body?.tools) && body.tools.length > 0; let eligibleTargets = [...orderedTargets]; + const compatFilterFailOpen = + config?.compatFilterFailOpen === true || + (settings as { compatFilterFailOpen?: unknown } | null | undefined)?.compatFilterFailOpen === + true; if (requestHasTools) { - const filtered = eligibleTargets.filter((target) => supportsToolCalling(target.modelStr)); + // Keep #5240 prompt-emulation providers (toolCalling:"emulated") even when + // registry/capability rows honestly report toolCalling:false. + const filtered = eligibleTargets.filter( + (target) => + supportsToolCalling(target.modelStr) || + providerSupportsEmulatedToolCalling(target.provider) + ); if (filtered.length > 0) { eligibleTargets = filtered; - } else { + } else if (compatFilterFailOpen) { log.warn( "COMBO", - "Auto strategy: all candidates filtered by tool-calling policy, falling back to full pool" + "Auto strategy: all candidates filtered by tool-calling policy, falling back to full pool (compatFilterFailOpen)" ); + } else { + // #8488: fail closed with an explicit compatibility error instead of + // re-admitting tool-incapable targets. + const toolCount = Array.isArray(body.tools) ? body.tools.length : 0; + return { + earlyResponse: errorResponseWithComboDiagnostics( + 400, + `No target in combo ${combo.name} supports tool calling; request carried ${toolCount} tools`, + { + poolSize: eligibleTargets.length, + attempted: 0, + excluded: eligibleTargets.map((target) => ({ + provider: target.provider, + model: target.modelStr, + reason: "tools", + })), + attemptOrder: [], + terminalReason: "capability_mismatch", + }, + { code: "capability_mismatch", type: "invalid_request_error" } + ), + }; } } @@ -141,12 +176,32 @@ export async function resolveAutoStrategyOrder( `Auto strategy: context-window filter kept ${filteredByContext.length}/${eligibleTargets.length} candidates (est. ${estimatedInputTokens} tokens)` ); eligibleTargets = filteredByContext; - } else { + } else if (compatFilterFailOpen) { log.warn( "COMBO", - `Auto strategy: all candidates filtered by context-window policy (est. ${estimatedInputTokens} tokens), falling back to full pool` + `Auto strategy: all candidates filtered by context-window policy (est. ${estimatedInputTokens} tokens), falling back to full pool (compatFilterFailOpen)` ); - // eligibleTargets intentionally unchanged — same fallback contract as tool-calling filter + } else { + // #8488: every candidate has a known limit below the estimate — surface + // context_length_exceeded rather than dispatching oversized targets. + return { + earlyResponse: errorResponseWithComboDiagnostics( + 400, + `Request requires approximately ${estimatedInputTokens} tokens, but every auto-strategy candidate in combo ${combo.name} has a smaller known context limit`, + { + poolSize: eligibleTargets.length, + attempted: 0, + excluded: eligibleTargets.map((target) => ({ + provider: target.provider, + model: target.modelStr, + reason: "context_window", + })), + attemptOrder: [], + terminalReason: "context_length_exceeded", + }, + { code: "context_length_exceeded", type: "invalid_request_error" } + ), + }; } eligibleTargets = await expandAutoComboCandidatePool(eligibleTargets, combo); diff --git a/tests/unit/8488-capability-filter-fail-closed.test.ts b/tests/unit/8488-capability-filter-fail-closed.test.ts new file mode 100644 index 0000000000..6e3807ce2b --- /dev/null +++ b/tests/unit/8488-capability-filter-fail-closed.test.ts @@ -0,0 +1,316 @@ +/** + * #8488 — capability filters fail closed when every candidate is incompatible. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-8488-compat-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const { saveModelsDevCapabilities } = await import("../../src/lib/modelsDevSync.ts"); +const { + filterTargetsByRequestCompatibility, + describeCapabilityFilterExhaustion, + providerSupportsEmulatedToolCalling, +} = await import("../../open-sse/services/combo/comboStructure.ts"); +const { resolveAutoStrategyOrder } = + await import("../../open-sse/services/combo/resolveAutoStrategy.ts"); + +function capabilityEntry(limit_context: number, overrides: Record = {}) { + return { + tool_call: null, + reasoning: null, + attachment: null, + structured_output: null, + temperature: null, + modalities_input: "[]", + modalities_output: "[]", + knowledge_cutoff: null, + release_date: null, + last_updated: null, + status: null, + family: null, + open_weights: null, + limit_context, + limit_input: null, + limit_output: null, + interleaved_field: null, + ...overrides, + }; +} + +function target(provider: string, modelStr: string) { + return { + kind: "model" as const, + stepId: "s1", + executionKey: `${provider}>${modelStr}`, + modelStr, + provider, + providerId: null, + connectionId: null, + weight: 1, + label: null, + }; +} + +const log = { + info() {}, + warn() {}, + error() {}, + debug() {}, +}; + +test.beforeEach(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("#8488 filter: some tool-capable targets kept (unchanged)", () => { + saveModelsDevCapabilities({ + openai: { + "with-tools": capabilityEntry(128000, { tool_call: true }), + "no-tools": capabilityEntry(128000, { tool_call: false }), + }, + }); + + const kept = filterTargetsByRequestCompatibility( + [target("openai", "openai/with-tools"), target("openai", "openai/no-tools")], + { + messages: [{ role: "user", content: "hi" }], + tools: [{ type: "function", function: { name: "lookup", parameters: {} } }], + }, + log + ); + assert.deepEqual( + kept.map((t) => t.modelStr), + ["openai/with-tools"] + ); +}); + +test("#8488 filter: zero tool-capable targets → empty (fail closed)", () => { + saveModelsDevCapabilities({ + openai: { + "no-tools-a": capabilityEntry(128000, { tool_call: false }), + "no-tools-b": capabilityEntry(128000, { tool_call: false }), + }, + }); + + const targets = [target("openai", "openai/no-tools-a"), target("openai", "openai/no-tools-b")]; + const body = { + messages: [{ role: "user", content: "hi" }], + tools: [{ type: "function", function: { name: "lookup", parameters: {} } }], + }; + const kept = filterTargetsByRequestCompatibility(targets, body, log); + assert.equal(kept.length, 0); + + const exhaustion = describeCapabilityFilterExhaustion(targets, body, "tools-combo"); + assert.ok(exhaustion); + assert.match(exhaustion!.message, /supports tool calling/i); + assert.equal(exhaustion!.terminalReason, "capability_mismatch"); + assert.ok(exhaustion!.excluded.some((e) => e.reason.includes("tools"))); +}); + +test("#8488 filter: chatgpt-web emulation providers stay eligible for tools (#5240)", () => { + // Registry honestly tags chatgpt-web models toolCalling:false; the prompt + // shim is what makes tools work. Fail-closed must not hard-reject them. + assert.equal(providerSupportsEmulatedToolCalling("chatgpt-web"), true); + assert.equal(providerSupportsEmulatedToolCalling("cgpt-web"), true); + assert.equal(providerSupportsEmulatedToolCalling("claude-web"), false); // toolCalling:"none" + assert.equal(providerSupportsEmulatedToolCalling("openai"), false); + + const kept = filterTargetsByRequestCompatibility( + [ + target("chatgpt-web", "chatgpt-web/gpt-5.5"), + target("chatgpt-web", "chatgpt-web/o3"), + ], + { + messages: [{ role: "user", content: "Use a tool." }], + tools: [{ type: "function", function: { name: "lookup", parameters: {} } }], + }, + log + ); + assert.equal(kept.length, 2); + assert.deepEqual( + kept.map((t) => t.modelStr), + ["chatgpt-web/gpt-5.5", "chatgpt-web/o3"] + ); + + const exhaustion = describeCapabilityFilterExhaustion( + [ + target("chatgpt-web", "chatgpt-web/gpt-5.5"), + target("chatgpt-web", "chatgpt-web/o3"), + ], + { + messages: [{ role: "user", content: "Use a tool." }], + tools: [{ type: "function", function: { name: "lookup", parameters: {} } }], + }, + "web-cookie-tools" + ); + assert.equal(exhaustion, null, "emulation-capable pool must not report capability_mismatch"); +}); + +test("#8488 auto: chatgpt-web emulation survives tool pre-filter (#5240)", async () => { + const result = await resolveAutoStrategyOrder({ + orderedTargets: [target("chatgpt-web", "chatgpt-web/gpt-5.5")] as never, + body: { + messages: [{ role: "user", content: "hi" }], + tools: [{ type: "function", function: { name: "lookup", parameters: {} } }], + }, + combo: { id: "c1", name: "auto-web-cookie", config: {} } as never, + settings: null, + config: {}, + relayOptions: null, + resilienceSettings: { quotaPreflight: { enabled: false } } as never, + log: log as never, + buildAutoCandidates: (async () => []) as never, + }); + + assert.ok(!("earlyResponse" in result), "must not 400 capability_mismatch for emulation providers"); + if ("orderedTargets" in result) { + assert.equal(result.orderedTargets.length, 1); + assert.equal(result.orderedTargets[0].modelStr, "chatgpt-web/gpt-5.5"); + } +}); + +test("#8488 filter: opt-in compatFilterFailOpen restores full pool", () => { + saveModelsDevCapabilities({ + openai: { + "no-tools-a": capabilityEntry(128000, { tool_call: false }), + "no-tools-b": capabilityEntry(128000, { tool_call: false }), + }, + }); + + const kept = filterTargetsByRequestCompatibility( + [target("openai", "openai/no-tools-a"), target("openai", "openai/no-tools-b")], + { + messages: [{ role: "user", content: "hi" }], + tools: [{ type: "function", function: { name: "lookup", parameters: {} } }], + }, + log, + "Context-aware fallback", + { failOpen: true } + ); + assert.equal(kept.length, 2); +}); + +test("#8488 filter: vision with no confirmed target → empty (fail closed)", () => { + saveModelsDevCapabilities({ + openai: { + "text-only": capabilityEntry(128000, { attachment: false, tool_call: true }), + }, + }); + + const kept = filterTargetsByRequestCompatibility( + [target("openai", "openai/text-only")], + { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "see?" }, + { type: "image_url", image_url: { url: "https://example.com/a.png" } }, + ], + }, + ], + }, + log + ); + assert.equal(kept.length, 0); +}); + +test("#8488 auto: tool pre-filter fail closed returns early 400", async () => { + saveModelsDevCapabilities({ + openai: { + "no-tools": capabilityEntry(128000, { tool_call: false }), + }, + }); + + const result = await resolveAutoStrategyOrder({ + orderedTargets: [target("openai", "openai/no-tools")] as never, + body: { + messages: [{ role: "user", content: "hi" }], + tools: [{ type: "function", function: { name: "lookup", parameters: {} } }], + }, + combo: { id: "c1", name: "auto-tools", config: {} } as never, + settings: null, + config: {}, + relayOptions: null, + resilienceSettings: { quotaPreflight: { enabled: false } } as never, + log: log as never, + buildAutoCandidates: (async () => []) as never, + }); + + assert.ok("earlyResponse" in result); + if ("earlyResponse" in result) { + assert.equal(result.earlyResponse.status, 400); + const body = await result.earlyResponse.json(); + assert.equal(body?.error?.code, "capability_mismatch"); + assert.match(String(body?.error?.message || ""), /supports tool calling/i); + } +}); + +test("#8488 auto: tool pre-filter fail-open opt-in keeps full pool", async () => { + saveModelsDevCapabilities({ + openai: { + "no-tools": capabilityEntry(128000, { tool_call: false }), + }, + }); + + const result = await resolveAutoStrategyOrder({ + orderedTargets: [target("openai", "openai/no-tools")] as never, + body: { + messages: [{ role: "user", content: "hi" }], + tools: [{ type: "function", function: { name: "lookup", parameters: {} } }], + }, + combo: { id: "c1", name: "auto-tools-open", config: { compatFilterFailOpen: true } } as never, + settings: null, + config: { compatFilterFailOpen: true }, + relayOptions: null, + resilienceSettings: { quotaPreflight: { enabled: false } } as never, + log: log as never, + buildAutoCandidates: (async () => []) as never, + }); + + assert.ok(!("earlyResponse" in result)); + if ("orderedTargets" in result) { + assert.equal(result.orderedTargets.length, 1); + } +}); + +test("#8488 auto: context pre-filter fail closed when all known limits too small", async () => { + saveModelsDevCapabilities({ + openai: { + tiny: capabilityEntry(100, { tool_call: true }), + }, + }); + + const hugePrompt = "x".repeat(4000); // ~1000 tokens at 4 chars/token + const result = await resolveAutoStrategyOrder({ + orderedTargets: [target("openai", "openai/tiny")] as never, + body: { messages: [{ role: "user", content: hugePrompt }] }, + combo: { id: "c1", name: "auto-ctx", config: {} } as never, + settings: null, + config: {}, + relayOptions: null, + resilienceSettings: { quotaPreflight: { enabled: false } } as never, + log: log as never, + buildAutoCandidates: (async () => []) as never, + }); + + assert.ok("earlyResponse" in result); + if ("earlyResponse" in result) { + assert.equal(result.earlyResponse.status, 400); + const body = await result.earlyResponse.json(); + assert.equal(body?.error?.code, "context_length_exceeded"); + } +}); diff --git a/tests/unit/combo-routing-engine.test.ts b/tests/unit/combo-routing-engine.test.ts index 9e2893d28b..55899ff073 100644 --- a/tests/unit/combo-routing-engine.test.ts +++ b/tests/unit/combo-routing-engine.test.ts @@ -1973,7 +1973,7 @@ test("handleComboChat skips tool, vision, and structured-output incompatible fal assert.deepEqual(calls, ["openai/compatible"]); }); -test("handleComboChat preserves strategy order when context-aware filtering rejects all targets", async () => { +test("handleComboChat fails closed when context-aware filtering rejects all targets (#8488)", async () => { saveModelsDevCapabilities({ openai: { "no-tools-a": capabilityEntry(128000, { tool_call: false }), @@ -1981,7 +1981,7 @@ test("handleComboChat preserves strategy order when context-aware filtering reje }, }); - const calls: any[] = []; + const calls: string[] = []; const result = await handleComboChat({ body: { messages: [{ role: "user", content: "Use a tool." }], @@ -1992,14 +1992,54 @@ test("handleComboChat preserves strategy order when context-aware filtering reje strategy: "priority", models: ["openai/no-tools-a", "openai/no-tools-b"], }, - handleSingleModel: async (_body: any, modelStr: any) => { + handleSingleModel: async (_body: Record, modelStr: string) => { calls.push(modelStr); return okResponse(); }, isModelAvailable: async () => true, log: createLog(), settings: null, - relayOptions: null as any, + relayOptions: null, + allCombos: null, + }); + + assert.equal(result.ok, false); + assert.equal(result.status, 400); + assert.deepEqual(calls, []); + const body = await result.json(); + assert.match(String(body?.error?.message || ""), /supports tool calling/i); + assert.equal(body?.error?.code, "capability_mismatch"); + assert.equal(body?.diagnostics?.terminalReason, "capability_mismatch"); +}); + +test("handleComboChat compatFilterFailOpen restores dispatch when all targets fail tools (#8488)", async () => { + saveModelsDevCapabilities({ + openai: { + "no-tools-a": capabilityEntry(128000, { tool_call: false }), + "no-tools-b": capabilityEntry(128000, { tool_call: false }), + }, + }); + + const calls: string[] = []; + const result = await handleComboChat({ + body: { + messages: [{ role: "user", content: "Use a tool." }], + tools: [{ type: "function", function: { name: "lookup", parameters: {} } }], + }, + combo: { + name: "context-aware-fail-open", + strategy: "priority", + models: ["openai/no-tools-a", "openai/no-tools-b"], + config: { compatFilterFailOpen: true }, + }, + handleSingleModel: async (_body: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + relayOptions: null, allCombos: null, });