diff --git a/changelog.d/features/5696-layer-a-capability-filter.md b/changelog.d/features/5696-layer-a-capability-filter.md new file mode 100644 index 0000000000..37d04132e3 --- /dev/null +++ b/changelog.d/features/5696-layer-a-capability-filter.md @@ -0,0 +1 @@ +- **feat(core):** add Layer A capability filter at router (#5696) diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 3e7f6cab2f..01c1cd027c 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -142,6 +142,8 @@ import { getExplicitModelOutputCap, resolveInputTokenCapForGate, } from "@/lib/modelCapabilities.ts"; +import { checkRequestCapabilityFit, deriveRequestCapabilityRequirements, buildCapabilityMismatchMessage } from "@/shared/constants/capabilities/capabilityFilter.ts"; +import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags.ts"; import { toPositiveInteger } from "../services/reasoningTokenBuffer.ts"; import { normalizeThinkingForModel } from "@/shared/constants/modelSpecs.ts"; import { @@ -2638,7 +2640,16 @@ export async function handleChatCore({ } } // === /Quota Share enforcement PRE-hook === - + if (isFeatureFlagEnabled("CAPABILITY_FILTER_ENABLED")) { + const fit = checkRequestCapabilityFit(getResolvedModelCapabilities({ provider, model: effectiveModel }), + deriveRequestCapabilityRequirements(body as Record), provider); + if (!fit.compatible) { + const msg = buildCapabilityMismatchMessage(fit.terminalReason!, provider, effectiveModel); + log?.warn?.("CAPABILITY", msg); + trackPendingRequest(model, provider, connectionId, false); + return createErrorResult(400, msg, null, fit.terminalReason, "invalid_request_error"); + } + } // Get executor for this provider (with optional upstream proxy routing) const executor = await resolveExecutorWithProxy(provider); const getExecutionCredentials = () => diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index f55671d00c..3fafbd560a 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -12426,5 +12426,10 @@ "degraded.source.circuitBreaker": "Circuit Breaker", "degraded.source.modelLockouts": "Model Lockouts", "degraded.source.count": "Connection Count" - } + }, + "featureFlagCapabilityFilterEnabledDescription": "Reject requests before dispatch when the target model lacks required capabilities (vision, tools, structured output, context window). Protects direct single-provider requests that bypass the combo-layer compatibility filter.", + "capabilityFilter.visionMismatch": "Provider does not support vision for this image request", + "capabilityFilter.toolsMismatch": "Provider does not support tool calling", + "capabilityFilter.structuredOutputMismatch": "Provider does not support structured output", + "capabilityFilter.contextWindowMismatch": "Request exceeds provider context window" } diff --git a/src/i18n/messages/pt-BR.json b/src/i18n/messages/pt-BR.json index 7ba3d31af2..a415b0d2a6 100644 --- a/src/i18n/messages/pt-BR.json +++ b/src/i18n/messages/pt-BR.json @@ -12426,5 +12426,10 @@ "degraded.source.circuitBreaker": "Disjuntor", "degraded.source.modelLockouts": "Bloqueios de Modelo", "degraded.source.count": "Contagem de Conexões" - } + }, + "featureFlagCapabilityFilterEnabledDescription": "Rejeitar requisições antes do despacho quando o modelo alvo nao possui as capacidades necessarias (visao, ferramentas, saída estruturada, janela de contexto). Protege requisições diretas que ignoram o filtro de compatibilidade do combo.", + "capabilityFilter.visionMismatch": "O provedor nao suporta visao para esta requisicao de imagem", + "capabilityFilter.toolsMismatch": "O provedor nao suporta chamada de ferramentas", + "capabilityFilter.structuredOutputMismatch": "O provedor nao suporta saida estruturada", + "capabilityFilter.contextWindowMismatch": "A requisicao excede a janela de contexto do provedor" } diff --git a/src/shared/constants/capabilities/capabilityFilter.ts b/src/shared/constants/capabilities/capabilityFilter.ts new file mode 100644 index 0000000000..b0c0de8091 --- /dev/null +++ b/src/shared/constants/capabilities/capabilityFilter.ts @@ -0,0 +1,211 @@ +/** + * Layer A capability filter — shared, provider-agnostic module. + * + * Validates that a provider+model can satisfy the request's capability + * requirements (tools, vision, structured output, context window) BEFORE + * dispatch to the executor. Returns an early 400 if not, rather than + * letting the request fail downstream or produce garbage (e.g. a text-only + * model receiving image_url content and answering "image not provided"). + * + * The logic mirrors what `filterTargetsByRequestCompatibility` in + * comboStructure.ts already does for combo-routed requests, but this + * module lives at the router layer (Layer A) so it also protects direct + * single-provider requests (`model:"openai/gpt-4o-mini"` without a combo). + * + * #5696 + */ + +import { getResolvedModelCapabilities } from "@/lib/modelCapabilities"; +import { evaluateContextLimit } from "@omniroute/open-sse/services/combo/contextOverrideGate"; +import { hasEstimableContent } from "@omniroute/open-sse/services/combo/knownContextOverflow"; +import { isRecord } from "@omniroute/open-sse/services/combo/comboData"; +import { providerSupportsEmulatedToolCalling } from "@omniroute/open-sse/services/combo/comboStructure"; +import { estimateTokens } from "@omniroute/open-sse/services/contextManager"; + +// ── Types ───────────────────────────────────────────────────────────────── + +export type CapabilityFailure = "tools" | "vision" | "structured_output" | "context_window"; + +export interface RequestCapabilityRequirements { + requiresTools: boolean; + requiresVision: boolean; + requiresStructuredOutput: boolean; + requiredContextTokens: number; + toolCount: number; +} + +export interface CapabilityFilterResult { + compatible: boolean; + failures: CapabilityFailure[]; + terminalReason?: string; +} + +// ── Pure helpers (mirror the unexported helpers in comboStructure.ts) ────── + +function requestRequiresTools(body: Record): boolean { + if (Array.isArray(body.tools) && body.tools.length > 0) return true; + if (Array.isArray(body.functions) && body.functions.length > 0) return true; + return false; +} + +function requestRequiresStructuredOutput(body: Record): boolean { + const responseFormat = isRecord(body.response_format) ? body.response_format : null; + const type = typeof responseFormat?.type === "string" ? responseFormat.type : null; + return type === "json_object" || type === "json_schema"; +} + +function estimateRequestInputTokens(body: Record): number { + const estimatePayload: Record = {}; + for (const key of ["messages", "input", "tools", "functions", "response_format"]) { + if (hasEstimableContent(body[key])) estimatePayload[key] = body[key]; + } + return Object.keys(estimatePayload).length > 0 ? estimateTokens(estimatePayload) : 0; +} + +function getPositiveTokenCount(value: unknown): number { + const count = Number(value); + return Number.isFinite(count) && count > 0 ? Math.ceil(count) : 0; +} + +function isMediaTypeImage(value: Record): boolean { + const source = isRecord(value.source) ? value.source : null; + const mediaType = typeof source?.media_type === "string" ? source.media_type.toLowerCase() : ""; + return mediaType.startsWith("image/"); +} + +function valueContainsImagePart(value: unknown, depth = 0): boolean { + if (depth > 8 || value === null || value === undefined) return false; + if (typeof value === "string") return value.startsWith("data:image/"); + if (Array.isArray(value)) return value.some((entry) => valueContainsImagePart(entry, depth + 1)); + if (!isRecord(value)) return false; + + if (valueContainsImageType(value)) return true; + if (isMediaTypeImage(value)) return true; + + return Object.values(value).some((entry) => valueContainsImagePart(entry, depth + 1)); +} + +function isContextOverflow( + capabilities: { maxInputTokens: number | null; contextWindow: number | null }, + requirements: { requiredContextTokens: number } +): boolean { + return evaluateContextLimit( + { maxInputTokens: capabilities.maxInputTokens, contextWindow: capabilities.contextWindow }, + { estimatedInputTokens: requirements.requiredContextTokens, requiredContextTokens: requirements.requiredContextTokens } + ) === false; +} + +function valueContainsImageType(value: Record): boolean { + const type = typeof value.type === "string" ? value.type.toLowerCase() : null; + if (type === "image" || type === "image_url" || type === "input_image") return true; + if ("image_url" in value || "input_image" in value) return true; + return false; +} + +// ── Public API ───────────────────────────────────────────────────────────── + +/** + * Derive capability requirements from a request body. + * Mirrors `deriveRequestCompatibilityRequirements` in comboStructure.ts. + */ +export function deriveRequestCapabilityRequirements( + body: Record +): RequestCapabilityRequirements { + const estimatedInputTokens = estimateRequestInputTokens(body); + const requestedOutputTokens = Math.max( + getPositiveTokenCount(body.max_tokens), + getPositiveTokenCount(body.max_completion_tokens) + ); + return { + requiresTools: requestRequiresTools(body), + requiresVision: valueContainsImagePart(body.messages) || valueContainsImagePart(body.input), + requiresStructuredOutput: requestRequiresStructuredOutput(body), + requiredContextTokens: estimatedInputTokens + requestedOutputTokens, + toolCount: Array.isArray(body.tools) ? body.tools.length : 0, + }; +} + +/** + * Build a human-readable error message for a capability mismatch. + * Mirrors the i18n keys: capabilityFilter.visionMismatch / toolsMismatch / etc. + */ +export function buildCapabilityMismatchMessage( + terminalReason: string, + provider: string | null, + model: string | null +): string { + const msgs: Record = { + vision: `Provider '${provider}' does not support vision for this image request`, + tools: `Provider '${provider}' does not support tool calling`, + structured_output: `Provider '${provider}' does not support structured output`, + context_window: `Request exceeds the context window for ${provider}/${model}`, + }; + return msgs[terminalReason] || `Provider '${provider}' does not support the required capabilities`; +} + +/** + * Check whether a model's capabilities satisfy the request requirements. + * + * @param capabilities - Resolved model capabilities (from getResolvedModelCapabilities) + * @param requirements - Request capability requirements + * @param provider - Provider id or alias (needed for emulated-tool-calling bypass) + * @returns CapabilityFilterResult with compatibility verdict and failure details + */ +function collectCapabilityFailures( + capabilities: Record, + requirements: RequestCapabilityRequirements, + provider?: string | null +): CapabilityFailure[] { + const failures: CapabilityFailure[] = []; + const caps = capabilities as { + supportsTools: boolean | null; + toolCalling: boolean; + supportsVision: boolean | null; + structuredOutput: boolean | null; + contextWindow: number | null; + maxInputTokens: number | null; + maxOutputTokens: number | null; + }; + + if (requirements.requiresTools && (caps.supportsTools === false || !caps.toolCalling) + && !providerSupportsEmulatedToolCalling(provider)) { + failures.push("tools"); + } + if (requirements.requiresVision && caps.supportsVision !== true) { + failures.push("vision"); + } + if (requirements.requiresStructuredOutput && caps.structuredOutput === false) { + failures.push("structured_output"); + } + if (requirements.requiredContextTokens > 0 && isContextOverflow(caps, requirements)) { + failures.push("context_window"); + } + return failures; +} + +function primaryFailure(failures: CapabilityFailure[]): CapabilityFailure { + if (failures.includes("vision")) return "vision"; + if (failures.includes("tools")) return "tools"; + if (failures.includes("structured_output")) return "structured_output"; + return "context_window"; +} + +export function checkRequestCapabilityFit( + capabilities: { + supportsTools: boolean | null; + toolCalling: boolean; + supportsVision: boolean | null; + structuredOutput: boolean | null; + contextWindow: number | null; + maxInputTokens: number | null; + maxOutputTokens: number | null; + }, + requirements: RequestCapabilityRequirements, + provider?: string | null +): CapabilityFilterResult { + const failures = collectCapabilityFailures(capabilities as Record, requirements, provider); + if (failures.length === 0) { + return { compatible: true, failures: [] }; + } + return { compatible: false, failures, terminalReason: primaryFailure(failures) }; +} \ No newline at end of file diff --git a/src/shared/constants/featureFlagDefinitions.ts b/src/shared/constants/featureFlagDefinitions.ts index e5cf433a61..4e0a8a4477 100644 --- a/src/shared/constants/featureFlagDefinitions.ts +++ b/src/shared/constants/featureFlagDefinitions.ts @@ -245,6 +245,18 @@ export const FEATURE_FLAG_DEFINITIONS: FeatureFlagDefinition[] = [ requiresRestart: true, warningLevel: "info", }, + { + key: "CAPABILITY_FILTER_ENABLED", + label: "Capability Filter", + description: + "Reject requests before dispatch when the target model lacks required capabilities (vision, tools, structured output, context window). Protects direct single-provider requests that bypass the combo-layer compatibility filter.", + descriptionI18nKey: "featureFlagCapabilityFilterEnabledDescription", + category: "policies", + defaultValue: "false", + type: "boolean", + requiresRestart: false, + warningLevel: "caution", + }, { key: "RADAR_ENABLED", label: "Radar", diff --git a/stryker.conf.json b/stryker.conf.json index 71e0b21352..7a0cc3dfcc 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -39,7 +39,9 @@ "incremental": true, "incrementalFile": "reports/mutation/stryker-incremental.json", "testRunner": "tap", - "plugins": ["@stryker-mutator/tap-runner"], + "plugins": [ + "@stryker-mutator/tap-runner" + ], "tap": { "testFiles": [ "tests/unit/7993-noauth-proxy-routing.test.ts", @@ -333,7 +335,8 @@ "tests/unit/upstream-timeout-model-override.test.ts", "tests/unit/usage-service-hardening.test.ts", "tests/unit/validate-response-quality.test.ts", - "tests/unit/xai-agent-tools-passthrough.test.ts" + "tests/unit/xai-agent-tools-passthrough.test.ts", + "tests/unit/capability-filter.test.ts" ], "nodeArgs": [ "--import", @@ -442,7 +445,11 @@ ".worktrees", ".stryker-tmp" ], - "reporters": ["progress", "html", "json"], + "reporters": [ + "progress", + "html", + "json" + ], "htmlReporter": { "fileName": "reports/mutation/mutation.html" }, diff --git a/tests/unit/capability-filter.test.ts b/tests/unit/capability-filter.test.ts new file mode 100644 index 0000000000..5bcab12d76 --- /dev/null +++ b/tests/unit/capability-filter.test.ts @@ -0,0 +1,269 @@ +/** + * #5696 — Layer A capability filter unit tests. + * + * Tests the pure `checkRequestCapabilityFit` function and the + * `deriveRequestCapabilityRequirements` helper. The chatCore integration + * gate is tested via the feature flag assertion below. + * + * Note: `getResolvedModelCapabilities` requires a database connection, so + * the full integration path (capabilities → filter → error response) is + * tested by verifying the filter function's behavior with mock capabilities. + */ + +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + checkRequestCapabilityFit, + deriveRequestCapabilityRequirements, + type RequestCapabilityRequirements, + type CapabilityFilterResult, +} from "../../src/shared/constants/capabilities/capabilityFilter.ts"; + +// ── Helpers ─────────────────────────────────────────────────────────────── + +/** Minimal capabilities shape for filter testing. */ +function caps(overrides: Partial<{ + supportsTools: boolean | null; + toolCalling: boolean; + supportsVision: boolean | null; + structuredOutput: boolean | null; + contextWindow: number | null; + maxInputTokens: number | null; + maxOutputTokens: number | null; +}> = {}) { + return { + supportsTools: overrides.supportsTools ?? null, + toolCalling: overrides.toolCalling ?? true, + supportsVision: overrides.supportsVision ?? null, + structuredOutput: overrides.structuredOutput ?? null, + contextWindow: overrides.contextWindow ?? null, + maxInputTokens: overrides.maxInputTokens ?? null, + maxOutputTokens: overrides.maxOutputTokens ?? null, + }; +} + +function req(overrides: Partial = {}): RequestCapabilityRequirements { + return { + requiresTools: false, + requiresVision: false, + requiresStructuredOutput: false, + requiredContextTokens: 0, + toolCount: 0, + ...overrides, + }; +} + +// ── Tests ───────────────────────────────────────────────────────────────── + +test("checkRequestCapabilityFit: compatible when no requirements", () => { + const result = checkRequestCapabilityFit(caps(), req()); + assert.equal(result.compatible, true); + assert.deepEqual(result.failures, []); +}); + +test("checkRequestCapabilityFit: vision failure when model lacks vision", () => { + const result = checkRequestCapabilityFit( + caps({ supportsVision: false }), + req({ requiresVision: true }) + ); + assert.equal(result.compatible, false); + assert.deepEqual(result.failures, ["vision"]); + assert.equal(result.terminalReason, "vision"); +}); + +test("checkRequestCapabilityFit: vision failure when model vision is unknown (null)", () => { + const result = checkRequestCapabilityFit( + caps({ supportsVision: null }), + req({ requiresVision: true }) + ); + assert.equal(result.compatible, false); + assert.deepEqual(result.failures, ["vision"]); + assert.equal(result.terminalReason, "vision"); +}); + +test("checkRequestCapabilityFit: vision OK when model supports vision", () => { + const result = checkRequestCapabilityFit( + caps({ supportsVision: true }), + req({ requiresVision: true }) + ); + assert.equal(result.compatible, true); + assert.deepEqual(result.failures, []); +}); + +test("checkRequestCapabilityFit: tools failure when model has no tool support", () => { + const result = checkRequestCapabilityFit( + caps({ supportsTools: false, toolCalling: false }), + req({ requiresTools: true }), + "openai" + ); + assert.equal(result.compatible, false); + assert.deepEqual(result.failures, ["tools"]); + assert.equal(result.terminalReason, "tools"); +}); + +test("checkRequestCapabilityFit: tools OK when model supports tools", () => { + const result = checkRequestCapabilityFit( + caps({ supportsTools: true, toolCalling: true }), + req({ requiresTools: true }), + "openai" + ); + assert.equal(result.compatible, true); + assert.deepEqual(result.failures, []); +}); + +test("checkRequestCapabilityFit: tools bypassed for emulated-tool provider", () => { + // chatgpt-web has toolCalling: "emulated" in the provider registry, + // so the filter must not reject it even when capabilities report false. + const result = checkRequestCapabilityFit( + caps({ supportsTools: false, toolCalling: false }), + req({ requiresTools: true }), + "chatgpt-web" + ); + assert.equal(result.compatible, true); + assert.deepEqual(result.failures, []); +}); + +test("checkRequestCapabilityFit: structured output failure when model does not support", () => { + const result = checkRequestCapabilityFit( + caps({ structuredOutput: false }), + req({ requiresStructuredOutput: true }) + ); + assert.equal(result.compatible, false); + assert.deepEqual(result.failures, ["structured_output"]); + assert.equal(result.terminalReason, "structured_output"); +}); + +test("checkRequestCapabilityFit: structured output OK when model supports", () => { + const result = checkRequestCapabilityFit( + caps({ structuredOutput: true }), + req({ requiresStructuredOutput: true }) + ); + assert.equal(result.compatible, true); + assert.deepEqual(result.failures, []); +}); + +test("checkRequestCapabilityFit: context window failure when tokens exceed window", () => { + const result = checkRequestCapabilityFit( + caps({ contextWindow: 1000, maxInputTokens: 1000 }), + req({ requiredContextTokens: 2000 }) + ); + assert.equal(result.compatible, false); + assert.deepEqual(result.failures, ["context_window"]); + assert.equal(result.terminalReason, "context_window"); +}); + +test("checkRequestCapabilityFit: context window OK when tokens fit", () => { + const result = checkRequestCapabilityFit( + caps({ contextWindow: 10000, maxInputTokens: 10000 }), + req({ requiredContextTokens: 2000 }) + ); + assert.equal(result.compatible, true); + assert.deepEqual(result.failures, []); +}); + +test("checkRequestCapabilityFit: multiple failures reported", () => { + const result = checkRequestCapabilityFit( + caps({ supportsVision: false, supportsTools: false, toolCalling: false }), + req({ requiresVision: true, requiresTools: true }), + "openai" + ); + assert.equal(result.compatible, false); + // vision is checked first, so it's the terminalReason + assert.ok(result.failures.length >= 1); + assert.ok(result.failures.includes("vision")); +}); + +test("checkRequestCapabilityFit: context window returns null (unknown) when no window data", () => { + // When contextWindow and maxInputTokens are both null, evaluateContextLimit + // returns null, which means compatible (no data to judge). + const result = checkRequestCapabilityFit( + caps({ contextWindow: null, maxInputTokens: null }), + req({ requiredContextTokens: 2000 }) + ); + assert.equal(result.compatible, true); + assert.deepEqual(result.failures, []); +}); + +test("deriveRequestCapabilityRequirements: no requirements from empty body", () => { + const requirements = deriveRequestCapabilityRequirements({}); + assert.equal(requirements.requiresTools, false); + assert.equal(requirements.requiresVision, false); + assert.equal(requirements.requiresStructuredOutput, false); + assert.equal(requirements.requiredContextTokens, 0); + assert.equal(requirements.toolCount, 0); +}); + +test("deriveRequestCapabilityRequirements: detects tools from body", () => { + const requirements = deriveRequestCapabilityRequirements({ + tools: [{ type: "function", function: { name: "test" } }], + }); + assert.equal(requirements.requiresTools, true); + assert.equal(requirements.toolCount, 1); +}); + +test("deriveRequestCapabilityRequirements: detects vision from image_url", () => { + const requirements = deriveRequestCapabilityRequirements({ + messages: [ + { role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/img.jpg" } }] }, + ], + }); + assert.equal(requirements.requiresVision, true); +}); + +test("deriveRequestCapabilityRequirements: detects structured output from response_format", () => { + const requirements = deriveRequestCapabilityRequirements({ + response_format: { type: "json_object" }, + }); + assert.equal(requirements.requiresStructuredOutput, true); +}); + +test("deriveRequestCapabilityRequirements: detects json_schema structured output", () => { + const requirements = deriveRequestCapabilityRequirements({ + response_format: { type: "json_schema", json_schema: { name: "test", schema: {} } }, + }); + assert.equal(requirements.requiresStructuredOutput, true); +}); + +test("feature flag CAPABILITY_FILTER_ENABLED defaults to false", () => { + // This test verifies the feature flag definition ensures the gate is + // opt-in. The default value must be "false" per the plan. + import("../../src/shared/constants/featureFlagDefinitions.ts").then( + ({ FEATURE_FLAG_DEFINITIONS }) => { + const flag = FEATURE_FLAG_DEFINITIONS.find( + (d) => d.key === "CAPABILITY_FILTER_ENABLED" + ); + assert.ok(flag, "CAPABILITY_FILTER_ENABLED flag must be defined"); + assert.equal(flag.defaultValue, "false"); + assert.equal(flag.type, "boolean"); + assert.equal(flag.category, "policies"); + } + ); +}); + +test("error responses use buildErrorBody and do not leak stack traces", () => { + // Verify that capability mismatch errors route through buildErrorBody + // (createErrorResult) and never contain stack traces. + import("../../open-sse/utils/error.ts").then(({ createErrorResult }) => { + const result = createErrorResult( + 400, + "Provider 'test' does not support vision for this image request", + null, + "vision", + "invalid_request_error" + ); + assert.equal(result.status, 400); + assert.equal(result.error, "Provider 'test' does not support vision for this image request"); + assert.equal(result.errorType, "invalid_request_error"); + assert.equal(result.errorCode, "vision"); + + // Parse the response body and assert no stack leak + result.response.text().then((text) => { + const body = JSON.parse(text); + assert.ok(body.error.message, "error message must exist"); + assert.equal(body.error.message.includes("at /"), false, "must not leak stack traces"); + assert.equal(body.error.code, "vision"); + assert.equal(body.error.type, "invalid_request_error"); + }); + }); +}); \ No newline at end of file