mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-08 08:12:20 +03:00
Compare commits
4 Commits
fix/9030-a
...
feat/5696-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4dbbaeb746 | ||
|
|
2404556a62 | ||
|
|
247a00ce66 | ||
|
|
ee94b0378d |
1
changelog.d/features/5696-layer-a-capability-filter.md
Normal file
1
changelog.d/features/5696-layer-a-capability-filter.md
Normal file
@@ -0,0 +1 @@
|
||||
- **feat(core):** add Layer A capability filter at router (#5696)
|
||||
@@ -350,7 +350,7 @@
|
||||
"open-sse/executors/deepseek-web.ts": 1148,
|
||||
"open-sse/executors/grok-web.ts": 1044,
|
||||
"open-sse/executors/muse-spark-web.ts": 1405,
|
||||
"open-sse/handlers/chatCore.ts": 5020,
|
||||
"open-sse/handlers/chatCore.ts": 5029,
|
||||
"open-sse/handlers/imageGeneration.ts": 3101,
|
||||
"open-sse/handlers/responseSanitizer.ts": 1115,
|
||||
"open-sse/handlers/search.ts": 1536,
|
||||
|
||||
@@ -134,6 +134,8 @@ import {
|
||||
getResolvedModelCapabilities,
|
||||
getExplicitModelOutputCap,
|
||||
} from "@/lib/modelCapabilities.ts";
|
||||
import { checkRequestCapabilityFit, deriveRequestCapabilityRequirements, buildCapabilityMismatchMessage } from "@/shared/constants/capabilities/capabilityFilter.ts";
|
||||
import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags.ts";
|
||||
import { toPositiveInteger } from "../services/reasoningTokenBuffer.ts";
|
||||
import { normalizeThinkingForModel } from "@/shared/constants/modelSpecs.ts";
|
||||
import {
|
||||
@@ -2605,7 +2607,16 @@ export async function handleChatCore({
|
||||
}
|
||||
}
|
||||
// === /Quota Share enforcement PRE-hook ===
|
||||
|
||||
if (isFeatureFlagEnabled("CAPABILITY_FILTER_ENABLED")) {
|
||||
const fit = checkRequestCapabilityFit(getResolvedModelCapabilities({ provider, model: effectiveModel }),
|
||||
deriveRequestCapabilityRequirements(body as Record<string, unknown>), provider);
|
||||
if (!fit.compatible) {
|
||||
const msg = buildCapabilityMismatchMessage(fit.terminalReason!, provider, effectiveModel);
|
||||
log?.warn?.("CAPABILITY", msg);
|
||||
trackPendingRequest(model, provider, connectionId, false);
|
||||
return createErrorResult(400, msg, null, fit.terminalReason, "invalid_request_error");
|
||||
}
|
||||
}
|
||||
// Get executor for this provider (with optional upstream proxy routing)
|
||||
const executor = await resolveExecutorWithProxy(provider);
|
||||
const getExecutionCredentials = () =>
|
||||
|
||||
@@ -938,6 +938,7 @@
|
||||
"featureFlagOmnirouteEmergencyFallbackDescription": "Route budget-exhausted requests to the emergency free fallback provider/model.",
|
||||
"featureFlagArenaEloSyncEnabledDescription": "Enable periodic Arena AI leaderboard ELO sync for model intelligence rankings.",
|
||||
"featureFlagExposeCcDiscoveryAliasesDescription": "Advertise claude/<provider>/<model> mirror ids on /v1/models so Claude Code gateway model discovery lists non-Claude models. Warning: doubles catalog entries for all clients when enabled globally.",
|
||||
"featureFlagCapabilityFilterEnabledDescription": "Reject requests before dispatch when the target model lacks required capabilities (vision, tools, structured output, context window). Protects direct single-provider requests that bypass the combo-layer compatibility filter.",
|
||||
"sidebar": {
|
||||
"home": "Home",
|
||||
"dashboard": "Dashboard",
|
||||
@@ -12220,5 +12221,9 @@
|
||||
"partnerLinkNote": "Partner link",
|
||||
"dismissAriaLabel": "Dismiss"
|
||||
},
|
||||
"featureFlagExposeFunctionalGatewayMirrorsDescription": "Advertise <gateway-alias>/<model> mirror ids on /v1/models for models whose canonical owner has no active credential but a passthrough gateway with an active credential routes them. Warning: adds catalog entries for all clients when enabled globally."
|
||||
"featureFlagExposeFunctionalGatewayMirrorsDescription": "Advertise <gateway-alias>/<model> mirror ids on /v1/models for models whose canonical owner has no active credential but a passthrough gateway with an active credential routes them. Warning: adds catalog entries for all clients when enabled globally.",
|
||||
"capabilityFilter.visionMismatch": "Provider does not support vision for this image request",
|
||||
"capabilityFilter.toolsMismatch": "Provider does not support tool calling",
|
||||
"capabilityFilter.structuredOutputMismatch": "Provider does not support structured output",
|
||||
"capabilityFilter.contextWindowMismatch": "Request exceeds provider context window"
|
||||
}
|
||||
|
||||
@@ -938,6 +938,7 @@
|
||||
"featureFlagOmnirouteEmergencyFallbackDescription": "Encaminhar solicitações com orçamento esgotado para o provedor/modelo de fallback gratuito de emergência.",
|
||||
"featureFlagArenaEloSyncEnabledDescription": "Habilitar sincronização periódica de ELO da tabela de classificação do Arena AI para rankings de inteligência de modelos.",
|
||||
"featureFlagExposeCcDiscoveryAliasesDescription": "Divulgar ids espelho claude/<provider>/<model> em /v1/models para que a descoberta de modelos do gateway Claude Code liste modelos não-Claude. Atenção: duplica as entradas do catálogo para todos os clientes quando ativado globalmente.",
|
||||
"featureFlagCapabilityFilterEnabledDescription": "Rejeitar requisicoes antes do despacho quando o modelo alvo nao possui as capacidades necessarias (visao, ferramentas, saida estruturada, janela de contexto). Protege requisicoes diretas que ignoram o filtro de compatibilidade do combo.",
|
||||
"sidebar": {
|
||||
"home": "Início",
|
||||
"dashboard": "Painel",
|
||||
@@ -12220,5 +12221,10 @@
|
||||
"partnerLinkNote": "Link de parceiro",
|
||||
"dismissAriaLabel": "Descartar"
|
||||
},
|
||||
"featureFlagExposeFunctionalGatewayMirrorsDescription": "__MISSING__:Advertise <gateway-alias>/<model> mirror ids on /v1/models for models whose canonical owner has no active credential but a passthrough gateway with an active credential routes them. Warning: adds catalog entries for all clients when enabled globally."
|
||||
"featureFlagExposeFunctionalGatewayMirrorsDescription": "__MISSING__:Advertise <gateway-alias>/<model> mirror ids on /v1/models for models whose canonical owner has no active credential but a passthrough gateway with an active credential routes them. Warning: adds catalog entries for all clients when enabled globally.",
|
||||
"featureFlagCapabilityFilterEnabledDescription": "Rejeitar requisições antes do despacho quando o modelo alvo nao possui as capacidades necessarias (visao, ferramentas, saída estruturada, janela de contexto). Protege requisições diretas que ignoram o filtro de compatibilidade do combo.",
|
||||
"capabilityFilter.visionMismatch": "O provedor nao suporta visao para esta requisicao de imagem",
|
||||
"capabilityFilter.toolsMismatch": "O provedor nao suporta chamada de ferramentas",
|
||||
"capabilityFilter.structuredOutputMismatch": "O provedor nao suporta saida estruturada",
|
||||
"capabilityFilter.contextWindowMismatch": "A requisicao excede a janela de contexto do provedor"
|
||||
}
|
||||
|
||||
211
src/shared/constants/capabilities/capabilityFilter.ts
Normal file
211
src/shared/constants/capabilities/capabilityFilter.ts
Normal file
@@ -0,0 +1,211 @@
|
||||
/**
|
||||
* Layer A capability filter — shared, provider-agnostic module.
|
||||
*
|
||||
* Validates that a provider+model can satisfy the request's capability
|
||||
* requirements (tools, vision, structured output, context window) BEFORE
|
||||
* dispatch to the executor. Returns an early 400 if not, rather than
|
||||
* letting the request fail downstream or produce garbage (e.g. a text-only
|
||||
* model receiving image_url content and answering "image not provided").
|
||||
*
|
||||
* The logic mirrors what `filterTargetsByRequestCompatibility` in
|
||||
* comboStructure.ts already does for combo-routed requests, but this
|
||||
* module lives at the router layer (Layer A) so it also protects direct
|
||||
* single-provider requests (`model:"openai/gpt-4o-mini"` without a combo).
|
||||
*
|
||||
* #5696
|
||||
*/
|
||||
|
||||
import { getResolvedModelCapabilities } from "@/lib/modelCapabilities";
|
||||
import { evaluateContextLimit } from "@omniroute/open-sse/services/combo/contextOverrideGate";
|
||||
import { hasEstimableContent } from "@omniroute/open-sse/services/combo/knownContextOverflow";
|
||||
import { isRecord } from "@omniroute/open-sse/services/combo/comboData";
|
||||
import { providerSupportsEmulatedToolCalling } from "@omniroute/open-sse/services/combo/comboStructure";
|
||||
import { estimateTokens } from "@omniroute/open-sse/services/contextManager";
|
||||
|
||||
// ── Types ─────────────────────────────────────────────────────────────────
|
||||
|
||||
export type CapabilityFailure = "tools" | "vision" | "structured_output" | "context_window";
|
||||
|
||||
export interface RequestCapabilityRequirements {
|
||||
requiresTools: boolean;
|
||||
requiresVision: boolean;
|
||||
requiresStructuredOutput: boolean;
|
||||
requiredContextTokens: number;
|
||||
toolCount: number;
|
||||
}
|
||||
|
||||
export interface CapabilityFilterResult {
|
||||
compatible: boolean;
|
||||
failures: CapabilityFailure[];
|
||||
terminalReason?: string;
|
||||
}
|
||||
|
||||
// ── Pure helpers (mirror the unexported helpers in comboStructure.ts) ──────
|
||||
|
||||
function requestRequiresTools(body: Record<string, unknown>): boolean {
|
||||
if (Array.isArray(body.tools) && body.tools.length > 0) return true;
|
||||
if (Array.isArray(body.functions) && body.functions.length > 0) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function requestRequiresStructuredOutput(body: Record<string, unknown>): boolean {
|
||||
const responseFormat = isRecord(body.response_format) ? body.response_format : null;
|
||||
const type = typeof responseFormat?.type === "string" ? responseFormat.type : null;
|
||||
return type === "json_object" || type === "json_schema";
|
||||
}
|
||||
|
||||
function estimateRequestInputTokens(body: Record<string, unknown>): number {
|
||||
const estimatePayload: Record<string, unknown> = {};
|
||||
for (const key of ["messages", "input", "tools", "functions", "response_format"]) {
|
||||
if (hasEstimableContent(body[key])) estimatePayload[key] = body[key];
|
||||
}
|
||||
return Object.keys(estimatePayload).length > 0 ? estimateTokens(estimatePayload) : 0;
|
||||
}
|
||||
|
||||
function getPositiveTokenCount(value: unknown): number {
|
||||
const count = Number(value);
|
||||
return Number.isFinite(count) && count > 0 ? Math.ceil(count) : 0;
|
||||
}
|
||||
|
||||
function isMediaTypeImage(value: Record<string, unknown>): boolean {
|
||||
const source = isRecord(value.source) ? value.source : null;
|
||||
const mediaType = typeof source?.media_type === "string" ? source.media_type.toLowerCase() : "";
|
||||
return mediaType.startsWith("image/");
|
||||
}
|
||||
|
||||
function valueContainsImagePart(value: unknown, depth = 0): boolean {
|
||||
if (depth > 8 || value === null || value === undefined) return false;
|
||||
if (typeof value === "string") return value.startsWith("data:image/");
|
||||
if (Array.isArray(value)) return value.some((entry) => valueContainsImagePart(entry, depth + 1));
|
||||
if (!isRecord(value)) return false;
|
||||
|
||||
if (valueContainsImageType(value)) return true;
|
||||
if (isMediaTypeImage(value)) return true;
|
||||
|
||||
return Object.values(value).some((entry) => valueContainsImagePart(entry, depth + 1));
|
||||
}
|
||||
|
||||
function isContextOverflow(
|
||||
capabilities: { maxInputTokens: number | null; contextWindow: number | null },
|
||||
requirements: { requiredContextTokens: number }
|
||||
): boolean {
|
||||
return evaluateContextLimit(
|
||||
{ maxInputTokens: capabilities.maxInputTokens, contextWindow: capabilities.contextWindow },
|
||||
{ estimatedInputTokens: requirements.requiredContextTokens, requiredContextTokens: requirements.requiredContextTokens }
|
||||
) === false;
|
||||
}
|
||||
|
||||
function valueContainsImageType(value: Record<string, unknown>): boolean {
|
||||
const type = typeof value.type === "string" ? value.type.toLowerCase() : null;
|
||||
if (type === "image" || type === "image_url" || type === "input_image") return true;
|
||||
if ("image_url" in value || "input_image" in value) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
// ── Public API ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Derive capability requirements from a request body.
|
||||
* Mirrors `deriveRequestCompatibilityRequirements` in comboStructure.ts.
|
||||
*/
|
||||
export function deriveRequestCapabilityRequirements(
|
||||
body: Record<string, unknown>
|
||||
): RequestCapabilityRequirements {
|
||||
const estimatedInputTokens = estimateRequestInputTokens(body);
|
||||
const requestedOutputTokens = Math.max(
|
||||
getPositiveTokenCount(body.max_tokens),
|
||||
getPositiveTokenCount(body.max_completion_tokens)
|
||||
);
|
||||
return {
|
||||
requiresTools: requestRequiresTools(body),
|
||||
requiresVision: valueContainsImagePart(body.messages) || valueContainsImagePart(body.input),
|
||||
requiresStructuredOutput: requestRequiresStructuredOutput(body),
|
||||
requiredContextTokens: estimatedInputTokens + requestedOutputTokens,
|
||||
toolCount: Array.isArray(body.tools) ? body.tools.length : 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a human-readable error message for a capability mismatch.
|
||||
* Mirrors the i18n keys: capabilityFilter.visionMismatch / toolsMismatch / etc.
|
||||
*/
|
||||
export function buildCapabilityMismatchMessage(
|
||||
terminalReason: string,
|
||||
provider: string | null,
|
||||
model: string | null
|
||||
): string {
|
||||
const msgs: Record<string, string> = {
|
||||
vision: `Provider '${provider}' does not support vision for this image request`,
|
||||
tools: `Provider '${provider}' does not support tool calling`,
|
||||
structured_output: `Provider '${provider}' does not support structured output`,
|
||||
context_window: `Request exceeds the context window for ${provider}/${model}`,
|
||||
};
|
||||
return msgs[terminalReason] || `Provider '${provider}' does not support the required capabilities`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether a model's capabilities satisfy the request requirements.
|
||||
*
|
||||
* @param capabilities - Resolved model capabilities (from getResolvedModelCapabilities)
|
||||
* @param requirements - Request capability requirements
|
||||
* @param provider - Provider id or alias (needed for emulated-tool-calling bypass)
|
||||
* @returns CapabilityFilterResult with compatibility verdict and failure details
|
||||
*/
|
||||
function collectCapabilityFailures(
|
||||
capabilities: Record<string, unknown>,
|
||||
requirements: RequestCapabilityRequirements,
|
||||
provider?: string | null
|
||||
): CapabilityFailure[] {
|
||||
const failures: CapabilityFailure[] = [];
|
||||
const caps = capabilities as {
|
||||
supportsTools: boolean | null;
|
||||
toolCalling: boolean;
|
||||
supportsVision: boolean | null;
|
||||
structuredOutput: boolean | null;
|
||||
contextWindow: number | null;
|
||||
maxInputTokens: number | null;
|
||||
maxOutputTokens: number | null;
|
||||
};
|
||||
|
||||
if (requirements.requiresTools && (caps.supportsTools === false || !caps.toolCalling)
|
||||
&& !providerSupportsEmulatedToolCalling(provider)) {
|
||||
failures.push("tools");
|
||||
}
|
||||
if (requirements.requiresVision && caps.supportsVision !== true) {
|
||||
failures.push("vision");
|
||||
}
|
||||
if (requirements.requiresStructuredOutput && caps.structuredOutput === false) {
|
||||
failures.push("structured_output");
|
||||
}
|
||||
if (requirements.requiredContextTokens > 0 && isContextOverflow(caps, requirements)) {
|
||||
failures.push("context_window");
|
||||
}
|
||||
return failures;
|
||||
}
|
||||
|
||||
function primaryFailure(failures: CapabilityFailure[]): CapabilityFailure {
|
||||
if (failures.includes("vision")) return "vision";
|
||||
if (failures.includes("tools")) return "tools";
|
||||
if (failures.includes("structured_output")) return "structured_output";
|
||||
return "context_window";
|
||||
}
|
||||
|
||||
export function checkRequestCapabilityFit(
|
||||
capabilities: {
|
||||
supportsTools: boolean | null;
|
||||
toolCalling: boolean;
|
||||
supportsVision: boolean | null;
|
||||
structuredOutput: boolean | null;
|
||||
contextWindow: number | null;
|
||||
maxInputTokens: number | null;
|
||||
maxOutputTokens: number | null;
|
||||
},
|
||||
requirements: RequestCapabilityRequirements,
|
||||
provider?: string | null
|
||||
): CapabilityFilterResult {
|
||||
const failures = collectCapabilityFailures(capabilities as Record<string, unknown>, requirements, provider);
|
||||
if (failures.length === 0) {
|
||||
return { compatible: true, failures: [] };
|
||||
}
|
||||
return { compatible: false, failures, terminalReason: primaryFailure(failures) };
|
||||
}
|
||||
@@ -233,6 +233,18 @@ export const FEATURE_FLAG_DEFINITIONS: FeatureFlagDefinition[] = [
|
||||
requiresRestart: true,
|
||||
warningLevel: "info",
|
||||
},
|
||||
{
|
||||
key: "CAPABILITY_FILTER_ENABLED",
|
||||
label: "Capability Filter",
|
||||
description:
|
||||
"Reject requests before dispatch when the target model lacks required capabilities (vision, tools, structured output, context window). Protects direct single-provider requests that bypass the combo-layer compatibility filter.",
|
||||
descriptionI18nKey: "featureFlagCapabilityFilterEnabledDescription",
|
||||
category: "policies",
|
||||
defaultValue: "false",
|
||||
type: "boolean",
|
||||
requiresRestart: false,
|
||||
warningLevel: "caution",
|
||||
},
|
||||
|
||||
// ──────────────── Runtime (15) ────────────────
|
||||
{
|
||||
|
||||
@@ -84,6 +84,7 @@
|
||||
"tests/unit/bug-7940-gemini-retrydelay.test.ts",
|
||||
"tests/unit/build/check-circular-deps.test.ts",
|
||||
"tests/unit/cache-sweeps.test.ts",
|
||||
"tests/unit/capability-filter.test.ts",
|
||||
"tests/unit/chat-adaptive-admission-binding.test.ts",
|
||||
"tests/unit/cc-bridge-openai-image-7777.test.ts",
|
||||
"tests/unit/cc-compatible-provider.test.ts",
|
||||
|
||||
269
tests/unit/capability-filter.test.ts
Normal file
269
tests/unit/capability-filter.test.ts
Normal file
@@ -0,0 +1,269 @@
|
||||
/**
|
||||
* #5696 — Layer A capability filter unit tests.
|
||||
*
|
||||
* Tests the pure `checkRequestCapabilityFit` function and the
|
||||
* `deriveRequestCapabilityRequirements` helper. The chatCore integration
|
||||
* gate is tested via the feature flag assertion below.
|
||||
*
|
||||
* Note: `getResolvedModelCapabilities` requires a database connection, so
|
||||
* the full integration path (capabilities → filter → error response) is
|
||||
* tested by verifying the filter function's behavior with mock capabilities.
|
||||
*/
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
checkRequestCapabilityFit,
|
||||
deriveRequestCapabilityRequirements,
|
||||
type RequestCapabilityRequirements,
|
||||
type CapabilityFilterResult,
|
||||
} from "../../src/shared/constants/capabilities/capabilityFilter.ts";
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
/** Minimal capabilities shape for filter testing. */
|
||||
function caps(overrides: Partial<{
|
||||
supportsTools: boolean | null;
|
||||
toolCalling: boolean;
|
||||
supportsVision: boolean | null;
|
||||
structuredOutput: boolean | null;
|
||||
contextWindow: number | null;
|
||||
maxInputTokens: number | null;
|
||||
maxOutputTokens: number | null;
|
||||
}> = {}) {
|
||||
return {
|
||||
supportsTools: overrides.supportsTools ?? null,
|
||||
toolCalling: overrides.toolCalling ?? true,
|
||||
supportsVision: overrides.supportsVision ?? null,
|
||||
structuredOutput: overrides.structuredOutput ?? null,
|
||||
contextWindow: overrides.contextWindow ?? null,
|
||||
maxInputTokens: overrides.maxInputTokens ?? null,
|
||||
maxOutputTokens: overrides.maxOutputTokens ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function req(overrides: Partial<RequestCapabilityRequirements> = {}): RequestCapabilityRequirements {
|
||||
return {
|
||||
requiresTools: false,
|
||||
requiresVision: false,
|
||||
requiresStructuredOutput: false,
|
||||
requiredContextTokens: 0,
|
||||
toolCount: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────
|
||||
|
||||
test("checkRequestCapabilityFit: compatible when no requirements", () => {
|
||||
const result = checkRequestCapabilityFit(caps(), req());
|
||||
assert.equal(result.compatible, true);
|
||||
assert.deepEqual(result.failures, []);
|
||||
});
|
||||
|
||||
test("checkRequestCapabilityFit: vision failure when model lacks vision", () => {
|
||||
const result = checkRequestCapabilityFit(
|
||||
caps({ supportsVision: false }),
|
||||
req({ requiresVision: true })
|
||||
);
|
||||
assert.equal(result.compatible, false);
|
||||
assert.deepEqual(result.failures, ["vision"]);
|
||||
assert.equal(result.terminalReason, "vision");
|
||||
});
|
||||
|
||||
test("checkRequestCapabilityFit: vision failure when model vision is unknown (null)", () => {
|
||||
const result = checkRequestCapabilityFit(
|
||||
caps({ supportsVision: null }),
|
||||
req({ requiresVision: true })
|
||||
);
|
||||
assert.equal(result.compatible, false);
|
||||
assert.deepEqual(result.failures, ["vision"]);
|
||||
assert.equal(result.terminalReason, "vision");
|
||||
});
|
||||
|
||||
test("checkRequestCapabilityFit: vision OK when model supports vision", () => {
|
||||
const result = checkRequestCapabilityFit(
|
||||
caps({ supportsVision: true }),
|
||||
req({ requiresVision: true })
|
||||
);
|
||||
assert.equal(result.compatible, true);
|
||||
assert.deepEqual(result.failures, []);
|
||||
});
|
||||
|
||||
test("checkRequestCapabilityFit: tools failure when model has no tool support", () => {
|
||||
const result = checkRequestCapabilityFit(
|
||||
caps({ supportsTools: false, toolCalling: false }),
|
||||
req({ requiresTools: true }),
|
||||
"openai"
|
||||
);
|
||||
assert.equal(result.compatible, false);
|
||||
assert.deepEqual(result.failures, ["tools"]);
|
||||
assert.equal(result.terminalReason, "tools");
|
||||
});
|
||||
|
||||
test("checkRequestCapabilityFit: tools OK when model supports tools", () => {
|
||||
const result = checkRequestCapabilityFit(
|
||||
caps({ supportsTools: true, toolCalling: true }),
|
||||
req({ requiresTools: true }),
|
||||
"openai"
|
||||
);
|
||||
assert.equal(result.compatible, true);
|
||||
assert.deepEqual(result.failures, []);
|
||||
});
|
||||
|
||||
test("checkRequestCapabilityFit: tools bypassed for emulated-tool provider", () => {
|
||||
// chatgpt-web has toolCalling: "emulated" in the provider registry,
|
||||
// so the filter must not reject it even when capabilities report false.
|
||||
const result = checkRequestCapabilityFit(
|
||||
caps({ supportsTools: false, toolCalling: false }),
|
||||
req({ requiresTools: true }),
|
||||
"chatgpt-web"
|
||||
);
|
||||
assert.equal(result.compatible, true);
|
||||
assert.deepEqual(result.failures, []);
|
||||
});
|
||||
|
||||
test("checkRequestCapabilityFit: structured output failure when model does not support", () => {
|
||||
const result = checkRequestCapabilityFit(
|
||||
caps({ structuredOutput: false }),
|
||||
req({ requiresStructuredOutput: true })
|
||||
);
|
||||
assert.equal(result.compatible, false);
|
||||
assert.deepEqual(result.failures, ["structured_output"]);
|
||||
assert.equal(result.terminalReason, "structured_output");
|
||||
});
|
||||
|
||||
test("checkRequestCapabilityFit: structured output OK when model supports", () => {
|
||||
const result = checkRequestCapabilityFit(
|
||||
caps({ structuredOutput: true }),
|
||||
req({ requiresStructuredOutput: true })
|
||||
);
|
||||
assert.equal(result.compatible, true);
|
||||
assert.deepEqual(result.failures, []);
|
||||
});
|
||||
|
||||
test("checkRequestCapabilityFit: context window failure when tokens exceed window", () => {
|
||||
const result = checkRequestCapabilityFit(
|
||||
caps({ contextWindow: 1000, maxInputTokens: 1000 }),
|
||||
req({ requiredContextTokens: 2000 })
|
||||
);
|
||||
assert.equal(result.compatible, false);
|
||||
assert.deepEqual(result.failures, ["context_window"]);
|
||||
assert.equal(result.terminalReason, "context_window");
|
||||
});
|
||||
|
||||
test("checkRequestCapabilityFit: context window OK when tokens fit", () => {
|
||||
const result = checkRequestCapabilityFit(
|
||||
caps({ contextWindow: 10000, maxInputTokens: 10000 }),
|
||||
req({ requiredContextTokens: 2000 })
|
||||
);
|
||||
assert.equal(result.compatible, true);
|
||||
assert.deepEqual(result.failures, []);
|
||||
});
|
||||
|
||||
test("checkRequestCapabilityFit: multiple failures reported", () => {
|
||||
const result = checkRequestCapabilityFit(
|
||||
caps({ supportsVision: false, supportsTools: false, toolCalling: false }),
|
||||
req({ requiresVision: true, requiresTools: true }),
|
||||
"openai"
|
||||
);
|
||||
assert.equal(result.compatible, false);
|
||||
// vision is checked first, so it's the terminalReason
|
||||
assert.ok(result.failures.length >= 1);
|
||||
assert.ok(result.failures.includes("vision"));
|
||||
});
|
||||
|
||||
test("checkRequestCapabilityFit: context window returns null (unknown) when no window data", () => {
|
||||
// When contextWindow and maxInputTokens are both null, evaluateContextLimit
|
||||
// returns null, which means compatible (no data to judge).
|
||||
const result = checkRequestCapabilityFit(
|
||||
caps({ contextWindow: null, maxInputTokens: null }),
|
||||
req({ requiredContextTokens: 2000 })
|
||||
);
|
||||
assert.equal(result.compatible, true);
|
||||
assert.deepEqual(result.failures, []);
|
||||
});
|
||||
|
||||
test("deriveRequestCapabilityRequirements: no requirements from empty body", () => {
|
||||
const requirements = deriveRequestCapabilityRequirements({});
|
||||
assert.equal(requirements.requiresTools, false);
|
||||
assert.equal(requirements.requiresVision, false);
|
||||
assert.equal(requirements.requiresStructuredOutput, false);
|
||||
assert.equal(requirements.requiredContextTokens, 0);
|
||||
assert.equal(requirements.toolCount, 0);
|
||||
});
|
||||
|
||||
test("deriveRequestCapabilityRequirements: detects tools from body", () => {
|
||||
const requirements = deriveRequestCapabilityRequirements({
|
||||
tools: [{ type: "function", function: { name: "test" } }],
|
||||
});
|
||||
assert.equal(requirements.requiresTools, true);
|
||||
assert.equal(requirements.toolCount, 1);
|
||||
});
|
||||
|
||||
test("deriveRequestCapabilityRequirements: detects vision from image_url", () => {
|
||||
const requirements = deriveRequestCapabilityRequirements({
|
||||
messages: [
|
||||
{ role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/img.jpg" } }] },
|
||||
],
|
||||
});
|
||||
assert.equal(requirements.requiresVision, true);
|
||||
});
|
||||
|
||||
test("deriveRequestCapabilityRequirements: detects structured output from response_format", () => {
|
||||
const requirements = deriveRequestCapabilityRequirements({
|
||||
response_format: { type: "json_object" },
|
||||
});
|
||||
assert.equal(requirements.requiresStructuredOutput, true);
|
||||
});
|
||||
|
||||
test("deriveRequestCapabilityRequirements: detects json_schema structured output", () => {
|
||||
const requirements = deriveRequestCapabilityRequirements({
|
||||
response_format: { type: "json_schema", json_schema: { name: "test", schema: {} } },
|
||||
});
|
||||
assert.equal(requirements.requiresStructuredOutput, true);
|
||||
});
|
||||
|
||||
test("feature flag CAPABILITY_FILTER_ENABLED defaults to false", () => {
|
||||
// This test verifies the feature flag definition ensures the gate is
|
||||
// opt-in. The default value must be "false" per the plan.
|
||||
import("../../src/shared/constants/featureFlagDefinitions.ts").then(
|
||||
({ FEATURE_FLAG_DEFINITIONS }) => {
|
||||
const flag = FEATURE_FLAG_DEFINITIONS.find(
|
||||
(d) => d.key === "CAPABILITY_FILTER_ENABLED"
|
||||
);
|
||||
assert.ok(flag, "CAPABILITY_FILTER_ENABLED flag must be defined");
|
||||
assert.equal(flag.defaultValue, "false");
|
||||
assert.equal(flag.type, "boolean");
|
||||
assert.equal(flag.category, "policies");
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test("error responses use buildErrorBody and do not leak stack traces", () => {
|
||||
// Verify that capability mismatch errors route through buildErrorBody
|
||||
// (createErrorResult) and never contain stack traces.
|
||||
import("../../open-sse/utils/error.ts").then(({ createErrorResult }) => {
|
||||
const result = createErrorResult(
|
||||
400,
|
||||
"Provider 'test' does not support vision for this image request",
|
||||
null,
|
||||
"vision",
|
||||
"invalid_request_error"
|
||||
);
|
||||
assert.equal(result.status, 400);
|
||||
assert.equal(result.error, "Provider 'test' does not support vision for this image request");
|
||||
assert.equal(result.errorType, "invalid_request_error");
|
||||
assert.equal(result.errorCode, "vision");
|
||||
|
||||
// Parse the response body and assert no stack leak
|
||||
result.response.text().then((text) => {
|
||||
const body = JSON.parse(text);
|
||||
assert.ok(body.error.message, "error message must exist");
|
||||
assert.equal(body.error.message.includes("at /"), false, "must not leak stack traces");
|
||||
assert.equal(body.error.code, "vision");
|
||||
assert.equal(body.error.type, "invalid_request_error");
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user