From 3c512a7bbae4d4aa069f1c2c2e16df305382005a Mon Sep 17 00:00:00 2001 From: Automation Date: Mon, 25 May 2026 18:25:19 +0200 Subject: [PATCH] fix(vision-bridge): process images when vision-capable model has combo mapping When a model-combo mapping routes a vision-capable model through a combo where some targets may NOT support vision, the vision bridge must process images so combo targets can describe them. Before: if body.model supports vision, the vision bridge skipped image processing entirely. Non-vision combo targets would receive raw images they can't handle. After: before skipping, check if the model has a model-combo mapping. If it does, process images through the vision bridge regardless of body.model's native vision support. - Add checkModelHasComboMapping() helper (dynamic import, failsafe) - Add checkModelHasComboMapping dep to VisionBridgeDependencies (testable) - Guardrail preCall: check combo mapping before early-return on vision support - Add VB-S11 / VB-S11b tests --- src/lib/guardrails/visionBridge.ts | 36 +++++++++++- tests/unit/guardrails/visionBridge.test.ts | 67 ++++++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/src/lib/guardrails/visionBridge.ts b/src/lib/guardrails/visionBridge.ts index 460d342471..402cf48560 100644 --- a/src/lib/guardrails/visionBridge.ts +++ b/src/lib/guardrails/visionBridge.ts @@ -18,6 +18,28 @@ import { isVisionBridgeForcedModel, } from "@/shared/constants/visionBridgeDefaults"; +/// Check if a model name has a model-combo mapping. +/// When a user sends `model: "gpt-4o"` with a model-combo mapping, +/// the actual execution model(s) might differ. Non-vision combo +/// targets would fail with images they can't handle, so the +/// vision bridge must process images even if body.model supports vision. +async function checkModelHasComboMapping(model: string): Promise { + try { + // 1. Check for exact combo name match + const { getComboByName } = await import("@/lib/localDb"); + const exactCombo = await getComboByName(model); + if (exactCombo) return true; + + // 2. Check for model-combo mapping (glob pattern match) + const { resolveComboForModel } = await import("@/lib/db/modelComboMappings"); + const mapping = await resolveComboForModel(model); + return mapping !== null; + } catch { + // Tables may not exist (pre-migration), or DB not initialized + return false; + } +} + export interface VisionBridgeDependencies { getSettings?: () => Promise>; callVisionModel?: ( @@ -25,6 +47,8 @@ export interface VisionBridgeDependencies { config: import("./visionBridgeHelpers").VisionModelConfig, apiKey?: string ) => Promise; + /** Skip real DB lookup — return true to test combo-mapping path, false for normal path. */ + checkModelHasComboMapping?: (model: string) => Promise; } export class VisionBridgeGuardrail extends BaseGuardrail { @@ -61,7 +85,17 @@ export class VisionBridgeGuardrail extends BaseGuardrail { // 4. Check if model supports vision const capabilities = getResolvedModelCapabilities(model); if (capabilities.supportsVision === true && !forceVisionBridge) { - return { block: false }; + // The request model supports vision natively, but check if a + // model-combo mapping routes this model through a combo where + // some targets may NOT support vision. In that case, the vision + // bridge must process images so combo targets can describe them. + const hasMapping = this.deps.checkModelHasComboMapping + ? await this.deps.checkModelHasComboMapping(model) + : await checkModelHasComboMapping(model); + if (!hasMapping) { + return { block: false }; + } + // Combo mapping found — fall through to process images } // 5. Get body and check for messages diff --git a/tests/unit/guardrails/visionBridge.test.ts b/tests/unit/guardrails/visionBridge.test.ts index 0925bbe8bc..c49c2d270e 100644 --- a/tests/unit/guardrails/visionBridge.test.ts +++ b/tests/unit/guardrails/visionBridge.test.ts @@ -38,6 +38,7 @@ function createGuardrail(options?: Parameters[0]) } return mockVisionResponse; }, + ...(options?.deps ?? {}), }, }); } @@ -501,3 +502,69 @@ test("VB-S10: returns meta with imagesProcessed count", async () => { assert.strictEqual(typeof meta.processingTimeMs, "number"); assert.strictEqual(meta.visionModel, "openai/gpt-4o-mini"); }); + +// ── VB-S11: Combo mapping forces vision processing despite vision-capable model ── + +test("VB-S11: processes images when vision-capable model has combo mapping", async () => { + mockVisionResponse = "A description from combo-mapped vision bridge"; + const guardrail = createGuardrail({ + deps: { + checkModelHasComboMapping: async (_model: string) => true, + }, + }); + + const payload = createPayload({ + model: "openai/gpt-4o", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What is this?" }, + { + type: "image_url", + image_url: { url: "https://example.com/image.png" }, + }, + ], + }, + ], + }); + + const startCallCount = visionCallCount; + const result = await guardrail.preCall(payload, createContext({ model: "openai/gpt-4o" })); + + // Vision bridge should have processed the image + assert.strictEqual(result.block, false); + assert.ok(visionCallCount > startCallCount, "Expected vision model to be called"); + assert.ok(result.modifiedPayload !== undefined, "Expected modifiedPayload when combo mapping forces vision bridge"); +}); + +test("VB-S11b: passthroughs when vision-capable model has NO combo mapping", async () => { + const guardrail = createGuardrail({ + deps: { + checkModelHasComboMapping: async (_model: string) => false, + }, + }); + + const payload = createPayload({ + model: "openai/gpt-4o", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What is this?" }, + { + type: "image_url", + image_url: { url: "https://example.com/image.png" }, + }, + ], + }, + ], + }); + + const result = await guardrail.preCall(payload, createContext({ model: "openai/gpt-4o" })); + + // Vision bridge should skip (passthrough) since model supports vision and no combo mapping + assert.strictEqual(result.block, false); + assert.strictEqual(result.modifiedPayload, undefined); + assert.strictEqual(visionCallCount, 0); +});