From 6fa748e32194c6dd3fe1df4038f3580f7dc71a8f Mon Sep 17 00:00:00 2001 From: Hernan Javier Ardila Sanchez Date: Mon, 20 Jul 2026 20:56:54 +0200 Subject: [PATCH] fix(vision-bridge): reroute auto/ prefix to vision model when images present (#7871) * fix(vision-bridge): reroute auto/ prefix to vision model when images present Rebuilt clean on release/v3.8.49 (the original branch forked from an old main and dragged ~70 unrelated files of base drift). Reconciled with the newer VibeProxy credential guards on the tip: the reroute now also fires for auto/ models, the keep-credentialed-model skip does not apply to auto (keeping auto would land on a text-only candidate), and the reroute-target credential guard is preserved. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> * test(guardrails): compact image_url literals to fit the 800-line test cap Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --------- Co-authored-by: herjarsa <204746071+herjarsa@users.noreply.github.com> Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- src/lib/guardrails/visionBridge.ts | 69 ++++++++++++++-------- tests/unit/guardrails/visionBridge.test.ts | 42 ++++++------- 2 files changed, 64 insertions(+), 47 deletions(-) diff --git a/src/lib/guardrails/visionBridge.ts b/src/lib/guardrails/visionBridge.ts index 1229b418b4..92cfa6f8c9 100644 --- a/src/lib/guardrails/visionBridge.ts +++ b/src/lib/guardrails/visionBridge.ts @@ -196,37 +196,48 @@ export class VisionBridgeGuardrail extends BaseGuardrail { return { block: false }; } - // 3b. Auto/ prefix → skip guardrail (auto-combo resolver handles vision-capable model selection) - if (model === "auto" || model.startsWith("auto/")) { - return { block: false }; - } + // 3b. Auto/ prefix — don't skip guardrail entirely. Images still need to be + // described or rerouted to a vision-capable model. The auto-combo resolver + // does NOT currently filter models by vision capability, so without the + // guardrail an image-bearing request assigned to a text-only model will + // fail upstream with "does not support images". + const isAuto = model === "auto" || model.startsWith("auto/"); - const forceVisionBridge = isVisionBridgeForcedModel(model); + // Declare before the conditional so they're available to the rest of preCall + let forceVisionBridge = false; + let comboVisionBridgeDecision: ComboVisionBridgeDecision | undefined; - // 4. Check if model supports vision - const capabilities = getResolvedModelCapabilities(model); - const comboVisionBridgeDecision = forceVisionBridge - ? "process" - : this.deps.checkModelHasComboMapping - ? (await this.deps.checkModelHasComboMapping(model)) - ? "process" - : "skip" - : await getComboVisionBridgeDecision(model); + if (!isAuto) { + forceVisionBridge = isVisionBridgeForcedModel(model); - if (comboVisionBridgeDecision === "skip") { - return { block: false }; - } + // 4. Check if model supports vision + const capabilities = getResolvedModelCapabilities(model); + comboVisionBridgeDecision = forceVisionBridge + ? "process" + : this.deps.checkModelHasComboMapping + ? (await this.deps.checkModelHasComboMapping(model)) + ? "process" + : "skip" + : await getComboVisionBridgeDecision(model); - if (capabilities.supportsVision === true && !forceVisionBridge) { - // The request model supports vision natively, but check if a - // model-combo mapping routes this model through a combo where - // some targets may NOT support vision. In that case, the vision - // bridge must process images so combo targets can describe them. - if (comboVisionBridgeDecision !== "process") { + if (comboVisionBridgeDecision === "skip") { return { block: false }; } - // Combo mapping found — fall through to process images + + if (capabilities?.supportsVision === true && !forceVisionBridge) { + // The request model supports vision natively, but check if a + // model-combo mapping routes this model through a combo where + // some targets may NOT support vision. In that case, the vision + // bridge must process images so combo targets can describe them. + if (comboVisionBridgeDecision !== "process") { + return { block: false }; + } + // Combo mapping found — fall through to process images + } } + // For auto models (isAuto=true), force remains false and combo decision + // remains undefined, which makes the reroute check on line ~189 treat it + // like a non-combo model — exactly what we want: reroute to a vision model. // 5. Get body and check for messages const body = payload as Record; @@ -266,12 +277,18 @@ export class VisionBridgeGuardrail extends BaseGuardrail { // exists, which produced: HTTP log zai → Guardrail reroute → opencode-zen 401 // "Missing API key" while the combo UI still showed body=zai. Fall through to // the image-describe path so the user's chosen model still answers. - if (comboVisionBridgeDecision === "not-combo" && !forceVisionBridge) { + // + // Reroute also fires for the auto/ prefix (isAuto): the auto-combo resolver + // does not filter candidates for vision capability, so an image-bearing + // request with model=auto would land on a text-only model (#7871). Keeping + // "auto" is never the answer there, so the keep-credentialed-model skip + // below does not apply to auto — only the reroute-target credential guard. + if ((comboVisionBridgeDecision === "not-combo" || isAuto) && !forceVisionBridge) { const checkCreds = this.deps.hasUsableCredentials ?? hasUsableCredentialsForModel; const originalUsable = await checkCreds(model); - if (originalUsable === true) { + if (originalUsable === true && !isAuto) { // Keep the credentialed model; describe images below if needed. context.log?.debug?.( "VISION_BRIDGE", diff --git a/tests/unit/guardrails/visionBridge.test.ts b/tests/unit/guardrails/visionBridge.test.ts index b4a8dcbef5..b1b0d9f7ec 100644 --- a/tests/unit/guardrails/visionBridge.test.ts +++ b/tests/unit/guardrails/visionBridge.test.ts @@ -107,10 +107,7 @@ test("VB-S05: passthroughs when visionBridgeEnabled is false", async () => { role: "user", content: [ { type: "text", text: "What is in this image?" }, - { - type: "image_url", - image_url: { url: "https://example.com/image.png" }, - }, + { type: "image_url", image_url: { url: "https://example.com/image.png" } }, ], }, ], @@ -237,7 +234,7 @@ test("VB-S04: passthroughs when messages array is empty", async () => { // ── VB-S12: Auto-prefix skip ──────────────────────────────────────────────── -test("VB-S12: skips guardrail for auto/ prefix model (auto/vision)", async () => { +test("VB-S12: reroutes auto/ prefix model to vision model (auto/vision)", async () => { const guardrail = createGuardrail(); const payload = createPayload({ @@ -247,10 +244,7 @@ test("VB-S12: skips guardrail for auto/ prefix model (auto/vision)", async () => role: "user", content: [ { type: "text", text: "What is in this image?" }, - { - type: "image_url", - image_url: { url: "https://example.com/image.png" }, - }, + { type: "image_url", image_url: { url: "https://example.com/image.png" } }, ], }, ], @@ -258,11 +252,17 @@ test("VB-S12: skips guardrail for auto/ prefix model (auto/vision)", async () => const result = await guardrail.preCall(payload, createContext({ model: "auto/vision" })); assert.strictEqual(result.block, false); - assert.strictEqual(result.modifiedPayload, undefined, "auto/vision should passthrough"); - assert.strictEqual(visionCallCount, 0, "should NOT call vision API for auto prefix"); + assert.ok(result.modifiedPayload, "auto/vision should reroute to vision model"); + assert.strictEqual( + result.modifiedPayload?.model, + "openai/gpt-4o-mini", + "should reroute to configured vision model" + ); + assert.strictEqual(result.meta?.rerouted, true, "rerouted meta should be set"); + assert.strictEqual(visionCallCount, 0, "should NOT call vision API (reroute, not describe)"); }); -test("VB-S12b: skips guardrail for bare auto prefix", async () => { +test("VB-S12b: reroutes auto prefix to best vision model when images present", async () => { const guardrail = createGuardrail(); const payload = createPayload({ @@ -272,10 +272,7 @@ test("VB-S12b: skips guardrail for bare auto prefix", async () => { role: "user", content: [ { type: "text", text: "What is in this image?" }, - { - type: "image_url", - image_url: { url: "https://example.com/image.png" }, - }, + { type: "image_url", image_url: { url: "https://example.com/image.png" } }, ], }, ], @@ -283,7 +280,13 @@ test("VB-S12b: skips guardrail for bare auto prefix", async () => { const result = await guardrail.preCall(payload, createContext({ model: "auto" })); assert.strictEqual(result.block, false); - assert.strictEqual(result.modifiedPayload, undefined, "auto should passthrough"); + assert.ok(result.modifiedPayload, "auto should reroute to vision model"); + assert.strictEqual( + result.modifiedPayload?.model, + "openai/gpt-4o-mini", + "should reroute to configured vision model" + ); + assert.strictEqual(result.meta?.rerouted, true, "rerouted meta should be set"); }); // ── VB-S01: Single image → reroute (individual non-vision model) ─────────── @@ -298,10 +301,7 @@ test("VB-S01: reroutes non-vision model with images to best vision model", async role: "user", content: [ { type: "text", text: "What is in this image?" }, - { - type: "image_url", - image_url: { url: "https://example.com/image.png" }, - }, + { type: "image_url", image_url: { url: "https://example.com/image.png" } }, ], }, ],