From dc185e5aab1593fa9c2d5102d55f6344f29d0a11 Mon Sep 17 00:00:00 2001 From: Aman <1402357+Zartharas@users.noreply.github.com> Date: Thu, 13 Aug 2026 04:53:27 -0600 Subject: [PATCH] fix(guardrails): support Responses input images in Vision Bridge (#10202) * fix(guardrails): bridge Responses input images * docs(changelog): add #10202 Vision Bridge fix fragment --- .../fixes/10202-responses-vision-bridge.md | 1 + src/lib/guardrails/visionBridge.ts | 18 +- src/lib/guardrails/visionBridgeHelpers.ts | 23 +- .../visionBridge-responses-9597.test.ts | 446 ++++++++++++++++++ 4 files changed, 480 insertions(+), 8 deletions(-) create mode 100644 changelog.d/fixes/10202-responses-vision-bridge.md create mode 100644 tests/unit/guardrails/visionBridge-responses-9597.test.ts diff --git a/changelog.d/fixes/10202-responses-vision-bridge.md b/changelog.d/fixes/10202-responses-vision-bridge.md new file mode 100644 index 0000000000..cb5ff02038 --- /dev/null +++ b/changelog.d/fixes/10202-responses-vision-bridge.md @@ -0,0 +1 @@ +- **fix(guardrails):** Vision Bridge handles OpenAI Responses `input`/`input_image` requests before combo vision filtering ([#10202](https://github.com/diegosouzapw/OmniRoute/pull/10202)) — thanks @Zartharas diff --git a/src/lib/guardrails/visionBridge.ts b/src/lib/guardrails/visionBridge.ts index 82c751c65d..4e569581e2 100644 --- a/src/lib/guardrails/visionBridge.ts +++ b/src/lib/guardrails/visionBridge.ts @@ -114,7 +114,11 @@ function extractLastUserText(messages: unknown[]): string | undefined { if (Array.isArray(message.content)) { for (const part of message.content) { const p = part as { type?: unknown; text?: unknown } | null | undefined; - if (p?.type === "text" && typeof p.text === "string" && p.text.trim()) { + if ( + (p?.type === "text" || p?.type === "input_text") && + typeof p.text === "string" && + p.text.trim() + ) { return p.text; } } @@ -211,10 +215,16 @@ export class VisionBridgeGuardrail extends BaseGuardrail { // remains undefined, which makes the reroute check on line ~189 treat it // like a non-combo model — exactly what we want: reroute to a vision model. - // 5. Get body and check for messages + // 5. Get body and normalize Chat Completions `messages` vs Responses `input`. + // Both containers carry role/content items and are supported by the shared + // media detector. Preserve the original wire container in modifiedPayload. const body = payload as Record; - const messages = body?.messages; - if (!Array.isArray(messages) || messages.length === 0) { + const messages = Array.isArray(body?.messages) + ? body.messages + : Array.isArray(body?.input) + ? body.input + : null; + if (!messages || messages.length === 0) { return { block: false }; } diff --git a/src/lib/guardrails/visionBridgeHelpers.ts b/src/lib/guardrails/visionBridgeHelpers.ts index 7841dd47db..dc1d729942 100644 --- a/src/lib/guardrails/visionBridgeHelpers.ts +++ b/src/lib/guardrails/visionBridgeHelpers.ts @@ -161,7 +161,9 @@ export interface RequestMessage { export type RequestContentPart = | { type: "text"; text: string } + | { type: "input_text"; text: string } | { type: "image_url"; image_url: { url: string; detail?: string } } + | { type: "input_image"; image_url: string; detail?: string } | { type: "image"; source: { type: "base64"; media_type: string; data: string } | { type: "url"; url: string }; @@ -815,6 +817,7 @@ async function callVisionModelSingle( export interface RequestBody { model?: string; messages?: RequestMessage[]; + input?: RequestMessage[]; [key: string]: unknown; } @@ -834,14 +837,23 @@ export function replaceImageParts( const result = structuredClone(body) as RequestBody; - if (!Array.isArray(result.messages)) { + const usesResponsesInput = !Array.isArray(result.messages) && Array.isArray(result.input); + const requestMessages = Array.isArray(result.messages) + ? result.messages + : usesResponsesInput + ? result.input + : null; + + if (!requestMessages) { return result; } + const replacementTextType: "text" | "input_text" = usesResponsesInput ? "input_text" : "text"; + let descriptionIndex = 0; - for (let msgIdx = 0; msgIdx < result.messages.length; msgIdx++) { - const message = result.messages[msgIdx]; + for (let msgIdx = 0; msgIdx < requestMessages.length; msgIdx++) { + const message = requestMessages[msgIdx]; if (!message || !Array.isArray(message.content)) { continue; } @@ -863,7 +875,10 @@ export function replaceImageParts( // image so a vision-capable upstream can still process it. newContent.push(part as RequestContentPart); } else { - newContent.push({ type: "text", text: description }); + newContent.push({ + type: replacementTextType, + text: description, + } as RequestContentPart); } } } else { diff --git a/tests/unit/guardrails/visionBridge-responses-9597.test.ts b/tests/unit/guardrails/visionBridge-responses-9597.test.ts new file mode 100644 index 0000000000..1ff788de00 --- /dev/null +++ b/tests/unit/guardrails/visionBridge-responses-9597.test.ts @@ -0,0 +1,446 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { VisionBridgeGuardrail } = await import("../../../src/lib/guardrails/visionBridge.ts"); +const { containsMediaKind } = await import("../../../open-sse/utils/mediaParts.ts"); + +import type { GuardrailContext } from "../../../src/lib/guardrails/base.ts"; +import type { VisionModelConfig } from "../../../src/lib/guardrails/visionBridgeHelpers.ts"; + +const IMAGE_DATA_URI = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + +test("#9597: Responses input/input_image is described before combo vision filtering", async () => { + let visionCallCount = 0; + let receivedImage = ""; + let receivedPrompt = ""; + + const guardrail = new VisionBridgeGuardrail({ + deps: { + getSettings: async () => ({ + modalityBridgeVisionEnabled: true, + modalityBridgeVisionMode: "describe", + modalityBridgeVisionModel: "openai/gpt-4o-mini", + modalityBridgeVisionTaskAware: true, + modalityBridgeVisionPrompt: "Describe this image concisely.", + modalityBridgeVisionTimeout: 30000, + modalityBridgeVisionMaxImages: 10, + modalityBridgeCacheEnabled: false, + }), + callVisionModel: async (imageDataUri: string, config: VisionModelConfig) => { + visionCallCount++; + receivedImage = imageDataUri; + receivedPrompt = config.prompt; + return "A green status badge reading PASS."; + }, + checkModelHasComboMapping: async () => true, + hasUsableCredentials: async () => true, + }, + }); + + const payload = { + model: "openai/gpt-4o", + input: [ + { + type: "message", + role: "user", + content: [ + { + type: "input_text", + text: "Read the status badge and report its text.", + }, + { + type: "input_image", + image_url: IMAGE_DATA_URI, + detail: "high", + }, + ], + }, + ], + stream: true, + }; + + const context = { + model: "openai/gpt-4o", + log: { + debug: () => undefined, + info: () => undefined, + warn: () => undefined, + }, + } as GuardrailContext; + + const result = await guardrail.preCall(payload, context); + + assert.equal(result.block, false); + assert.equal( + visionCallCount, + 1, + "Responses input/input_image should invoke the configured vision model once" + ); + assert.equal(receivedImage, IMAGE_DATA_URI); + assert.match( + receivedPrompt, + /Read the status badge and report its text/, + "task-aware prompting should read Responses input_text" + ); + + assert.ok(result.modifiedPayload, "Responses describe mode should return a transformed payload"); + + const modified = result.modifiedPayload as { + model?: string; + messages?: unknown; + input: Array<{ + role?: string; + content: Array<{ + type?: string; + text?: string; + image_url?: unknown; + }>; + }>; + }; + + assert.equal( + modified.model, + payload.model, + "describe mode must preserve the requested answer model" + ); + + assert.equal( + "messages" in modified, + false, + "Vision Bridge must preserve the native Responses request shape" + ); + + const content = modified.input[0]?.content ?? []; + + assert.equal( + content.some((part) => part.type === "input_image"), + false, + "raw input_image must be removed before combo compatibility filtering" + ); + + assert.equal( + content.some((part) => part.type === "text"), + false, + "Responses payload must not receive Chat-format text parts" + ); + + assert.equal(content[0]?.type, "input_text"); + assert.equal(content[0]?.text, "Read the status badge and report its text."); + + assert.equal(content[1]?.type, "input_text", "image description must use Responses input_text"); + + assert.match(content[1]?.text ?? "", /PASS/, "vision description should replace the image"); + + assert.equal( + containsMediaKind(modified.input, "image"), + false, + "shared combo media detector must see no image after Vision Bridge" + ); + + assert.equal( + JSON.stringify(modified).includes(IMAGE_DATA_URI), + false, + "raw image bytes must not reach the text-only combo target" + ); +}); + +function settings9597(): Record { + return { + modalityBridgeVisionEnabled: true, + modalityBridgeVisionMode: "describe", + modalityBridgeVisionModel: "openai/gpt-4o-mini", + modalityBridgeVisionTaskAware: true, + modalityBridgeVisionPrompt: "Describe this image concisely.", + modalityBridgeVisionTimeout: 30000, + modalityBridgeVisionMaxImages: 10, + modalityBridgeCacheEnabled: false, + }; +} + +function context9597(model: string): GuardrailContext { + return { + model, + log: { + debug: () => undefined, + info: () => undefined, + warn: () => undefined, + }, + } as GuardrailContext; +} + +test("#9597 matrix: Responses input without images remains untouched", async () => { + let visionCallCount = 0; + + const guardrail = new VisionBridgeGuardrail({ + deps: { + getSettings: async () => settings9597(), + callVisionModel: async () => { + visionCallCount++; + return "unexpected"; + }, + checkModelHasComboMapping: async () => true, + hasUsableCredentials: async () => true, + }, + }); + + const payload = { + model: "glm5.2", + input: [ + { + type: "message", + role: "user", + content: [ + { + type: "input_text", + text: "This request contains no image.", + }, + ], + }, + ], + stream: true, + }; + + const result = await guardrail.preCall(payload, context9597(payload.model)); + + assert.equal(result.block, false); + assert.equal(result.modifiedPayload, undefined); + assert.equal(visionCallCount, 0); +}); + +test("#9597 matrix: Chat Completions image path remains Chat-shaped", async () => { + let visionCallCount = 0; + + const guardrail = new VisionBridgeGuardrail({ + deps: { + getSettings: async () => settings9597(), + callVisionModel: async () => { + visionCallCount++; + return "A blue status badge."; + }, + checkModelHasComboMapping: async () => true, + hasUsableCredentials: async () => true, + }, + }); + + const payload = { + model: "glm5.2", + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "Describe the badge.", + }, + { + type: "image_url", + image_url: { + url: IMAGE_DATA_URI, + }, + }, + ], + }, + ], + }; + + const result = await guardrail.preCall(payload, context9597(payload.model)); + + assert.equal(result.block, false); + assert.equal(visionCallCount, 1); + assert.ok(result.modifiedPayload); + + const modified = result.modifiedPayload as { + model?: string; + input?: unknown; + messages: Array<{ + content: Array<{ + type?: string; + text?: string; + image_url?: unknown; + }>; + }>; + }; + + assert.equal(modified.model, payload.model); + assert.equal("input" in modified, false); + + const content = modified.messages[0]?.content ?? []; + + assert.deepEqual( + content.map((part) => part.type), + ["text", "text"] + ); + assert.equal(content[0]?.text, "Describe the badge."); + assert.match(content[1]?.text ?? "", /blue status badge/); + assert.equal(containsMediaKind(modified.messages, "image"), false); +}); + +test("#9597 matrix: Responses combo describe failure never leaks the raw image", async () => { + let visionCallCount = 0; + + const guardrail = new VisionBridgeGuardrail({ + deps: { + getSettings: async () => settings9597(), + callVisionModel: async () => { + visionCallCount++; + throw new Error("synthetic vision failure"); + }, + checkModelHasComboMapping: async () => true, + hasUsableCredentials: async () => true, + }, + }); + + const payload = { + model: "glm5.2", + input: [ + { + type: "message", + role: "user", + content: [ + { + type: "input_text", + text: "Read this image.", + }, + { + type: "input_image", + image_url: IMAGE_DATA_URI, + detail: "high", + }, + ], + }, + ], + }; + + const result = await guardrail.preCall(payload, context9597(payload.model)); + + assert.equal(result.block, false); + assert.equal(visionCallCount, 1); + assert.ok(result.modifiedPayload); + + const modified = result.modifiedPayload as { + model?: string; + messages?: unknown; + input: Array<{ + content: Array<{ + type?: string; + text?: string; + }>; + }>; + }; + + assert.equal(modified.model, payload.model); + assert.equal("messages" in modified, false); + + const content = modified.input[0]?.content ?? []; + + assert.equal( + content.some((part) => part.type === "input_image"), + false + ); + assert.equal(content[1]?.type, "input_text"); + assert.match(content[1]?.text ?? "", /unavailable/); + assert.equal(containsMediaKind(modified.input, "image"), false); + assert.equal(JSON.stringify(modified).includes(IMAGE_DATA_URI), false); +}); + +test("#9597 matrix: bridge transformation clears the real fail-closed combo vision gate", async () => { + const { + deriveRequestCompatibilityRequirements, + describeCapabilityFilterExhaustion, + filterTargetsByRequestCompatibility, + } = await import("../../../open-sse/services/combo/comboStructure.ts"); + + const target = { + kind: "model" as const, + stepId: "text-only", + executionKey: "text-only", + modelStr: "conol-web/deepseek/deepseek-v4-pro", + provider: "conol-web", + providerId: null, + connectionId: null, + weight: 0, + label: null, + }; + + const comboLog = { + info: () => undefined, + warn: () => undefined, + debug: () => undefined, + }; + + const rawPayload = { + model: "glm5.2", + input: [ + { + type: "message", + role: "user", + content: [ + { + type: "input_text", + text: "Read the badge.", + }, + { + type: "input_image", + image_url: IMAGE_DATA_URI, + detail: "high", + }, + ], + }, + ], + }; + + assert.equal(deriveRequestCompatibilityRequirements(rawPayload).requiresVision, true); + + const rawFiltered = filterTargetsByRequestCompatibility([target], rawPayload, comboLog); + + assert.equal( + rawFiltered.length, + 0, + "the existing fail-closed combo filter must still reject the raw image request" + ); + + const rawExhaustion = describeCapabilityFilterExhaustion([target], rawPayload, "glm5.2"); + + assert.ok(rawExhaustion); + assert.equal(rawExhaustion.terminalReason, "capability_mismatch"); + assert.match(rawExhaustion.message, /confirmed vision support/); + + const guardrail = new VisionBridgeGuardrail({ + deps: { + getSettings: async () => settings9597(), + callVisionModel: async (_imageDataUri: string, _config: VisionModelConfig) => + "A green badge reading PASS.", + checkModelHasComboMapping: async () => true, + hasUsableCredentials: async () => true, + }, + }); + + const result = await guardrail.preCall(rawPayload, context9597(rawPayload.model)); + + assert.equal(result.block, false); + assert.ok(result.modifiedPayload); + + const modified = result.modifiedPayload as Record; + + assert.equal( + deriveRequestCompatibilityRequirements(modified).requiresVision, + false, + "Vision Bridge must remove the image requirement before combo filtering" + ); + + const filteredAfterBridge = filterTargetsByRequestCompatibility([target], modified, comboLog); + + assert.equal(filteredAfterBridge.length, 1); + assert.equal(filteredAfterBridge[0]?.modelStr, target.modelStr); + + const exhaustionAfterBridge = describeCapabilityFilterExhaustion([target], modified, "glm5.2"); + + assert.equal( + exhaustionAfterBridge, + null, + "the transformed request must not produce capability_mismatch" + ); +}); + +/* 9597-MATRIX-END */