diff --git a/docs/security/GUARDRAILS.md b/docs/security/GUARDRAILS.md index a199f2af0b..f20cb80527 100644 --- a/docs/security/GUARDRAILS.md +++ b/docs/security/GUARDRAILS.md @@ -93,6 +93,19 @@ describe prompt, steering the description toward what the user actually asked (codex-vision-proxy pattern) and asking the vision model to transcribe visible text. With the flag off — or no user text — the base prompt is used unchanged. +The describe self-loop's own OpenAI-compatible request (`callVisionModelSingle()` +in `visionBridgeHelpers.ts`) always requests `image_url.detail: "high"` — +unconditionally, for every caller/provider, not gated on any client signal. +Low-detail sampling degrades OCR accuracy for exactly the text-transcription +task this prompt asks for, so the describe call itself always asks for high +detail regardless of what detail level the original inbound request used. This +only affects the internal describe request body; it does not change how +OmniRoute forwards the caller's own `image_url.detail` on the primary request — +that default is applied separately, and only for detected OpenCode clients, in +`defaultImageDetail()` (`open-sse/handlers/chatCore/upstreamBody.ts`). The +Anthropic wire-format branch of the describe self-loop has no `detail` field +and is unaffected by either default. + #### Describe output cap (`modalityBridgeVisionMaxChars`) | Key | Default | Range | diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 67185c80d1..351453c5b0 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -2892,6 +2892,7 @@ export async function handleChatCore({ credentials, log, bypassDefaultToolLimit: isOpencodeClient, + isOpencodeClient, }); updatePendingScope(pendingScope, { diff --git a/open-sse/handlers/chatCore/upstreamBody.ts b/open-sse/handlers/chatCore/upstreamBody.ts index f2d8368c8c..52d1ddcc1b 100644 --- a/open-sse/handlers/chatCore/upstreamBody.ts +++ b/open-sse/handlers/chatCore/upstreamBody.ts @@ -87,6 +87,76 @@ function truncateToolList( return bodyToSend; } +// OpenCode's AI SDK file-part serializer omits `image_url.detail`, which makes wide, text-dense +// screenshots fall back to low-detail vision sampling upstream. Gated on `isOpencodeClient` (the +// request's User-Agent / `x-opencode-*` header signal, not the `provider` field — `provider` is +// the upstream target and can be anything regardless of which client sent the request) so this +// override doesn't change the detail default for non-OpenCode callers on any provider. +function defaultImageDetail(bodyToSend: Body, isOpencodeClient: boolean): Body { + if (!isOpencodeClient) return bodyToSend; + + let nextBody = bodyToSend; + + if (Array.isArray(bodyToSend.messages)) { + const messages = bodyToSend.messages.map((message) => { + if (!message || typeof message !== "object" || Array.isArray(message)) return message; + const messageRecord = message as Record; + if (!Array.isArray(messageRecord.content)) return message; + + let changed = false; + const content = messageRecord.content.map((part) => { + if (!part || typeof part !== "object" || Array.isArray(part)) return part; + const partRecord = part as Record; + const imageUrl = partRecord.image_url; + if ( + partRecord.type !== "image_url" || + !imageUrl || + typeof imageUrl !== "object" || + Array.isArray(imageUrl) + ) { + return part; + } + + const imageUrlRecord = imageUrl as Record; + if (imageUrlRecord.detail !== undefined) return part; + changed = true; + return { ...partRecord, image_url: { ...imageUrlRecord, detail: "high" } }; + }); + + return changed ? { ...messageRecord, content } : message; + }); + + if (messages.some((message, index) => message !== bodyToSend.messages?.[index])) { + nextBody = { ...nextBody, messages }; + } + } + + if (Array.isArray(bodyToSend.input)) { + const input = bodyToSend.input.map((item) => { + if (!item || typeof item !== "object" || Array.isArray(item)) return item; + const itemRecord = item as Record; + if (!Array.isArray(itemRecord.content)) return item; + + let changed = false; + const content = itemRecord.content.map((part) => { + if (!part || typeof part !== "object" || Array.isArray(part)) return part; + const partRecord = part as Record; + if (partRecord.type !== "input_image" || partRecord.detail !== undefined) return part; + changed = true; + return { ...partRecord, detail: "high" }; + }); + + return changed ? { ...itemRecord, content } : item; + }); + + if (input.some((item, index) => item !== bodyToSend.input?.[index])) { + nextBody = { ...nextBody, input }; + } + } + + return nextBody; +} + // Inject prompt_cache_key only for providers that support it. async function injectPromptCacheKey( bodyToSend: Body, @@ -117,6 +187,7 @@ export async function prepareUpstreamBody(opts: { targetFormat: string; credentials: CredentialsLike; bypassDefaultToolLimit?: boolean; + isOpencodeClient?: boolean; log?: LoggerLike; }): Promise { const { @@ -126,6 +197,7 @@ export async function prepareUpstreamBody(opts: { targetFormat, credentials, bypassDefaultToolLimit = false, + isOpencodeClient = false, log, } = opts; @@ -157,6 +229,7 @@ export async function prepareUpstreamBody(opts: { model: payloadRuleModel, log, }); + bodyToSend = defaultImageDetail(bodyToSend, isOpencodeClient); bodyToSend = truncateToolList(bodyToSend, provider, bypassDefaultToolLimit ?? false, log); const connectionCacheOverride = resolveConnectionCacheOverride(credentials?.providerSpecificData); bodyToSend = await injectPromptCacheKey( diff --git a/src/lib/guardrails/visionBridgeHelpers.ts b/src/lib/guardrails/visionBridgeHelpers.ts index 9310cef608..bd90380613 100644 --- a/src/lib/guardrails/visionBridgeHelpers.ts +++ b/src/lib/guardrails/visionBridgeHelpers.ts @@ -788,10 +788,26 @@ async function callVisionModelSingle( role: "user", content: [ { + // Global, not OpenCode-scoped: this is OmniRoute's own internal + // describe self-loop (VisionBridgeGuardrail), called for every + // caller/provider when the target model lacks vision support — + // there is no client-identity signal at this layer to gate on + // (unlike the OpenCode-only `isOpencodeClient` default in + // `chatCore/upstreamBody.ts`, which forwards the *caller's own* + // image_url.detail and is deliberately scoped). "high" is + // requested unconditionally because the describe prompt asks + // the vision model to transcribe visible text + // (`modalityBridgeVisionTaskAware`, see docs/security/GUARDRAILS.md) + // — low-detail sampling degrades OCR accuracy for every + // describe call, not just OpenCode-originated ones. This path + // only affects the OpenAI-compatible wire format branch; the + // Anthropic branch above has no `detail` concept and is + // unaffected (see the "unaffected for Anthropic" compat + // assertion in visionBridgeHelpers.callVisionModel.test.ts). type: "image_url", image_url: { url: normalizedImageInput, - detail: "low", + detail: "high", }, }, { type: "text", text: config.prompt }, diff --git a/tests/unit/chatcore-upstream-body.test.ts b/tests/unit/chatcore-upstream-body.test.ts index 685a545fb2..2f1fe135a9 100644 --- a/tests/unit/chatcore-upstream-body.test.ts +++ b/tests/unit/chatcore-upstream-body.test.ts @@ -50,6 +50,83 @@ test("leaves the model untouched when it already matches", async () => { assert.equal(out.model, "model-a"); }); +test("defaults OpenAI image inputs to high detail for OpenCode clients without overriding explicit detail", async () => { + const out = await prepareUpstreamBody({ + translatedBody: { + model: "model-a", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Read this screenshot" }, + { type: "image_url", image_url: { url: "data:image/png;base64,test" } }, + { + type: "image_url", + image_url: { url: "data:image/png;base64,test", detail: "low" }, + }, + ], + }, + ], + }, + modelToCall: "model-a", + provider: "opencode-zen", + targetFormat: FORMATS.OPENAI, + credentials: null, + isOpencodeClient: true, + }); + + const content = ( + out.messages as Array<{ content: Array<{ image_url?: { detail?: string } }> }> + )[0].content; + assert.equal(content[1].image_url?.detail, "high"); + assert.equal(content[2].image_url?.detail, "low"); +}); + +test("defaults Responses input images to high detail for OpenCode clients", async () => { + const out = await prepareUpstreamBody({ + translatedBody: { + model: "model-a", + input: [ + { + role: "user", + content: [{ type: "input_image", image_url: "data:image/png;base64,test" }], + }, + ], + }, + modelToCall: "model-a", + provider: "opencode-zen", + targetFormat: FORMATS.OPENAI_RESPONSES, + credentials: null, + isOpencodeClient: true, + }); + + const content = (out.input as Array<{ content: Array<{ detail?: string }> }>)[0].content; + assert.equal(content[0].detail, "high"); +}); + +test("leaves image detail untouched for non-OpenCode clients on the same provider", async () => { + const out = await prepareUpstreamBody({ + translatedBody: { + model: "model-a", + messages: [ + { + role: "user", + content: [{ type: "image_url", image_url: { url: "data:image/png;base64,test" } }], + }, + ], + }, + modelToCall: "model-a", + provider: "opencode-zen", + targetFormat: FORMATS.OPENAI, + credentials: null, + }); + + const content = ( + out.messages as Array<{ content: Array<{ image_url?: { detail?: string } }> }> + )[0].content; + assert.equal(content[0].image_url?.detail, undefined); +}); + test("strips Codex GPT-5 verbosity after routing resolves to opencode-go/GLM", async () => { const translatedBody = { model: "glm-5.2", diff --git a/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts b/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts index 820e139f51..0303a9a885 100644 --- a/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts +++ b/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts @@ -306,7 +306,7 @@ test("callVisionModel uses correct request body format", async () => { }; assert.strictEqual(imagePart.type, "image_url"); assert.strictEqual(imagePart.image_url.url, imageUri); - assert.strictEqual(imagePart.image_url.detail, "low"); + assert.strictEqual(imagePart.image_url.detail, "high"); // Second content is text prompt const textPart = message.content[1] as { type: string; text: string }; @@ -358,10 +358,17 @@ test("callVisionModel fetches remote images before Anthropic requests", async () assert.strictEqual(fetchCalls[1].url, "https://api.anthropic.com/v1/messages"); const anthropicBody = JSON.parse(fetchCalls[1].init?.body as string); - const imageSource = anthropicBody.messages[0].content[0].source; + const imagePart = anthropicBody.messages[0].content[0]; + const imageSource = imagePart.source; assert.strictEqual(imageSource.type, "base64"); assert.strictEqual(imageSource.media_type, "image/png"); assert.strictEqual(imageSource.data, Buffer.from("cat-image-bytes").toString("base64")); + // Compatibility guard for the global (not OpenCode-scoped) `detail: "high"` + // default added to the OpenAI-compatible describe path: Anthropic's wire + // format has no `detail` concept, so the describe self-loop must not leak + // an OpenAI-only field into the Anthropic request body. + assert.strictEqual(imagePart.detail, undefined); + assert.strictEqual(imageSource.detail, undefined); } finally { globalThis.fetch = originalFetch; }