diff --git a/docs/security/GUARDRAILS.md b/docs/security/GUARDRAILS.md index a6e5511935..74784cb7b8 100644 --- a/docs/security/GUARDRAILS.md +++ b/docs/security/GUARDRAILS.md @@ -107,6 +107,26 @@ fragment the cache. Failed describes are never cached. Settings: | `modalityBridgeCacheTtlMinutes` | `60` | 1–1440 | | `modalityBridgeCacheMaxEntries` | `200` | 10–5000 | +#### Remote image normalization (self-loop describe/base64 fetch) + +When the bridge fetches a **remote** image itself — the Anthropic describe +self-call and the claude-wire-format base64 conversion +(`ensureBase64ImagesForClaudeWire`), both via +`fetchRemoteImageAsDataUri()` in `visionBridgeHelpers.ts` — the resulting data +URI is passed through `normalizeDataUri()` +(`open-sse/utils/imageNormalize.ts`) before being embedded in the vision-model +request. Oversized images are downscaled to a **2048px long edge** (matching +the resize cap OpenAI/Anthropic already apply server-side), which cuts +upload bytes/latency without changing what the vision model sees. Resizing +uses `sharp`, loaded via dynamic import: on a platform where its native +binary fails to load, `normalizeDataUri()` **never throws** — it falls back +to a passthrough of the original bytes, so the describe/base64-conversion +path always keeps working. Non-image bytes (a fetch that did not return a +decodable image) are also passed through untouched. This normalization is +scoped to images the bridge fetches for its own self-call — it is never +applied to the caller's raw passthrough payload, consistent with the +opt-in-only mutation principle (Hard Rule #20). + #### Settings schema + migration The new `modalityBridge*` keys are Zod-validated in `updateSettingsSchema` diff --git a/src/lib/guardrails/visionBridgeHelpers.ts b/src/lib/guardrails/visionBridgeHelpers.ts index dc1d729942..e8b0b7e3bc 100644 --- a/src/lib/guardrails/visionBridgeHelpers.ts +++ b/src/lib/guardrails/visionBridgeHelpers.ts @@ -2,6 +2,7 @@ * Vision Bridge helper functions for image processing. */ import { detectMediaParts, type MediaPart } from "@omniroute/open-sse/utils/mediaParts"; +import { normalizeDataUri } from "@omniroute/open-sse/utils/imageNormalize"; import { fetchRemoteImage } from "@/shared/network/remoteImageFetch"; import { getRuntimePorts } from "@/lib/runtime/ports"; import { resolveSelfLoopBearer } from "@/shared/middleware/chatBodyAdmission"; @@ -314,7 +315,12 @@ async function fetchRemoteImageAsDataUri( fetchImpl, }); const mediaType = remoteImage.contentType.split(";")[0]?.trim() || "image/png"; - return `data:${mediaType};base64,${remoteImage.buffer.toString("base64")}`; + const dataUri = `data:${mediaType};base64,${remoteImage.buffer.toString("base64")}`; + // Downscale to the long-edge cap before handing the image to the vision + // model self-call — scoped to this bridge-fetched image only, never the + // user's raw passthrough payload (opt-in principle, HR#20). + // `normalizeDataUri` never throws and is a passthrough for non-image bytes. + return normalizeDataUri(dataUri); } async function normalizeVisionImageInput( diff --git a/tests/unit/vision-bridge-image-normalize.test.ts b/tests/unit/vision-bridge-image-normalize.test.ts new file mode 100644 index 0000000000..dc8fd4896f --- /dev/null +++ b/tests/unit/vision-bridge-image-normalize.test.ts @@ -0,0 +1,95 @@ +/** + * Task B2: the vision bridge self-loop fetches a remote image and hands it + * to the vision model as a data URI (`fetchRemoteImageAsDataUri`, + * `src/lib/guardrails/visionBridgeHelpers.ts`). That fetched image must be + * normalized (long-edge cap 2048, `@omniroute/open-sse/utils/imageNormalize`) + * before being embedded — the same treatment `normalizeDataUri` already + * gives any other image, now applied to remote fetches performed by the + * bridge itself. Scope: ONLY this self-call path, never the user's raw + * passthrough payload (HR#20 opt-in principle). + * + * `ensureBase64ImagesForClaudeWire` is the exported entry point that reaches + * the private `fetchRemoteImageAsDataUri` — it resolves every non-data-URI + * image part of a claude-wire-format request via that same fetch helper, so + * it is the smallest public surface to exercise the fetch → normalize path + * with dependency-injected `fetchImpl` (mirrors the DI pattern used by + * `tests/unit/vision-bridge-describe-cache.test.ts` and + * `tests/unit/remote-image-fetch.test.ts`). + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { ensureBase64ImagesForClaudeWire } from "../../src/lib/guardrails/visionBridgeHelpers.ts"; + +// zai speaks the claude wire format (open-sse/config/providers/registry/zai/index.ts), +// so `isClaudeWireFormatModel` routes it through the base64 self-fetch path. +const CLAUDE_WIRE_MODEL = "zai/glm-4.6"; + +function bodyWithRemoteImage(url: string) { + return { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "describe this" }, + { type: "image_url", image_url: { url } }, + ], + }, + ], + }; +} + +test("remote image fetched for the claude-wire self-call is downscaled to the long-edge cap", async (t) => { + let sharp: typeof import("sharp"); + try { + sharp = (await import("sharp")).default as never; + } catch { + t.skip("sharp not installed"); + return; + } + const big = await sharp({ create: { width: 4096, height: 100, channels: 3, background: "#fff" } }) + .png() + .toBuffer(); + + const fetchImpl = (async () => + new Response(big, { + status: 200, + headers: { "content-type": "image/png" }, + })) as unknown as typeof fetch; + + const result = await ensureBase64ImagesForClaudeWire( + bodyWithRemoteImage("https://example.com/big.png"), + CLAUDE_WIRE_MODEL, + fetchImpl + ); + + const imagePart = (result.messages?.[0]?.content as Array<{ image_url?: { url: string } }>)[1]; + const dataUri = imagePart?.image_url?.url ?? ""; + assert.match(dataUri, /^data:image\/png;base64,/); + + const b64 = dataUri.split(",")[1] ?? ""; + const decoded = Buffer.from(b64, "base64"); + const meta = await sharp(decoded).metadata(); + assert.ok((meta.width ?? 0) <= 2048, `expected width <= 2048, got ${meta.width}`); + assert.notEqual(meta.width, 4096, "image must have been downscaled, not left at 4096"); +}); + +test("remote non-image bytes pass through untouched (fail-open, no normalization)", async () => { + const junk = Buffer.from("not-an-image-at-all"); + + const fetchImpl = (async () => + new Response(junk, { + status: 200, + headers: { "content-type": "application/octet-stream" }, + })) as unknown as typeof fetch; + + const result = await ensureBase64ImagesForClaudeWire( + bodyWithRemoteImage("https://example.com/junk.bin"), + CLAUDE_WIRE_MODEL, + fetchImpl + ); + + const imagePart = (result.messages?.[0]?.content as Array<{ image_url?: { url: string } }>)[1]; + const dataUri = imagePart?.image_url?.url ?? ""; + assert.equal(dataUri, `data:application/octet-stream;base64,${junk.toString("base64")}`); +});