mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-14 03:02:14 +03:00
feat(bridge): normalize fetched images before vision describe self-call
Route the bridge's own fetchRemoteImageAsDataUri() output through normalizeDataUri() (long-edge cap 2048) before handing it to the vision model — matches the resize cap OpenAI/Anthropic already apply, cutting upload bytes/latency. Scoped to the bridge's self-fetched images only, never the user's raw passthrough payload (HR#20 opt-in principle).
This commit is contained in:
@@ -107,6 +107,26 @@ fragment the cache. Failed describes are never cached. Settings:
|
||||
| `modalityBridgeCacheTtlMinutes` | `60` | 1–1440 |
|
||||
| `modalityBridgeCacheMaxEntries` | `200` | 10–5000 |
|
||||
|
||||
#### Remote image normalization (self-loop describe/base64 fetch)
|
||||
|
||||
When the bridge fetches a **remote** image itself — the Anthropic describe
|
||||
self-call and the claude-wire-format base64 conversion
|
||||
(`ensureBase64ImagesForClaudeWire`), both via
|
||||
`fetchRemoteImageAsDataUri()` in `visionBridgeHelpers.ts` — the resulting data
|
||||
URI is passed through `normalizeDataUri()`
|
||||
(`open-sse/utils/imageNormalize.ts`) before being embedded in the vision-model
|
||||
request. Oversized images are downscaled to a **2048px long edge** (matching
|
||||
the resize cap OpenAI/Anthropic already apply server-side), which cuts
|
||||
upload bytes/latency without changing what the vision model sees. Resizing
|
||||
uses `sharp`, loaded via dynamic import: on a platform where its native
|
||||
binary fails to load, `normalizeDataUri()` **never throws** — it falls back
|
||||
to a passthrough of the original bytes, so the describe/base64-conversion
|
||||
path always keeps working. Non-image bytes (a fetch that did not return a
|
||||
decodable image) are also passed through untouched. This normalization is
|
||||
scoped to images the bridge fetches for its own self-call — it is never
|
||||
applied to the caller's raw passthrough payload, consistent with the
|
||||
opt-in-only mutation principle (Hard Rule #20).
|
||||
|
||||
#### Settings schema + migration
|
||||
|
||||
The new `modalityBridge*` keys are Zod-validated in `updateSettingsSchema`
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
* Vision Bridge helper functions for image processing.
|
||||
*/
|
||||
import { detectMediaParts, type MediaPart } from "@omniroute/open-sse/utils/mediaParts";
|
||||
import { normalizeDataUri } from "@omniroute/open-sse/utils/imageNormalize";
|
||||
import { fetchRemoteImage } from "@/shared/network/remoteImageFetch";
|
||||
import { getRuntimePorts } from "@/lib/runtime/ports";
|
||||
import { resolveSelfLoopBearer } from "@/shared/middleware/chatBodyAdmission";
|
||||
@@ -314,7 +315,12 @@ async function fetchRemoteImageAsDataUri(
|
||||
fetchImpl,
|
||||
});
|
||||
const mediaType = remoteImage.contentType.split(";")[0]?.trim() || "image/png";
|
||||
return `data:${mediaType};base64,${remoteImage.buffer.toString("base64")}`;
|
||||
const dataUri = `data:${mediaType};base64,${remoteImage.buffer.toString("base64")}`;
|
||||
// Downscale to the long-edge cap before handing the image to the vision
|
||||
// model self-call — scoped to this bridge-fetched image only, never the
|
||||
// user's raw passthrough payload (opt-in principle, HR#20).
|
||||
// `normalizeDataUri` never throws and is a passthrough for non-image bytes.
|
||||
return normalizeDataUri(dataUri);
|
||||
}
|
||||
|
||||
async function normalizeVisionImageInput(
|
||||
|
||||
95
tests/unit/vision-bridge-image-normalize.test.ts
Normal file
95
tests/unit/vision-bridge-image-normalize.test.ts
Normal file
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* Task B2: the vision bridge self-loop fetches a remote image and hands it
|
||||
* to the vision model as a data URI (`fetchRemoteImageAsDataUri`,
|
||||
* `src/lib/guardrails/visionBridgeHelpers.ts`). That fetched image must be
|
||||
* normalized (long-edge cap 2048, `@omniroute/open-sse/utils/imageNormalize`)
|
||||
* before being embedded — the same treatment `normalizeDataUri` already
|
||||
* gives any other image, now applied to remote fetches performed by the
|
||||
* bridge itself. Scope: ONLY this self-call path, never the user's raw
|
||||
* passthrough payload (HR#20 opt-in principle).
|
||||
*
|
||||
* `ensureBase64ImagesForClaudeWire` is the exported entry point that reaches
|
||||
* the private `fetchRemoteImageAsDataUri` — it resolves every non-data-URI
|
||||
* image part of a claude-wire-format request via that same fetch helper, so
|
||||
* it is the smallest public surface to exercise the fetch → normalize path
|
||||
* with dependency-injected `fetchImpl` (mirrors the DI pattern used by
|
||||
* `tests/unit/vision-bridge-describe-cache.test.ts` and
|
||||
* `tests/unit/remote-image-fetch.test.ts`).
|
||||
*/
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { ensureBase64ImagesForClaudeWire } from "../../src/lib/guardrails/visionBridgeHelpers.ts";
|
||||
|
||||
// zai speaks the claude wire format (open-sse/config/providers/registry/zai/index.ts),
|
||||
// so `isClaudeWireFormatModel` routes it through the base64 self-fetch path.
|
||||
const CLAUDE_WIRE_MODEL = "zai/glm-4.6";
|
||||
|
||||
function bodyWithRemoteImage(url: string) {
|
||||
return {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "describe this" },
|
||||
{ type: "image_url", image_url: { url } },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
test("remote image fetched for the claude-wire self-call is downscaled to the long-edge cap", async (t) => {
|
||||
let sharp: typeof import("sharp");
|
||||
try {
|
||||
sharp = (await import("sharp")).default as never;
|
||||
} catch {
|
||||
t.skip("sharp not installed");
|
||||
return;
|
||||
}
|
||||
const big = await sharp({ create: { width: 4096, height: 100, channels: 3, background: "#fff" } })
|
||||
.png()
|
||||
.toBuffer();
|
||||
|
||||
const fetchImpl = (async () =>
|
||||
new Response(big, {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" },
|
||||
})) as unknown as typeof fetch;
|
||||
|
||||
const result = await ensureBase64ImagesForClaudeWire(
|
||||
bodyWithRemoteImage("https://example.com/big.png"),
|
||||
CLAUDE_WIRE_MODEL,
|
||||
fetchImpl
|
||||
);
|
||||
|
||||
const imagePart = (result.messages?.[0]?.content as Array<{ image_url?: { url: string } }>)[1];
|
||||
const dataUri = imagePart?.image_url?.url ?? "";
|
||||
assert.match(dataUri, /^data:image\/png;base64,/);
|
||||
|
||||
const b64 = dataUri.split(",")[1] ?? "";
|
||||
const decoded = Buffer.from(b64, "base64");
|
||||
const meta = await sharp(decoded).metadata();
|
||||
assert.ok((meta.width ?? 0) <= 2048, `expected width <= 2048, got ${meta.width}`);
|
||||
assert.notEqual(meta.width, 4096, "image must have been downscaled, not left at 4096");
|
||||
});
|
||||
|
||||
test("remote non-image bytes pass through untouched (fail-open, no normalization)", async () => {
|
||||
const junk = Buffer.from("not-an-image-at-all");
|
||||
|
||||
const fetchImpl = (async () =>
|
||||
new Response(junk, {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/octet-stream" },
|
||||
})) as unknown as typeof fetch;
|
||||
|
||||
const result = await ensureBase64ImagesForClaudeWire(
|
||||
bodyWithRemoteImage("https://example.com/junk.bin"),
|
||||
CLAUDE_WIRE_MODEL,
|
||||
fetchImpl
|
||||
);
|
||||
|
||||
const imagePart = (result.messages?.[0]?.content as Array<{ image_url?: { url: string } }>)[1];
|
||||
const dataUri = imagePart?.image_url?.url ?? "";
|
||||
assert.equal(dataUri, `data:application/octet-stream;base64,${junk.toString("base64")}`);
|
||||
});
|
||||
Reference in New Issue
Block a user