Compare commits

...

3 Commits

Author SHA1 Message Date
Xiangzhe
f139a107f2 test(bridge): height-dominant long-edge coverage
Add a 100x4096 PNG case to image-normalize.test.ts alongside the existing
width-dominant one, so normalizeImageBuffer's long-edge cap is proven on
both axes.
2026-08-13 16:41:55 -03:00
Xiangzhe
89dee4a2df feat(bridge): normalize fetched images before vision describe self-call
Route the bridge's own fetchRemoteImageAsDataUri() output through
normalizeDataUri() (long-edge cap 2048) before handing it to the vision
model — matches the resize cap OpenAI/Anthropic already apply, cutting
upload bytes/latency. Scoped to the bridge's self-fetched images only,
never the user's raw passthrough payload (HR#20 opt-in principle).
2026-08-13 16:41:50 -03:00
Xiangzhe
e04a91aa18 feat(bridge): optional-sharp image normalization util (long-edge 2048) 2026-08-13 16:18:33 -03:00
5 changed files with 236 additions and 1 deletions

View File

@@ -107,6 +107,26 @@ fragment the cache. Failed describes are never cached. Settings:
| `modalityBridgeCacheTtlMinutes` | `60` | 11440 |
| `modalityBridgeCacheMaxEntries` | `200` | 105000 |
#### Remote image normalization (self-loop describe/base64 fetch)
When the bridge fetches a **remote** image itself — the Anthropic describe
self-call and the claude-wire-format base64 conversion
(`ensureBase64ImagesForClaudeWire`), both via
`fetchRemoteImageAsDataUri()` in `visionBridgeHelpers.ts` — the resulting data
URI is passed through `normalizeDataUri()`
(`open-sse/utils/imageNormalize.ts`) before being embedded in the vision-model
request. Oversized images are downscaled to a **2048px long edge** (matching
the resize cap OpenAI/Anthropic already apply server-side), which cuts
upload bytes/latency without changing what the vision model sees. Resizing
uses `sharp`, loaded via dynamic import: on a platform where its native
binary fails to load, `normalizeDataUri()` **never throws** — it falls back
to a passthrough of the original bytes, so the describe/base64-conversion
path always keeps working. Non-image bytes (a fetch that did not return a
decodable image) are also passed through untouched. This normalization is
scoped to images the bridge fetches for its own self-call — it is never
applied to the caller's raw passthrough payload, consistent with the
opt-in-only mutation principle (Hard Rule #20).
#### Settings schema + migration
The new `modalityBridge*` keys are Zod-validated in `updateSettingsSchema`

View File

@@ -0,0 +1,62 @@
/**
* Optional-sharp image normalization.
*
* Rationale (migrated from freellmapi `server/src/lib/image-normalize.ts:40-58`):
* OpenAI resizes images to a long-edge cap of 2048px server-side, Anthropic applies
* a similar cap. Downscaling client-side before upload reduces tokens/latency without
* changing model behavior. `sharp` is loaded via dynamic import so that a platform
* where its native binary fails to load never crashes the request path — it just
* falls back to a passthrough (original buffer, unresized).
*/
const DEFAULT_MAX_LONG_EDGE = 2048;
type SharpModule = typeof import("sharp");
let sharpPromise: Promise<SharpModule | null> | null = null;
async function loadSharp(): Promise<SharpModule | null> {
if (!sharpPromise) {
sharpPromise = import("sharp").then((m) => (m.default ?? m) as SharpModule).catch(() => null);
}
return sharpPromise;
}
export async function normalizeImageBuffer(
input: Buffer,
opts?: { maxLongEdge?: number }
): Promise<{ buffer: Buffer; mime: string | null; resized: boolean }> {
const maxLongEdge = opts?.maxLongEdge ?? DEFAULT_MAX_LONG_EDGE;
const sharp = await loadSharp();
if (!sharp) return { buffer: input, mime: null, resized: false };
try {
const img = sharp(input, { failOn: "error" });
const meta = await img.metadata();
const long = Math.max(meta.width ?? 0, meta.height ?? 0);
if (!long || long <= maxLongEdge) {
return { buffer: input, mime: meta.format ? `image/${meta.format}` : null, resized: false };
}
const buffer = await img
.resize({ width: maxLongEdge, height: maxLongEdge, fit: "inside", withoutEnlargement: true })
.toBuffer();
return { buffer, mime: meta.format ? `image/${meta.format}` : null, resized: true };
} catch {
return { buffer: input, mime: null, resized: false };
}
}
export async function normalizeDataUri(
dataUri: string,
opts?: { maxLongEdge?: number }
): Promise<string> {
try {
const match = /^data:([^;,]+);base64,(.*)$/s.exec(dataUri);
if (!match) return dataUri;
const input = Buffer.from(match[2], "base64");
if (!input.length) return dataUri;
const out = await normalizeImageBuffer(input, opts);
if (!out.resized) return dataUri;
return `data:${match[1]};base64,${out.buffer.toString("base64")}`;
} catch {
return dataUri;
}
}

View File

@@ -2,6 +2,7 @@
* Vision Bridge helper functions for image processing.
*/
import { detectMediaParts, type MediaPart } from "@omniroute/open-sse/utils/mediaParts";
import { normalizeDataUri } from "@omniroute/open-sse/utils/imageNormalize";
import { fetchRemoteImage } from "@/shared/network/remoteImageFetch";
import { getRuntimePorts } from "@/lib/runtime/ports";
import { resolveSelfLoopBearer } from "@/shared/middleware/chatBodyAdmission";
@@ -314,7 +315,12 @@ async function fetchRemoteImageAsDataUri(
fetchImpl,
});
const mediaType = remoteImage.contentType.split(";")[0]?.trim() || "image/png";
return `data:${mediaType};base64,${remoteImage.buffer.toString("base64")}`;
const dataUri = `data:${mediaType};base64,${remoteImage.buffer.toString("base64")}`;
// Downscale to the long-edge cap before handing the image to the vision
// model self-call — scoped to this bridge-fetched image only, never the
// user's raw passthrough payload (opt-in principle, HR#20).
// `normalizeDataUri` never throws and is a passthrough for non-image bytes.
return normalizeDataUri(dataUri);
}
async function normalizeVisionImageInput(

View File

@@ -0,0 +1,52 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { normalizeImageBuffer, normalizeDataUri } from "../../open-sse/utils/imageNormalize.ts";
test("passthrough when input is not a decodable image (sharp absent or garbage bytes)", async () => {
const junk = Buffer.from("not-an-image");
const out = await normalizeImageBuffer(junk);
assert.equal(out.resized, false);
assert.ok(out.buffer.equals(junk));
});
test("normalizeDataUri never throws and preserves the uri on failure", async () => {
const uri = "data:image/png;base64,%%%broken%%%";
assert.equal(await normalizeDataUri(uri), uri);
});
// Só roda quando sharp estiver instalado (optionalDependency presente no devbox):
test("downscales a large PNG to the long-edge cap when sharp is available", async (t) => {
let sharp: typeof import("sharp");
try {
sharp = (await import("sharp")).default as never;
} catch {
t.skip("sharp not installed");
return;
}
const big = await sharp({ create: { width: 4096, height: 100, channels: 3, background: "#fff" } })
.png()
.toBuffer();
const out = await normalizeImageBuffer(big, { maxLongEdge: 2048 });
assert.equal(out.resized, true);
const meta = await sharp(out.buffer).metadata();
assert.equal(meta.width, 2048);
});
test("downscales a height-dominant PNG to the long-edge cap on the height axis", async (t) => {
let sharp: typeof import("sharp");
try {
sharp = (await import("sharp")).default as never;
} catch {
t.skip("sharp not installed");
return;
}
const tall = await sharp({
create: { width: 100, height: 4096, channels: 3, background: "#fff" },
})
.png()
.toBuffer();
const out = await normalizeImageBuffer(tall, { maxLongEdge: 2048 });
assert.equal(out.resized, true);
const meta = await sharp(out.buffer).metadata();
assert.equal(meta.height, 2048);
});

View File

@@ -0,0 +1,95 @@
/**
* Task B2: the vision bridge self-loop fetches a remote image and hands it
* to the vision model as a data URI (`fetchRemoteImageAsDataUri`,
* `src/lib/guardrails/visionBridgeHelpers.ts`). That fetched image must be
* normalized (long-edge cap 2048, `@omniroute/open-sse/utils/imageNormalize`)
* before being embedded — the same treatment `normalizeDataUri` already
* gives any other image, now applied to remote fetches performed by the
* bridge itself. Scope: ONLY this self-call path, never the user's raw
* passthrough payload (HR#20 opt-in principle).
*
* `ensureBase64ImagesForClaudeWire` is the exported entry point that reaches
* the private `fetchRemoteImageAsDataUri` — it resolves every non-data-URI
* image part of a claude-wire-format request via that same fetch helper, so
* it is the smallest public surface to exercise the fetch → normalize path
* with dependency-injected `fetchImpl` (mirrors the DI pattern used by
* `tests/unit/vision-bridge-describe-cache.test.ts` and
* `tests/unit/remote-image-fetch.test.ts`).
*/
import { test } from "node:test";
import assert from "node:assert/strict";
import { ensureBase64ImagesForClaudeWire } from "../../src/lib/guardrails/visionBridgeHelpers.ts";
// zai speaks the claude wire format (open-sse/config/providers/registry/zai/index.ts),
// so `isClaudeWireFormatModel` routes it through the base64 self-fetch path.
const CLAUDE_WIRE_MODEL = "zai/glm-4.6";
function bodyWithRemoteImage(url: string) {
return {
messages: [
{
role: "user",
content: [
{ type: "text", text: "describe this" },
{ type: "image_url", image_url: { url } },
],
},
],
};
}
test("remote image fetched for the claude-wire self-call is downscaled to the long-edge cap", async (t) => {
let sharp: typeof import("sharp");
try {
sharp = (await import("sharp")).default as never;
} catch {
t.skip("sharp not installed");
return;
}
const big = await sharp({ create: { width: 4096, height: 100, channels: 3, background: "#fff" } })
.png()
.toBuffer();
const fetchImpl = (async () =>
new Response(big, {
status: 200,
headers: { "content-type": "image/png" },
})) as unknown as typeof fetch;
const result = await ensureBase64ImagesForClaudeWire(
bodyWithRemoteImage("https://example.com/big.png"),
CLAUDE_WIRE_MODEL,
fetchImpl
);
const imagePart = (result.messages?.[0]?.content as Array<{ image_url?: { url: string } }>)[1];
const dataUri = imagePart?.image_url?.url ?? "";
assert.match(dataUri, /^data:image\/png;base64,/);
const b64 = dataUri.split(",")[1] ?? "";
const decoded = Buffer.from(b64, "base64");
const meta = await sharp(decoded).metadata();
assert.ok((meta.width ?? 0) <= 2048, `expected width <= 2048, got ${meta.width}`);
assert.notEqual(meta.width, 4096, "image must have been downscaled, not left at 4096");
});
test("remote non-image bytes pass through untouched (fail-open, no normalization)", async () => {
const junk = Buffer.from("not-an-image-at-all");
const fetchImpl = (async () =>
new Response(junk, {
status: 200,
headers: { "content-type": "application/octet-stream" },
})) as unknown as typeof fetch;
const result = await ensureBase64ImagesForClaudeWire(
bodyWithRemoteImage("https://example.com/junk.bin"),
CLAUDE_WIRE_MODEL,
fetchImpl
);
const imagePart = (result.messages?.[0]?.content as Array<{ image_url?: { url: string } }>)[1];
const dataUri = imagePart?.image_url?.url ?? "";
assert.equal(dataUri, `data:application/octet-stream;base64,${junk.toString("base64")}`);
});