From b1d710d45b0eff94e105e8dbbf58575319339406 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Sat, 15 Aug 2026 17:44:47 -0300 Subject: [PATCH] fix(video-bridge): route captions through provider connections --- src/lib/guardrails/videoBridge.ts | 4 +++ src/lib/guardrails/visionBridgeHelpers.ts | 20 +++++++++---- tests/unit/guardrails/videoBridge.test.ts | 12 ++++++-- ...isionBridgeHelpers.callVisionModel.test.ts | 29 +++++++++++++++++++ 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/src/lib/guardrails/videoBridge.ts b/src/lib/guardrails/videoBridge.ts index ed9ecc6747..2a6de89191 100644 --- a/src/lib/guardrails/videoBridge.ts +++ b/src/lib/guardrails/videoBridge.ts @@ -1,3 +1,5 @@ +import { fetch as undiciFetch } from "undici"; + import { getSettings as defaultGetSettings } from "@/lib/db/settings"; import { getResolvedModelCapabilities } from "@/lib/modelCapabilities"; import { @@ -235,8 +237,10 @@ export class VideoBridgeGuardrail extends BaseGuardrail { producerModel = model; }, prompt, + routeThroughOmniRoute: true, signal, timeoutMs: runtime.timeoutMs, + fetchImpl: undiciFetch as unknown as typeof fetch, }); successfulModels.add(producerModel); if (key && cache) cache.setEntry(key, { value: caption, producerModel }); diff --git a/src/lib/guardrails/visionBridgeHelpers.ts b/src/lib/guardrails/visionBridgeHelpers.ts index 4d2f6e1804..9310cef608 100644 --- a/src/lib/guardrails/visionBridgeHelpers.ts +++ b/src/lib/guardrails/visionBridgeHelpers.ts @@ -346,6 +346,8 @@ export interface VisionModelConfig { prompt: string; timeoutMs: number; maxImages: number; + /** Route catalog models through OmniRoute so provider connections remain authoritative. */ + routeThroughOmniRoute?: boolean; /** Optional parent deadline/abort propagated by multi-step media bridges. */ signal?: AbortSignal; /** Injectable fetch (tests). Defaults to undici fetch to bypass the runtime's hooked global fetch. */ @@ -655,7 +657,8 @@ async function callVisionModelSingle( // body reaches the backend as a data URI (the OpenAI→claude translator only // preserves data URIs as base64; remote URLs become source.url which these // backends reject). - const isAnthropic = config.model.startsWith("anthropic/"); + const routeThroughOmniRoute = config.routeThroughOmniRoute === true; + const isAnthropic = !routeThroughOmniRoute && config.model.startsWith("anthropic/"); const requiresBase64 = isAnthropic || isClaudeWireFormatModel(config.model); try { @@ -721,15 +724,18 @@ async function callVisionModelSingle( // VISION_BRIDGE_BASE_URL so the vision-bridge call can be routed through // OmniRoute itself or any other OpenAI-compatible endpoint instead of // hardcoded api.openai.com. - const baseUrl = resolveVisionBridgeBaseUrl(config.model); + const baseUrl = routeThroughOmniRoute + ? `http://localhost:${getRuntimePorts().port}/v1` + : resolveVisionBridgeBaseUrl(config.model); // When routing through the OmniRoute self-loop (non-standard provider), // keep the full provider-prefixed model ID so OmniRoute can resolve the // correct provider backend. Only strip the prefix for direct OpenAI calls. const useFullModelId = - baseUrl.startsWith("http://localhost") && - config.model.includes("/") && - !config.model.startsWith("openai/"); + routeThroughOmniRoute || + (baseUrl.startsWith("http://localhost") && + config.model.includes("/") && + !config.model.startsWith("openai/")); const requestModel = useFullModelId ? config.model : modelName; // Build headers with optional recursion guard for self-loop calls. @@ -749,7 +755,9 @@ async function callVisionModelSingle( Authorization: `Bearer ${selfLoopApiKey}`, }; if (useFullModelId) { - headers["x-omniroute-disabled-guardrails"] = "vision-bridge"; + headers["x-omniroute-disabled-guardrails"] = routeThroughOmniRoute + ? "vision-bridge,video-bridge" + : "vision-bridge"; // Internal self-loop sub-request: the parent request already holds the // single heavyweight admission lease (`CHAT_MAX_HEAVY_IN_FLIGHT=1`), so a // large base64-image describe body would be rejected with 503 diff --git a/tests/unit/guardrails/videoBridge.test.ts b/tests/unit/guardrails/videoBridge.test.ts index 4ba5d336f4..0fcf2172df 100644 --- a/tests/unit/guardrails/videoBridge.test.ts +++ b/tests/unit/guardrails/videoBridge.test.ts @@ -240,6 +240,8 @@ test("maxVideos preserves excess raw video only when target video support is unk test("empty Video and Vision model settings use the Vision auto-router and report the effective model", async () => { let selectedFixedModel: string | undefined; let calledModel = ""; + let routedThroughOmniRoute = false; + let injectedFetch = false; const bridge = new VideoBridgeGuardrail({ deps: { getSettings: async () => ({ @@ -259,6 +261,8 @@ test("empty Video and Vision model settings use the Vision auto-router and repor }), callVisionModel: async (_image, config) => { calledModel = config.model; + routedThroughOmniRoute = config.routeThroughOmniRoute === true; + injectedFetch = typeof config.fetchImpl === "function"; return "a safe observation"; }, }, @@ -266,6 +270,8 @@ test("empty Video and Vision model settings use the Vision auto-router and repor const result = await bridge.preCall(payload(), {}); assert.equal(selectedFixedModel, undefined); assert.equal(calledModel, "google/gemini-2.5-flash"); + assert.equal(routedThroughOmniRoute, true); + assert.equal(injectedFetch, true); assert.equal(result.meta?.videoModel, "google/gemini-2.5-flash"); assert.ok(result.modifiedPayload); }); @@ -347,10 +353,10 @@ test("real primary failure reports and caches the successful fallback model iden const fetchImpl: typeof fetch = async (_input, init) => { const body = JSON.parse(String(init?.body)) as { model: string }; attemptedModels.push(body.model); - if (body.model === "gpt-4o-mini") { + if (body.model === primary) { return new Response("primary unavailable", { status: 503 }); } - return Response.json({ content: [{ type: "text", text: "fallback observation" }] }); + return Response.json({ choices: [{ message: { content: "fallback observation" } }] }); }; const bridge = new VideoBridgeGuardrail({ deps: { @@ -384,7 +390,7 @@ test("real primary failure reports and caches the successful fallback model iden const first = await bridge.preCall(payload(), {}); const second = await bridge.preCall(payload(), {}); - assert.deepEqual(attemptedModels, ["gpt-4o-mini", "claude-fable-5"]); + assert.deepEqual(attemptedModels, [primary, fallback]); assert.equal(first.meta?.videoModel, fallback, "meta must name the successful fallback"); assert.equal(second.meta?.videoModel, fallback, "cache hit must retain the producer identity"); assert.equal(second.meta?.cacheHits, 1); diff --git a/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts b/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts index bdbef94807..820e139f51 100644 --- a/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts +++ b/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts @@ -93,6 +93,35 @@ test("callVisionModel returns description on success", async () => { } }); +test("callVisionModel can route a catalog model through the OmniRoute self-loop", async () => { + let capturedUrl = ""; + let capturedBody: Record = {}; + let capturedHeaders: Record = {}; + const fetchImpl: typeof fetch = async (input, init) => { + capturedUrl = String(input); + capturedBody = JSON.parse(String(init?.body)); + capturedHeaders = (init?.headers ?? {}) as Record; + return Response.json({ choices: [{ message: { content: "GREEN_SCENE_2" } }] }); + }; + + const result = await callVisionModel("data:image/png;base64,iVBORw0KGgo", { + model: "openai/gpt-4o-mini", + prompt: "Describe this frame", + timeoutMs: 30000, + maxImages: 1, + routeThroughOmniRoute: true, + fetchImpl, + }); + + const url = new URL(capturedUrl); + assert.equal(url.hostname, "localhost"); + assert.equal(url.pathname, "/v1/chat/completions"); + assert.equal(capturedBody.model, "openai/gpt-4o-mini"); + assert.equal(capturedHeaders["x-omniroute-admission-bypass"], "internal"); + assert.match(capturedHeaders["x-omniroute-disabled-guardrails"], /video-bridge/); + assert.equal(result, "GREEN_SCENE_2"); +}); + test("callVisionModel throws on HTTP error", async () => { const mockResponse = { ok: false,