fix(video-bridge): route captions through provider connections

This commit is contained in:
Diego Rodrigues de Sa e Souza
2026-08-15 17:44:47 -03:00
parent 382b2fba26
commit b1d710d45b
4 changed files with 56 additions and 9 deletions

View File

@@ -1,3 +1,5 @@
import { fetch as undiciFetch } from "undici";
import { getSettings as defaultGetSettings } from "@/lib/db/settings";
import { getResolvedModelCapabilities } from "@/lib/modelCapabilities";
import {
@@ -235,8 +237,10 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
producerModel = model;
},
prompt,
routeThroughOmniRoute: true,
signal,
timeoutMs: runtime.timeoutMs,
fetchImpl: undiciFetch as unknown as typeof fetch,
});
successfulModels.add(producerModel);
if (key && cache) cache.setEntry(key, { value: caption, producerModel });

View File

@@ -346,6 +346,8 @@ export interface VisionModelConfig {
prompt: string;
timeoutMs: number;
maxImages: number;
/** Route catalog models through OmniRoute so provider connections remain authoritative. */
routeThroughOmniRoute?: boolean;
/** Optional parent deadline/abort propagated by multi-step media bridges. */
signal?: AbortSignal;
/** Injectable fetch (tests). Defaults to undici fetch to bypass the runtime's hooked global fetch. */
@@ -655,7 +657,8 @@ async function callVisionModelSingle(
// body reaches the backend as a data URI (the OpenAI→claude translator only
// preserves data URIs as base64; remote URLs become source.url which these
// backends reject).
const isAnthropic = config.model.startsWith("anthropic/");
const routeThroughOmniRoute = config.routeThroughOmniRoute === true;
const isAnthropic = !routeThroughOmniRoute && config.model.startsWith("anthropic/");
const requiresBase64 = isAnthropic || isClaudeWireFormatModel(config.model);
try {
@@ -721,15 +724,18 @@ async function callVisionModelSingle(
// VISION_BRIDGE_BASE_URL so the vision-bridge call can be routed through
// OmniRoute itself or any other OpenAI-compatible endpoint instead of
// hardcoded api.openai.com.
const baseUrl = resolveVisionBridgeBaseUrl(config.model);
const baseUrl = routeThroughOmniRoute
? `http://localhost:${getRuntimePorts().port}/v1`
: resolveVisionBridgeBaseUrl(config.model);
// When routing through the OmniRoute self-loop (non-standard provider),
// keep the full provider-prefixed model ID so OmniRoute can resolve the
// correct provider backend. Only strip the prefix for direct OpenAI calls.
const useFullModelId =
baseUrl.startsWith("http://localhost") &&
config.model.includes("/") &&
!config.model.startsWith("openai/");
routeThroughOmniRoute ||
(baseUrl.startsWith("http://localhost") &&
config.model.includes("/") &&
!config.model.startsWith("openai/"));
const requestModel = useFullModelId ? config.model : modelName;
// Build headers with optional recursion guard for self-loop calls.
@@ -749,7 +755,9 @@ async function callVisionModelSingle(
Authorization: `Bearer ${selfLoopApiKey}`,
};
if (useFullModelId) {
headers["x-omniroute-disabled-guardrails"] = "vision-bridge";
headers["x-omniroute-disabled-guardrails"] = routeThroughOmniRoute
? "vision-bridge,video-bridge"
: "vision-bridge";
// Internal self-loop sub-request: the parent request already holds the
// single heavyweight admission lease (`CHAT_MAX_HEAVY_IN_FLIGHT=1`), so a
// large base64-image describe body would be rejected with 503

View File

@@ -240,6 +240,8 @@ test("maxVideos preserves excess raw video only when target video support is unk
test("empty Video and Vision model settings use the Vision auto-router and report the effective model", async () => {
let selectedFixedModel: string | undefined;
let calledModel = "";
let routedThroughOmniRoute = false;
let injectedFetch = false;
const bridge = new VideoBridgeGuardrail({
deps: {
getSettings: async () => ({
@@ -259,6 +261,8 @@ test("empty Video and Vision model settings use the Vision auto-router and repor
}),
callVisionModel: async (_image, config) => {
calledModel = config.model;
routedThroughOmniRoute = config.routeThroughOmniRoute === true;
injectedFetch = typeof config.fetchImpl === "function";
return "a safe observation";
},
},
@@ -266,6 +270,8 @@ test("empty Video and Vision model settings use the Vision auto-router and repor
const result = await bridge.preCall(payload(), {});
assert.equal(selectedFixedModel, undefined);
assert.equal(calledModel, "google/gemini-2.5-flash");
assert.equal(routedThroughOmniRoute, true);
assert.equal(injectedFetch, true);
assert.equal(result.meta?.videoModel, "google/gemini-2.5-flash");
assert.ok(result.modifiedPayload);
});
@@ -347,10 +353,10 @@ test("real primary failure reports and caches the successful fallback model iden
const fetchImpl: typeof fetch = async (_input, init) => {
const body = JSON.parse(String(init?.body)) as { model: string };
attemptedModels.push(body.model);
if (body.model === "gpt-4o-mini") {
if (body.model === primary) {
return new Response("primary unavailable", { status: 503 });
}
return Response.json({ content: [{ type: "text", text: "fallback observation" }] });
return Response.json({ choices: [{ message: { content: "fallback observation" } }] });
};
const bridge = new VideoBridgeGuardrail({
deps: {
@@ -384,7 +390,7 @@ test("real primary failure reports and caches the successful fallback model iden
const first = await bridge.preCall(payload(), {});
const second = await bridge.preCall(payload(), {});
assert.deepEqual(attemptedModels, ["gpt-4o-mini", "claude-fable-5"]);
assert.deepEqual(attemptedModels, [primary, fallback]);
assert.equal(first.meta?.videoModel, fallback, "meta must name the successful fallback");
assert.equal(second.meta?.videoModel, fallback, "cache hit must retain the producer identity");
assert.equal(second.meta?.cacheHits, 1);

View File

@@ -93,6 +93,35 @@ test("callVisionModel returns description on success", async () => {
}
});
test("callVisionModel can route a catalog model through the OmniRoute self-loop", async () => {
let capturedUrl = "";
let capturedBody: Record<string, unknown> = {};
let capturedHeaders: Record<string, string> = {};
const fetchImpl: typeof fetch = async (input, init) => {
capturedUrl = String(input);
capturedBody = JSON.parse(String(init?.body));
capturedHeaders = (init?.headers ?? {}) as Record<string, string>;
return Response.json({ choices: [{ message: { content: "GREEN_SCENE_2" } }] });
};
const result = await callVisionModel("data:image/png;base64,iVBORw0KGgo", {
model: "openai/gpt-4o-mini",
prompt: "Describe this frame",
timeoutMs: 30000,
maxImages: 1,
routeThroughOmniRoute: true,
fetchImpl,
});
const url = new URL(capturedUrl);
assert.equal(url.hostname, "localhost");
assert.equal(url.pathname, "/v1/chat/completions");
assert.equal(capturedBody.model, "openai/gpt-4o-mini");
assert.equal(capturedHeaders["x-omniroute-admission-bypass"], "internal");
assert.match(capturedHeaders["x-omniroute-disabled-guardrails"], /video-bridge/);
assert.equal(result, "GREEN_SCENE_2");
});
test("callVisionModel throws on HTTP error", async () => {
const mockResponse = {
ok: false,