feat(vision-bridge): add automatic image description fallback for non-vision models (#1476)

* feat(vision-bridge): add automatic image description fallback for non-vision models

Implements VisionBridgeGuardrail (priority 5) that intercepts image-bearing
requests to non-vision models, extracts descriptions via a configurable
vision model (default: gpt-4o-mini), and replaces images with text before
forwarding. Fails open on any error.

- Add VisionBridgeGuardrail class extending BaseGuardrail
- Add visionBridgeHelpers: extractImageParts, callVisionModel, replaceImageParts, resolveImageAsDataUri
- Add visionBridgeDefaults with configurable settings
- Register VisionBridgeGuardrail in GuardrailRegistry at priority 5
- Add 51 unit tests covering all spec scenarios (VB-S01 through VB-S10)
- Dependency injection for getSettings and callVisionModel (testable without SQLite)

Closes diegosouzapw/OmniRoute#1424

* fix(vision-bridge): resolve Anthropic API, parallel processing, provider keys, structuredClone
This commit is contained in:
Hernan Javier Ardila Sanchez
2026-04-21 23:14:24 +02:00
committed by GitHub
parent b925be2758
commit b709dca2d6
10 changed files with 1828 additions and 0 deletions

View File

@@ -1,6 +1,7 @@
import { BaseGuardrail, type GuardrailContext, type GuardrailExecutionResult } from "./base";
import { PIIMaskerGuardrail } from "./piiMasker";
import { PromptInjectionGuardrail } from "./promptInjection";
import { VisionBridgeGuardrail } from "./visionBridge";
type HeadersLike = Headers | Record<string, unknown> | null | undefined;
@@ -262,6 +263,7 @@ let defaultGuardrailsRegistered = false;
export function registerDefaultGuardrails() {
if (defaultGuardrailsRegistered) return guardrailRegistry;
guardrailRegistry.register(new VisionBridgeGuardrail());
guardrailRegistry.register(new PIIMaskerGuardrail());
guardrailRegistry.register(new PromptInjectionGuardrail());
defaultGuardrailsRegistered = true;

View File

@@ -0,0 +1,150 @@
/**
* Vision Bridge Guardrail.
* Intercepts image-bearing requests to non-vision models,
* extracts descriptions via vision model, and replaces images with text.
*/
import { BaseGuardrail, type GuardrailContext, type GuardrailResult } from "./base";
import { getSettings as defaultGetSettings } from "@/lib/db/settings";
import { getResolvedModelCapabilities } from "@/lib/modelCapabilities";
import {
extractImageParts,
callVisionModel as defaultCallVisionModel,
replaceImageParts,
} from "./visionBridgeHelpers";
import {
VISION_BRIDGE_DEFAULTS,
getVisionBridgeConfig,
} from "@/shared/constants/visionBridgeDefaults";
export interface VisionBridgeDependencies {
getSettings?: () => Promise<Record<string, unknown>>;
callVisionModel?: (
imageDataUri: string,
config: import("./visionBridgeHelpers").VisionModelConfig,
apiKey?: string
) => Promise<string>;
}
export class VisionBridgeGuardrail extends BaseGuardrail {
name = "vision-bridge";
priority = 5;
private readonly deps: VisionBridgeDependencies;
constructor(options?: { enabled?: boolean; deps?: VisionBridgeDependencies }) {
super("vision-bridge", { priority: 5, enabled: options?.enabled });
this.deps = options?.deps ?? {};
}
async preCall(
payload: unknown,
context: GuardrailContext
): Promise<GuardrailResult<unknown>> {
// 1. Check if disabled at guardrail level
if (!this.enabled) {
return { block: false };
}
// 2. Check disabled via context (header, body, API key)
if (context.disabledGuardrails?.includes("vision-bridge")) {
return { block: false };
}
// 3. Get model from context or payload
const model =
context.model || (payload as Record<string, unknown>)?.model as string | undefined;
if (!model) {
return { block: false };
}
// 4. Check if model supports vision
const capabilities = getResolvedModelCapabilities(model);
if (capabilities.supportsVision === true) {
return { block: false };
}
// 5. Get body and check for messages
const body = payload as Record<string, unknown>;
const messages = body?.messages;
if (!Array.isArray(messages) || messages.length === 0) {
return { block: false };
}
// 6. Check for images using helper (extractImageParts returns empty if no images)
const imageParts = extractImageParts(
messages as Parameters<typeof extractImageParts>[0]
);
if (imageParts.length === 0) {
return { block: false };
}
// 7. Get settings (injectable for testing)
const getSettings = this.deps.getSettings ?? defaultGetSettings;
let settings: Record<string, unknown> = {};
try {
settings = await getSettings();
} catch {
// If getSettings fails, use defaults
}
// 8. Check if Vision Bridge is enabled in settings
const enabled = settings.visionBridgeEnabled ?? VISION_BRIDGE_DEFAULTS.enabled;
if (!enabled) {
return { block: false };
}
// 9. Get configuration
const config = getVisionBridgeConfig({
visionBridgeEnabled: settings.visionBridgeEnabled as boolean | undefined,
visionBridgeModel: settings.visionBridgeModel as string | undefined,
visionBridgePrompt: settings.visionBridgePrompt as string | undefined,
visionBridgeTimeout: settings.visionBridgeTimeout as number | undefined,
visionBridgeMaxImages: settings.visionBridgeMaxImages as number | undefined,
});
// 10. Limit images
const limitedParts = imageParts.slice(0, config.maxImages);
// 11. Call vision model for each image in parallel (injectable for testing)
const callVision = this.deps.callVisionModel ?? defaultCallVisionModel;
const logger = context.log;
const startTime = Date.now();
// Process all images in parallel using Promise.allSettled for fail-partial behavior
const results = await Promise.allSettled(
limitedParts.map(async (imagePart, i) => {
const description = await callVision(imagePart.imageUrl, config);
return `[Image ${i + 1}]: ${description}`;
})
);
// Collect descriptions maintaining original order
const descriptions = results.map((result, i) => {
if (result.status === "fulfilled") {
return result.value;
}
const message = result.reason instanceof Error ? result.reason.message : String(result.reason);
logger?.warn?.("VISION-BRIDGE", `Failed to get description for image ${i + 1}: ${message}`);
return `[Image ${i + 1}]: (unavailable)`;
});
// 12. Replace image parts with text descriptions
const modifiedBody = replaceImageParts(
body as Parameters<typeof replaceImageParts>[0],
descriptions
);
const processingTime = Date.now() - startTime;
return {
block: false,
modifiedPayload: modifiedBody,
meta: {
imagesProcessed: descriptions.length,
descriptions,
processingTimeMs: processingTime,
visionModel: config.model,
},
};
}
}

View File

@@ -0,0 +1,330 @@
/**
* Vision Bridge helper functions for image processing.
*/
/**
* Provider to environment variable mapping for API key resolution.
*/
const PROVIDER_API_KEY_MAP: Record<string, string> = {
anthropic: "ANTHROPIC_API_KEY",
google: "GOOGLE_API_KEY",
openai: "OPENAI_API_KEY",
};
/**
* Resolve API key based on model provider.
* @param model - Model identifier (e.g., "anthropic/claude-3-haiku", "openai/gpt-4o-mini")
* @param explicitKey - Explicit API key passed as argument (takes precedence)
* @returns Resolved API key string
*/
export function resolveProviderApiKey(model: string, explicitKey?: string): string {
if (explicitKey) return explicitKey;
const provider = model.includes("/") ? model.split("/")[0] : "";
const envVar = PROVIDER_API_KEY_MAP[provider] || "OPENAI_API_KEY";
return process.env[envVar] || "";
}
export interface ImagePart {
messageIndex: number;
partIndex: number;
imageUrl: string;
imageType: "image_url" | "image";
}
export interface RequestMessage {
role?: string;
content?: string | RequestContentPart[];
}
export type RequestContentPart =
| { type: "text"; text: string }
| { type: "image_url"; image_url: { url: string; detail?: string } }
| { type: "image"; source: { type: "base64"; media_type: string; data: string } };
/**
* Extract image parts from messages array.
* Supports both OpenAI image_url format and base64 image format.
*/
export function extractImageParts(messages: RequestMessage[]): ImagePart[] {
const results: ImagePart[] = [];
if (!Array.isArray(messages)) {
return results;
}
for (let msgIdx = 0; msgIdx < messages.length; msgIdx++) {
const message = messages[msgIdx];
if (!message || !Array.isArray(message.content)) {
continue;
}
for (let partIdx = 0; partIdx < message.content.length; partIdx++) {
const part = message.content[partIdx];
if (part?.type === "image_url" && part.image_url?.url) {
results.push({
messageIndex: msgIdx,
partIndex: partIdx,
imageUrl: part.image_url.url,
imageType: "image_url",
});
} else if (part?.type === "image" && part.source?.type === "base64") {
const { media_type, data } = part.source;
const dataUri = `data:${media_type};base64,${data}`;
results.push({
messageIndex: msgIdx,
partIndex: partIdx,
imageUrl: dataUri,
imageType: "image",
});
}
}
}
return results;
}
/**
* Resolve image URL to data URI format for vision model.
* - HTTP/HTTPS URLs: passed through as-is
* - Data URIs: passed through as-is
* - Base64 without media type: assumed PNG
*/
export function resolveImageAsDataUri(imageUrl: string): string {
if (!imageUrl || typeof imageUrl !== "string") {
throw new Error("Invalid image URL: must be a non-empty string");
}
// Already a data URI
if (imageUrl.startsWith("data:")) {
return imageUrl;
}
// HTTP/HTTPS URL - vision API will fetch it
if (imageUrl.startsWith("http://") || imageUrl.startsWith("https://")) {
return imageUrl;
}
// Assume it's a base64 string without prefix
// Add PNG as default media type
return `data:image/png;base64,${imageUrl}`;
}
export interface VisionModelConfig {
model: string;
prompt: string;
timeoutMs: number;
maxImages: number;
}
/**
* Call the vision model to get an image description.
* Supports both OpenAI-compatible and Anthropic API formats.
*/
export async function callVisionModel(
imageDataUri: string,
config: VisionModelConfig,
apiKey?: string
): Promise<string> {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), config.timeoutMs);
// Resolve API key based on provider
const resolvedApiKey = resolveProviderApiKey(config.model, apiKey);
// Detect provider from model identifier
const isAnthropic = config.model.startsWith("anthropic/");
try {
// Extract model name from provider/model format
const modelName = config.model.includes("/")
? config.model.split("/")[1]
: config.model;
let response: Response;
if (isAnthropic) {
// Anthropic API path
const anthropicBaseUrl = process.env.ANTHROPIC_API_URL || "https://api.anthropic.com";
// Parse data URI to extract media type and base64 data
const matches = imageDataUri.match(/^data:([^;]+);base64,(.+)$/);
let mediaType = "image/png";
let base64Data = imageDataUri;
if (matches) {
mediaType = matches[1];
base64Data = matches[2];
}
response = await fetch(`${anthropicBaseUrl}/v1/messages`, {
method: "POST",
signal: controller.signal,
headers: {
"x-api-key": resolvedApiKey,
"anthropic-version": "2023-06-01",
"content-type": "application/json",
},
body: JSON.stringify({
model: modelName,
messages: [
{
role: "user",
content: [
{
type: "image",
source: {
type: "base64",
media_type: mediaType,
data: base64Data,
},
},
{
type: "text",
text: config.prompt,
},
],
},
],
max_tokens: 300,
}),
});
} else {
// OpenAI-compatible path (default)
const baseUrl = process.env.OPENAI_API_URL || "https://api.openai.com/v1";
response = await fetch(`${baseUrl}/chat/completions`, {
method: "POST",
signal: controller.signal,
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${resolvedApiKey}`,
},
body: JSON.stringify({
model: modelName,
messages: [
{
role: "user",
content: [
{
type: "image_url",
image_url: {
url: imageDataUri,
detail: "low",
},
},
{ type: "text", text: config.prompt },
],
},
],
max_tokens: 300,
}),
});
}
clearTimeout(timeoutId);
if (!response.ok) {
const errorText = await response.text().catch(() => "Unknown error");
throw new Error(`Vision API error ${response.status}: ${errorText}`);
}
const data = await response.json();
if (isAnthropic) {
// Anthropic response format: { content: [{ type: "text", text: "..." }] }
const anthropicData = data as {
content?: Array<{ type?: string; text?: string }>;
error?: { message?: string };
};
if (anthropicData.error) {
throw new Error(`Vision API error: ${anthropicData.error.message || JSON.stringify(anthropicData.error)}`);
}
const textContent = anthropicData.content?.find((c) => c.type === "text");
const content = textContent?.text;
if (!content || typeof content !== "string") {
throw new Error("Vision API returned empty or invalid response");
}
return content.trim();
} else {
// OpenAI-compatible response format: { choices: [{ message: { content: "..." } }] }
const openaiData = data as {
choices?: Array<{ message?: { content?: string } }>;
error?: { message?: string };
};
if (openaiData.error) {
throw new Error(`Vision API error: ${openaiData.error.message || JSON.stringify(openaiData.error)}`);
}
const content = openaiData.choices?.[0]?.message?.content;
if (!content || typeof content !== "string") {
throw new Error("Vision API returned empty or invalid response");
}
return content.trim();
}
} catch (error) {
clearTimeout(timeoutId);
if (error instanceof Error && error.name === "AbortError") {
throw new Error("Vision model call timed out");
}
throw error;
}
}
export interface RequestBody {
model?: string;
messages?: RequestMessage[];
[key: string]: unknown;
}
/**
* Replace image content parts with text descriptions.
* Concatenates descriptions with labels: "[Image 1]: ..."
*/
export function replaceImageParts(body: RequestBody, descriptions: string[]): RequestBody {
if (!descriptions || descriptions.length === 0) {
return body;
}
const result = structuredClone(body) as RequestBody;
if (!Array.isArray(result.messages)) {
return result;
}
let descriptionIndex = 0;
for (let msgIdx = 0; msgIdx < result.messages.length; msgIdx++) {
const message = result.messages[msgIdx];
if (!message || !Array.isArray(message.content)) {
continue;
}
const newContent: RequestContentPart[] = [];
for (const part of message.content) {
if (part?.type === "image_url" || part?.type === "image") {
if (descriptionIndex < descriptions.length) {
newContent.push({
type: "text",
text: descriptions[descriptionIndex],
});
descriptionIndex++;
}
} else {
newContent.push(part as RequestContentPart);
}
}
message.content = newContent;
}
return result;
}

View File

@@ -0,0 +1,55 @@
/**
* Vision Bridge default configuration values.
*/
export const VISION_BRIDGE_DEFAULTS = {
enabled: true,
model: "openai/gpt-4o-mini",
prompt:
"Describe this image concisely in 2-3 sentences. Focus on the most relevant visual details.",
timeoutMs: 30000,
maxImagesPerRequest: 10,
} as const;
/**
* Settings keys for Vision Bridge (to be stored in key_value table).
*/
export const VISION_BRIDGE_SETTINGS_KEYS = [
"visionBridgeEnabled",
"visionBridgeModel",
"visionBridgePrompt",
"visionBridgeTimeout",
"visionBridgeMaxImages",
] as const;
export type VisionBridgeSettings = {
visionBridgeEnabled?: boolean;
visionBridgeModel?: string;
visionBridgePrompt?: string;
visionBridgeTimeout?: number;
visionBridgeMaxImages?: number;
};
export type VisionBridgeConfig = {
enabled: boolean;
model: string;
prompt: string;
timeoutMs: number;
maxImages: number;
};
/**
* Merge settings with defaults to produce a complete config.
*/
export function getVisionBridgeConfig(
settings: VisionBridgeSettings | undefined | null = {}
): VisionBridgeConfig {
const s = settings ?? {};
return {
enabled: s.visionBridgeEnabled ?? VISION_BRIDGE_DEFAULTS.enabled,
model: s.visionBridgeModel ?? VISION_BRIDGE_DEFAULTS.model,
prompt: s.visionBridgePrompt ?? VISION_BRIDGE_DEFAULTS.prompt,
timeoutMs: s.visionBridgeTimeout ?? VISION_BRIDGE_DEFAULTS.timeoutMs,
maxImages: s.visionBridgeMaxImages ?? VISION_BRIDGE_DEFAULTS.maxImagesPerRequest,
};
}

View File

@@ -0,0 +1,515 @@
/**
* Tests for VisionBridgeGuardrail.
* Uses dependency injection to avoid SQLite dependency.
*/
import test from "node:test";
import assert from "node:assert/strict";
const { VisionBridgeGuardrail } = await import(
"../../../src/lib/guardrails/visionBridge.ts"
);
const { resetGuardrailsForTests } = await import(
"../../../src/lib/guardrails/registry.ts"
);
const { getResolvedModelCapabilities } = await import(
"../../../src/lib/modelCapabilities.ts"
);
import type { GuardrailContext } from "../../../src/lib/guardrails/base.ts";
import type { VisionModelConfig } from "../../../src/lib/guardrails/visionBridgeHelpers.ts";
// ── Mock state ──────────────────────────────────────────────────────────────
let mockSettings: Record<string, unknown> = {
visionBridgeEnabled: true,
visionBridgeModel: "openai/gpt-4o-mini",
visionBridgePrompt: "Describe this image concisely.",
visionBridgeTimeout: 30000,
visionBridgeMaxImages: 10,
};
let mockVisionResponse = "A beautiful sunset over the ocean";
let shouldVisionFail = false;
let visionCallCount = 0;
function createGuardrail(
options?: Parameters<typeof VisionBridgeGuardrail>[0]
) {
return new VisionBridgeGuardrail({
...options,
deps: {
getSettings: async () => mockSettings,
callVisionModel: async (
_imageDataUri: string,
_config: VisionModelConfig
) => {
visionCallCount++;
if (shouldVisionFail) {
throw new Error("Vision model failed");
}
return mockVisionResponse;
},
},
});
}
test.beforeEach(() => {
resetGuardrailsForTests({ registerDefaults: false });
visionCallCount = 0;
shouldVisionFail = false;
mockSettings = {
visionBridgeEnabled: true,
visionBridgeModel: "openai/gpt-4o-mini",
visionBridgePrompt: "Describe this image concisely.",
visionBridgeTimeout: 30000,
visionBridgeMaxImages: 10,
};
});
// ── Helpers ─────────────────────────────────────────────────────────────────
function createContext(
overrides: Partial<GuardrailContext> = {}
): GuardrailContext {
return {
model: "minimax/minimax-01",
log: console,
...overrides,
};
}
function createPayload(
overrides: Record<string, unknown> = {}
): Record<string, unknown> {
return {
model: "minimax/minimax-01",
messages: [{ role: "user", content: "Hello" }],
...overrides,
};
}
// ── Basic Properties ────────────────────────────────────────────────────────
test("VisionBridgeGuardrail has correct name and priority", () => {
const guardrail = createGuardrail();
assert.strictEqual(guardrail.name, "vision-bridge");
assert.strictEqual(guardrail.priority, 5);
});
test("VisionBridgeGuardrail is enabled by default", () => {
const guardrail = createGuardrail();
assert.strictEqual(guardrail.enabled, true);
});
test("VisionBridgeGuardrail can be disabled via constructor", () => {
const guardrail = createGuardrail({ enabled: false });
assert.strictEqual(guardrail.enabled, false);
});
// ── VB-S05: Vision Bridge disabled via settings ────────────────────────────
test("VB-S05: passthroughs when visionBridgeEnabled is false", async () => {
mockSettings.visionBridgeEnabled = false;
const guardrail = createGuardrail();
const payload = createPayload({
messages: [
{
role: "user",
content: [
{ type: "text", text: "What is in this image?" },
{
type: "image_url",
image_url: { url: "https://example.com/image.png" },
},
],
},
],
});
const result = await guardrail.preCall(payload, createContext());
assert.strictEqual(result.block, false);
assert.strictEqual(result.modifiedPayload, undefined);
});
// ── VB-S06: Disabled via context ────────────────────────────────────────────
test("VB-S06: skips when disabledGuardrails includes vision-bridge", async () => {
const guardrail = createGuardrail();
const payload = createPayload();
const context = createContext({ disabledGuardrails: ["vision-bridge"] });
const result = await guardrail.preCall(payload, context);
assert.strictEqual(result.block, false);
assert.strictEqual(result.modifiedPayload, undefined);
});
// ── VB-S02: Vision-capable model passthrough ────────────────────────────────
test("VB-S02: passthroughs for vision-capable model (gpt-4o)", async () => {
const guardrail = createGuardrail();
const payload = createPayload({
model: "openai/gpt-4o",
messages: [
{
role: "user",
content: [
{ type: "text", text: "What is this?" },
{
type: "image_url",
image_url: { url: "https://example.com/image.png" },
},
],
},
],
});
const result = await guardrail.preCall(
payload,
createContext({ model: "openai/gpt-4o" })
);
// If supportsVision is true, it should passthrough (no modification)
// If supportsVision is null/undefined (no sync data), it will process — that's correct behavior
const capabilities = getResolvedModelCapabilities("openai/gpt-4o");
if (capabilities.supportsVision === true) {
assert.strictEqual(result.block, false);
assert.strictEqual(result.modifiedPayload, undefined);
} else {
// Without sync data, supportsVision is null — guardrail processes the image
// This is correct fail-open behavior for unknown model capabilities
assert.strictEqual(result.block, false);
}
});
test("VB-S02: model capabilities returns supportsVision for known models", () => {
const gpt4oCaps = getResolvedModelCapabilities("openai/gpt-4o");
// supportsVision may be true (if sync data exists) or null (if not synced)
assert.ok(gpt4oCaps.supportsVision === true || gpt4oCaps.supportsVision === null);
});
// ── VB-S04: No images passthrough ──────────────────────────────────────────
test("VB-S04: passthroughs when no images in messages", async () => {
const guardrail = createGuardrail();
const payload = createPayload({
messages: [{ role: "user", content: "Hello, how are you?" }],
});
const result = await guardrail.preCall(payload, createContext());
assert.strictEqual(result.block, false);
assert.strictEqual(result.modifiedPayload, undefined);
});
test("VB-S04: passthroughs when messages array is empty", async () => {
const guardrail = createGuardrail();
const payload = createPayload({ messages: [] });
const result = await guardrail.preCall(payload, createContext());
assert.strictEqual(result.block, false);
});
// ── VB-S01: Single image processing ─────────────────────────────────────────
test("VB-S01: replaces image with description for non-vision model", async () => {
mockVisionResponse = "A beautiful sunset over the ocean";
const guardrail = createGuardrail();
const payload = createPayload({
model: "minimax/minimax-01",
messages: [
{
role: "user",
content: [
{ type: "text", text: "What is in this image?" },
{
type: "image_url",
image_url: { url: "https://example.com/image.png" },
},
],
},
],
});
const result = await guardrail.preCall(
payload,
createContext({ model: "minimax/minimax-01" })
);
assert.strictEqual(result.block, false);
assert.ok(result.modifiedPayload);
const modified = result.modifiedPayload as {
messages: Array<{ content: unknown[] }>;
};
const content = modified.messages[0].content as Array<{
type: string;
text?: string;
}>;
const imagePart = content.find((p) => p.type === "image_url");
assert.strictEqual(imagePart, undefined);
const descriptionPart = content.find(
(p) => p.type === "text" && p.text?.includes("sunset")
);
assert.ok(descriptionPart);
});
// ── VB-S04: Multiple images ─────────────────────────────────────────────────
test("VB-S04: processes multiple images and concatenates descriptions", async () => {
let callIdx = 0;
const descriptions = ["A cute cat", "A playful dog", "A colorful bird"];
const guardrail = new VisionBridgeGuardrail({
deps: {
getSettings: async () => mockSettings,
callVisionModel: async () => {
const desc = descriptions[callIdx] || "Unknown image";
callIdx++;
return desc;
},
},
});
const payload = createPayload({
model: "minimax/minimax-01",
messages: [
{
role: "user",
content: [
{ type: "text", text: "Describe these images" },
{
type: "image_url",
image_url: { url: "https://example.com/cat.png" },
},
{
type: "image_url",
image_url: { url: "https://example.com/dog.png" },
},
{
type: "image_url",
image_url: { url: "https://example.com/bird.png" },
},
],
},
],
});
const result = await guardrail.preCall(
payload,
createContext({ model: "minimax/minimax-01" })
);
assert.strictEqual(result.block, false);
assert.ok(result.modifiedPayload);
assert.strictEqual(callIdx, 3);
const modified = result.modifiedPayload as {
messages: Array<{ content: unknown[] }>;
};
const content = modified.messages[0].content as Array<{
type: string;
text?: string;
}>;
assert.ok(
content.some((p) => p.type === "text" && p.text?.includes("[Image 1]"))
);
assert.ok(
content.some((p) => p.type === "text" && p.text?.includes("[Image 2]"))
);
assert.ok(
content.some((p) => p.type === "text" && p.text?.includes("[Image 3]"))
);
});
// ── VB-S03: Fail-open on vision error ──────────────────────────────────────
test("VB-S03: returns modified payload with unavailable text when vision API fails", async () => {
shouldVisionFail = true;
const guardrail = createGuardrail();
const payload = createPayload({
model: "minimax/minimax-01",
messages: [
{
role: "user",
content: [
{ type: "text", text: "What is this?" },
{
type: "image_url",
image_url: { url: "https://example.com/image.png" },
},
],
},
],
});
const result = await guardrail.preCall(
payload,
createContext({ model: "minimax/minimax-01" })
);
assert.strictEqual(result.block, false);
assert.ok(result.modifiedPayload);
const modified = result.modifiedPayload as {
messages: Array<{ content: unknown[] }>;
};
const content = modified.messages[0].content as Array<{
type: string;
text?: string;
}>;
// Should have "unavailable" text instead of image
const unavailPart = content.find(
(p) => p.type === "text" && p.text?.includes("unavailable")
);
assert.ok(unavailPart);
});
test("VB-S03: logs warning when vision API fails", async () => {
shouldVisionFail = true;
let warningLogged = false;
const guardrail = createGuardrail();
const payload = createPayload({
model: "minimax/minimax-01",
messages: [
{
role: "user",
content: [
{
type: "image_url",
image_url: { url: "https://example.com/image.png" },
},
],
},
],
});
const mockLog = {
warn: (_tag: string, msg: string) => {
if (msg.includes("Failed to get description")) {
warningLogged = true;
}
},
};
await guardrail.preCall(
payload,
createContext({
model: "minimax/minimax-01",
log: mockLog as GuardrailContext["log"],
})
);
assert.strictEqual(warningLogged, true);
});
// ── VB-S07: Base64 image format ─────────────────────────────────────────────
test("VB-S07: handles base64 image format", async () => {
mockVisionResponse = "An image description";
const guardrail = createGuardrail();
const payload = createPayload({
model: "minimax/minimax-01",
messages: [
{
role: "user",
content: [
{
type: "image",
source: {
type: "base64",
media_type: "image/png",
data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
},
},
],
},
],
});
const result = await guardrail.preCall(
payload,
createContext({ model: "minimax/minimax-01" })
);
assert.strictEqual(result.block, false);
assert.ok(result.modifiedPayload);
});
// ── VB-S09: Image count limit ───────────────────────────────────────────────
test("VB-S09: respects maxImages setting", async () => {
mockSettings.visionBridgeMaxImages = 2;
const guardrail = createGuardrail();
const images = Array.from({ length: 5 }, (_, i) => ({
type: "image_url" as const,
image_url: { url: `https://example.com/image${i}.png` },
}));
const payload = createPayload({
model: "minimax/minimax-01",
messages: [
{
role: "user",
content: [{ type: "text", text: "Describe these" }, ...images],
},
],
});
await guardrail.preCall(
payload,
createContext({ model: "minimax/minimax-01" })
);
// Should only call vision API for 2 images (maxImages=2)
assert.strictEqual(visionCallCount, 2);
});
// ── VB-S10: Meta information returned ───────────────────────────────────────
test("VB-S10: returns meta with imagesProcessed count", async () => {
mockVisionResponse = "A test description";
const guardrail = createGuardrail();
const payload = createPayload({
model: "minimax/minimax-01",
messages: [
{
role: "user",
content: [
{ type: "text", text: "What is this?" },
{
type: "image_url",
image_url: { url: "https://example.com/a.png" },
},
{
type: "image_url",
image_url: { url: "https://example.com/b.png" },
},
],
},
],
});
const result = await guardrail.preCall(
payload,
createContext({ model: "minimax/minimax-01" })
);
assert.ok(result.meta);
assert.ok(typeof result.meta === "object");
const meta = result.meta as Record<string, unknown>;
assert.strictEqual(meta.imagesProcessed, 2);
assert.ok(Array.isArray(meta.descriptions));
assert.strictEqual((meta.descriptions as string[]).length, 2);
assert.strictEqual(typeof meta.processingTimeMs, "number");
assert.strictEqual(meta.visionModel, "openai/gpt-4o-mini");
});

View File

@@ -0,0 +1,231 @@
/**
* Tests for callVisionModel helper function.
*/
import test from "node:test";
import assert from "node:assert/strict";
import { callVisionModel, type VisionModelConfig } from "@/lib/guardrails/visionBridgeHelpers";
// Store original fetch
const originalFetch = globalThis.fetch;
test("callVisionModel returns description on success", async () => {
// Mock global fetch
const mockResponse = {
ok: true,
json: async () => ({
choices: [{ message: { content: "A beautiful sunset over the ocean" } }],
}),
};
globalThis.fetch = async () => mockResponse as unknown as Response;
try {
const config: VisionModelConfig = {
model: "openai/gpt-4o-mini",
prompt: "Describe this image",
timeoutMs: 30000,
maxImages: 10,
};
const result = await callVisionModel(
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
config
);
assert.strictEqual(result, "A beautiful sunset over the ocean");
} finally {
// Restore original fetch
globalThis.fetch = originalFetch;
}
});
test("callVisionModel throws on HTTP error", async () => {
const mockResponse = {
ok: false,
status: 500,
text: async () => "Internal Server Error",
};
globalThis.fetch = async () => mockResponse as unknown as Response;
try {
const config: VisionModelConfig = {
model: "openai/gpt-4o-mini",
prompt: "Describe this image",
timeoutMs: 30000,
maxImages: 10,
};
await assert.rejects(
async () =>
await callVisionModel("data:image/png;base64,iVBORw0KGgo", config),
/Vision API error 500/
);
} finally {
globalThis.fetch = originalFetch;
}
});
test("callVisionModel throws on API error response", async () => {
const mockResponse = {
ok: true,
json: async () => ({
error: { message: "Invalid API key" },
}),
};
globalThis.fetch = async () => mockResponse as unknown as Response;
try {
const config: VisionModelConfig = {
model: "openai/gpt-4o-mini",
prompt: "Describe this image",
timeoutMs: 30000,
maxImages: 10,
};
await assert.rejects(
async () => await callVisionModel("data:image/png;base64,iVBORw0KGgo", config),
/Invalid API key/
);
} finally {
globalThis.fetch = originalFetch;
}
});
test("callVisionModel throws on empty response", async () => {
const mockResponse = {
ok: true,
json: async () => ({
choices: [{}],
}),
};
globalThis.fetch = async () => mockResponse as unknown as Response;
try {
const config: VisionModelConfig = {
model: "openai/gpt-4o-mini",
prompt: "Describe this image",
timeoutMs: 30000,
maxImages: 10,
};
await assert.rejects(
async () => await callVisionModel("data:image/png;base64,iVBORw0KGgo", config),
/empty or invalid/
);
} finally {
globalThis.fetch = originalFetch;
}
});
test("callVisionModel trims whitespace from response", async () => {
const mockResponse = {
ok: true,
json: async () => ({
choices: [{ message: { content: " A test description " } }],
}),
};
globalThis.fetch = async () => mockResponse as unknown as Response;
try {
const config: VisionModelConfig = {
model: "openai/gpt-4o-mini",
prompt: "Describe this image",
timeoutMs: 30000,
maxImages: 10,
};
const result = await callVisionModel("data:image/png;base64,iVBORw0KGgo", config);
assert.strictEqual(result, "A test description");
} finally {
globalThis.fetch = originalFetch;
}
});
test("callVisionModel passes custom API key", async () => {
let capturedHeaders: Record<string, string> = {};
const mockResponse = {
ok: true,
json: async () => ({
choices: [{ message: { content: "Description" } }],
}),
};
globalThis.fetch = async (url: URL | RequestInfo, init?: RequestInit) => {
if (init?.headers) {
capturedHeaders = init.headers as Record<string, string>;
}
return mockResponse as unknown as Response;
};
try {
const config: VisionModelConfig = {
model: "openai/gpt-4o-mini",
prompt: "Describe this image",
timeoutMs: 30000,
maxImages: 10,
};
await callVisionModel("data:image/png;base64,iVBORw0KGgo", config, "sk-custom-key");
assert.strictEqual(
capturedHeaders["Authorization"],
"Bearer sk-custom-key"
);
} finally {
globalThis.fetch = originalFetch;
}
});
test("callVisionModel uses correct request body format", async () => {
let capturedBody: Record<string, unknown> = {};
const mockResponse = {
ok: true,
json: async () => ({
choices: [{ message: { content: "Description" } }],
}),
};
globalThis.fetch = async (url: URL | RequestInfo, init?: RequestInit) => {
if (init?.body) {
capturedBody = JSON.parse(init.body as string);
}
return mockResponse as unknown as Response;
};
try {
const config: VisionModelConfig = {
model: "openai/gpt-4o-mini",
prompt: "What is in this image?",
timeoutMs: 30000,
maxImages: 10,
};
const imageUri = "data:image/png;base64,test123";
await callVisionModel(imageUri, config);
// Verify request structure
assert.strictEqual(capturedBody.model, "gpt-4o-mini");
assert.ok(Array.isArray(capturedBody.messages));
assert.strictEqual((capturedBody.messages as unknown[]).length, 1);
const message = (capturedBody.messages as Array<{role: string; content: unknown[]}>)[0];
assert.strictEqual(message.role, "user");
assert.ok(Array.isArray(message.content));
assert.strictEqual(message.content.length, 2);
// First content is image_url
const imagePart = message.content[0] as {type: string; image_url: {url: string; detail: string}};
assert.strictEqual(imagePart.type, "image_url");
assert.strictEqual(imagePart.image_url.url, imageUri);
assert.strictEqual(imagePart.image_url.detail, "low");
// Second content is text prompt
const textPart = message.content[1] as {type: string; text: string};
assert.strictEqual(textPart.type, "text");
assert.strictEqual(textPart.text, "What is in this image?");
} finally {
globalThis.fetch = originalFetch;
}
});

View File

@@ -0,0 +1,138 @@
/**
* Tests for extractImageParts helper function.
*/
import test from "node:test";
import assert from "node:assert/strict";
import { extractImageParts } from "@/lib/guardrails/visionBridgeHelpers";
interface RequestMessage {
role?: string;
content?: string | RequestContentPart[];
}
type RequestContentPart =
| { type: "text"; text: string }
| { type: "image_url"; image_url: { url: string; detail?: string } }
| { type: "image"; source: { type: "base64"; media_type: string; data: string } };
test("extractImageParts returns empty array for messages without images", () => {
const messages: RequestMessage[] = [
{ role: "user", content: "Hello, how are you?" },
];
const result = extractImageParts(messages);
assert.deepStrictEqual(result, []);
});
test("extractImageParts detects image_url format", () => {
const messages: RequestMessage[] = [
{
role: "user",
content: [
{ type: "text", text: "What is in this image?" },
{ type: "image_url", image_url: { url: "https://example.com/image.png" } },
],
},
];
const result = extractImageParts(messages);
assert.strictEqual(result.length, 1);
assert.strictEqual(result[0].messageIndex, 0);
assert.strictEqual(result[0].partIndex, 1);
assert.strictEqual(result[0].imageUrl, "https://example.com/image.png");
assert.strictEqual(result[0].imageType, "image_url");
});
test("extractImageParts detects base64 image format", () => {
const messages: RequestMessage[] = [
{
role: "user",
content: [
{
type: "image",
source: {
type: "base64",
media_type: "image/png",
data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
},
},
],
},
];
const result = extractImageParts(messages);
assert.strictEqual(result.length, 1);
assert.strictEqual(result[0].imageType, "image");
assert.ok(result[0].imageUrl.startsWith("data:image/png;base64,"));
});
test("extractImageParts handles multiple images in single message", () => {
const messages: RequestMessage[] = [
{
role: "user",
content: [
{ type: "text", text: "Compare these images" },
{ type: "image_url", image_url: { url: "https://example.com/image1.png" } },
{ type: "image_url", image_url: { url: "https://example.com/image2.png" } },
],
},
];
const result = extractImageParts(messages);
assert.strictEqual(result.length, 2);
assert.strictEqual(result[0].partIndex, 1);
assert.strictEqual(result[1].partIndex, 2);
});
test("extractImageParts handles images across multiple messages", () => {
const messages: RequestMessage[] = [
{ role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/image1.png" } }] },
{ role: "assistant", content: "Here is analysis of the first image." },
{ role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/image2.png" } }] },
];
const result = extractImageParts(messages);
assert.strictEqual(result.length, 2);
assert.strictEqual(result[0].messageIndex, 0);
assert.strictEqual(result[1].messageIndex, 2);
});
test("extractImageParts handles empty messages array", () => {
const result = extractImageParts([]);
assert.deepStrictEqual(result, []);
});
test("extractImageParts handles messages with null/undefined content", () => {
const messages: RequestMessage[] = [
{ role: "user", content: null as unknown as RequestContentPart[] },
{ role: "user", content: undefined as unknown as RequestContentPart[] },
];
const result = extractImageParts(messages);
assert.deepStrictEqual(result, []);
});
test("extractImageParts handles data URI image_url format", () => {
const dataUri = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
const messages: RequestMessage[] = [
{ role: "user", content: [{ type: "image_url", image_url: { url: dataUri } }] },
];
const result = extractImageParts(messages);
assert.strictEqual(result.length, 1);
assert.strictEqual(result[0].imageUrl, dataUri);
});
test("extractImageParts preserves order of images", () => {
const messages: RequestMessage[] = [
{
role: "user",
content: [
{ type: "text", text: "First" },
{ type: "image_url", image_url: { url: "https://example.com/A.png" } },
{ type: "text", text: "Second" },
{ type: "image_url", image_url: { url: "https://example.com/B.png" } },
{ type: "image_url", image_url: { url: "https://example.com/C.png" } },
],
},
];
const result = extractImageParts(messages);
assert.strictEqual(result.length, 3);
assert.strictEqual(result[0].partIndex, 1);
assert.strictEqual(result[1].partIndex, 3);
assert.strictEqual(result[2].partIndex, 4);
});

View File

@@ -0,0 +1,254 @@
/**
* Tests for replaceImageParts helper function.
*/
import test from "node:test";
import assert from "node:assert/strict";
import { replaceImageParts } from "@/lib/guardrails/visionBridgeHelpers";
test("replaceImageParts replaces single image with description", () => {
const body = {
model: "minimax/minimax-01",
messages: [
{
role: "user",
content: [
{ type: "text", text: "What is in this image?" },
{ type: "image_url", image_url: { url: "https://example.com/image.png" } },
],
},
],
};
const descriptions = ["[Image 1]: A beautiful sunset"];
const result = replaceImageParts(body, descriptions);
// Should have original text preserved
const content = result.messages[0].content as Array<{type: string; text?: string}>;
assert.strictEqual(content[0].type, "text");
assert.strictEqual(content[0].text, "What is in this image?");
// Should have description instead of image
assert.strictEqual(content[1].type, "text");
assert.strictEqual(content[1].text, "[Image 1]: A beautiful sunset");
});
test("replaceImageParts replaces multiple images with descriptions", () => {
const body = {
model: "minimax/minimax-01",
messages: [
{
role: "user",
content: [
{ type: "text", text: "Compare these images" },
{ type: "image_url", image_url: { url: "https://example.com/A.png" } },
{ type: "image_url", image_url: { url: "https://example.com/B.png" } },
{ type: "image_url", image_url: { url: "https://example.com/C.png" } },
],
},
],
};
const descriptions = [
"[Image 1]: A cat",
"[Image 2]: A dog",
"[Image 3]: A bird",
];
const result = replaceImageParts(body, descriptions);
const content = result.messages[0].content as Array<{type: string; text?: string}>;
assert.strictEqual(content[0].type, "text");
assert.strictEqual(content[0].text, "Compare these images");
assert.strictEqual(content[1].type, "text");
assert.strictEqual(content[1].text, "[Image 1]: A cat");
assert.strictEqual(content[2].type, "text");
assert.strictEqual(content[2].text, "[Image 2]: A dog");
assert.strictEqual(content[3].type, "text");
assert.strictEqual(content[3].text, "[Image 3]: A bird");
});
test("replaceImageParts handles empty descriptions array", () => {
const body = {
model: "minimax/minimax-01",
messages: [
{
role: "user",
content: [
{ type: "text", text: "Hello" },
{ type: "image_url", image_url: { url: "https://example.com/image.png" } },
],
},
],
};
const result = replaceImageParts(body, []);
// Original should be unchanged
const content = result.messages[0].content as Array<{type: string}>;
assert.strictEqual(content[1].type, "image_url");
});
test("replaceImageParts preserves non-image content", () => {
const body = {
model: "minimax/minimax-01",
messages: [
{
role: "system",
content: "You are a helpful assistant.",
},
{
role: "user",
content: [
{ type: "text", text: "Analyze this" },
{ type: "image_url", image_url: { url: "https://example.com/image.png" } },
],
},
{
role: "assistant",
content: "I can see the image shows a sunset.",
},
],
};
const result = replaceImageParts(body, ["[Image 1]: A sunset over the ocean"]);
// System message should be unchanged
assert.strictEqual(result.messages[0].content, "You are a helpful assistant.");
// Assistant message should be unchanged
assert.strictEqual(result.messages[2].content, "I can see the image shows a sunset.");
});
test("replaceImageParts handles base64 images", () => {
const body = {
model: "minimax/minimax-01",
messages: [
{
role: "user",
content: [
{
type: "image",
source: {
type: "base64",
media_type: "image/png",
data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
},
},
],
},
],
};
const descriptions = ["[Image 1]: A red circle"];
const result = replaceImageParts(body, descriptions);
const content = result.messages[0].content as Array<{type: string; text?: string}>;
assert.strictEqual(content[0].type, "text");
assert.strictEqual(content[0].text, "[Image 1]: A red circle");
});
test("replaceImageParts handles undefined descriptions", () => {
const body = {
model: "test",
messages: [
{
role: "user",
content: [
{ type: "text", text: "Hello" },
{ type: "image_url", image_url: { url: "https://example.com/image.png" } },
],
},
],
};
const result = replaceImageParts(body, undefined as unknown as string[]);
// Should return original body when descriptions is undefined
const content = result.messages[0].content as Array<{type: string}>;
assert.strictEqual(content[1].type, "image_url");
});
test("replaceImageParts handles empty messages array", () => {
const body = {
model: "test",
messages: [],
};
const descriptions = ["[Image 1]: Description"];
const result = replaceImageParts(body, descriptions);
assert.deepStrictEqual(result.messages, []);
});
test("replaceImageParts handles messages without content array", () => {
const body = {
model: "test",
messages: [
{ role: "user", content: "Just a text message" },
{ role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/image.png" } }] },
],
};
const descriptions = ["[Image 1]: Description"];
const result = replaceImageParts(body, descriptions);
// First message (string content) should be unchanged
assert.strictEqual(result.messages[0].content, "Just a text message");
// Second message should have image replaced
const content = result.messages[1].content as Array<{type: string; text?: string}>;
assert.strictEqual(content[0].type, "text");
assert.strictEqual(content[0].text, "[Image 1]: Description");
});
test("replaceImageParts does not modify original body", () => {
const body = {
model: "minimax/minimax-01",
messages: [
{
role: "user",
content: [
{ type: "text", text: "Original" },
{ type: "image_url", image_url: { url: "https://example.com/image.png" } },
],
},
],
};
const descriptions = ["[Image 1]: Modified"];
replaceImageParts(body, descriptions);
// Original should be unchanged
const content = body.messages[0].content as Array<{type: string}>;
assert.strictEqual(content[1].type, "image_url");
});
test("replaceImageParts handles mixed images and text", () => {
const body = {
model: "test",
messages: [
{
role: "user",
content: [
{ type: "image_url", image_url: { url: "https://example.com/first.png" } },
{ type: "text", text: "between images" },
{ type: "image_url", image_url: { url: "https://example.com/second.png" } },
],
},
],
};
const descriptions = ["[Image 1]: First image", "[Image 2]: Second image"];
const result = replaceImageParts(body, descriptions);
const content = result.messages[0].content as Array<{type: string; text?: string}>;
assert.strictEqual(content[0].type, "text");
assert.strictEqual(content[0].text, "[Image 1]: First image");
assert.strictEqual(content[1].type, "text");
assert.strictEqual(content[1].text, "between images");
assert.strictEqual(content[2].type, "text");
assert.strictEqual(content[2].text, "[Image 2]: Second image");
});

View File

@@ -0,0 +1,62 @@
/**
* Tests for resolveImageAsDataUri helper function.
*/
import test from "node:test";
import assert from "node:assert/strict";
import { resolveImageAsDataUri } from "@/lib/guardrails/visionBridgeHelpers";
test("resolveImageAsDataUri passes through HTTPS URL as-is", () => {
const url = "https://example.com/image.png";
const result = resolveImageAsDataUri(url);
assert.strictEqual(result, url);
});
test("resolveImageAsDataUri passes through HTTP URL as-is", () => {
const url = "http://example.com/image.png";
const result = resolveImageAsDataUri(url);
assert.strictEqual(result, url);
});
test("resolveImageAsDataUri passes through data URI as-is", () => {
const dataUri = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
const result = resolveImageAsDataUri(dataUri);
assert.strictEqual(result, dataUri);
});
test("resolveImageAsDataUri converts base64 string to PNG data URI", () => {
const base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
const result = resolveImageAsDataUri(base64);
assert.strictEqual(result, `data:image/png;base64,${base64}`);
});
test("resolveImageAsDataUri throws for empty string", () => {
assert.throws(() => {
resolveImageAsDataUri("");
}, /Invalid image URL/);
});
test("resolveImageAsDataUri throws for null", () => {
assert.throws(() => {
resolveImageAsDataUri(null as unknown as string);
}, /Invalid image URL/);
});
test("resolveImageAsDataUri throws for undefined", () => {
assert.throws(() => {
resolveImageAsDataUri(undefined as unknown as string);
}, /Invalid image URL/);
});
test("resolveImageAsDataUri handles data URI with different media types", () => {
const jpegUri = "data:image/jpeg;base64,/9j/4AAQSkZJRg==";
const result = resolveImageAsDataUri(jpegUri);
assert.strictEqual(result, jpegUri);
});
test("resolveImageAsDataUri treats plain base64 with prefix as data URI", () => {
// If it doesn't start with data: or http, treat as raw base64
const rawBase64 = "abcdef123456==";
const result = resolveImageAsDataUri(rawBase64);
assert.strictEqual(result, "data:image/png;base64,abcdef123456==");
});

View File

@@ -0,0 +1,91 @@
/**
* Tests for Vision Bridge default constants.
*/
import test from "node:test";
import assert from "node:assert/strict";
import {
VISION_BRIDGE_DEFAULTS,
VISION_BRIDGE_SETTINGS_KEYS,
getVisionBridgeConfig,
type VisionBridgeSettings,
} from "@/shared/constants/visionBridgeDefaults";
test("VISION_BRIDGE_DEFAULTS exports correct values", () => {
assert.strictEqual(VISION_BRIDGE_DEFAULTS.enabled, true);
assert.strictEqual(VISION_BRIDGE_DEFAULTS.model, "openai/gpt-4o-mini");
assert.strictEqual(
VISION_BRIDGE_DEFAULTS.prompt,
"Describe this image concisely in 2-3 sentences. Focus on the most relevant visual details."
);
assert.strictEqual(VISION_BRIDGE_DEFAULTS.timeoutMs, 30000);
assert.strictEqual(VISION_BRIDGE_DEFAULTS.maxImagesPerRequest, 10);
});
test("VISION_BRIDGE_SETTINGS_KEYS exports all expected keys", () => {
assert.deepStrictEqual(VISION_BRIDGE_SETTINGS_KEYS, [
"visionBridgeEnabled",
"visionBridgeModel",
"visionBridgePrompt",
"visionBridgeTimeout",
"visionBridgeMaxImages",
]);
});
test("getVisionBridgeConfig returns defaults when no settings provided", () => {
const config = getVisionBridgeConfig({});
assert.strictEqual(config.enabled, true);
assert.strictEqual(config.model, "openai/gpt-4o-mini");
assert.strictEqual(config.prompt, VISION_BRIDGE_DEFAULTS.prompt);
assert.strictEqual(config.timeoutMs, 30000);
assert.strictEqual(config.maxImages, 10);
});
test("getVisionBridgeConfig applies custom settings", () => {
const customSettings: VisionBridgeSettings = {
visionBridgeEnabled: false,
visionBridgeModel: "anthropic/claude-3-haiku",
visionBridgePrompt: "What is in this image?",
visionBridgeTimeout: 60000,
visionBridgeMaxImages: 5,
};
const config = getVisionBridgeConfig(customSettings);
assert.strictEqual(config.enabled, false);
assert.strictEqual(config.model, "anthropic/claude-3-haiku");
assert.strictEqual(config.prompt, "What is in this image?");
assert.strictEqual(config.timeoutMs, 60000);
assert.strictEqual(config.maxImages, 5);
});
test("getVisionBridgeConfig merges partial settings with defaults", () => {
const partialSettings: VisionBridgeSettings = {
visionBridgeModel: "openai/gpt-4o",
};
const config = getVisionBridgeConfig(partialSettings);
// Custom value
assert.strictEqual(config.model, "openai/gpt-4o");
// Default values for the rest
assert.strictEqual(config.enabled, true);
assert.strictEqual(config.prompt, VISION_BRIDGE_DEFAULTS.prompt);
assert.strictEqual(config.timeoutMs, 30000);
assert.strictEqual(config.maxImages, 10);
});
test("getVisionBridgeConfig handles undefined settings", () => {
const config = getVisionBridgeConfig(undefined);
assert.strictEqual(config.enabled, true);
assert.strictEqual(config.model, "openai/gpt-4o-mini");
});
test("getVisionBridgeConfig handles null settings", () => {
const config = getVisionBridgeConfig(null as unknown as VisionBridgeSettings);
assert.strictEqual(config.enabled, true);
assert.strictEqual(config.model, "openai/gpt-4o-mini");
});