diff --git a/src/lib/guardrails/registry.ts b/src/lib/guardrails/registry.ts index 24ed1dc156..b90cd2147e 100644 --- a/src/lib/guardrails/registry.ts +++ b/src/lib/guardrails/registry.ts @@ -1,6 +1,7 @@ import { BaseGuardrail, type GuardrailContext, type GuardrailExecutionResult } from "./base"; import { PIIMaskerGuardrail } from "./piiMasker"; import { PromptInjectionGuardrail } from "./promptInjection"; +import { VisionBridgeGuardrail } from "./visionBridge"; type HeadersLike = Headers | Record | null | undefined; @@ -262,6 +263,7 @@ let defaultGuardrailsRegistered = false; export function registerDefaultGuardrails() { if (defaultGuardrailsRegistered) return guardrailRegistry; + guardrailRegistry.register(new VisionBridgeGuardrail()); guardrailRegistry.register(new PIIMaskerGuardrail()); guardrailRegistry.register(new PromptInjectionGuardrail()); defaultGuardrailsRegistered = true; diff --git a/src/lib/guardrails/visionBridge.ts b/src/lib/guardrails/visionBridge.ts new file mode 100644 index 0000000000..f9bcac35ca --- /dev/null +++ b/src/lib/guardrails/visionBridge.ts @@ -0,0 +1,150 @@ +/** + * Vision Bridge Guardrail. + * Intercepts image-bearing requests to non-vision models, + * extracts descriptions via vision model, and replaces images with text. + */ + +import { BaseGuardrail, type GuardrailContext, type GuardrailResult } from "./base"; +import { getSettings as defaultGetSettings } from "@/lib/db/settings"; +import { getResolvedModelCapabilities } from "@/lib/modelCapabilities"; +import { + extractImageParts, + callVisionModel as defaultCallVisionModel, + replaceImageParts, +} from "./visionBridgeHelpers"; +import { + VISION_BRIDGE_DEFAULTS, + getVisionBridgeConfig, +} from "@/shared/constants/visionBridgeDefaults"; + +export interface VisionBridgeDependencies { + getSettings?: () => Promise>; + callVisionModel?: ( + imageDataUri: string, + config: import("./visionBridgeHelpers").VisionModelConfig, + apiKey?: string + ) => Promise; +} + +export class VisionBridgeGuardrail extends BaseGuardrail { + name = "vision-bridge"; + priority = 5; + + private readonly deps: VisionBridgeDependencies; + + constructor(options?: { enabled?: boolean; deps?: VisionBridgeDependencies }) { + super("vision-bridge", { priority: 5, enabled: options?.enabled }); + this.deps = options?.deps ?? {}; + } + + async preCall( + payload: unknown, + context: GuardrailContext + ): Promise> { + // 1. Check if disabled at guardrail level + if (!this.enabled) { + return { block: false }; + } + + // 2. Check disabled via context (header, body, API key) + if (context.disabledGuardrails?.includes("vision-bridge")) { + return { block: false }; + } + + // 3. Get model from context or payload + const model = + context.model || (payload as Record)?.model as string | undefined; + if (!model) { + return { block: false }; + } + + // 4. Check if model supports vision + const capabilities = getResolvedModelCapabilities(model); + if (capabilities.supportsVision === true) { + return { block: false }; + } + + // 5. Get body and check for messages + const body = payload as Record; + const messages = body?.messages; + if (!Array.isArray(messages) || messages.length === 0) { + return { block: false }; + } + + // 6. Check for images using helper (extractImageParts returns empty if no images) + const imageParts = extractImageParts( + messages as Parameters[0] + ); + if (imageParts.length === 0) { + return { block: false }; + } + + // 7. Get settings (injectable for testing) + const getSettings = this.deps.getSettings ?? defaultGetSettings; + let settings: Record = {}; + try { + settings = await getSettings(); + } catch { + // If getSettings fails, use defaults + } + + // 8. Check if Vision Bridge is enabled in settings + const enabled = settings.visionBridgeEnabled ?? VISION_BRIDGE_DEFAULTS.enabled; + if (!enabled) { + return { block: false }; + } + + // 9. Get configuration + const config = getVisionBridgeConfig({ + visionBridgeEnabled: settings.visionBridgeEnabled as boolean | undefined, + visionBridgeModel: settings.visionBridgeModel as string | undefined, + visionBridgePrompt: settings.visionBridgePrompt as string | undefined, + visionBridgeTimeout: settings.visionBridgeTimeout as number | undefined, + visionBridgeMaxImages: settings.visionBridgeMaxImages as number | undefined, + }); + + // 10. Limit images + const limitedParts = imageParts.slice(0, config.maxImages); + + // 11. Call vision model for each image in parallel (injectable for testing) + const callVision = this.deps.callVisionModel ?? defaultCallVisionModel; + const logger = context.log; + const startTime = Date.now(); + + // Process all images in parallel using Promise.allSettled for fail-partial behavior + const results = await Promise.allSettled( + limitedParts.map(async (imagePart, i) => { + const description = await callVision(imagePart.imageUrl, config); + return `[Image ${i + 1}]: ${description}`; + }) + ); + + // Collect descriptions maintaining original order + const descriptions = results.map((result, i) => { + if (result.status === "fulfilled") { + return result.value; + } + const message = result.reason instanceof Error ? result.reason.message : String(result.reason); + logger?.warn?.("VISION-BRIDGE", `Failed to get description for image ${i + 1}: ${message}`); + return `[Image ${i + 1}]: (unavailable)`; + }); + + // 12. Replace image parts with text descriptions + const modifiedBody = replaceImageParts( + body as Parameters[0], + descriptions + ); + const processingTime = Date.now() - startTime; + + return { + block: false, + modifiedPayload: modifiedBody, + meta: { + imagesProcessed: descriptions.length, + descriptions, + processingTimeMs: processingTime, + visionModel: config.model, + }, + }; + } +} \ No newline at end of file diff --git a/src/lib/guardrails/visionBridgeHelpers.ts b/src/lib/guardrails/visionBridgeHelpers.ts new file mode 100644 index 0000000000..c0d28376b6 --- /dev/null +++ b/src/lib/guardrails/visionBridgeHelpers.ts @@ -0,0 +1,330 @@ +/** + * Vision Bridge helper functions for image processing. + */ + +/** + * Provider to environment variable mapping for API key resolution. + */ +const PROVIDER_API_KEY_MAP: Record = { + anthropic: "ANTHROPIC_API_KEY", + google: "GOOGLE_API_KEY", + openai: "OPENAI_API_KEY", +}; + +/** + * Resolve API key based on model provider. + * @param model - Model identifier (e.g., "anthropic/claude-3-haiku", "openai/gpt-4o-mini") + * @param explicitKey - Explicit API key passed as argument (takes precedence) + * @returns Resolved API key string + */ +export function resolveProviderApiKey(model: string, explicitKey?: string): string { + if (explicitKey) return explicitKey; + const provider = model.includes("/") ? model.split("/")[0] : ""; + const envVar = PROVIDER_API_KEY_MAP[provider] || "OPENAI_API_KEY"; + return process.env[envVar] || ""; +} + +export interface ImagePart { + messageIndex: number; + partIndex: number; + imageUrl: string; + imageType: "image_url" | "image"; +} + +export interface RequestMessage { + role?: string; + content?: string | RequestContentPart[]; +} + +export type RequestContentPart = + | { type: "text"; text: string } + | { type: "image_url"; image_url: { url: string; detail?: string } } + | { type: "image"; source: { type: "base64"; media_type: string; data: string } }; + +/** + * Extract image parts from messages array. + * Supports both OpenAI image_url format and base64 image format. + */ +export function extractImageParts(messages: RequestMessage[]): ImagePart[] { + const results: ImagePart[] = []; + + if (!Array.isArray(messages)) { + return results; + } + + for (let msgIdx = 0; msgIdx < messages.length; msgIdx++) { + const message = messages[msgIdx]; + if (!message || !Array.isArray(message.content)) { + continue; + } + + for (let partIdx = 0; partIdx < message.content.length; partIdx++) { + const part = message.content[partIdx]; + + if (part?.type === "image_url" && part.image_url?.url) { + results.push({ + messageIndex: msgIdx, + partIndex: partIdx, + imageUrl: part.image_url.url, + imageType: "image_url", + }); + } else if (part?.type === "image" && part.source?.type === "base64") { + const { media_type, data } = part.source; + const dataUri = `data:${media_type};base64,${data}`; + results.push({ + messageIndex: msgIdx, + partIndex: partIdx, + imageUrl: dataUri, + imageType: "image", + }); + } + } + } + + return results; +} + +/** + * Resolve image URL to data URI format for vision model. + * - HTTP/HTTPS URLs: passed through as-is + * - Data URIs: passed through as-is + * - Base64 without media type: assumed PNG + */ +export function resolveImageAsDataUri(imageUrl: string): string { + if (!imageUrl || typeof imageUrl !== "string") { + throw new Error("Invalid image URL: must be a non-empty string"); + } + + // Already a data URI + if (imageUrl.startsWith("data:")) { + return imageUrl; + } + + // HTTP/HTTPS URL - vision API will fetch it + if (imageUrl.startsWith("http://") || imageUrl.startsWith("https://")) { + return imageUrl; + } + + // Assume it's a base64 string without prefix + // Add PNG as default media type + return `data:image/png;base64,${imageUrl}`; +} + +export interface VisionModelConfig { + model: string; + prompt: string; + timeoutMs: number; + maxImages: number; +} + +/** + * Call the vision model to get an image description. + * Supports both OpenAI-compatible and Anthropic API formats. + */ +export async function callVisionModel( + imageDataUri: string, + config: VisionModelConfig, + apiKey?: string +): Promise { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), config.timeoutMs); + + // Resolve API key based on provider + const resolvedApiKey = resolveProviderApiKey(config.model, apiKey); + + // Detect provider from model identifier + const isAnthropic = config.model.startsWith("anthropic/"); + + try { + // Extract model name from provider/model format + const modelName = config.model.includes("/") + ? config.model.split("/")[1] + : config.model; + + let response: Response; + + if (isAnthropic) { + // Anthropic API path + const anthropicBaseUrl = process.env.ANTHROPIC_API_URL || "https://api.anthropic.com"; + + // Parse data URI to extract media type and base64 data + const matches = imageDataUri.match(/^data:([^;]+);base64,(.+)$/); + let mediaType = "image/png"; + let base64Data = imageDataUri; + + if (matches) { + mediaType = matches[1]; + base64Data = matches[2]; + } + + response = await fetch(`${anthropicBaseUrl}/v1/messages`, { + method: "POST", + signal: controller.signal, + headers: { + "x-api-key": resolvedApiKey, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + body: JSON.stringify({ + model: modelName, + messages: [ + { + role: "user", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: mediaType, + data: base64Data, + }, + }, + { + type: "text", + text: config.prompt, + }, + ], + }, + ], + max_tokens: 300, + }), + }); + } else { + // OpenAI-compatible path (default) + const baseUrl = process.env.OPENAI_API_URL || "https://api.openai.com/v1"; + + response = await fetch(`${baseUrl}/chat/completions`, { + method: "POST", + signal: controller.signal, + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${resolvedApiKey}`, + }, + body: JSON.stringify({ + model: modelName, + messages: [ + { + role: "user", + content: [ + { + type: "image_url", + image_url: { + url: imageDataUri, + detail: "low", + }, + }, + { type: "text", text: config.prompt }, + ], + }, + ], + max_tokens: 300, + }), + }); + } + + clearTimeout(timeoutId); + + if (!response.ok) { + const errorText = await response.text().catch(() => "Unknown error"); + throw new Error(`Vision API error ${response.status}: ${errorText}`); + } + + const data = await response.json(); + + if (isAnthropic) { + // Anthropic response format: { content: [{ type: "text", text: "..." }] } + const anthropicData = data as { + content?: Array<{ type?: string; text?: string }>; + error?: { message?: string }; + }; + + if (anthropicData.error) { + throw new Error(`Vision API error: ${anthropicData.error.message || JSON.stringify(anthropicData.error)}`); + } + + const textContent = anthropicData.content?.find((c) => c.type === "text"); + const content = textContent?.text; + if (!content || typeof content !== "string") { + throw new Error("Vision API returned empty or invalid response"); + } + + return content.trim(); + } else { + // OpenAI-compatible response format: { choices: [{ message: { content: "..." } }] } + const openaiData = data as { + choices?: Array<{ message?: { content?: string } }>; + error?: { message?: string }; + }; + + if (openaiData.error) { + throw new Error(`Vision API error: ${openaiData.error.message || JSON.stringify(openaiData.error)}`); + } + + const content = openaiData.choices?.[0]?.message?.content; + if (!content || typeof content !== "string") { + throw new Error("Vision API returned empty or invalid response"); + } + + return content.trim(); + } + } catch (error) { + clearTimeout(timeoutId); + + if (error instanceof Error && error.name === "AbortError") { + throw new Error("Vision model call timed out"); + } + + throw error; + } +} + +export interface RequestBody { + model?: string; + messages?: RequestMessage[]; + [key: string]: unknown; +} + +/** + * Replace image content parts with text descriptions. + * Concatenates descriptions with labels: "[Image 1]: ..." + */ +export function replaceImageParts(body: RequestBody, descriptions: string[]): RequestBody { + if (!descriptions || descriptions.length === 0) { + return body; + } + + const result = structuredClone(body) as RequestBody; + + if (!Array.isArray(result.messages)) { + return result; + } + + let descriptionIndex = 0; + + for (let msgIdx = 0; msgIdx < result.messages.length; msgIdx++) { + const message = result.messages[msgIdx]; + if (!message || !Array.isArray(message.content)) { + continue; + } + + const newContent: RequestContentPart[] = []; + + for (const part of message.content) { + if (part?.type === "image_url" || part?.type === "image") { + if (descriptionIndex < descriptions.length) { + newContent.push({ + type: "text", + text: descriptions[descriptionIndex], + }); + descriptionIndex++; + } + } else { + newContent.push(part as RequestContentPart); + } + } + + message.content = newContent; + } + + return result; +} diff --git a/src/shared/constants/visionBridgeDefaults.ts b/src/shared/constants/visionBridgeDefaults.ts new file mode 100644 index 0000000000..18e069d6d3 --- /dev/null +++ b/src/shared/constants/visionBridgeDefaults.ts @@ -0,0 +1,55 @@ +/** + * Vision Bridge default configuration values. + */ + +export const VISION_BRIDGE_DEFAULTS = { + enabled: true, + model: "openai/gpt-4o-mini", + prompt: + "Describe this image concisely in 2-3 sentences. Focus on the most relevant visual details.", + timeoutMs: 30000, + maxImagesPerRequest: 10, +} as const; + +/** + * Settings keys for Vision Bridge (to be stored in key_value table). + */ +export const VISION_BRIDGE_SETTINGS_KEYS = [ + "visionBridgeEnabled", + "visionBridgeModel", + "visionBridgePrompt", + "visionBridgeTimeout", + "visionBridgeMaxImages", +] as const; + +export type VisionBridgeSettings = { + visionBridgeEnabled?: boolean; + visionBridgeModel?: string; + visionBridgePrompt?: string; + visionBridgeTimeout?: number; + visionBridgeMaxImages?: number; +}; + +export type VisionBridgeConfig = { + enabled: boolean; + model: string; + prompt: string; + timeoutMs: number; + maxImages: number; +}; + +/** + * Merge settings with defaults to produce a complete config. + */ +export function getVisionBridgeConfig( + settings: VisionBridgeSettings | undefined | null = {} +): VisionBridgeConfig { + const s = settings ?? {}; + return { + enabled: s.visionBridgeEnabled ?? VISION_BRIDGE_DEFAULTS.enabled, + model: s.visionBridgeModel ?? VISION_BRIDGE_DEFAULTS.model, + prompt: s.visionBridgePrompt ?? VISION_BRIDGE_DEFAULTS.prompt, + timeoutMs: s.visionBridgeTimeout ?? VISION_BRIDGE_DEFAULTS.timeoutMs, + maxImages: s.visionBridgeMaxImages ?? VISION_BRIDGE_DEFAULTS.maxImagesPerRequest, + }; +} \ No newline at end of file diff --git a/tests/unit/guardrails/visionBridge.test.ts b/tests/unit/guardrails/visionBridge.test.ts new file mode 100644 index 0000000000..550dc63469 --- /dev/null +++ b/tests/unit/guardrails/visionBridge.test.ts @@ -0,0 +1,515 @@ +/** + * Tests for VisionBridgeGuardrail. + * Uses dependency injection to avoid SQLite dependency. + */ + +import test from "node:test"; +import assert from "node:assert/strict"; + +const { VisionBridgeGuardrail } = await import( + "../../../src/lib/guardrails/visionBridge.ts" +); +const { resetGuardrailsForTests } = await import( + "../../../src/lib/guardrails/registry.ts" +); +const { getResolvedModelCapabilities } = await import( + "../../../src/lib/modelCapabilities.ts" +); +import type { GuardrailContext } from "../../../src/lib/guardrails/base.ts"; +import type { VisionModelConfig } from "../../../src/lib/guardrails/visionBridgeHelpers.ts"; + +// ── Mock state ────────────────────────────────────────────────────────────── + +let mockSettings: Record = { + visionBridgeEnabled: true, + visionBridgeModel: "openai/gpt-4o-mini", + visionBridgePrompt: "Describe this image concisely.", + visionBridgeTimeout: 30000, + visionBridgeMaxImages: 10, +}; + +let mockVisionResponse = "A beautiful sunset over the ocean"; +let shouldVisionFail = false; +let visionCallCount = 0; + +function createGuardrail( + options?: Parameters[0] +) { + return new VisionBridgeGuardrail({ + ...options, + deps: { + getSettings: async () => mockSettings, + callVisionModel: async ( + _imageDataUri: string, + _config: VisionModelConfig + ) => { + visionCallCount++; + if (shouldVisionFail) { + throw new Error("Vision model failed"); + } + return mockVisionResponse; + }, + }, + }); +} + +test.beforeEach(() => { + resetGuardrailsForTests({ registerDefaults: false }); + visionCallCount = 0; + shouldVisionFail = false; + mockSettings = { + visionBridgeEnabled: true, + visionBridgeModel: "openai/gpt-4o-mini", + visionBridgePrompt: "Describe this image concisely.", + visionBridgeTimeout: 30000, + visionBridgeMaxImages: 10, + }; +}); + +// ── Helpers ───────────────────────────────────────────────────────────────── + +function createContext( + overrides: Partial = {} +): GuardrailContext { + return { + model: "minimax/minimax-01", + log: console, + ...overrides, + }; +} + +function createPayload( + overrides: Record = {} +): Record { + return { + model: "minimax/minimax-01", + messages: [{ role: "user", content: "Hello" }], + ...overrides, + }; +} + +// ── Basic Properties ──────────────────────────────────────────────────────── + +test("VisionBridgeGuardrail has correct name and priority", () => { + const guardrail = createGuardrail(); + assert.strictEqual(guardrail.name, "vision-bridge"); + assert.strictEqual(guardrail.priority, 5); +}); + +test("VisionBridgeGuardrail is enabled by default", () => { + const guardrail = createGuardrail(); + assert.strictEqual(guardrail.enabled, true); +}); + +test("VisionBridgeGuardrail can be disabled via constructor", () => { + const guardrail = createGuardrail({ enabled: false }); + assert.strictEqual(guardrail.enabled, false); +}); + +// ── VB-S05: Vision Bridge disabled via settings ──────────────────────────── + +test("VB-S05: passthroughs when visionBridgeEnabled is false", async () => { + mockSettings.visionBridgeEnabled = false; + const guardrail = createGuardrail(); + + const payload = createPayload({ + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What is in this image?" }, + { + type: "image_url", + image_url: { url: "https://example.com/image.png" }, + }, + ], + }, + ], + }); + + const result = await guardrail.preCall(payload, createContext()); + assert.strictEqual(result.block, false); + assert.strictEqual(result.modifiedPayload, undefined); +}); + +// ── VB-S06: Disabled via context ──────────────────────────────────────────── + +test("VB-S06: skips when disabledGuardrails includes vision-bridge", async () => { + const guardrail = createGuardrail(); + const payload = createPayload(); + const context = createContext({ disabledGuardrails: ["vision-bridge"] }); + + const result = await guardrail.preCall(payload, context); + assert.strictEqual(result.block, false); + assert.strictEqual(result.modifiedPayload, undefined); +}); + +// ── VB-S02: Vision-capable model passthrough ──────────────────────────────── + +test("VB-S02: passthroughs for vision-capable model (gpt-4o)", async () => { + const guardrail = createGuardrail(); + const payload = createPayload({ + model: "openai/gpt-4o", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What is this?" }, + { + type: "image_url", + image_url: { url: "https://example.com/image.png" }, + }, + ], + }, + ], + }); + + const result = await guardrail.preCall( + payload, + createContext({ model: "openai/gpt-4o" }) + ); + + // If supportsVision is true, it should passthrough (no modification) + // If supportsVision is null/undefined (no sync data), it will process — that's correct behavior + const capabilities = getResolvedModelCapabilities("openai/gpt-4o"); + if (capabilities.supportsVision === true) { + assert.strictEqual(result.block, false); + assert.strictEqual(result.modifiedPayload, undefined); + } else { + // Without sync data, supportsVision is null — guardrail processes the image + // This is correct fail-open behavior for unknown model capabilities + assert.strictEqual(result.block, false); + } +}); + +test("VB-S02: model capabilities returns supportsVision for known models", () => { + const gpt4oCaps = getResolvedModelCapabilities("openai/gpt-4o"); + // supportsVision may be true (if sync data exists) or null (if not synced) + assert.ok(gpt4oCaps.supportsVision === true || gpt4oCaps.supportsVision === null); +}); + +// ── VB-S04: No images passthrough ────────────────────────────────────────── + +test("VB-S04: passthroughs when no images in messages", async () => { + const guardrail = createGuardrail(); + const payload = createPayload({ + messages: [{ role: "user", content: "Hello, how are you?" }], + }); + + const result = await guardrail.preCall(payload, createContext()); + assert.strictEqual(result.block, false); + assert.strictEqual(result.modifiedPayload, undefined); +}); + +test("VB-S04: passthroughs when messages array is empty", async () => { + const guardrail = createGuardrail(); + const payload = createPayload({ messages: [] }); + const result = await guardrail.preCall(payload, createContext()); + assert.strictEqual(result.block, false); +}); + +// ── VB-S01: Single image processing ───────────────────────────────────────── + +test("VB-S01: replaces image with description for non-vision model", async () => { + mockVisionResponse = "A beautiful sunset over the ocean"; + const guardrail = createGuardrail(); + + const payload = createPayload({ + model: "minimax/minimax-01", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What is in this image?" }, + { + type: "image_url", + image_url: { url: "https://example.com/image.png" }, + }, + ], + }, + ], + }); + + const result = await guardrail.preCall( + payload, + createContext({ model: "minimax/minimax-01" }) + ); + + assert.strictEqual(result.block, false); + assert.ok(result.modifiedPayload); + + const modified = result.modifiedPayload as { + messages: Array<{ content: unknown[] }>; + }; + const content = modified.messages[0].content as Array<{ + type: string; + text?: string; + }>; + + const imagePart = content.find((p) => p.type === "image_url"); + assert.strictEqual(imagePart, undefined); + + const descriptionPart = content.find( + (p) => p.type === "text" && p.text?.includes("sunset") + ); + assert.ok(descriptionPart); +}); + +// ── VB-S04: Multiple images ───────────────────────────────────────────────── + +test("VB-S04: processes multiple images and concatenates descriptions", async () => { + let callIdx = 0; + const descriptions = ["A cute cat", "A playful dog", "A colorful bird"]; + + const guardrail = new VisionBridgeGuardrail({ + deps: { + getSettings: async () => mockSettings, + callVisionModel: async () => { + const desc = descriptions[callIdx] || "Unknown image"; + callIdx++; + return desc; + }, + }, + }); + + const payload = createPayload({ + model: "minimax/minimax-01", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Describe these images" }, + { + type: "image_url", + image_url: { url: "https://example.com/cat.png" }, + }, + { + type: "image_url", + image_url: { url: "https://example.com/dog.png" }, + }, + { + type: "image_url", + image_url: { url: "https://example.com/bird.png" }, + }, + ], + }, + ], + }); + + const result = await guardrail.preCall( + payload, + createContext({ model: "minimax/minimax-01" }) + ); + + assert.strictEqual(result.block, false); + assert.ok(result.modifiedPayload); + assert.strictEqual(callIdx, 3); + + const modified = result.modifiedPayload as { + messages: Array<{ content: unknown[] }>; + }; + const content = modified.messages[0].content as Array<{ + type: string; + text?: string; + }>; + + assert.ok( + content.some((p) => p.type === "text" && p.text?.includes("[Image 1]")) + ); + assert.ok( + content.some((p) => p.type === "text" && p.text?.includes("[Image 2]")) + ); + assert.ok( + content.some((p) => p.type === "text" && p.text?.includes("[Image 3]")) + ); +}); + +// ── VB-S03: Fail-open on vision error ────────────────────────────────────── + +test("VB-S03: returns modified payload with unavailable text when vision API fails", async () => { + shouldVisionFail = true; + const guardrail = createGuardrail(); + + const payload = createPayload({ + model: "minimax/minimax-01", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What is this?" }, + { + type: "image_url", + image_url: { url: "https://example.com/image.png" }, + }, + ], + }, + ], + }); + + const result = await guardrail.preCall( + payload, + createContext({ model: "minimax/minimax-01" }) + ); + + assert.strictEqual(result.block, false); + assert.ok(result.modifiedPayload); + + const modified = result.modifiedPayload as { + messages: Array<{ content: unknown[] }>; + }; + const content = modified.messages[0].content as Array<{ + type: string; + text?: string; + }>; + + // Should have "unavailable" text instead of image + const unavailPart = content.find( + (p) => p.type === "text" && p.text?.includes("unavailable") + ); + assert.ok(unavailPart); +}); + +test("VB-S03: logs warning when vision API fails", async () => { + shouldVisionFail = true; + let warningLogged = false; + const guardrail = createGuardrail(); + + const payload = createPayload({ + model: "minimax/minimax-01", + messages: [ + { + role: "user", + content: [ + { + type: "image_url", + image_url: { url: "https://example.com/image.png" }, + }, + ], + }, + ], + }); + + const mockLog = { + warn: (_tag: string, msg: string) => { + if (msg.includes("Failed to get description")) { + warningLogged = true; + } + }, + }; + + await guardrail.preCall( + payload, + createContext({ + model: "minimax/minimax-01", + log: mockLog as GuardrailContext["log"], + }) + ); + + assert.strictEqual(warningLogged, true); +}); + +// ── VB-S07: Base64 image format ───────────────────────────────────────────── + +test("VB-S07: handles base64 image format", async () => { + mockVisionResponse = "An image description"; + const guardrail = createGuardrail(); + + const payload = createPayload({ + model: "minimax/minimax-01", + messages: [ + { + role: "user", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + }, + }, + ], + }, + ], + }); + + const result = await guardrail.preCall( + payload, + createContext({ model: "minimax/minimax-01" }) + ); + + assert.strictEqual(result.block, false); + assert.ok(result.modifiedPayload); +}); + +// ── VB-S09: Image count limit ─────────────────────────────────────────────── + +test("VB-S09: respects maxImages setting", async () => { + mockSettings.visionBridgeMaxImages = 2; + const guardrail = createGuardrail(); + + const images = Array.from({ length: 5 }, (_, i) => ({ + type: "image_url" as const, + image_url: { url: `https://example.com/image${i}.png` }, + })); + + const payload = createPayload({ + model: "minimax/minimax-01", + messages: [ + { + role: "user", + content: [{ type: "text", text: "Describe these" }, ...images], + }, + ], + }); + + await guardrail.preCall( + payload, + createContext({ model: "minimax/minimax-01" }) + ); + + // Should only call vision API for 2 images (maxImages=2) + assert.strictEqual(visionCallCount, 2); +}); + +// ── VB-S10: Meta information returned ─────────────────────────────────────── + +test("VB-S10: returns meta with imagesProcessed count", async () => { + mockVisionResponse = "A test description"; + const guardrail = createGuardrail(); + + const payload = createPayload({ + model: "minimax/minimax-01", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What is this?" }, + { + type: "image_url", + image_url: { url: "https://example.com/a.png" }, + }, + { + type: "image_url", + image_url: { url: "https://example.com/b.png" }, + }, + ], + }, + ], + }); + + const result = await guardrail.preCall( + payload, + createContext({ model: "minimax/minimax-01" }) + ); + + assert.ok(result.meta); + assert.ok(typeof result.meta === "object"); + + const meta = result.meta as Record; + assert.strictEqual(meta.imagesProcessed, 2); + assert.ok(Array.isArray(meta.descriptions)); + assert.strictEqual((meta.descriptions as string[]).length, 2); + assert.strictEqual(typeof meta.processingTimeMs, "number"); + assert.strictEqual(meta.visionModel, "openai/gpt-4o-mini"); +}); \ No newline at end of file diff --git a/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts b/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts new file mode 100644 index 0000000000..dd241214e1 --- /dev/null +++ b/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts @@ -0,0 +1,231 @@ +/** + * Tests for callVisionModel helper function. + */ + +import test from "node:test"; +import assert from "node:assert/strict"; +import { callVisionModel, type VisionModelConfig } from "@/lib/guardrails/visionBridgeHelpers"; + +// Store original fetch +const originalFetch = globalThis.fetch; + +test("callVisionModel returns description on success", async () => { + // Mock global fetch + const mockResponse = { + ok: true, + json: async () => ({ + choices: [{ message: { content: "A beautiful sunset over the ocean" } }], + }), + }; + globalThis.fetch = async () => mockResponse as unknown as Response; + + try { + const config: VisionModelConfig = { + model: "openai/gpt-4o-mini", + prompt: "Describe this image", + timeoutMs: 30000, + maxImages: 10, + }; + + const result = await callVisionModel( + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + config + ); + + assert.strictEqual(result, "A beautiful sunset over the ocean"); + } finally { + // Restore original fetch + globalThis.fetch = originalFetch; + } +}); + +test("callVisionModel throws on HTTP error", async () => { + const mockResponse = { + ok: false, + status: 500, + text: async () => "Internal Server Error", + }; + globalThis.fetch = async () => mockResponse as unknown as Response; + + try { + const config: VisionModelConfig = { + model: "openai/gpt-4o-mini", + prompt: "Describe this image", + timeoutMs: 30000, + maxImages: 10, + }; + + await assert.rejects( + async () => + await callVisionModel("data:image/png;base64,iVBORw0KGgo", config), + /Vision API error 500/ + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("callVisionModel throws on API error response", async () => { + const mockResponse = { + ok: true, + json: async () => ({ + error: { message: "Invalid API key" }, + }), + }; + globalThis.fetch = async () => mockResponse as unknown as Response; + + try { + const config: VisionModelConfig = { + model: "openai/gpt-4o-mini", + prompt: "Describe this image", + timeoutMs: 30000, + maxImages: 10, + }; + + await assert.rejects( + async () => await callVisionModel("data:image/png;base64,iVBORw0KGgo", config), + /Invalid API key/ + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("callVisionModel throws on empty response", async () => { + const mockResponse = { + ok: true, + json: async () => ({ + choices: [{}], + }), + }; + globalThis.fetch = async () => mockResponse as unknown as Response; + + try { + const config: VisionModelConfig = { + model: "openai/gpt-4o-mini", + prompt: "Describe this image", + timeoutMs: 30000, + maxImages: 10, + }; + + await assert.rejects( + async () => await callVisionModel("data:image/png;base64,iVBORw0KGgo", config), + /empty or invalid/ + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("callVisionModel trims whitespace from response", async () => { + const mockResponse = { + ok: true, + json: async () => ({ + choices: [{ message: { content: " A test description " } }], + }), + }; + globalThis.fetch = async () => mockResponse as unknown as Response; + + try { + const config: VisionModelConfig = { + model: "openai/gpt-4o-mini", + prompt: "Describe this image", + timeoutMs: 30000, + maxImages: 10, + }; + + const result = await callVisionModel("data:image/png;base64,iVBORw0KGgo", config); + assert.strictEqual(result, "A test description"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("callVisionModel passes custom API key", async () => { + let capturedHeaders: Record = {}; + + const mockResponse = { + ok: true, + json: async () => ({ + choices: [{ message: { content: "Description" } }], + }), + }; + + globalThis.fetch = async (url: URL | RequestInfo, init?: RequestInit) => { + if (init?.headers) { + capturedHeaders = init.headers as Record; + } + return mockResponse as unknown as Response; + }; + + try { + const config: VisionModelConfig = { + model: "openai/gpt-4o-mini", + prompt: "Describe this image", + timeoutMs: 30000, + maxImages: 10, + }; + + await callVisionModel("data:image/png;base64,iVBORw0KGgo", config, "sk-custom-key"); + + assert.strictEqual( + capturedHeaders["Authorization"], + "Bearer sk-custom-key" + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("callVisionModel uses correct request body format", async () => { + let capturedBody: Record = {}; + + const mockResponse = { + ok: true, + json: async () => ({ + choices: [{ message: { content: "Description" } }], + }), + }; + + globalThis.fetch = async (url: URL | RequestInfo, init?: RequestInit) => { + if (init?.body) { + capturedBody = JSON.parse(init.body as string); + } + return mockResponse as unknown as Response; + }; + + try { + const config: VisionModelConfig = { + model: "openai/gpt-4o-mini", + prompt: "What is in this image?", + timeoutMs: 30000, + maxImages: 10, + }; + + const imageUri = "data:image/png;base64,test123"; + await callVisionModel(imageUri, config); + + // Verify request structure + assert.strictEqual(capturedBody.model, "gpt-4o-mini"); + assert.ok(Array.isArray(capturedBody.messages)); + assert.strictEqual((capturedBody.messages as unknown[]).length, 1); + + const message = (capturedBody.messages as Array<{role: string; content: unknown[]}>)[0]; + assert.strictEqual(message.role, "user"); + assert.ok(Array.isArray(message.content)); + assert.strictEqual(message.content.length, 2); + + // First content is image_url + const imagePart = message.content[0] as {type: string; image_url: {url: string; detail: string}}; + assert.strictEqual(imagePart.type, "image_url"); + assert.strictEqual(imagePart.image_url.url, imageUri); + assert.strictEqual(imagePart.image_url.detail, "low"); + + // Second content is text prompt + const textPart = message.content[1] as {type: string; text: string}; + assert.strictEqual(textPart.type, "text"); + assert.strictEqual(textPart.text, "What is in this image?"); + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/tests/unit/guardrails/visionBridgeHelpers.extractImageParts.test.ts b/tests/unit/guardrails/visionBridgeHelpers.extractImageParts.test.ts new file mode 100644 index 0000000000..e061548c45 --- /dev/null +++ b/tests/unit/guardrails/visionBridgeHelpers.extractImageParts.test.ts @@ -0,0 +1,138 @@ +/** + * Tests for extractImageParts helper function. + */ + +import test from "node:test"; +import assert from "node:assert/strict"; +import { extractImageParts } from "@/lib/guardrails/visionBridgeHelpers"; + +interface RequestMessage { + role?: string; + content?: string | RequestContentPart[]; +} + +type RequestContentPart = + | { type: "text"; text: string } + | { type: "image_url"; image_url: { url: string; detail?: string } } + | { type: "image"; source: { type: "base64"; media_type: string; data: string } }; + +test("extractImageParts returns empty array for messages without images", () => { + const messages: RequestMessage[] = [ + { role: "user", content: "Hello, how are you?" }, + ]; + const result = extractImageParts(messages); + assert.deepStrictEqual(result, []); +}); + +test("extractImageParts detects image_url format", () => { + const messages: RequestMessage[] = [ + { + role: "user", + content: [ + { type: "text", text: "What is in this image?" }, + { type: "image_url", image_url: { url: "https://example.com/image.png" } }, + ], + }, + ]; + const result = extractImageParts(messages); + assert.strictEqual(result.length, 1); + assert.strictEqual(result[0].messageIndex, 0); + assert.strictEqual(result[0].partIndex, 1); + assert.strictEqual(result[0].imageUrl, "https://example.com/image.png"); + assert.strictEqual(result[0].imageType, "image_url"); +}); + +test("extractImageParts detects base64 image format", () => { + const messages: RequestMessage[] = [ + { + role: "user", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + }, + }, + ], + }, + ]; + const result = extractImageParts(messages); + assert.strictEqual(result.length, 1); + assert.strictEqual(result[0].imageType, "image"); + assert.ok(result[0].imageUrl.startsWith("data:image/png;base64,")); +}); + +test("extractImageParts handles multiple images in single message", () => { + const messages: RequestMessage[] = [ + { + role: "user", + content: [ + { type: "text", text: "Compare these images" }, + { type: "image_url", image_url: { url: "https://example.com/image1.png" } }, + { type: "image_url", image_url: { url: "https://example.com/image2.png" } }, + ], + }, + ]; + const result = extractImageParts(messages); + assert.strictEqual(result.length, 2); + assert.strictEqual(result[0].partIndex, 1); + assert.strictEqual(result[1].partIndex, 2); +}); + +test("extractImageParts handles images across multiple messages", () => { + const messages: RequestMessage[] = [ + { role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/image1.png" } }] }, + { role: "assistant", content: "Here is analysis of the first image." }, + { role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/image2.png" } }] }, + ]; + const result = extractImageParts(messages); + assert.strictEqual(result.length, 2); + assert.strictEqual(result[0].messageIndex, 0); + assert.strictEqual(result[1].messageIndex, 2); +}); + +test("extractImageParts handles empty messages array", () => { + const result = extractImageParts([]); + assert.deepStrictEqual(result, []); +}); + +test("extractImageParts handles messages with null/undefined content", () => { + const messages: RequestMessage[] = [ + { role: "user", content: null as unknown as RequestContentPart[] }, + { role: "user", content: undefined as unknown as RequestContentPart[] }, + ]; + const result = extractImageParts(messages); + assert.deepStrictEqual(result, []); +}); + +test("extractImageParts handles data URI image_url format", () => { + const dataUri = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; + const messages: RequestMessage[] = [ + { role: "user", content: [{ type: "image_url", image_url: { url: dataUri } }] }, + ]; + const result = extractImageParts(messages); + assert.strictEqual(result.length, 1); + assert.strictEqual(result[0].imageUrl, dataUri); +}); + +test("extractImageParts preserves order of images", () => { + const messages: RequestMessage[] = [ + { + role: "user", + content: [ + { type: "text", text: "First" }, + { type: "image_url", image_url: { url: "https://example.com/A.png" } }, + { type: "text", text: "Second" }, + { type: "image_url", image_url: { url: "https://example.com/B.png" } }, + { type: "image_url", image_url: { url: "https://example.com/C.png" } }, + ], + }, + ]; + const result = extractImageParts(messages); + assert.strictEqual(result.length, 3); + assert.strictEqual(result[0].partIndex, 1); + assert.strictEqual(result[1].partIndex, 3); + assert.strictEqual(result[2].partIndex, 4); +}); diff --git a/tests/unit/guardrails/visionBridgeHelpers.replaceImageParts.test.ts b/tests/unit/guardrails/visionBridgeHelpers.replaceImageParts.test.ts new file mode 100644 index 0000000000..263e3dbfb3 --- /dev/null +++ b/tests/unit/guardrails/visionBridgeHelpers.replaceImageParts.test.ts @@ -0,0 +1,254 @@ +/** + * Tests for replaceImageParts helper function. + */ + +import test from "node:test"; +import assert from "node:assert/strict"; +import { replaceImageParts } from "@/lib/guardrails/visionBridgeHelpers"; + +test("replaceImageParts replaces single image with description", () => { + const body = { + model: "minimax/minimax-01", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What is in this image?" }, + { type: "image_url", image_url: { url: "https://example.com/image.png" } }, + ], + }, + ], + }; + + const descriptions = ["[Image 1]: A beautiful sunset"]; + + const result = replaceImageParts(body, descriptions); + + // Should have original text preserved + const content = result.messages[0].content as Array<{type: string; text?: string}>; + assert.strictEqual(content[0].type, "text"); + assert.strictEqual(content[0].text, "What is in this image?"); + + // Should have description instead of image + assert.strictEqual(content[1].type, "text"); + assert.strictEqual(content[1].text, "[Image 1]: A beautiful sunset"); +}); + +test("replaceImageParts replaces multiple images with descriptions", () => { + const body = { + model: "minimax/minimax-01", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Compare these images" }, + { type: "image_url", image_url: { url: "https://example.com/A.png" } }, + { type: "image_url", image_url: { url: "https://example.com/B.png" } }, + { type: "image_url", image_url: { url: "https://example.com/C.png" } }, + ], + }, + ], + }; + + const descriptions = [ + "[Image 1]: A cat", + "[Image 2]: A dog", + "[Image 3]: A bird", + ]; + + const result = replaceImageParts(body, descriptions); + + const content = result.messages[0].content as Array<{type: string; text?: string}>; + assert.strictEqual(content[0].type, "text"); + assert.strictEqual(content[0].text, "Compare these images"); + assert.strictEqual(content[1].type, "text"); + assert.strictEqual(content[1].text, "[Image 1]: A cat"); + assert.strictEqual(content[2].type, "text"); + assert.strictEqual(content[2].text, "[Image 2]: A dog"); + assert.strictEqual(content[3].type, "text"); + assert.strictEqual(content[3].text, "[Image 3]: A bird"); +}); + +test("replaceImageParts handles empty descriptions array", () => { + const body = { + model: "minimax/minimax-01", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Hello" }, + { type: "image_url", image_url: { url: "https://example.com/image.png" } }, + ], + }, + ], + }; + + const result = replaceImageParts(body, []); + + // Original should be unchanged + const content = result.messages[0].content as Array<{type: string}>; + assert.strictEqual(content[1].type, "image_url"); +}); + +test("replaceImageParts preserves non-image content", () => { + const body = { + model: "minimax/minimax-01", + messages: [ + { + role: "system", + content: "You are a helpful assistant.", + }, + { + role: "user", + content: [ + { type: "text", text: "Analyze this" }, + { type: "image_url", image_url: { url: "https://example.com/image.png" } }, + ], + }, + { + role: "assistant", + content: "I can see the image shows a sunset.", + }, + ], + }; + + const result = replaceImageParts(body, ["[Image 1]: A sunset over the ocean"]); + + // System message should be unchanged + assert.strictEqual(result.messages[0].content, "You are a helpful assistant."); + + // Assistant message should be unchanged + assert.strictEqual(result.messages[2].content, "I can see the image shows a sunset."); +}); + +test("replaceImageParts handles base64 images", () => { + const body = { + model: "minimax/minimax-01", + messages: [ + { + role: "user", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + }, + }, + ], + }, + ], + }; + + const descriptions = ["[Image 1]: A red circle"]; + + const result = replaceImageParts(body, descriptions); + + const content = result.messages[0].content as Array<{type: string; text?: string}>; + assert.strictEqual(content[0].type, "text"); + assert.strictEqual(content[0].text, "[Image 1]: A red circle"); +}); + +test("replaceImageParts handles undefined descriptions", () => { + const body = { + model: "test", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Hello" }, + { type: "image_url", image_url: { url: "https://example.com/image.png" } }, + ], + }, + ], + }; + + const result = replaceImageParts(body, undefined as unknown as string[]); + + // Should return original body when descriptions is undefined + const content = result.messages[0].content as Array<{type: string}>; + assert.strictEqual(content[1].type, "image_url"); +}); + +test("replaceImageParts handles empty messages array", () => { + const body = { + model: "test", + messages: [], + }; + + const descriptions = ["[Image 1]: Description"]; + const result = replaceImageParts(body, descriptions); + + assert.deepStrictEqual(result.messages, []); +}); + +test("replaceImageParts handles messages without content array", () => { + const body = { + model: "test", + messages: [ + { role: "user", content: "Just a text message" }, + { role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/image.png" } }] }, + ], + }; + + const descriptions = ["[Image 1]: Description"]; + const result = replaceImageParts(body, descriptions); + + // First message (string content) should be unchanged + assert.strictEqual(result.messages[0].content, "Just a text message"); + + // Second message should have image replaced + const content = result.messages[1].content as Array<{type: string; text?: string}>; + assert.strictEqual(content[0].type, "text"); + assert.strictEqual(content[0].text, "[Image 1]: Description"); +}); + +test("replaceImageParts does not modify original body", () => { + const body = { + model: "minimax/minimax-01", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Original" }, + { type: "image_url", image_url: { url: "https://example.com/image.png" } }, + ], + }, + ], + }; + + const descriptions = ["[Image 1]: Modified"]; + replaceImageParts(body, descriptions); + + // Original should be unchanged + const content = body.messages[0].content as Array<{type: string}>; + assert.strictEqual(content[1].type, "image_url"); +}); + +test("replaceImageParts handles mixed images and text", () => { + const body = { + model: "test", + messages: [ + { + role: "user", + content: [ + { type: "image_url", image_url: { url: "https://example.com/first.png" } }, + { type: "text", text: "between images" }, + { type: "image_url", image_url: { url: "https://example.com/second.png" } }, + ], + }, + ], + }; + + const descriptions = ["[Image 1]: First image", "[Image 2]: Second image"]; + const result = replaceImageParts(body, descriptions); + + const content = result.messages[0].content as Array<{type: string; text?: string}>; + assert.strictEqual(content[0].type, "text"); + assert.strictEqual(content[0].text, "[Image 1]: First image"); + assert.strictEqual(content[1].type, "text"); + assert.strictEqual(content[1].text, "between images"); + assert.strictEqual(content[2].type, "text"); + assert.strictEqual(content[2].text, "[Image 2]: Second image"); +}); diff --git a/tests/unit/guardrails/visionBridgeHelpers.resolveImageAsDataUri.test.ts b/tests/unit/guardrails/visionBridgeHelpers.resolveImageAsDataUri.test.ts new file mode 100644 index 0000000000..af47178c88 --- /dev/null +++ b/tests/unit/guardrails/visionBridgeHelpers.resolveImageAsDataUri.test.ts @@ -0,0 +1,62 @@ +/** + * Tests for resolveImageAsDataUri helper function. + */ + +import test from "node:test"; +import assert from "node:assert/strict"; +import { resolveImageAsDataUri } from "@/lib/guardrails/visionBridgeHelpers"; + +test("resolveImageAsDataUri passes through HTTPS URL as-is", () => { + const url = "https://example.com/image.png"; + const result = resolveImageAsDataUri(url); + assert.strictEqual(result, url); +}); + +test("resolveImageAsDataUri passes through HTTP URL as-is", () => { + const url = "http://example.com/image.png"; + const result = resolveImageAsDataUri(url); + assert.strictEqual(result, url); +}); + +test("resolveImageAsDataUri passes through data URI as-is", () => { + const dataUri = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; + const result = resolveImageAsDataUri(dataUri); + assert.strictEqual(result, dataUri); +}); + +test("resolveImageAsDataUri converts base64 string to PNG data URI", () => { + const base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; + const result = resolveImageAsDataUri(base64); + assert.strictEqual(result, `data:image/png;base64,${base64}`); +}); + +test("resolveImageAsDataUri throws for empty string", () => { + assert.throws(() => { + resolveImageAsDataUri(""); + }, /Invalid image URL/); +}); + +test("resolveImageAsDataUri throws for null", () => { + assert.throws(() => { + resolveImageAsDataUri(null as unknown as string); + }, /Invalid image URL/); +}); + +test("resolveImageAsDataUri throws for undefined", () => { + assert.throws(() => { + resolveImageAsDataUri(undefined as unknown as string); + }, /Invalid image URL/); +}); + +test("resolveImageAsDataUri handles data URI with different media types", () => { + const jpegUri = "data:image/jpeg;base64,/9j/4AAQSkZJRg=="; + const result = resolveImageAsDataUri(jpegUri); + assert.strictEqual(result, jpegUri); +}); + +test("resolveImageAsDataUri treats plain base64 with prefix as data URI", () => { + // If it doesn't start with data: or http, treat as raw base64 + const rawBase64 = "abcdef123456=="; + const result = resolveImageAsDataUri(rawBase64); + assert.strictEqual(result, "data:image/png;base64,abcdef123456=="); +}); diff --git a/tests/unit/visionBridgeDefaults.test.ts b/tests/unit/visionBridgeDefaults.test.ts new file mode 100644 index 0000000000..2d631e05de --- /dev/null +++ b/tests/unit/visionBridgeDefaults.test.ts @@ -0,0 +1,91 @@ +/** + * Tests for Vision Bridge default constants. + */ + +import test from "node:test"; +import assert from "node:assert/strict"; +import { + VISION_BRIDGE_DEFAULTS, + VISION_BRIDGE_SETTINGS_KEYS, + getVisionBridgeConfig, + type VisionBridgeSettings, +} from "@/shared/constants/visionBridgeDefaults"; + +test("VISION_BRIDGE_DEFAULTS exports correct values", () => { + assert.strictEqual(VISION_BRIDGE_DEFAULTS.enabled, true); + assert.strictEqual(VISION_BRIDGE_DEFAULTS.model, "openai/gpt-4o-mini"); + assert.strictEqual( + VISION_BRIDGE_DEFAULTS.prompt, + "Describe this image concisely in 2-3 sentences. Focus on the most relevant visual details." + ); + assert.strictEqual(VISION_BRIDGE_DEFAULTS.timeoutMs, 30000); + assert.strictEqual(VISION_BRIDGE_DEFAULTS.maxImagesPerRequest, 10); +}); + +test("VISION_BRIDGE_SETTINGS_KEYS exports all expected keys", () => { + assert.deepStrictEqual(VISION_BRIDGE_SETTINGS_KEYS, [ + "visionBridgeEnabled", + "visionBridgeModel", + "visionBridgePrompt", + "visionBridgeTimeout", + "visionBridgeMaxImages", + ]); +}); + +test("getVisionBridgeConfig returns defaults when no settings provided", () => { + const config = getVisionBridgeConfig({}); + + assert.strictEqual(config.enabled, true); + assert.strictEqual(config.model, "openai/gpt-4o-mini"); + assert.strictEqual(config.prompt, VISION_BRIDGE_DEFAULTS.prompt); + assert.strictEqual(config.timeoutMs, 30000); + assert.strictEqual(config.maxImages, 10); +}); + +test("getVisionBridgeConfig applies custom settings", () => { + const customSettings: VisionBridgeSettings = { + visionBridgeEnabled: false, + visionBridgeModel: "anthropic/claude-3-haiku", + visionBridgePrompt: "What is in this image?", + visionBridgeTimeout: 60000, + visionBridgeMaxImages: 5, + }; + + const config = getVisionBridgeConfig(customSettings); + + assert.strictEqual(config.enabled, false); + assert.strictEqual(config.model, "anthropic/claude-3-haiku"); + assert.strictEqual(config.prompt, "What is in this image?"); + assert.strictEqual(config.timeoutMs, 60000); + assert.strictEqual(config.maxImages, 5); +}); + +test("getVisionBridgeConfig merges partial settings with defaults", () => { + const partialSettings: VisionBridgeSettings = { + visionBridgeModel: "openai/gpt-4o", + }; + + const config = getVisionBridgeConfig(partialSettings); + + // Custom value + assert.strictEqual(config.model, "openai/gpt-4o"); + // Default values for the rest + assert.strictEqual(config.enabled, true); + assert.strictEqual(config.prompt, VISION_BRIDGE_DEFAULTS.prompt); + assert.strictEqual(config.timeoutMs, 30000); + assert.strictEqual(config.maxImages, 10); +}); + +test("getVisionBridgeConfig handles undefined settings", () => { + const config = getVisionBridgeConfig(undefined); + + assert.strictEqual(config.enabled, true); + assert.strictEqual(config.model, "openai/gpt-4o-mini"); +}); + +test("getVisionBridgeConfig handles null settings", () => { + const config = getVisionBridgeConfig(null as unknown as VisionBridgeSettings); + + assert.strictEqual(config.enabled, true); + assert.strictEqual(config.model, "openai/gpt-4o-mini"); +});