From 71c85f31cd3d9ed4df6cec67815338faff602075 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Sat, 8 Aug 2026 01:59:11 -0300 Subject: [PATCH] =?UTF-8?q?feat(guardrails):=20modality=20bridge=20core=20?= =?UTF-8?q?=E2=80=94=20vision=20mode/task-aware/cache/input=5Fimage=20+=20?= =?UTF-8?q?modalityBridge=20settings=20(#9759)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(sse): unified media-part detection helper (image+audio, input_image) * refactor(guardrails): extractImageParts/comboStructure delegate to unified media detector * fix(sse): media detector — audio parts no longer shadow sibling/nested image indicators * fix(guardrails): close extract↔replace contract for input_image (allowlist + splice) * perf(guardrails): skip media traversal when bridge disabled; short-circuit combo image check * feat(guardrails): in-memory LRU bridge cache (sha256 keyed) * feat(settings): modalityBridge* schema with legacy visionBridge* fallback * feat(db): migrate visionBridge* settings to modalityBridge* (idempotent) * refactor(guardrails): harden bridge cache key/config + settings resolution (review minors) * feat(guardrails): vision bridge mode selector (auto/describe/reroute) short-circuit * feat(guardrails): task-aware vision description prompt (default on) * feat(guardrails): describe-path cache integration * docs(guardrails): review polish — cache-key coupling notes + helper header * feat(guardrails): in-memory bridge stats + modality-bridge response header * feat(api): modality bridge stats endpoint + header wiring in chat handler * docs(guardrails): document modality bridge mode/task-aware/cache/header + stats endpoint * chore: untrack _tasks symlink (inherited from base tip; blocks pre-commit tracked-artifacts gate) * fix(db): renumber modality bridge migration 139->140 (base renumbered ccr_blocks to 139) * docs(guardrails): migration filename touch-up 139->140 * docs(db): stale comment touch-ups after 139->140 renumber and #9688 landing * fix(db): renumber modality bridge migration 140->141 (base renumbered connection_runtime_state to 140) * test(db): migration test titles 139->141 --------- Co-authored-by: diegosouzapw --- docs/openapi.yaml | 13 + docs/reference/API_REFERENCE.md | 1 + docs/security/GUARDRAILS.md | 124 +++++++-- open-sse/services/combo/comboStructure.ts | 25 +- open-sse/utils/mediaParts.ts | 250 ++++++++++++++++++ src/app/api/modality-bridge/stats/route.ts | 21 ++ .../141_modality_bridge_settings.sql | 30 +++ .../guardrails/modalityBridge/bridgeCache.ts | 82 ++++++ .../guardrails/modalityBridge/bridgeStats.ts | 72 +++++ src/lib/guardrails/visionBridge.ts | 119 +++++++-- src/lib/guardrails/visionBridgeHelpers.ts | 97 +++---- .../constants/modalityBridgeDefaults.ts | 85 ++++++ src/shared/validation/settingsSchemas.ts | 16 ++ src/sse/handlers/chat.ts | 21 +- src/sse/handlers/chatHelpers.ts | 25 ++ tests/unit/helpers/decollidedMigrationsDir.ts | 79 ++++++ tests/unit/media-parts.test.ts | 220 +++++++++++++++ tests/unit/modality-bridge-cache.test.ts | 63 +++++ tests/unit/modality-bridge-header.test.ts | 116 ++++++++ ...modality-bridge-settings-migration.test.ts | 92 +++++++ tests/unit/modality-bridge-settings.test.ts | 96 +++++++ .../unit/vision-bridge-describe-cache.test.ts | 102 +++++++ tests/unit/vision-bridge-mode.test.ts | 138 ++++++++++ tests/unit/vision-bridge-task-aware.test.ts | 115 ++++++++ 24 files changed, 1886 insertions(+), 116 deletions(-) create mode 100644 open-sse/utils/mediaParts.ts create mode 100644 src/app/api/modality-bridge/stats/route.ts create mode 100644 src/lib/db/migrations/141_modality_bridge_settings.sql create mode 100644 src/lib/guardrails/modalityBridge/bridgeCache.ts create mode 100644 src/lib/guardrails/modalityBridge/bridgeStats.ts create mode 100644 src/shared/constants/modalityBridgeDefaults.ts create mode 100644 tests/unit/helpers/decollidedMigrationsDir.ts create mode 100644 tests/unit/media-parts.test.ts create mode 100644 tests/unit/modality-bridge-cache.test.ts create mode 100644 tests/unit/modality-bridge-header.test.ts create mode 100644 tests/unit/modality-bridge-settings-migration.test.ts create mode 100644 tests/unit/modality-bridge-settings.test.ts create mode 100644 tests/unit/vision-bridge-describe-cache.test.ts create mode 100644 tests/unit/vision-bridge-mode.test.ts create mode 100644 tests/unit/vision-bridge-task-aware.test.ts diff --git a/docs/openapi.yaml b/docs/openapi.yaml index c4a6ee5420..27e1d37cc8 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -5294,6 +5294,19 @@ paths: "200": description: Caches cleared + /api/modality-bridge/stats: + get: + tags: [System] + summary: Get Modality Bridge telemetry + description: In-memory per-modality bridge counters (bridged, cacheHits, failures, lastUsedAt). Counters reset on process restart. + security: + - ManagementSessionAuth: [] + responses: + "200": + description: Per-modality bridge stats (vision, audio) + "401": + description: Unauthorized + /api/cache/stats: get: tags: [System] diff --git a/docs/reference/API_REFERENCE.md b/docs/reference/API_REFERENCE.md index 3f3e6467a0..096c3e8428 100644 --- a/docs/reference/API_REFERENCE.md +++ b/docs/reference/API_REFERENCE.md @@ -573,6 +573,7 @@ Response example: | `/api/rate-limits` | GET | Per-account rate limits | | `/api/monitoring/health` | GET | Health check + provider summary (`catalogCount`, `configuredCount`, `activeCount`, `monitoredCount`) | | `/api/cache/stats` | GET/DELETE | Cache stats / clear | +| `/api/modality-bridge/stats` | GET | In-memory Modality Bridge telemetry — per-modality `bridged`/`cacheHits`/`failures`/`lastUsedAt` counters (reset on restart; management auth) | ### Backup & Export/Import diff --git a/docs/security/GUARDRAILS.md b/docs/security/GUARDRAILS.md index 8b900359db..309e5ff7b5 100644 --- a/docs/security/GUARDRAILS.md +++ b/docs/security/GUARDRAILS.md @@ -1,13 +1,13 @@ --- title: "Guardrails" -version: 3.8.40 -lastUpdated: 2026-06-28 +version: 3.8.50 +lastUpdated: 2026-08-07 --- # Guardrails > **Source of truth:** `src/lib/guardrails/` -> **Last updated:** 2026-06-28 — v3.8.40 (injection-guard coverage + 16 KB scan bound + red-team) +> **Last updated:** 2026-08-07 — v3.8.50 (Modality Bridge PR-1: mode selector, task-aware prompt, describe cache, transparency header + stats) Guardrails enforce safety, policy, and content transformations at the boundary between OmniRoute and upstream providers. Each guardrail can inspect (and @@ -32,31 +32,107 @@ The registry auto-loads four guardrails in priority order on import Lower priority numbers run **first**. -### Vision Bridge (`visionBridge.ts`) +### Vision Bridge (`visionBridge.ts`) — Modality Bridge PR-1 -Intercepts image-bearing requests aimed at **non-vision models** and replaces -the image parts with text descriptions produced by a configurable vision model -before the upstream call. This lets text-only providers transparently handle +Intercepts image-bearing requests aimed at **non-vision models** and either +reroutes the whole request to a vision-capable model or replaces the image +parts with text descriptions produced by a configurable vision model before +the upstream call. This lets text-only providers transparently handle multimodal payloads. Flow: 1. Skip if the target model already supports vision (unless it appears in the forced-bridge list `isVisionBridgeForcedModel`). -2. Extract image parts via `extractImageParts(messages)`. Skip if none. - `extractImageParts` recognizes all three image shapes: OpenAI `image_url`, - Anthropic base64 `source.type:"base64"`, and Anthropic URL - `source.type:"url"` — so Claude-Code-compatible clients (e.g. Zoo Code) - sending `{ type: "image", source: { type: "url", url } }` are described - instead of silently dropped. -3. Load runtime config from `getSettings()` (`visionBridgeEnabled`, - `visionBridgeModel`, `visionBridgePrompt`, `visionBridgeTimeout`, - `visionBridgeMaxImages`). -4. Cap images at `maxImages`, call the vision model **in parallel** - (`Promise.allSettled`), and inject `[Image N]: ` text parts - in their place — failed images become `[Image N]: (unavailable)`. -5. Return `modifiedPayload` + meta (`imagesProcessed`, `processingTimeMs`, - `visionModel`). +2. Extract image parts via `extractImageParts(messages)` + (`visionBridgeHelpers.ts`), which delegates to the **unified media + detector** `detectMediaParts()` in `open-sse/utils/mediaParts.ts` — the + single source of truth shared with the combo compatibility filter. + Extraction is allowlisted to top-level parts of the shapes + `replaceImageParts` can splice back (the extract↔replace contract): OpenAI + `image_url`, Anthropic base64 `source.type:"base64"`, Anthropic URL + `source.type:"url"`, and Responses API `input_image`. Nested hits and + indicator-only shapes are combo-filter material and are never extracted. + Skip if none found. +3. Resolve runtime config via `resolveVisionBridgeRuntimeSettings()` + (`src/shared/constants/modalityBridgeDefaults.ts`): new `modalityBridge*` + settings keys win; legacy `visionBridge*` keys remain a **one-cycle + fallback** (rollback window). Skip before any media traversal when the + bridge is disabled. +4. Mode selector (`modalityBridgeVisionMode`, see table below) decides + reroute vs describe. Reroute returns `modifiedPayload` with only `model` + swapped, plus meta `{ rerouted, fromModel, toModel, imagesKept }`. +5. Describe path: cap images at `maxImages`, compose the task-aware prompt, + consult the describe cache, call the vision model **in parallel** + (`Promise.allSettled`), and inject `[Image N]: ` text parts in + their place. A failed describe yields `null` and the original image part is + **preserved** (#4012) — except on the combo describe path when every + describe failed, where a confirmed non-vision upstream gets an + `(unavailable — no vision-capable provider connected)` stub instead (#8430). +6. Return `modifiedPayload` + meta (`imagesProcessed`, `descriptions`, + `processingTimeMs`, `visionModel`). + +#### Mode selector (`modalityBridgeVisionMode`) + +| Mode | Default | Behavior | +| ---------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `auto` | ✔ | Legacy heuristic, untouched (#6640/#7204): non-combo/`auto/` models reroute to the best vision model unless the original model already has usable credentials (then describe); combo targets always describe. | +| `describe` | | Always describe — the reroute block is skipped entirely; the user's chosen model always answers. | +| `reroute` | | Force reroute: the keep-credentialed-model guard is bypassed. The reroute-**target** credential guard still applies — when no usable vision target exists, the request falls through to describe so raw images never reach a text-only backend (#8430). | + +Forced modes short-circuit **before** the auto heuristic runs; `auto` behavior +is byte-identical to the pre-PR-1 guardrail. + +#### Task-aware describe prompt (`modalityBridgeVisionTaskAware`) + +Default **true**. `composeVisionPrompt()` (`visionBridgeHelpers.ts`) appends +the text of the **last user message** (truncated to 500 chars) to the base +describe prompt, steering the description toward what the user actually asked +(codex-vision-proxy pattern) and asking the vision model to transcribe visible +text. With the flag off — or no user text — the base prompt is used unchanged. + +#### Describe cache (`modalityBridge/bridgeCache.ts`) + +In-memory LRU + TTL cache for describe outputs, shared process-wide. +Key = `sha256(imageRef + composedPrompt + configuredBridgeModel)` with +length-prefix framing (no field-boundary collisions). The model component is +the **configured** bridge model, not the model that actually answered — +`callVisionModel` may fall back internally, and keying per attempt would +fragment the cache. Failed describes are never cached. Settings: + +| Key | Default | Range | +| ------------------------------- | ------- | ------- | +| `modalityBridgeCacheEnabled` | `true` | — | +| `modalityBridgeCacheTtlMinutes` | `60` | 1–1440 | +| `modalityBridgeCacheMaxEntries` | `200` | 10–5000 | + +#### Settings schema + migration + +The new `modalityBridge*` keys are Zod-validated in `updateSettingsSchema` +(`src/shared/validation/settingsSchemas.ts`): `modalityBridgeVisionEnabled`, +`modalityBridgeVisionMode`, `modalityBridgeVisionModel`, +`modalityBridgeVisionTaskAware`, `modalityBridgeVisionPrompt`, +`modalityBridgeVisionTimeout`, `modalityBridgeVisionMaxImages`, the +`modalityBridgeCache*` trio, and the PR-3-reserved `modalityBridgeAudio*` +group. Migration `141_modality_bridge_settings.sql` copies existing legacy +`visionBridge*` values to the matching new keys (idempotent, never overwrites +an operator-set `modalityBridge*` value); the legacy keys stay accepted as a +read fallback for one release cycle. + +#### Transparency header + stats + +Describe-transformed responses carry +`x-omniroute-modality-bridge: image->text;model=;parts=` +(built by `buildModalityBridgeHeader()` in `modalityBridge/bridgeStats.ts`, +stamped by `withModalityBridgeHeader()` in `src/sse/handlers/chatHelpers.ts`). +Rerouted requests get **no** header — the payload was untouched and the model +swap is already visible in the response body's `model` field. + +`GET /api/modality-bridge/stats` (management auth, same tier as +`GET /api/settings`) returns the in-memory per-modality counters +`{ bridged, cacheHits, failures, lastUsedAt }` for `vision` (and the +PR-3-reserved `audio`). Counters reset on process restart by design +(telemetry, not accounting). **Self-loop admission bypass:** when the describe call routes through OmniRoute's own `/v1` self-loop (non-standard provider model), the sub-request sends @@ -67,8 +143,10 @@ operator-configured `OMNIROUTE_API_KEY` / `ROUTER_API_KEY` env key (#1350) so is only honored for those exact credentials, so external clients cannot use the header to skip admission. -Defaults live in `src/shared/constants/visionBridgeDefaults.ts`. The guardrail -exposes a `deps` constructor option so tests can inject fake `getSettings` and +Legacy defaults live in `src/shared/constants/visionBridgeDefaults.ts`; the +new mode/task-aware/cache defaults and the settings resolver live in +`src/shared/constants/modalityBridgeDefaults.ts`. The guardrail exposes a +`deps` constructor option so tests can inject fake `getSettings` and `callVisionModel` implementations. ### PII Masker (`piiMasker.ts`) diff --git a/open-sse/services/combo/comboStructure.ts b/open-sse/services/combo/comboStructure.ts index 176f41a99f..aa6ff9e596 100644 --- a/open-sse/services/combo/comboStructure.ts +++ b/open-sse/services/combo/comboStructure.ts @@ -18,6 +18,7 @@ import { getHiddenModelsByProvider } from "../../../src/lib/db/models"; import { getComboModelString, normalizeComboStep } from "../../../src/lib/combos/steps.ts"; import { getProviderByAlias, getProviderById } from "../../../src/shared/constants/providers.ts"; import { estimateTokens } from "../contextManager.ts"; +import { containsMediaKind } from "../../utils/mediaParts.ts"; import { getResolvedModelCapabilities } from "../modelCapabilities.ts"; import { parseModel, stripContextWindowSuffix } from "../model.ts"; import { dedupeTargetsByExecutionKey, isRecord } from "./comboData.ts"; @@ -483,21 +484,15 @@ function estimateRequestInputTokens(body: Record): number { return Object.keys(estimatePayload).length > 0 ? estimateTokens(estimatePayload) : 0; } -function valueContainsImagePart(value: unknown, depth = 0): boolean { - if (depth > 8 || value === null || value === undefined) return false; - if (typeof value === "string") return value.startsWith("data:image/"); - if (Array.isArray(value)) return value.some((entry) => valueContainsImagePart(entry, depth + 1)); - if (!isRecord(value)) return false; - - const type = typeof value.type === "string" ? value.type.toLowerCase() : null; - if (type === "image" || type === "image_url" || type === "input_image") return true; - if ("image_url" in value || "input_image" in value) return true; - - const source = isRecord(value.source) ? value.source : null; - const mediaType = typeof source?.media_type === "string" ? source.media_type.toLowerCase() : ""; - if (mediaType.startsWith("image/")) return true; - - return Object.values(value).some((entry) => valueContainsImagePart(entry, depth + 1)); +function valueContainsImagePart(value: unknown): boolean { + // Delegates to the unified media detector (open-sse/utils/mediaParts.ts) — + // single source of truth shared with the vision-bridge guardrail. The + // detector keeps this filter's legacy permissive matches (image-ish `type` + // in any casing, bare `image_url`/`input_image` keys, source.media_type + // image/*, bare data:image strings, recursion capped at depth 8) via + // "image_indicator" parts. containsMediaKind short-circuits on the first + // hit — this runs on every request, so no full-part collection here. + return containsMediaKind([{ content: [value] }], "image"); } export function deriveRequestCompatibilityRequirements( diff --git a/open-sse/utils/mediaParts.ts b/open-sse/utils/mediaParts.ts new file mode 100644 index 0000000000..1a52ef003d --- /dev/null +++ b/open-sse/utils/mediaParts.ts @@ -0,0 +1,250 @@ +/** + * Unified media-part detection for request messages. + * Single source of truth shared by the vision/audio bridge guardrails (src/) + * and the combo compatibility filter (open-sse/) — the two previously kept + * divergent copies (guardrail missed input_image; combo saw it). + */ +export type MediaKind = "image" | "audio"; + +export interface MediaPart { + kind: MediaKind; + /** URL, data URI, or base64 payload reference for the media content. */ + ref: string; + /** + * Location of the top-level content part this hit belongs to. For nested + * hits (`nested: true`) these indexes point at the CONTAINER part — the + * entry of `message.content` under which the media was found — not at the + * media object itself. + */ + messageIndex: number; + partIndex: number; + /** + * True when the media was found below the top level of the content part + * (inside another object/array, e.g. an image nested in an audio payload + * or a data URI inside a text field). Splice-style consumers can only + * replace top-level parts, so they must skip nested hits. + */ + nested: boolean; + /** Original wire shape, for callers that need format-specific handling. */ + shape: + | "image_url" + | "image_base64" + | "image_source_url" + | "input_image" + | "data_uri_string" + | "input_audio" + | "audio_url" + /** Audio detected via `source.media_type: audio/*` (no explicit type). */ + | "audio_source" + /** + * Combo-parity indicator: the value looks like an image part (image-ish + * `type` in any casing, a bare `image_url`/`input_image` key, or a + * `source.media_type` of image/*) but carries no extractable ref — `ref` + * may be "". Boolean callers (combo compatibility filter) count it; + * ref-consuming callers (vision bridge) must skip empty refs. + */ + | "image_indicator"; +} + +const MAX_DEPTH = 8; + +interface DetectCtx { + out: MediaPart[]; + messageIndex: number; + partIndex: number; + /** When set, `found` flips true on the first part of this kind (early exit). */ + stopAtKind?: MediaKind; + found?: boolean; +} + +/** Extract a URL from either a bare string or a `{ url }` object. */ +function urlFrom(raw: unknown): string | undefined { + if (typeof raw === "string") return raw; + const url = (raw as Record | undefined)?.url; + return typeof url === "string" ? url : undefined; +} + +function pushPart( + ctx: DetectCtx, + kind: MediaKind, + ref: string, + shape: MediaPart["shape"], + depth: number +): void { + ctx.out.push({ + kind, + ref, + messageIndex: ctx.messageIndex, + partIndex: ctx.partIndex, + nested: depth > 0, + shape, + }); + if (ctx.stopAtKind === kind) ctx.found = true; +} + +/** Strict image shapes with an extractable ref. Returns true when one was pushed. */ +function inspectImageShapes( + obj: Record, + type: string | undefined, + ctx: DetectCtx, + depth: number +): boolean { + if (type === "image_url" || type === "input_image") { + const url = urlFrom(obj.image_url); + if (url) { + pushPart(ctx, "image", url, type === "input_image" ? "input_image" : "image_url", depth); + return true; + } + } + if (type === "image") { + const source = obj.source as Record | undefined; + if (source?.type === "base64" && typeof source.data === "string") { + const media = typeof source.media_type === "string" ? source.media_type : "image/png"; + pushPart(ctx, "image", `data:${media};base64,${source.data}`, "image_base64", depth); + return true; + } + // Non-empty url required: an empty `source.url` is not an extractable image + // (mirrors the guardrail's historical `if (url)` guard). + if (source?.type === "url" && typeof source.url === "string" && source.url) { + pushPart(ctx, "image", source.url, "image_source_url", depth); + return true; + } + } + return false; +} + +/** + * Audio shapes. Returns true when a part was pushed (at most one per object). + * Callers must NOT early-return on audio: the same object can also carry + * image indicators or nest image parts inside its payload. + */ +function inspectAudioShapes( + obj: Record, + type: string | undefined, + mediaType: unknown, + ctx: DetectCtx, + depth: number +): boolean { + if (type === "input_audio") { + const audio = obj.input_audio as Record | undefined; + if (typeof audio?.data === "string") { + pushPart(ctx, "audio", audio.data, "input_audio", depth); + return true; + } + } + if (type === "audio_url") { + const url = urlFrom(obj.audio_url); + if (url) { + pushPart(ctx, "audio", url, "audio_url", depth); + return true; + } + } + if (typeof mediaType === "string" && mediaType.startsWith("audio/")) { + const data = (obj.source as Record).data; + if (typeof data === "string") { + pushPart(ctx, "audio", data, "audio_source", depth); + return true; + } + } + return false; +} + +/** + * Combo-parity image indicators: the legacy valueContainsImagePart + * (comboStructure) matched image-ish `type` names case-insensitively, bare + * `image_url`/`input_image` keys, and `source.media_type` image/* — all + * without needing an extractable ref. Emit an indicator part (ref + * best-effort, possibly "") so boolean callers keep seeing those requests as + * vision requests. Returns true when one was pushed. + */ +function inspectImageIndicators( + obj: Record, + type: string | undefined, + mediaType: unknown, + ctx: DetectCtx, + depth: number +): boolean { + const lowerType = type?.toLowerCase(); + const looksLikeImage = + lowerType === "image" || + lowerType === "image_url" || + lowerType === "input_image" || + "image_url" in obj || + "input_image" in obj; + const imageMediaType = + typeof mediaType === "string" && mediaType.toLowerCase().startsWith("image/"); + if (!looksLikeImage && !imageMediaType) return false; + pushPart(ctx, "image", urlFrom(obj.image_url ?? obj.input_image) ?? "", "image_indicator", depth); + return true; +} + +function inspect(value: unknown, ctx: DetectCtx, depth: number): void { + if (ctx.found || depth > MAX_DEPTH || value == null) return; + if (typeof value === "string") { + if (value.startsWith("data:image/")) pushPart(ctx, "image", value, "data_uri_string", depth); + return; + } + if (Array.isArray(value)) { + for (const entry of value) { + inspect(entry, ctx, depth + 1); + if (ctx.found) return; + } + return; + } + if (typeof value !== "object") return; + const obj = value as Record; + const type = typeof obj.type === "string" ? obj.type : undefined; + + if (inspectImageShapes(obj, type, ctx, depth)) return; + + const mediaType = (obj.source as Record | undefined)?.media_type; + // Audio does not early-return: the same object can also carry image + // indicators (bare `image_url`/`input_image` keys the legacy combo filter + // matched) or nest image parts inside its payload. + inspectAudioShapes(obj, type, mediaType, ctx, depth); + if (ctx.found) return; + if (inspectImageIndicators(obj, type, mediaType, ctx, depth)) return; + for (const nested of Object.values(obj)) { + inspect(nested, ctx, depth + 1); + if (ctx.found) return; + } +} + +export function detectMediaParts( + messages: ReadonlyArray<{ role?: string; content?: unknown }> | undefined | null +): MediaPart[] { + const out: MediaPart[] = []; + if (!Array.isArray(messages)) return out; + for (let messageIndex = 0; messageIndex < messages.length; messageIndex++) { + const content = messages[messageIndex]?.content; + if (!Array.isArray(content)) continue; + for (let partIndex = 0; partIndex < content.length; partIndex++) { + inspect(content[partIndex], { out, messageIndex, partIndex }, 0); + } + } + return out; +} + +/** + * Early-exit presence check: returns true as soon as the FIRST part of the + * requested kind is found, without collecting the full part list or finishing + * the traversal. Prefer this on hot paths (e.g. the combo compatibility + * filter runs on every request) over `detectMediaParts(...).some(...)`. + */ +export function containsMediaKind( + messages: ReadonlyArray<{ role?: string; content?: unknown }> | undefined | null, + kind: MediaKind +): boolean { + if (!Array.isArray(messages)) return false; + const out: MediaPart[] = []; + for (let messageIndex = 0; messageIndex < messages.length; messageIndex++) { + const content = messages[messageIndex]?.content; + if (!Array.isArray(content)) continue; + for (let partIndex = 0; partIndex < content.length; partIndex++) { + const ctx: DetectCtx = { out, messageIndex, partIndex, stopAtKind: kind }; + inspect(content[partIndex], ctx, 0); + if (ctx.found) return true; + } + } + return false; +} diff --git a/src/app/api/modality-bridge/stats/route.ts b/src/app/api/modality-bridge/stats/route.ts new file mode 100644 index 0000000000..6dfa6c1e19 --- /dev/null +++ b/src/app/api/modality-bridge/stats/route.ts @@ -0,0 +1,21 @@ +import { NextResponse } from "next/server"; +import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; +import { getBridgeStats } from "@/lib/guardrails/modalityBridge/bridgeStats"; + +/** + * GET /api/modality-bridge/stats — read-only, in-memory Modality Bridge + * telemetry (per-modality bridged/cacheHits/failures/lastUsedAt counters). + * Same MANAGEMENT auth tier as GET /api/settings (routeGuard default — + * intentionally NOT local-only: harmless read-only telemetry, no side effects). + * Counters reset on process restart; force-dynamic + no-store so the dashboard + * always sees live values. + */ +export const dynamic = "force-dynamic"; +export const revalidate = 0; + +export async function GET(request: Request) { + const authError = await requireManagementAuth(request); + if (authError) return authError; + + return NextResponse.json(getBridgeStats(), { headers: { "Cache-Control": "no-store" } }); +} diff --git a/src/lib/db/migrations/141_modality_bridge_settings.sql b/src/lib/db/migrations/141_modality_bridge_settings.sql new file mode 100644 index 0000000000..651f08d5b6 --- /dev/null +++ b/src/lib/db/migrations/141_modality_bridge_settings.sql @@ -0,0 +1,30 @@ +-- 141_modality_bridge_settings.sql +-- Modality Bridge (PR-1): copy legacy visionBridge* settings to the new modalityBridge* keys. +-- Legacy keys are left in place for one release cycle (rollback window). +-- Idempotent: each INSERT only fires when the new key does not exist yet, so an +-- operator-set modalityBridge* value is never overwritten and re-runs are no-ops. + +INSERT INTO key_value (namespace, key, value) +SELECT 'settings', 'modalityBridgeVisionEnabled', value FROM key_value +WHERE namespace = 'settings' AND key = 'visionBridgeEnabled' + AND NOT EXISTS (SELECT 1 FROM key_value WHERE namespace = 'settings' AND key = 'modalityBridgeVisionEnabled'); + +INSERT INTO key_value (namespace, key, value) +SELECT 'settings', 'modalityBridgeVisionModel', value FROM key_value +WHERE namespace = 'settings' AND key = 'visionBridgeModel' + AND NOT EXISTS (SELECT 1 FROM key_value WHERE namespace = 'settings' AND key = 'modalityBridgeVisionModel'); + +INSERT INTO key_value (namespace, key, value) +SELECT 'settings', 'modalityBridgeVisionPrompt', value FROM key_value +WHERE namespace = 'settings' AND key = 'visionBridgePrompt' + AND NOT EXISTS (SELECT 1 FROM key_value WHERE namespace = 'settings' AND key = 'modalityBridgeVisionPrompt'); + +INSERT INTO key_value (namespace, key, value) +SELECT 'settings', 'modalityBridgeVisionTimeout', value FROM key_value +WHERE namespace = 'settings' AND key = 'visionBridgeTimeout' + AND NOT EXISTS (SELECT 1 FROM key_value WHERE namespace = 'settings' AND key = 'modalityBridgeVisionTimeout'); + +INSERT INTO key_value (namespace, key, value) +SELECT 'settings', 'modalityBridgeVisionMaxImages', value FROM key_value +WHERE namespace = 'settings' AND key = 'visionBridgeMaxImages' + AND NOT EXISTS (SELECT 1 FROM key_value WHERE namespace = 'settings' AND key = 'modalityBridgeVisionMaxImages'); diff --git a/src/lib/guardrails/modalityBridge/bridgeCache.ts b/src/lib/guardrails/modalityBridge/bridgeCache.ts new file mode 100644 index 0000000000..e0a10429e6 --- /dev/null +++ b/src/lib/guardrails/modalityBridge/bridgeCache.ts @@ -0,0 +1,82 @@ +/** + * Modality Bridge description cache (PR-1). + * + * In-memory LRU + TTL cache for bridge outputs (image/audio descriptions), + * keyed by sha256(contentRef + prompt + model). Avoids re-describing the + * same media with the same prompt/model within the configured TTL. + */ +import { createHash } from "node:crypto"; + +import type { VisionBridgeRuntimeSettings } from "@/shared/constants/modalityBridgeDefaults"; + +export function bridgeCacheKey(contentRef: string, prompt: string, model: string): string { + // Length-prefix framing: hashing the byte lengths first makes the field + // boundaries unambiguous, so ("ab","c") can never collide with ("a","bc"). + return createHash("sha256") + .update(`${Buffer.byteLength(contentRef)}:${Buffer.byteLength(prompt)}:`) + .update(contentRef) + .update(prompt) + .update(model) + .digest("hex"); +} + +export interface BridgeCacheOptions { + maxEntries: number; + ttlMs: number; + /** Injectable clock for tests. */ + now?: () => number; +} + +export class BridgeCache { + private readonly entries = new Map(); + + constructor(private readonly opts: BridgeCacheOptions) {} + + get(key: string): string | undefined { + const hit = this.entries.get(key); + if (!hit) return undefined; + const now = (this.opts.now ?? Date.now)(); + if (hit.expiresAt <= now) { + this.entries.delete(key); + return undefined; + } + // Map preserves insertion order — re-insert to mark as most-recently-used. + this.entries.delete(key); + this.entries.set(key, hit); + return hit.value; + } + + set(key: string, value: string): void { + const now = (this.opts.now ?? Date.now)(); + this.entries.delete(key); + this.entries.set(key, { value, expiresAt: now + this.opts.ttlMs }); + while (this.entries.size > this.opts.maxEntries) { + const oldest = this.entries.keys().next().value; + if (oldest === undefined) break; + this.entries.delete(oldest); + } + } + + get size(): number { + return this.entries.size; + } +} + +/** Process-wide singleton used by the bridges; recreated when config changes. */ +let shared: { cache: BridgeCache; ttlMs: number; maxEntries: number } | null = null; + +export function getSharedBridgeCache(ttlMs: number, maxEntries: number): BridgeCache { + if (!shared || shared.ttlMs !== ttlMs || shared.maxEntries !== maxEntries) { + shared = { cache: new BridgeCache({ maxEntries, ttlMs }), ttlMs, maxEntries }; + } + return shared.cache; +} + +/** + * Single conversion point from runtime settings to the shared cache: every + * bridge (vision, audio) goes through here so the minutes→ms conversion can + * never diverge between callers and thrash the singleton on each request. + */ +export function getSharedBridgeCacheFor(settings: VisionBridgeRuntimeSettings): BridgeCache { + return getSharedBridgeCache(settings.cacheTtlMinutes * 60_000, settings.cacheMaxEntries); +} diff --git a/src/lib/guardrails/modalityBridge/bridgeStats.ts b/src/lib/guardrails/modalityBridge/bridgeStats.ts new file mode 100644 index 0000000000..052c73fabe --- /dev/null +++ b/src/lib/guardrails/modalityBridge/bridgeStats.ts @@ -0,0 +1,72 @@ +/** + * Modality Bridge stats + response transparency header (PR-1 Task 9). + * + * In-memory, process-global counters for bridge activity ("vision" today, + * "audio" reserved for PR-3) plus the builder for the + * `x-omniroute-modality-bridge` response header, which tells clients that + * their request payload was transparently transformed (image→text describe). + * Reroutes do NOT get a header — the payload was untouched, only the model + * changed, and that is already visible in the response body's `model` field. + * + * Counters reset on process restart by design (telemetry, not accounting). + */ + +export interface BridgeModalityStats { + bridged: number; + cacheHits: number; + failures: number; + lastUsedAt: string | null; +} + +const stats: Record<"vision" | "audio", BridgeModalityStats> = { + vision: { bridged: 0, cacheHits: 0, failures: 0, lastUsedAt: null }, + audio: { bridged: 0, cacheHits: 0, failures: 0, lastUsedAt: null }, +}; + +export function recordBridgeUse( + kind: "vision" | "audio", + opts: { cacheHit?: boolean; failure?: boolean } = {} +): void { + const s = stats[kind]; + s.bridged += 1; + if (opts.cacheHit) s.cacheHits += 1; + if (opts.failure) s.failures += 1; + s.lastUsedAt = new Date().toISOString(); +} + +export function getBridgeStats(): Record<"vision" | "audio", BridgeModalityStats> { + return structuredClone(stats); +} + +/** + * Structural subset of GuardrailExecutionResult (src/lib/guardrails/base.ts) — + * only the fields the header builder reads, so callers can pass the registry's + * `results` array directly without a type dependency on the guardrail core. + */ +interface GuardrailMetaEntry { + guardrail: string; + meta?: Record | null; +} + +/** Response header value for a describe-bridged request; null when untouched. */ +export function buildModalityBridgeHeader(results: GuardrailMetaEntry[]): string | null { + const segments: string[] = []; + for (const r of results) { + const meta = r.meta ?? {}; + if ( + r.guardrail === "vision-bridge" && + typeof meta.imagesProcessed === "number" && + !meta.rerouted + ) { + segments.push( + `image->text;model=${String(meta.visionModel ?? "unknown")};parts=${meta.imagesProcessed}` + ); + } + if (r.guardrail === "audio-bridge" && typeof meta.clipsProcessed === "number") { + segments.push( + `audio->text;model=${String(meta.sttModel ?? "unknown")};parts=${meta.clipsProcessed}` + ); + } + } + return segments.length ? segments.join(", ") : null; +} diff --git a/src/lib/guardrails/visionBridge.ts b/src/lib/guardrails/visionBridge.ts index 33801b4810..f48ebd72b8 100644 --- a/src/lib/guardrails/visionBridge.ts +++ b/src/lib/guardrails/visionBridge.ts @@ -11,14 +11,17 @@ import { getResolvedModelCapabilities } from "@/lib/modelCapabilities"; import { extractImageParts, callVisionModel as defaultCallVisionModel, + composeVisionPrompt, replaceImageParts, } from "./visionBridgeHelpers"; import { - VISION_BRIDGE_DEFAULTS, getVisionBridgeConfig, isVisionBridgeForcedModel, } from "@/shared/constants/visionBridgeDefaults"; +import { resolveVisionBridgeRuntimeSettings } from "@/shared/constants/modalityBridgeDefaults"; import { getBestVisionModel } from "./visionBridgeRouter"; +import { bridgeCacheKey, getSharedBridgeCacheFor } from "./modalityBridge/bridgeCache"; +import { recordBridgeUse } from "./modalityBridge/bridgeStats"; import { isProviderConnectionUsable, hasUsableCredentialsForModel, @@ -94,6 +97,30 @@ async function getComboVisionBridgeDecision(model: string): Promise= 0; i--) { + const message = messages[i] as { role?: unknown; content?: unknown } | null | undefined; + if (message?.role !== "user") continue; + if (typeof message.content === "string" && message.content.trim()) { + return message.content; + } + if (Array.isArray(message.content)) { + for (const part of message.content) { + const p = part as { type?: unknown; text?: unknown } | null | undefined; + if (p?.type === "text" && typeof p.text === "string" && p.text.trim()) { + return p.text; + } + } + } + } + return undefined; +} + export interface VisionBridgeDependencies { getSettings?: () => Promise>; callVisionModel?: ( @@ -189,13 +216,7 @@ export class VisionBridgeGuardrail extends BaseGuardrail { return { block: false }; } - // 6. Check for images using helper (extractImageParts returns empty if no images) - const imageParts = extractImageParts(messages as Parameters[0]); - if (imageParts.length === 0) { - return { block: false }; - } - - // 7. Get settings (injectable for testing) + // 6. Get settings (injectable for testing) const getSettings = this.deps.getSettings ?? defaultGetSettings; let settings: Record = {}; try { @@ -204,9 +225,18 @@ export class VisionBridgeGuardrail extends BaseGuardrail { // If getSettings fails, use defaults } - // 8. Check if Vision Bridge is enabled in settings - const enabled = settings.visionBridgeEnabled ?? VISION_BRIDGE_DEFAULTS.enabled; - if (!enabled) { + // 7. Resolve runtime settings (new modalityBridge* keys win; legacy + // visionBridge* keys stay a one-cycle fallback) and check enabled — + // BEFORE any media traversal, so a disabled bridge never pays the + // per-request deep scan of every message content part. + const runtime = resolveVisionBridgeRuntimeSettings(settings); + if (!runtime.enabled) { + return { block: false }; + } + + // 8. Check for images using helper (extractImageParts returns empty if no images) + const imageParts = extractImageParts(messages as Parameters[0]); + if (imageParts.length === 0) { return { block: false }; } @@ -226,9 +256,16 @@ export class VisionBridgeGuardrail extends BaseGuardrail { // request with model=auto would land on a text-only model (#7871). Keeping // "auto" is never the answer there, so the keep-credentialed-model skip // below does not apply to auto — only the reroute-target credential guard. - if ((comboVisionBridgeDecision === "not-combo" || isAuto) && !forceVisionBridge) { + const rerouteEligible = + (comboVisionBridgeDecision === "not-combo" || isAuto) && !forceVisionBridge; + // Forced modes short-circuit BEFORE the auto heuristic (#6640/#7204 untouched): + // - "describe" skips the whole reroute block → straight to the describe path. + // - "reroute" skips only the keep-credentialed-model guard; the reroute-target + // credential guard still applies, and with no usable target it falls through + // to describe (raw images must never reach a text-only backend — #8430). + if (rerouteEligible && runtime.mode !== "describe") { const checkCreds = this.deps.hasUsableCredentials ?? hasUsableCredentialsForModel; - const originalUsable = await checkCreds(model); + const originalUsable = runtime.mode === "reroute" ? false : await checkCreds(model); if (originalUsable === true && !isAuto) { // Keep the credentialed model; describe images below if needed. @@ -238,14 +275,17 @@ export class VisionBridgeGuardrail extends BaseGuardrail { ); } else { // Honor an explicit operator override from the Vision Bridge settings tab - // (settings.visionBridgeModel) as the fixed reroute target, for consistency - // with the combo/describe path below (step 10) which always honors it via - // getVisionBridgeConfig. When unset, auto-select the fastest available - // vision-capable model from available providers. - const configuredModel = - typeof settings.visionBridgeModel === "string" && settings.visionBridgeModel.trim() - ? settings.visionBridgeModel.trim() - : undefined; + // as the fixed reroute target, for consistency with the combo/describe + // path below (step 10) which always honors it via getVisionBridgeConfig. + // New modalityBridgeVisionModel wins over legacy visionBridgeModel (same + // precedence as resolveVisionBridgeRuntimeSettings — runtime.model can't + // be used here because it backfills the default and this path must + // auto-select the fastest available vision-capable model when unset). + const rawConfiguredModel = [ + settings.modalityBridgeVisionModel, + settings.visionBridgeModel, + ].find((value): value is string => typeof value === "string" && value.trim().length > 0); + const configuredModel = rawConfiguredModel?.trim(); // Propagate the same resolved credential check used by the adjacent // checkCreds() calls above/below (#8430) — without this, the router // falls back to the real DB-backed hasUsableCredentialsForModel and @@ -284,13 +324,15 @@ export class VisionBridgeGuardrail extends BaseGuardrail { // Fall through: describe images as text (or no-op if describe path can't run) } - // 10. Get configuration + // 10. Get configuration — fed from the resolved runtime values so the new + // modalityBridge* keys are honored; getVisionBridgeConfig keeps producing + // the same VisionModelConfig shape callVisionModel expects. const config = getVisionBridgeConfig({ - visionBridgeEnabled: settings.visionBridgeEnabled as boolean | undefined, - visionBridgeModel: settings.visionBridgeModel as string | undefined, - visionBridgePrompt: settings.visionBridgePrompt as string | undefined, - visionBridgeTimeout: settings.visionBridgeTimeout as number | undefined, - visionBridgeMaxImages: settings.visionBridgeMaxImages as number | undefined, + visionBridgeEnabled: runtime.enabled, + visionBridgeModel: runtime.model, + visionBridgePrompt: runtime.prompt, + visionBridgeTimeout: runtime.timeoutMs, + visionBridgeMaxImages: runtime.maxImages, }); // 11. Limit images @@ -301,10 +343,30 @@ export class VisionBridgeGuardrail extends BaseGuardrail { const logger = context.log; const startTime = Date.now(); + // Task-aware focus hint: append the LAST user question so the description + // targets what the user actually asked instead of a generic caption. + const lastUserText = extractLastUserText(messages); + const composedPrompt = composeVisionPrompt(config.prompt, lastUserText, runtime.taskAware); + const describeConfig = { ...config, prompt: composedPrompt }; + + // Shared describe cache (sha256 of contentRef+prompt+model): the same image + // with the same prompt/model is described once per TTL. Failures are never + // cached — a throw inside the map happens before the cache write. The model + // component is the CONFIGURED bridge model (config.model — the bridge-config + // identity), not the model that actually produced the description: + // callVisionModel may fall back internally to another vision model, and + // keying by attempt would fragment the cache and leak router state into the + // key. Intentional and stable — do not "fix" this to key per attempt. + const cache = runtime.cacheEnabled ? getSharedBridgeCacheFor(runtime) : null; + // Process all images in parallel using Promise.allSettled for fail-partial behavior const results = await Promise.allSettled( limitedParts.map(async (imagePart, i) => { - const description = await callVision(imagePart.imageUrl, config); + const key = cache ? bridgeCacheKey(imagePart.imageUrl, composedPrompt, config.model) : null; + const cached = key && cache ? cache.get(key) : undefined; + const description = cached ?? (await callVision(imagePart.imageUrl, describeConfig)); + if (cached === undefined && key && cache) cache.set(key, description); + recordBridgeUse("vision", { cacheHit: cached !== undefined }); return `[Image ${i + 1}]: ${description}`; }) ); @@ -320,6 +382,7 @@ export class VisionBridgeGuardrail extends BaseGuardrail { const message = result.reason instanceof Error ? result.reason.message : String(result.reason); logger?.warn?.("VISION-BRIDGE", `Failed to get description for image ${i + 1}: ${message}`); + recordBridgeUse("vision", { failure: true }); return null; }); diff --git a/src/lib/guardrails/visionBridgeHelpers.ts b/src/lib/guardrails/visionBridgeHelpers.ts index b8376103fd..b8eb432017 100644 --- a/src/lib/guardrails/visionBridgeHelpers.ts +++ b/src/lib/guardrails/visionBridgeHelpers.ts @@ -1,6 +1,7 @@ /** * Vision Bridge helper functions for image processing. */ +import { detectMediaParts, type MediaPart } from "@omniroute/open-sse/utils/mediaParts"; import { fetchRemoteImage } from "@/shared/network/remoteImageFetch"; import { getRuntimePorts } from "@/lib/runtime/ports"; import { resolveSelfLoopBearer } from "@/shared/middleware/chatBodyAdmission"; @@ -118,53 +119,37 @@ export type RequestContentPart = * vision-bridge guardrail, so the image was silently dropped by a text-only * executor instead of being described. */ +/** + * Shapes `replaceImageParts` knows how to splice: top-level content parts + * whose `type` is `image_url`, `image`, or `input_image`. Everything else the + * detector reports (nested hits, `data_uri_string`, `image_indicator`) is + * combo-filter material only — extracting it would desync the positional + * description consumption in visionBridge (descriptions would shift onto the + * wrong images). + */ +const REPLACEABLE_IMAGE_SHAPES: ReadonlySet = new Set([ + "image_url", + "image_base64", + "image_source_url", + "input_image", +]); + export function extractImageParts(messages: RequestMessage[]): ImagePart[] { - const results: ImagePart[] = []; - - if (!Array.isArray(messages)) { - return results; - } - - for (let msgIdx = 0; msgIdx < messages.length; msgIdx++) { - const message = messages[msgIdx]; - if (!message || !Array.isArray(message.content)) { - continue; - } - - for (let partIdx = 0; partIdx < message.content.length; partIdx++) { - const part = message.content[partIdx]; - - if (part?.type === "image_url" && part.image_url?.url) { - results.push({ - messageIndex: msgIdx, - partIndex: partIdx, - imageUrl: part.image_url.url, - imageType: "image_url", - }); - } else if (part?.type === "image" && part.source?.type === "base64") { - const { media_type, data } = part.source; - const dataUri = `data:${media_type};base64,${data}`; - results.push({ - messageIndex: msgIdx, - partIndex: partIdx, - imageUrl: dataUri, - imageType: "image", - }); - } else if (part?.type === "image" && part.source?.type === "url") { - const url = part.source.url; - if (url) { - results.push({ - messageIndex: msgIdx, - partIndex: partIdx, - imageUrl: url, - imageType: "url", - }); - } - } - } - } - - return results; + // Delegates to the unified detector (open-sse/utils/mediaParts.ts) so the + // guardrail and the combo compatibility filter share one source of truth. + // Extraction is ALLOWLISTED to top-level (non-nested) parts whose shape + // replaceImageParts can splice back — the extract↔replace contract: every + // extracted part MUST be replaceable, in the same order, or the positional + // descriptions shift onto the wrong images. + return detectMediaParts(messages) + .filter((p) => p.kind === "image" && !p.nested && REPLACEABLE_IMAGE_SHAPES.has(p.shape)) + .map((p) => ({ + messageIndex: p.messageIndex, + partIndex: p.partIndex, + imageUrl: p.ref, + imageType: + p.shape === "image_base64" ? "image" : p.shape === "image_source_url" ? "url" : "image_url", + })); } /** @@ -223,6 +208,19 @@ export interface VisionModelConfig { maxImages: number; } +/** Task-aware focus hint (codex-vision-proxy pattern): steer the description + * toward what the user actually asked, instead of a generic caption. */ +export function composeVisionPrompt( + basePrompt: string, + lastUserText: string | undefined, + taskAware: boolean +): string { + const text = (lastUserText ?? "").trim(); + if (!taskAware || !text) return basePrompt; + const hint = text.length > 500 ? `${text.slice(0, 500)}…` : text; + return `${basePrompt}\n\nThe user asked: "${hint}". Focus your description on what is relevant to answering this, and transcribe any text visible in the image.`; +} + /** * Call the vision model to get an image description. * Supports both OpenAI-compatible and Anthropic API formats. @@ -703,7 +701,12 @@ export function replaceImageParts( const newContent: RequestContentPart[] = []; for (const part of message.content) { - if (part?.type === "image_url" || part?.type === "image") { + // `input_image` (Responses API) is read through a widened type: it is + // not part of the historical RequestContentPart union but MUST be + // replaceable — extractImageParts allowlists it, and every extracted + // part needs a matching splice here (extract↔replace contract). + const partType = (part as { type?: string } | null | undefined)?.type; + if (partType === "image_url" || partType === "image" || partType === "input_image") { if (descriptionIndex < descriptions.length) { const description = descriptions[descriptionIndex]; descriptionIndex++; diff --git a/src/shared/constants/modalityBridgeDefaults.ts b/src/shared/constants/modalityBridgeDefaults.ts new file mode 100644 index 0000000000..336d45b280 --- /dev/null +++ b/src/shared/constants/modalityBridgeDefaults.ts @@ -0,0 +1,85 @@ +/** + * Modality Bridge default configuration values (PR-1). + * + * New `modalityBridge*` settings keys supersede the legacy `visionBridge*` + * keys; the legacy keys stay accepted as a fallback for one release cycle + * (rollback window) and are resolved here in a single place. + */ +import { VISION_BRIDGE_DEFAULTS } from "./visionBridgeDefaults"; + +export type VisionBridgeMode = "auto" | "describe" | "reroute"; + +export const MODALITY_BRIDGE_DEFAULTS = { + visionMode: "auto" as VisionBridgeMode, + visionTaskAware: true, + cacheEnabled: true, + cacheTtlMinutes: 60, + cacheMaxEntries: 200, + audioEnabled: true, + audioModel: "", + audioTimeoutMs: 60000, + audioMaxClips: 3, +} as const; + +export interface VisionBridgeRuntimeSettings { + enabled: boolean; + mode: VisionBridgeMode; + model: string; + taskAware: boolean; + prompt: string; + timeoutMs: number; + maxImages: number; + cacheEnabled: boolean; + cacheTtlMinutes: number; + cacheMaxEntries: number; +} + +// Typed candidate pickers: a stored value of the wrong type (e.g. the string +// "off" in a boolean field) is skipped so the next candidate/default wins. +function pickBoolean(...values: unknown[]): boolean | undefined { + for (const v of values) if (typeof v === "boolean") return v; + return undefined; +} + +function pickNumber(...values: unknown[]): number | undefined { + for (const v of values) if (typeof v === "number" && Number.isFinite(v)) return v; + return undefined; +} + +function pickString(...values: unknown[]): string | undefined { + for (const v of values) if (typeof v === "string") return v; + return undefined; +} + +/** New modalityBridge* keys win; legacy visionBridge* keys are a one-cycle fallback. */ +export function resolveVisionBridgeRuntimeSettings( + settings: Record | null | undefined +): VisionBridgeRuntimeSettings { + const s = settings ?? {}; + const mode = pickString(s.modalityBridgeVisionMode); + return { + enabled: + pickBoolean(s.modalityBridgeVisionEnabled, s.visionBridgeEnabled) ?? + VISION_BRIDGE_DEFAULTS.enabled, + mode: mode === "describe" || mode === "reroute" ? mode : MODALITY_BRIDGE_DEFAULTS.visionMode, + model: + pickString(s.modalityBridgeVisionModel, s.visionBridgeModel) ?? VISION_BRIDGE_DEFAULTS.model, + taskAware: + pickBoolean(s.modalityBridgeVisionTaskAware) ?? MODALITY_BRIDGE_DEFAULTS.visionTaskAware, + prompt: + pickString(s.modalityBridgeVisionPrompt, s.visionBridgePrompt) ?? + VISION_BRIDGE_DEFAULTS.prompt, + timeoutMs: + pickNumber(s.modalityBridgeVisionTimeout, s.visionBridgeTimeout) ?? + VISION_BRIDGE_DEFAULTS.timeoutMs, + maxImages: + pickNumber(s.modalityBridgeVisionMaxImages, s.visionBridgeMaxImages) ?? + VISION_BRIDGE_DEFAULTS.maxImagesPerRequest, + cacheEnabled: + pickBoolean(s.modalityBridgeCacheEnabled) ?? MODALITY_BRIDGE_DEFAULTS.cacheEnabled, + cacheTtlMinutes: + pickNumber(s.modalityBridgeCacheTtlMinutes) ?? MODALITY_BRIDGE_DEFAULTS.cacheTtlMinutes, + cacheMaxEntries: + pickNumber(s.modalityBridgeCacheMaxEntries) ?? MODALITY_BRIDGE_DEFAULTS.cacheMaxEntries, + }; +} diff --git a/src/shared/validation/settingsSchemas.ts b/src/shared/validation/settingsSchemas.ts index 1a56f651e5..bd4dea2a1d 100644 --- a/src/shared/validation/settingsSchemas.ts +++ b/src/shared/validation/settingsSchemas.ts @@ -330,6 +330,22 @@ export const updateSettingsSchema = z.object({ visionBridgePrompt: z.string().max(5000).optional(), visionBridgeTimeout: z.number().int().min(1000).max(300000).optional(), visionBridgeMaxImages: z.number().int().min(1).max(20).optional(), + // Modality Bridge settings (new schema — visionBridge* keys above are the + // deprecated legacy aliases, kept accepted for one release cycle) + modalityBridgeVisionEnabled: z.boolean().optional(), + modalityBridgeVisionMode: z.enum(["auto", "describe", "reroute"]).optional(), + modalityBridgeVisionModel: z.string().max(200).optional(), + modalityBridgeVisionTaskAware: z.boolean().optional(), + modalityBridgeVisionPrompt: z.string().max(5000).optional(), + modalityBridgeVisionTimeout: z.number().int().min(1000).max(300000).optional(), + modalityBridgeVisionMaxImages: z.number().int().min(1).max(20).optional(), + modalityBridgeAudioEnabled: z.boolean().optional(), + modalityBridgeAudioModel: z.string().max(200).optional(), + modalityBridgeAudioTimeout: z.number().int().min(1000).max(300000).optional(), + modalityBridgeAudioMaxClips: z.number().int().min(1).max(10).optional(), + modalityBridgeCacheEnabled: z.boolean().optional(), + modalityBridgeCacheTtlMinutes: z.number().int().min(1).max(1440).optional(), + modalityBridgeCacheMaxEntries: z.number().int().min(10).max(5000).optional(), // Missing settings lkgpEnabled: z.boolean().optional(), // #1311: echo the requested alias/combo name in the response model field (opt-in) diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 99878c615c..2f1aa3b45f 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -77,7 +77,9 @@ import { withSessionHeader, withSelectedConnectionHeader, withCorrelationId, + withModalityBridgeHeader, } from "./chatHelpers"; +import { buildModalityBridgeHeader } from "@/lib/guardrails/modalityBridge/bridgeStats"; import { isAntigravityMissingProjectError, PROVIDER_BREAKER_FAILURE_STATUSES, @@ -544,6 +546,10 @@ async function handleChatImplementation( isModelAllowedForKey, log, })); + // Modality Bridge transparency (Task 9): non-null only when a pre-call bridge + // guardrail transformed the payload (describe path) — stamped on the main + // success exits below via withModalityBridgeHeader(). + const modalityBridgeHeader = buildModalityBridgeHeader(preCallGuardrails.results); telemetry.endPhase(); // T08: per-key active session limit (0 = unlimited). @@ -907,7 +913,10 @@ async function handleChatImplementation( if (fallbackResponse.ok) { log.info("GLOBAL_FALLBACK", `Global fallback ${fallbackModel} succeeded`); recordTelemetry(telemetry); - return withSessionHeader(fallbackResponse, sessionId); + return withModalityBridgeHeader( + withSessionHeader(fallbackResponse, sessionId), + modalityBridgeHeader + ); } log.warn( "GLOBAL_FALLBACK", @@ -944,7 +953,10 @@ async function handleChatImplementation( }); } catch {} } - return withCorrelationId(withSessionHeader(response, sessionId), reqId); + return withModalityBridgeHeader( + withCorrelationId(withSessionHeader(response, sessionId), reqId), + modalityBridgeHeader + ); } telemetry.endPhase(); @@ -986,7 +998,10 @@ async function handleChatImplementation( false ); recordTelemetry(telemetry); - return withCorrelationId(withSessionHeader(response, sessionId), reqId); + return withModalityBridgeHeader( + withCorrelationId(withSessionHeader(response, sessionId), reqId), + modalityBridgeHeader + ); } export const handleChat = chatAdmission.withChatAdmission(handleChatImplementation); diff --git a/src/sse/handlers/chatHelpers.ts b/src/sse/handlers/chatHelpers.ts index 80f8685b1f..c94f055241 100644 --- a/src/sse/handlers/chatHelpers.ts +++ b/src/sse/handlers/chatHelpers.ts @@ -908,6 +908,31 @@ export function withCorrelationId(response: Response, correlationId: string | nu } } +/** + * Modality Bridge transparency (PR-1 Task 9): stamp the + * `x-omniroute-modality-bridge` header on responses whose request payload was + * transparently transformed (e.g. image→text describe). `value` comes from + * buildModalityBridgeHeader(); null (untouched/rerouted request) is a no-op. + * Same try-set/clone-fallback shape as withSessionHeader — the clone reuses + * `response.body`, so SSE streams pass through untouched. + */ +export function withModalityBridgeHeader(response: Response, value: string | null): Response { + if (!response || !value) return response; + + try { + response.headers.set("x-omniroute-modality-bridge", value); + return response; + } catch { + const cloned = new Response(response.body, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); + cloned.headers.set("x-omniroute-modality-bridge", value); + return cloned; + } +} + export function withSelectedConnectionHeader( response: Response, connectionId: string | null | undefined diff --git a/tests/unit/helpers/decollidedMigrationsDir.ts b/tests/unit/helpers/decollidedMigrationsDir.ts new file mode 100644 index 0000000000..00d0f42d8f --- /dev/null +++ b/tests/unit/helpers/decollidedMigrationsDir.ts @@ -0,0 +1,79 @@ +/** + * Test-only workaround for the inherited base-red "Migration version collision + * detected" on release/v3.8.50 (originally the `134_ccr_blocks.sql` + + * `134_proxy_logs_egress_ip.sql` pair, fixed by #9688; the surviving pair is + * `135_connection_runtime_state.sql` + `135_migrate_model_capability_max_token.sql`, + * fix #9676 in flight). Any test that exercises a code path opening + * the DB (e.g. `VisionBridgeGuardrail.preCall` → `getResolvedModelCapabilities` + * → `getDbInstance`) dies at migration-file scan time, BEFORE the code under + * test runs — making TDD on those paths impossible until the base is fixed. + * + * Base-red tracking: issue #9679; remaining fix PR in flight: #9676 (#9688 + * already landed). Once the base has no duplicate prefixes the copy and + * this degrades to a plain pass-through copy — at that point this helper (and + * its callsites) can be removed. Grep trigger: 9679 / 9676 / 9688. + * + * This helper copies the real migrations into a temp dir, renumbering any file + * whose numeric prefix duplicates an earlier one to a fresh (max+1) version, + * and points `OMNIROUTE_MIGRATIONS_DIR` (supported operator env var — see + * `src/lib/db/migrationRunner.ts::resolveMigrationsDir`) at the copy. On the + * FRESH per-process test DATA_DIR (tests/_setup/isolateDataDir.ts) the schema + * CONTENT applied is byte-identical, but note the renumbering does shift the + * displaced duplicate to the END of the migration ORDER (it runs after every + * lower-numbered file instead of at its original slot). Harmless for the + * current colliding pair — and strictly better than the crash — but not + * literally "the same run" as production. + * + * MUST be called before the first `getDbInstance()` in the process (i.e. at + * test-file top level, before any `preCall`). An explicitly configured + * `OMNIROUTE_MIGRATIONS_DIR` always wins. + */ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +export function useDecollidedMigrationsDir(): void { + if (process.env.OMNIROUTE_MIGRATIONS_DIR) return; + + const realDir = path.resolve("src/lib/db/migrations"); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-test-migrations-")); + + const files = fs + .readdirSync(realDir, { withFileTypes: true }) + .filter((entry) => entry.isFile()) + .map((entry) => entry.name) + .sort(); + + let maxVersion = 0; + for (const file of files) { + const match = file.match(/^(\d+)_/); + if (match) maxVersion = Math.max(maxVersion, Number.parseInt(match[1], 10)); + } + + const seenVersions = new Set(); + for (const file of files) { + const match = file.match(/^(\d+)_(.*)$/); + let target = file; + if (match) { + const version = Number.parseInt(match[1], 10); + if (seenVersions.has(version)) { + // Collision: move the later duplicate to a fresh version slot. + maxVersion += 1; + target = `${maxVersion}_${match[2]}`; + } else { + seenVersions.add(version); + } + } + fs.copyFileSync(path.join(realDir, file), path.join(tmp, target)); + } + + process.env.OMNIROUTE_MIGRATIONS_DIR = tmp; + + process.on("exit", () => { + try { + fs.rmSync(tmp, { recursive: true, force: true }); + } catch { + // Best-effort cleanup — the OS reaps its temp dir eventually. + } + }); +} diff --git a/tests/unit/media-parts.test.ts b/tests/unit/media-parts.test.ts new file mode 100644 index 0000000000..1facb94fc6 --- /dev/null +++ b/tests/unit/media-parts.test.ts @@ -0,0 +1,220 @@ +import { test } from "node:test"; +import assert from "node:assert"; +import { containsMediaKind, detectMediaParts } from "../../open-sse/utils/mediaParts"; +import { extractImageParts, replaceImageParts } from "../../src/lib/guardrails/visionBridgeHelpers"; + +const msg = (content: unknown) => [{ role: "user", content }]; + +test("detects OpenAI image_url part", () => { + const parts = detectMediaParts( + msg([{ type: "image_url", image_url: { url: "https://x/i.png" } }]) + ); + assert.equal(parts.length, 1); + assert.equal(parts[0].kind, "image"); + assert.equal(parts[0].ref, "https://x/i.png"); +}); + +test("detects Anthropic base64 image source", () => { + const parts = detectMediaParts( + msg([{ type: "image", source: { type: "base64", media_type: "image/png", data: "AAA" } }]) + ); + assert.equal(parts[0].ref, "data:image/png;base64,AAA"); +}); + +test("detects Responses-API input_image", () => { + const parts = detectMediaParts(msg([{ type: "input_image", image_url: "https://x/i.png" }])); + assert.equal(parts.length, 1); + assert.equal(parts[0].kind, "image"); +}); + +test("detects bare data:image string in content array", () => { + const parts = detectMediaParts(msg([{ type: "text", text: "data:image/jpeg;base64,QUJD" }])); + assert.equal(parts.length, 1); +}); + +test("detects input_audio and audio_url as audio kind", () => { + const parts = detectMediaParts( + msg([ + { type: "input_audio", input_audio: { data: "QUJD", format: "wav" } }, + { type: "audio_url", audio_url: { url: "https://x/a.mp3" } }, + ]) + ); + assert.deepEqual( + parts.map((p) => p.kind), + ["audio", "audio"] + ); +}); + +test("recursion depth capped at 8", () => { + let nested: Record = { type: "image_url", image_url: { url: "https://x" } }; + for (let i = 0; i < 10; i++) nested = { wrap: nested }; + assert.equal(detectMediaParts(msg([nested])).length, 0); +}); + +test("string content and empty messages yield []", () => { + assert.deepEqual(detectMediaParts([{ role: "user", content: "oi" }]), []); + assert.deepEqual(detectMediaParts([]), []); +}); + +test("combo-parity: image indicators without extractable refs are still detected", () => { + // Bare image_url key without a `type` (legacy combo matched by key presence). + const bareKey = detectMediaParts(msg([{ image_url: { url: "https://x/i.png" } }])); + assert.equal(bareKey.length, 1); + assert.equal(bareKey[0].kind, "image"); + assert.equal(bareKey[0].ref, "https://x/i.png"); + + // `type: "image"` with no usable source (legacy combo matched by type name). + const bareType = detectMediaParts(msg([{ type: "image" }])); + assert.equal(bareType.length, 1); + assert.equal(bareType[0].shape, "image_indicator"); + assert.equal(bareType[0].ref, ""); + + // source.media_type image/* on an untyped part. + const bySourceMedia = detectMediaParts( + msg([{ source: { media_type: "image/JPEG", data: "AAA" } }]) + ); + assert.equal(bySourceMedia.length, 1); + assert.equal(bySourceMedia[0].kind, "image"); + + // Case-insensitive type match (legacy combo lowercased `type`). + const upperType = detectMediaParts(msg([{ type: "IMAGE_URL", image_url: { url: "https://x" } }])); + assert.equal(upperType.length, 1); + assert.equal(upperType[0].ref, "https://x"); +}); + +test("audio part does not shadow a bare image_url key on the same object", () => { + const parts = detectMediaParts( + msg([{ type: "input_audio", input_audio: { data: "QUJD" }, image_url: "https://x/i.png" }]) + ); + assert.deepEqual(parts.map((p) => p.kind).sort(), ["audio", "image"]); + assert.equal(parts.find((p) => p.kind === "image")?.ref, "https://x/i.png"); +}); + +test("audio_url part does not shadow a bare input_image key on the same object", () => { + const parts = detectMediaParts( + msg([{ type: "audio_url", audio_url: "https://x/a.mp3", input_image: "https://x/i.png" }]) + ); + assert.deepEqual(parts.map((p) => p.kind).sort(), ["audio", "image"]); +}); + +test("audio media_type part still recurses into sibling values (combo parity)", () => { + const parts = detectMediaParts( + msg([{ source: { media_type: "audio/mp3", data: "QUJD" }, sibling: { type: "image" } }]) + ); + assert.equal(parts.filter((p) => p.kind === "audio").length, 1); + assert.equal(parts.filter((p) => p.kind === "image").length, 1); +}); + +test("image nested inside input_audio payload is still detected", () => { + const parts = detectMediaParts( + msg([ + { + type: "input_audio", + input_audio: { + data: "QUJD", + cover: { type: "image_url", image_url: { url: "https://x/c.png" } }, + }, + }, + ]) + ); + assert.equal(parts.filter((p) => p.kind === "audio").length, 1); + assert.ok(parts.some((p) => p.kind === "image" && p.ref === "https://x/c.png")); +}); + +test("bare source.media_type audio/* yields a single audio_source part", () => { + const parts = detectMediaParts(msg([{ source: { media_type: "audio/wav", data: "QUJD" } }])); + assert.equal(parts.length, 1); + assert.equal(parts[0].kind, "audio"); + assert.equal(parts[0].shape, "audio_source"); + assert.equal(parts[0].ref, "QUJD"); +}); + +test("containsMediaKind early-exit agrees with detectMediaParts presence", () => { + const withImage = msg([ + { type: "text", text: "hi" }, + { type: "image_url", image_url: { url: "https://x/i.png" } }, + ]); + assert.equal(containsMediaKind(withImage, "image"), true); + assert.equal(containsMediaKind(withImage, "audio"), false); + + const withAudio = msg([{ type: "input_audio", input_audio: { data: "QUJD", format: "wav" } }]); + assert.equal(containsMediaKind(withAudio, "audio"), true); + assert.equal(containsMediaKind(withAudio, "image"), false); + + // Nested/indicator hits count for presence (combo parity). + assert.equal(containsMediaKind(msg([{ image_url: "https://x" }]), "image"), true); + assert.equal(containsMediaKind([{ role: "user", content: "oi" }], "image"), false); + assert.equal(containsMediaKind(undefined, "image"), false); +}); + +test("extractImageParts skips indicator parts without extractable refs", () => { + assert.deepEqual( + extractImageParts([{ role: "user", content: [{ type: "image" }] } as never]), + [] + ); +}); + +test("extractImageParts now sees input_image (Responses API)", () => { + const parts = extractImageParts([ + { role: "user", content: [{ type: "input_image", image_url: "https://x/i.png" }] } as never, + ]); + assert.equal(parts.length, 1); + assert.equal(parts[0].imageUrl, "https://x/i.png"); +}); + +test("extract→replace round-trip: input_image does not shift sibling descriptions", () => { + const body = { + messages: [ + { + role: "user", + content: [ + { type: "input_image", image_url: "https://x/a.png" }, + { type: "image_url", image_url: { url: "https://x/b.png" } }, + ], + }, + ], + }; + const extracted = extractImageParts(body.messages as never); + assert.equal(extracted.length, 2); + assert.equal(extracted[0].imageUrl, "https://x/a.png"); + assert.equal(extracted[1].imageUrl, "https://x/b.png"); + + const replaced = replaceImageParts(body as never, ["DA", "DB"]); + const content = (replaced.messages as Array<{ content: unknown }>)[0].content; + assert.deepEqual(content, [ + { type: "text", text: "DA" }, + { type: "text", text: "DB" }, + ]); +}); + +test("extractImageParts does not extract a data URI embedded in a text part", () => { + const parts = extractImageParts([ + { + role: "user", + content: [{ type: "text", text: "data:image/png;base64,QUJD" }], + } as never, + ]); + assert.deepEqual(parts, []); +}); + +test("extractImageParts skips nested and indicator-only detections", () => { + const parts = extractImageParts([ + { + role: "user", + content: [ + // Image nested inside an audio payload: detector reports it (combo needs + // it) but the replacer cannot splice it — must not be extracted. + { + type: "input_audio", + input_audio: { + data: "QUJD", + cover: { type: "image_url", image_url: { url: "https://x/c.png" } }, + }, + }, + // Bare image_url key without type: indicator shape, not replaceable. + { image_url: "https://x/bare.png" }, + ], + } as never, + ]); + assert.deepEqual(parts, []); +}); diff --git a/tests/unit/modality-bridge-cache.test.ts b/tests/unit/modality-bridge-cache.test.ts new file mode 100644 index 0000000000..0ebb81b9e0 --- /dev/null +++ b/tests/unit/modality-bridge-cache.test.ts @@ -0,0 +1,63 @@ +import { test } from "node:test"; +import assert from "node:assert"; + +import { + BridgeCache, + bridgeCacheKey, + getSharedBridgeCacheFor, +} from "../../src/lib/guardrails/modalityBridge/bridgeCache.ts"; +import { resolveVisionBridgeRuntimeSettings } from "../../src/shared/constants/modalityBridgeDefaults.ts"; + +test("key is stable sha256 of content+prompt+model", () => { + const a = bridgeCacheKey("data:image/png;base64,AAA", "describe", "gpt-4o-mini"); + const b = bridgeCacheKey("data:image/png;base64,AAA", "describe", "gpt-4o-mini"); + assert.equal(a, b); + assert.match(a, /^[a-f0-9]{64}$/); + assert.notEqual(a, bridgeCacheKey("data:image/png;base64,AAA", "other", "gpt-4o-mini")); +}); + +test("key framing prevents boundary-shift collisions between fields", () => { + // Without length-prefix framing ("ab","c") and ("a","bc") would hash the + // same concatenated bytes. + assert.notEqual(bridgeCacheKey("ab", "c", "m"), bridgeCacheKey("a", "bc", "m")); + assert.notEqual(bridgeCacheKey("x", "yz", "m"), bridgeCacheKey("x", "y", "zm")); +}); + +test("get/set roundtrip and TTL expiry", () => { + let now = 1000; + const cache = new BridgeCache({ maxEntries: 10, ttlMs: 500, now: () => now }); + cache.set("k1", "desc"); + assert.equal(cache.get("k1"), "desc"); + now = 1600; + assert.equal(cache.get("k1"), undefined); +}); + +test("LRU evicts oldest when full", () => { + const cache = new BridgeCache({ maxEntries: 2, ttlMs: 60000, now: () => 0 }); + cache.set("a", "1"); + cache.set("b", "2"); + cache.get("a"); + cache.set("c", "3"); + assert.equal(cache.get("b"), undefined); + assert.equal(cache.get("a"), "1"); +}); + +test("getSharedBridgeCacheFor reuses the instance for the same config and recreates on change", () => { + const base = resolveVisionBridgeRuntimeSettings({}); + const a = getSharedBridgeCacheFor(base); + const b = getSharedBridgeCacheFor({ ...base }); + assert.equal(a, b); + + const c = getSharedBridgeCacheFor({ ...base, cacheTtlMinutes: base.cacheTtlMinutes + 5 }); + assert.notEqual(c, a); + + const d = getSharedBridgeCacheFor({ ...base, cacheTtlMinutes: base.cacheTtlMinutes + 5 }); + assert.equal(d, c); + + const e = getSharedBridgeCacheFor({ + ...base, + cacheTtlMinutes: base.cacheTtlMinutes + 5, + cacheMaxEntries: base.cacheMaxEntries + 1, + }); + assert.notEqual(e, d); +}); diff --git a/tests/unit/modality-bridge-header.test.ts b/tests/unit/modality-bridge-header.test.ts new file mode 100644 index 0000000000..0635e2521b --- /dev/null +++ b/tests/unit/modality-bridge-header.test.ts @@ -0,0 +1,116 @@ +/** + * Modality Bridge stats + transparency header (PR-1 Task 9): + * - buildModalityBridgeHeader() derives the `x-omniroute-modality-bridge` + * response header value from pre-call guardrail results (describe path only — + * reroute and untouched requests get no header). + * - recordBridgeUse()/getBridgeStats() keep in-memory per-modality counters. + * + * Stats and the describe cache are PROCESS-GLOBAL: assertions use >= against + * captured before-values and every guardrail case uses a unique image payload + * (`auto/` model + mode "describe" keeps the flow DB-free — same recipe as + * tests/unit/vision-bridge-describe-cache.test.ts). + */ +import { test } from "node:test"; +import assert from "node:assert"; + +import { + buildModalityBridgeHeader, + recordBridgeUse, + getBridgeStats, +} from "../../src/lib/guardrails/modalityBridge/bridgeStats.ts"; +import { VisionBridgeGuardrail } from "../../src/lib/guardrails/visionBridge.ts"; + +test("header built from vision-bridge describe meta", () => { + const h = buildModalityBridgeHeader([ + { guardrail: "vision-bridge", meta: { imagesProcessed: 2, visionModel: "openai/gpt-4o-mini" } }, + ]); + assert.equal(h, "image->text;model=openai/gpt-4o-mini;parts=2"); +}); + +test("no header for reroute or untouched requests", () => { + assert.equal( + buildModalityBridgeHeader([{ guardrail: "vision-bridge", meta: { rerouted: true } }]), + null + ); + assert.equal(buildModalityBridgeHeader([]), null); +}); + +test("audio-bridge meta produces the audio segment (PR-3 forward-compat)", () => { + const h = buildModalityBridgeHeader([ + { guardrail: "vision-bridge", meta: { imagesProcessed: 1, visionModel: "m1" } }, + { guardrail: "audio-bridge", meta: { clipsProcessed: 3, sttModel: "m2" } }, + ]); + assert.equal(h, "image->text;model=m1;parts=1, audio->text;model=m2;parts=3"); +}); + +test("stats counters accumulate", () => { + recordBridgeUse("vision", { cacheHit: true }); + const s = getBridgeStats(); + assert.ok(s.vision.bridged >= 1); + assert.ok(s.vision.cacheHits >= 1); +}); + +// ── Guardrail-level wiring: describe path bumps the vision counters ───────── + +function statsGuardrail(counter: { calls: number }, behavior?: { failAlways?: boolean }) { + return new VisionBridgeGuardrail({ + deps: { + getSettings: async () => ({ modalityBridgeVisionMode: "describe" }), + callVisionModel: async () => { + counter.calls++; + if (behavior?.failAlways) throw new Error("describe indisponível"); + return "uma descrição da imagem"; + }, + hasUsableCredentials: async () => null, + }, + }); +} + +/** Unique per-test payload — the test name lands inside the base64 content. */ +function bodyWithImage(uniqueRef: string): Record { + return { + model: "auto/bridge-stats", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "o que há na imagem?" }, + { + type: "image_url", + image_url: { + url: `data:image/png;base64,${Buffer.from(uniqueRef).toString("base64")}`, + }, + }, + ], + }, + ], + }; +} + +const context = { model: "auto/bridge-stats", log: console }; + +test("describe path records bridged use; identical repeat counts a cache hit", async () => { + const before = getBridgeStats().vision; + const counter = { calls: 0 }; + const guardrail = statsGuardrail(counter); + + await guardrail.preCall(bodyWithImage("bridge-stats-hit-test"), context); + const afterFirst = getBridgeStats().vision; + assert.ok(afterFirst.bridged >= before.bridged + 1, "successful describe must count as bridged"); + assert.ok(typeof afterFirst.lastUsedAt === "string", "lastUsedAt must be stamped"); + + await guardrail.preCall(bodyWithImage("bridge-stats-hit-test"), context); + const afterSecond = getBridgeStats().vision; + assert.equal(counter.calls, 1, "second identical request must be served from the cache"); + assert.ok(afterSecond.bridged >= afterFirst.bridged + 1, "cache-served describe still bridged"); + assert.ok(afterSecond.cacheHits >= afterFirst.cacheHits + 1, "cache hit must be counted"); +}); + +test("failed describe records a failure", async () => { + const before = getBridgeStats().vision; + const guardrail = statsGuardrail({ calls: 0 }, { failAlways: true }); + + await guardrail.preCall(bodyWithImage("bridge-stats-failure-test"), context); + const after = getBridgeStats().vision; + assert.ok(after.failures >= before.failures + 1, "failed describe must count as failure"); +}); diff --git a/tests/unit/modality-bridge-settings-migration.test.ts b/tests/unit/modality-bridge-settings-migration.test.ts new file mode 100644 index 0000000000..84e2a7c2e3 --- /dev/null +++ b/tests/unit/modality-bridge-settings-migration.test.ts @@ -0,0 +1,92 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import Database from "better-sqlite3"; + +const MIGRATION_PATH = path.resolve("src/lib/db/migrations/141_modality_bridge_settings.sql"); +const INITIAL_SCHEMA_PATH = path.resolve("src/lib/db/migrations/001_initial_schema.sql"); + +/** Extract the REAL key_value DDL from the initial schema so the seed can never drift. */ +function keyValueTableDdl(): string { + const schema = fs.readFileSync(INITIAL_SCHEMA_PATH, "utf8"); + const match = schema.match(/CREATE TABLE IF NOT EXISTS key_value \([\s\S]*?\);/); + assert.ok(match, "key_value CREATE TABLE block not found in 001_initial_schema.sql"); + return match[0]; +} + +function createSeededDb(): InstanceType { + const db = new Database(":memory:"); + db.exec(keyValueTableDdl()); + return db; +} + +function getSetting(db: InstanceType, key: string): string | undefined { + const row = db + .prepare("SELECT value FROM key_value WHERE namespace = 'settings' AND key = ?") + .get(key) as { value: string } | undefined; + return row?.value; +} + +test("migration 141 copies legacy visionBridge* settings to modalityBridge* keys", () => { + const sql = fs.readFileSync(MIGRATION_PATH, "utf8"); + const db = createSeededDb(); + try { + const seed = db.prepare( + "INSERT INTO key_value (namespace, key, value) VALUES ('settings', ?, ?)" + ); + seed.run("visionBridgeEnabled", "true"); + seed.run("visionBridgeModel", "openai/gpt-4o-mini"); + seed.run("visionBridgePrompt", "legacy prompt"); + seed.run("visionBridgeTimeout", "45000"); + seed.run("visionBridgeMaxImages", "6"); + + db.exec(sql); + + assert.equal(getSetting(db, "modalityBridgeVisionEnabled"), "true"); + assert.equal(getSetting(db, "modalityBridgeVisionModel"), "openai/gpt-4o-mini"); + assert.equal(getSetting(db, "modalityBridgeVisionPrompt"), "legacy prompt"); + assert.equal(getSetting(db, "modalityBridgeVisionTimeout"), "45000"); + assert.equal(getSetting(db, "modalityBridgeVisionMaxImages"), "6"); + // Legacy keys stay untouched (one-cycle rollback window). + assert.equal(getSetting(db, "visionBridgeEnabled"), "true"); + } finally { + db.close(); + } +}); + +test("migration 141 is idempotent and never overwrites an existing new key", () => { + const sql = fs.readFileSync(MIGRATION_PATH, "utf8"); + const db = createSeededDb(); + try { + const seed = db.prepare( + "INSERT INTO key_value (namespace, key, value) VALUES ('settings', ?, ?)" + ); + seed.run("visionBridgeModel", "openai/gpt-4o-mini"); + // Operator already set the new key — the migration must not clobber it. + seed.run("modalityBridgeVisionModel", "gemini/gemini-2.5-flash"); + + db.exec(sql); + db.exec(sql); // re-run: idempotent + + assert.equal(getSetting(db, "modalityBridgeVisionModel"), "gemini/gemini-2.5-flash"); + const count = db + .prepare("SELECT COUNT(*) AS n FROM key_value WHERE namespace = 'settings'") + .get() as { n: number }; + assert.equal(count.n, 2); + } finally { + db.close(); + } +}); + +test("migration 141 is a no-op on a database without legacy settings", () => { + const sql = fs.readFileSync(MIGRATION_PATH, "utf8"); + const db = createSeededDb(); + try { + db.exec(sql); + const count = db.prepare("SELECT COUNT(*) AS n FROM key_value").get() as { n: number }; + assert.equal(count.n, 0); + } finally { + db.close(); + } +}); diff --git a/tests/unit/modality-bridge-settings.test.ts b/tests/unit/modality-bridge-settings.test.ts new file mode 100644 index 0000000000..cf5d71dd2e --- /dev/null +++ b/tests/unit/modality-bridge-settings.test.ts @@ -0,0 +1,96 @@ +import { test } from "node:test"; +import assert from "node:assert"; + +import { + resolveVisionBridgeRuntimeSettings, + MODALITY_BRIDGE_DEFAULTS, +} from "../../src/shared/constants/modalityBridgeDefaults.ts"; +import { updateSettingsSchema } from "../../src/shared/validation/settingsSchemas.ts"; + +test("new keys win over legacy keys", () => { + const s = resolveVisionBridgeRuntimeSettings({ + modalityBridgeVisionEnabled: false, + visionBridgeEnabled: true, + modalityBridgeVisionModel: "gemini/gemini-2.5-flash", + visionBridgeModel: "openai/gpt-4o-mini", + }); + assert.equal(s.enabled, false); + assert.equal(s.model, "gemini/gemini-2.5-flash"); +}); + +test("legacy keys used as fallback when new keys absent (rollback 1 ciclo)", () => { + const s = resolveVisionBridgeRuntimeSettings({ + visionBridgeEnabled: false, + visionBridgePrompt: "legacy prompt", + }); + assert.equal(s.enabled, false); + assert.equal(s.prompt, "legacy prompt"); +}); + +test("defaults: mode auto, taskAware true, cache on", () => { + const s = resolveVisionBridgeRuntimeSettings({}); + assert.equal(s.mode, "auto"); + assert.equal(s.taskAware, true); + assert.equal(s.cacheEnabled, true); + assert.equal(s.cacheTtlMinutes, MODALITY_BRIDGE_DEFAULTS.cacheTtlMinutes); +}); + +test("wrong-typed stored values are skipped in favor of the next candidate/default", () => { + const s = resolveVisionBridgeRuntimeSettings({ + modalityBridgeVisionEnabled: "off", // string in a boolean field + modalityBridgeVisionTimeout: "9000", // string in a number field + modalityBridgeVisionModel: 42, // number in a string field + }); + assert.equal(s.enabled, true); // falls through to VISION_BRIDGE_DEFAULTS.enabled + assert.equal(s.timeoutMs, 30000); + assert.equal(s.model, "openai/gpt-4o-mini"); + + // A wrong-typed NEW key must still fall through to a well-typed LEGACY key. + const fallback = resolveVisionBridgeRuntimeSettings({ + modalityBridgeVisionEnabled: "off", + visionBridgeEnabled: false, + }); + assert.equal(fallback.enabled, false); +}); + +test("Modality Bridge settings are accepted by the settings PATCH schema", () => { + const validation = updateSettingsSchema.safeParse({ + modalityBridgeVisionEnabled: true, + modalityBridgeVisionMode: "describe", + modalityBridgeVisionModel: "gemini/gemini-2.5-flash", + modalityBridgeVisionTaskAware: false, + modalityBridgeVisionPrompt: "Describe the image contents briefly.", + modalityBridgeVisionTimeout: 45000, + modalityBridgeVisionMaxImages: 6, + modalityBridgeAudioEnabled: true, + modalityBridgeAudioModel: "openai/gpt-4o-mini-transcribe", + modalityBridgeAudioTimeout: 60000, + modalityBridgeAudioMaxClips: 3, + modalityBridgeCacheEnabled: true, + modalityBridgeCacheTtlMinutes: 60, + modalityBridgeCacheMaxEntries: 200, + }); + + assert.equal(validation.success, true); +}); + +test("Modality Bridge settings keep numeric bounds enforced (each field individually)", () => { + const invalidByField: Record = { + modalityBridgeVisionTimeout: 999999, + modalityBridgeVisionMaxImages: 0, + modalityBridgeAudioMaxClips: 11, + modalityBridgeCacheTtlMinutes: 0, + modalityBridgeCacheMaxEntries: 9, + }; + for (const [field, value] of Object.entries(invalidByField)) { + const validation = updateSettingsSchema.safeParse({ [field]: value }); + assert.equal(validation.success, false, `${field}=${value} should be rejected`); + } +}); + +test("Modality Bridge vision mode rejects values outside the enum", () => { + const validation = updateSettingsSchema.safeParse({ + modalityBridgeVisionMode: "invalid-mode", + }); + assert.equal(validation.success, false); +}); diff --git a/tests/unit/vision-bridge-describe-cache.test.ts b/tests/unit/vision-bridge-describe-cache.test.ts new file mode 100644 index 0000000000..0fbd1d701b --- /dev/null +++ b/tests/unit/vision-bridge-describe-cache.test.ts @@ -0,0 +1,102 @@ +/** + * Describe-path cache integration (Modality Bridge PR-1): the describe loop + * consults the shared BridgeCache (sha256 of contentRef+prompt+model) so the + * same image with the same prompt/model is described once per TTL. Failures + * are never cached. Opt-out via `modalityBridgeCacheEnabled: false`. + * + * The shared cache is PROCESS-WIDE — every test uses a unique image payload so + * tests cannot cross-contaminate each other's keys. Guardrail cases use + * `model: "auto/..."` + `mode: "describe"` so the flow is DB-free. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { VisionBridgeGuardrail } from "../../src/lib/guardrails/visionBridge.ts"; + +function cacheGuardrail( + settings: Record, + counter: { calls: number }, + behavior?: { failFirstCall?: boolean } +): InstanceType { + return new VisionBridgeGuardrail({ + deps: { + getSettings: async () => ({ modalityBridgeVisionMode: "describe", ...settings }), + callVisionModel: async () => { + counter.calls++; + if (behavior?.failFirstCall && counter.calls === 1) { + throw new Error("primeiro describe falhou"); + } + return "uma descrição da imagem"; + }, + hasUsableCredentials: async () => null, + }, + }); +} + +/** Unique per-test payload — the test name lands inside the base64 content. */ +function bodyWithImage(uniqueRef: string): Record { + return { + model: "auto/describe-cache", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "o que há na imagem?" }, + { + type: "image_url", + image_url: { + url: `data:image/png;base64,${Buffer.from(uniqueRef).toString("base64")}`, + }, + }, + ], + }, + ], + }; +} + +const context = { model: "auto/describe-cache", log: console }; + +test("same image+prompt+model described twice → single upstream call (cache hit)", async () => { + const counter = { calls: 0 }; + const guardrail = cacheGuardrail({}, counter); + + const first = await guardrail.preCall(bodyWithImage("cache-hit-test"), context); + assert.equal((first.meta ?? {}).imagesProcessed, 1); + assert.equal(counter.calls, 1); + + const second = await guardrail.preCall(bodyWithImage("cache-hit-test"), context); + assert.equal((second.meta ?? {}).imagesProcessed, 1, "cached describe still replaces the image"); + assert.equal(counter.calls, 1, "second identical request must be served from the cache"); + + const descriptions = (second.meta ?? {}).descriptions as string[]; + assert.ok( + descriptions?.[0]?.includes("uma descrição da imagem"), + "cached description must be spliced into the payload" + ); +}); + +test("modalityBridgeCacheEnabled=false → every request hits the vision model", async () => { + const counter = { calls: 0 }; + const guardrail = cacheGuardrail({ modalityBridgeCacheEnabled: false }, counter); + + await guardrail.preCall(bodyWithImage("cache-disabled-test"), context); + await guardrail.preCall(bodyWithImage("cache-disabled-test"), context); + assert.equal(counter.calls, 2, "disabled cache must not dedupe describe calls"); +}); + +test("failed describe is NOT cached — the next request retries upstream", async () => { + const counter = { calls: 0 }; + const guardrail = cacheGuardrail({}, counter, { failFirstCall: true }); + + await guardrail.preCall(bodyWithImage("failure-not-cached-test"), context); + assert.equal(counter.calls, 1); + + const second = await guardrail.preCall(bodyWithImage("failure-not-cached-test"), context); + assert.equal(counter.calls, 2, "failure must not be cached; retry must reach upstream"); + + const descriptions = (second.meta ?? {}).descriptions as string[]; + assert.ok( + descriptions?.[0]?.includes("uma descrição da imagem"), + "successful retry description must be used" + ); +}); diff --git a/tests/unit/vision-bridge-mode.test.ts b/tests/unit/vision-bridge-mode.test.ts new file mode 100644 index 0000000000..ff745dd1a5 --- /dev/null +++ b/tests/unit/vision-bridge-mode.test.ts @@ -0,0 +1,138 @@ +/** + * Vision Bridge mode selector (auto | describe | reroute) — Modality Bridge PR-1. + * + * The forced modes short-circuit BEFORE the auto reroute×describe heuristic, so + * the #6640/#7204/#7871/#8430 contracts stay untouched in "auto" (the default): + * - "describe": never whole-request-reroutes — straight to the describe path. + * - "reroute": skips only the keep-credentialed-model guard; the reroute-target + * credential guard still applies, and with no usable target it falls back to + * describe (raw images must never reach a text-only backend — #8430). + * + * Uses dependency injection for settings/vision calls/credentials. The model + * capability lookup inside preCall still opens the real (isolated) SQLite DB, + * which on the current base dies on the inherited 134 migration collision — + * hence the decollided-migrations helper below. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { useDecollidedMigrationsDir } from "./helpers/decollidedMigrationsDir.ts"; + +useDecollidedMigrationsDir(); + +const { VisionBridgeGuardrail } = await import("../../src/lib/guardrails/visionBridge.ts"); + +const TEXT_ONLY_MODEL = "some/text-only-model"; + +/** + * Unique per-test payload: the describe path caches by image+prompt+model + * (Task 8), so reusing the same data URI across tests would turn a later + * describe into a cache hit and hide the upstream call being asserted. + */ +function imageBody(uniqueRef: string): Record { + return { + model: TEXT_ONLY_MODEL, + messages: [ + { + role: "user", + content: [ + { type: "text", text: "o que há na imagem?" }, + { + type: "image_url", + image_url: { + url: `data:image/png;base64,${Buffer.from(uniqueRef).toString("base64")}`, + }, + }, + ], + }, + ], + }; +} + +function metaOf(result: { meta?: Record | null }): Record { + return result.meta ?? {}; +} + +test("mode=describe: never reroutes even when a reroute target exists", async () => { + const guardrail = new VisionBridgeGuardrail({ + deps: { + getSettings: async () => ({ + modalityBridgeVisionMode: "describe", + // A configured vision model — in auto/reroute this would be a valid + // fixed reroute target (credentials indeterminate → fail-open #8430). + modalityBridgeVisionModel: "openai/gpt-4o-mini", + }), + callVisionModel: async () => "uma foto de um gato", + hasUsableCredentials: async () => null, + }, + }); + + const body = imageBody("mode-describe-test"); + const result = await guardrail.preCall(body, { model: TEXT_ONLY_MODEL, log: console }); + + const meta = metaOf(result); + assert.notEqual(meta.rerouted, true, "describe mode must never whole-request-reroute"); + assert.equal(meta.imagesProcessed, 1, "the image must be described instead"); +}); + +test("mode=reroute: falls back to describe when no reroute target has credentials", async () => { + const describeCalls: string[] = []; + const guardrail = new VisionBridgeGuardrail({ + deps: { + getSettings: async () => ({ modalityBridgeVisionMode: "reroute" }), + callVisionModel: async () => { + describeCalls.push("describe"); + return "desc"; + }, + // Every model confirmed unusable — no reroute target can win (#8430). + hasUsableCredentials: async () => false, + }, + }); + + const body = imageBody("mode-reroute-fallback-test"); + const result = await guardrail.preCall(body, { model: TEXT_ONLY_MODEL, log: console }); + + const meta = metaOf(result); + assert.notEqual(meta.rerouted, true, "must not reroute to a target without credentials"); + assert.ok(describeCalls.length >= 1, "deveria ter caído para o caminho de descrição"); +}); + +test("mode=reroute: forces reroute where auto mode would keep the credentialed model", async () => { + const guardrail = new VisionBridgeGuardrail({ + deps: { + getSettings: async () => ({ + modalityBridgeVisionMode: "reroute", + modalityBridgeVisionModel: "openai/gpt-4o-mini", + }), + callVisionModel: async () => "desc", + // Original model IS credentialed (auto mode would keep it, #7204); + // reroute target indeterminate → fail-open proceeds (#8430). + hasUsableCredentials: async (model: string) => (model === TEXT_ONLY_MODEL ? true : null), + }, + }); + + const body = imageBody("mode-reroute-forces-test"); + const result = await guardrail.preCall(body, { model: TEXT_ONLY_MODEL, log: console }); + + const meta = metaOf(result); + assert.equal(meta.rerouted, true, "reroute mode must skip the keep-credentialed-model guard"); + assert.equal(meta.toModel, "openai/gpt-4o-mini"); + assert.equal(meta.fromModel, TEXT_ONLY_MODEL); +}); + +test("mode=auto (default): credentialed model is described, not hijacked", async () => { + const guardrail = new VisionBridgeGuardrail({ + deps: { + getSettings: async () => ({}), + callVisionModel: async () => "desc", + hasUsableCredentials: async () => true, + }, + }); + + const body = imageBody("mode-auto-default-test"); + const result = await guardrail.preCall(body, { model: TEXT_ONLY_MODEL, log: console }); + + const meta = metaOf(result); + assert.notEqual(meta.rerouted, true, "auto mode keeps the credentialed model (#7204)"); + assert.equal(meta.imagesProcessed, 1, "images are described for the kept model"); +}); diff --git a/tests/unit/vision-bridge-task-aware.test.ts b/tests/unit/vision-bridge-task-aware.test.ts new file mode 100644 index 0000000000..2c5a7fc408 --- /dev/null +++ b/tests/unit/vision-bridge-task-aware.test.ts @@ -0,0 +1,115 @@ +/** + * Task-aware vision description prompt (codex-vision-proxy pattern) — + * Modality Bridge PR-1. The describe path appends the user's last question as + * a focus hint so the vision model describes what is relevant to answering it + * instead of producing a generic caption. Default ON; disabled via + * `modalityBridgeVisionTaskAware: false`. + * + * Guardrail-level cases use `model: "auto/..."` + `mode: "describe"` so the + * whole flow is DB-free (the auto prefix skips the capability/combo lookups + * that open SQLite, and the forced describe mode skips the reroute block). + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { composeVisionPrompt } from "../../src/lib/guardrails/visionBridgeHelpers.ts"; +import { VisionBridgeGuardrail } from "../../src/lib/guardrails/visionBridge.ts"; +import type { VisionModelConfig } from "../../src/lib/guardrails/visionBridgeHelpers.ts"; + +// ── composeVisionPrompt (pure) ────────────────────────────────────────────── + +test("appends user focus hint when taskAware", () => { + const p = composeVisionPrompt("Describe the image.", "qual o erro no screenshot?", true); + assert.ok(p.startsWith("Describe the image.")); + assert.ok(p.includes("qual o erro no screenshot?")); +}); + +test("no hint when disabled or no user text", () => { + assert.equal(composeVisionPrompt("Base.", "pergunta", false), "Base."); + assert.equal(composeVisionPrompt("Base.", undefined, true), "Base."); + assert.equal(composeVisionPrompt("Base.", " ", true), "Base."); +}); + +test("hint truncated to 500 chars", () => { + const p = composeVisionPrompt("Base.", "x".repeat(2000), true); + assert.ok(p.length < 700, `expected truncated prompt, got length ${p.length}`); + assert.ok(p.includes("x".repeat(500))); + assert.ok(!p.includes("x".repeat(501))); +}); + +// ── Guardrail describe path wiring ────────────────────────────────────────── + +function describeGuardrail( + settings: Record, + capturedPrompts: string[] +): InstanceType { + return new VisionBridgeGuardrail({ + deps: { + getSettings: async () => ({ modalityBridgeVisionMode: "describe", ...settings }), + callVisionModel: async (_imageDataUri: string, config: VisionModelConfig) => { + capturedPrompts.push(config.prompt); + return "descrição"; + }, + hasUsableCredentials: async () => null, + }, + }); +} + +/** + * Unique per-test payload: the describe path caches by image+prompt+model + * (Task 8), so reusing the same data URI across tests would make a later + * describe a cache hit and hide the upstream call whose prompt is asserted. + */ +function autoImageBody(uniqueRef: string, userText: string): Record { + return { + model: "auto/task-aware", + messages: [ + { + role: "user", + content: [ + { type: "text", text: userText }, + { + type: "image_url", + image_url: { + url: `data:image/png;base64,${Buffer.from(uniqueRef).toString("base64")}`, + }, + }, + ], + }, + ], + }; +} + +test("describe call prompt contains the last user question (taskAware default on)", async () => { + const prompts: string[] = []; + const guardrail = describeGuardrail({}, prompts); + + const result = await guardrail.preCall( + autoImageBody("task-aware-default-on-test", "qual o erro no screenshot?"), + { model: "auto/task-aware", log: console } + ); + + assert.equal((result.meta ?? {}).imagesProcessed, 1); + assert.equal(prompts.length, 1); + assert.ok( + prompts[0].includes("qual o erro no screenshot?"), + `prompt should carry the user question, got: ${prompts[0]}` + ); +}); + +test("modalityBridgeVisionTaskAware=false keeps the base prompt untouched", async () => { + const prompts: string[] = []; + const guardrail = describeGuardrail( + { modalityBridgeVisionTaskAware: false, modalityBridgeVisionPrompt: "Base prompt." }, + prompts + ); + + const result = await guardrail.preCall( + autoImageBody("task-aware-disabled-test", "pergunta que não deve vazar"), + { model: "auto/task-aware", log: console } + ); + + assert.equal((result.meta ?? {}).imagesProcessed, 1); + assert.equal(prompts.length, 1); + assert.equal(prompts[0], "Base prompt."); +});