mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-10 17:22:17 +03:00
Merge remote-tracking branch 'origin/release/v3.8.50' into feat/radar-key-input
This commit is contained in:
diegosouzapw
@@ -5294,6 +5294,19 @@ paths:
|
||||
"200":
|
||||
description: Caches cleared
|
||||
|
||||
/api/modality-bridge/stats:
|
||||
get:
|
||||
tags: [System]
|
||||
summary: Get Modality Bridge telemetry
|
||||
description: In-memory per-modality bridge counters (bridged, cacheHits, failures, lastUsedAt). Counters reset on process restart.
|
||||
security:
|
||||
- ManagementSessionAuth: []
|
||||
responses:
|
||||
"200":
|
||||
description: Per-modality bridge stats (vision, audio)
|
||||
"401":
|
||||
description: Unauthorized
|
||||
|
||||
/api/cache/stats:
|
||||
get:
|
||||
tags: [System]
|
||||
|
||||
@@ -573,6 +573,7 @@ Response example:
|
||||
| `/api/rate-limits` | GET | Per-account rate limits |
|
||||
| `/api/monitoring/health` | GET | Health check + provider summary (`catalogCount`, `configuredCount`, `activeCount`, `monitoredCount`) |
|
||||
| `/api/cache/stats` | GET/DELETE | Cache stats / clear |
|
||||
| `/api/modality-bridge/stats` | GET | In-memory Modality Bridge telemetry — per-modality `bridged`/`cacheHits`/`failures`/`lastUsedAt` counters (reset on restart; management auth) |
|
||||
|
||||
### Backup & Export/Import
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
---
|
||||
title: "Guardrails"
|
||||
version: 3.8.40
|
||||
lastUpdated: 2026-06-28
|
||||
version: 3.8.50
|
||||
lastUpdated: 2026-08-07
|
||||
---
|
||||
|
||||
# Guardrails
|
||||
|
||||
> **Source of truth:** `src/lib/guardrails/`
|
||||
> **Last updated:** 2026-06-28 — v3.8.40 (injection-guard coverage + 16 KB scan bound + red-team)
|
||||
> **Last updated:** 2026-08-07 — v3.8.50 (Modality Bridge PR-1: mode selector, task-aware prompt, describe cache, transparency header + stats)
|
||||
|
||||
Guardrails enforce safety, policy, and content transformations at the boundary
|
||||
between OmniRoute and upstream providers. Each guardrail can inspect (and
|
||||
@@ -32,31 +32,107 @@ The registry auto-loads four guardrails in priority order on import
|
||||
|
||||
Lower priority numbers run **first**.
|
||||
|
||||
### Vision Bridge (`visionBridge.ts`)
|
||||
### Vision Bridge (`visionBridge.ts`) — Modality Bridge PR-1
|
||||
|
||||
Intercepts image-bearing requests aimed at **non-vision models** and replaces
|
||||
the image parts with text descriptions produced by a configurable vision model
|
||||
before the upstream call. This lets text-only providers transparently handle
|
||||
Intercepts image-bearing requests aimed at **non-vision models** and either
|
||||
reroutes the whole request to a vision-capable model or replaces the image
|
||||
parts with text descriptions produced by a configurable vision model before
|
||||
the upstream call. This lets text-only providers transparently handle
|
||||
multimodal payloads.
|
||||
|
||||
Flow:
|
||||
|
||||
1. Skip if the target model already supports vision (unless it appears in the
|
||||
forced-bridge list `isVisionBridgeForcedModel`).
|
||||
2. Extract image parts via `extractImageParts(messages)`. Skip if none.
|
||||
`extractImageParts` recognizes all three image shapes: OpenAI `image_url`,
|
||||
Anthropic base64 `source.type:"base64"`, and Anthropic URL
|
||||
`source.type:"url"` — so Claude-Code-compatible clients (e.g. Zoo Code)
|
||||
sending `{ type: "image", source: { type: "url", url } }` are described
|
||||
instead of silently dropped.
|
||||
3. Load runtime config from `getSettings()` (`visionBridgeEnabled`,
|
||||
`visionBridgeModel`, `visionBridgePrompt`, `visionBridgeTimeout`,
|
||||
`visionBridgeMaxImages`).
|
||||
4. Cap images at `maxImages`, call the vision model **in parallel**
|
||||
(`Promise.allSettled`), and inject `[Image N]: <description>` text parts
|
||||
in their place — failed images become `[Image N]: (unavailable)`.
|
||||
5. Return `modifiedPayload` + meta (`imagesProcessed`, `processingTimeMs`,
|
||||
`visionModel`).
|
||||
2. Extract image parts via `extractImageParts(messages)`
|
||||
(`visionBridgeHelpers.ts`), which delegates to the **unified media
|
||||
detector** `detectMediaParts()` in `open-sse/utils/mediaParts.ts` — the
|
||||
single source of truth shared with the combo compatibility filter.
|
||||
Extraction is allowlisted to top-level parts of the shapes
|
||||
`replaceImageParts` can splice back (the extract↔replace contract): OpenAI
|
||||
`image_url`, Anthropic base64 `source.type:"base64"`, Anthropic URL
|
||||
`source.type:"url"`, and Responses API `input_image`. Nested hits and
|
||||
indicator-only shapes are combo-filter material and are never extracted.
|
||||
Skip if none found.
|
||||
3. Resolve runtime config via `resolveVisionBridgeRuntimeSettings()`
|
||||
(`src/shared/constants/modalityBridgeDefaults.ts`): new `modalityBridge*`
|
||||
settings keys win; legacy `visionBridge*` keys remain a **one-cycle
|
||||
fallback** (rollback window). Skip before any media traversal when the
|
||||
bridge is disabled.
|
||||
4. Mode selector (`modalityBridgeVisionMode`, see table below) decides
|
||||
reroute vs describe. Reroute returns `modifiedPayload` with only `model`
|
||||
swapped, plus meta `{ rerouted, fromModel, toModel, imagesKept }`.
|
||||
5. Describe path: cap images at `maxImages`, compose the task-aware prompt,
|
||||
consult the describe cache, call the vision model **in parallel**
|
||||
(`Promise.allSettled`), and inject `[Image N]: <description>` text parts in
|
||||
their place. A failed describe yields `null` and the original image part is
|
||||
**preserved** (#4012) — except on the combo describe path when every
|
||||
describe failed, where a confirmed non-vision upstream gets an
|
||||
`(unavailable — no vision-capable provider connected)` stub instead (#8430).
|
||||
6. Return `modifiedPayload` + meta (`imagesProcessed`, `descriptions`,
|
||||
`processingTimeMs`, `visionModel`).
|
||||
|
||||
#### Mode selector (`modalityBridgeVisionMode`)
|
||||
|
||||
| Mode | Default | Behavior |
|
||||
| ---------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `auto` | ✔ | Legacy heuristic, untouched (#6640/#7204): non-combo/`auto/` models reroute to the best vision model unless the original model already has usable credentials (then describe); combo targets always describe. |
|
||||
| `describe` | | Always describe — the reroute block is skipped entirely; the user's chosen model always answers. |
|
||||
| `reroute` | | Force reroute: the keep-credentialed-model guard is bypassed. The reroute-**target** credential guard still applies — when no usable vision target exists, the request falls through to describe so raw images never reach a text-only backend (#8430). |
|
||||
|
||||
Forced modes short-circuit **before** the auto heuristic runs; `auto` behavior
|
||||
is byte-identical to the pre-PR-1 guardrail.
|
||||
|
||||
#### Task-aware describe prompt (`modalityBridgeVisionTaskAware`)
|
||||
|
||||
Default **true**. `composeVisionPrompt()` (`visionBridgeHelpers.ts`) appends
|
||||
the text of the **last user message** (truncated to 500 chars) to the base
|
||||
describe prompt, steering the description toward what the user actually asked
|
||||
(codex-vision-proxy pattern) and asking the vision model to transcribe visible
|
||||
text. With the flag off — or no user text — the base prompt is used unchanged.
|
||||
|
||||
#### Describe cache (`modalityBridge/bridgeCache.ts`)
|
||||
|
||||
In-memory LRU + TTL cache for describe outputs, shared process-wide.
|
||||
Key = `sha256(imageRef + composedPrompt + configuredBridgeModel)` with
|
||||
length-prefix framing (no field-boundary collisions). The model component is
|
||||
the **configured** bridge model, not the model that actually answered —
|
||||
`callVisionModel` may fall back internally, and keying per attempt would
|
||||
fragment the cache. Failed describes are never cached. Settings:
|
||||
|
||||
| Key | Default | Range |
|
||||
| ------------------------------- | ------- | ------- |
|
||||
| `modalityBridgeCacheEnabled` | `true` | — |
|
||||
| `modalityBridgeCacheTtlMinutes` | `60` | 1–1440 |
|
||||
| `modalityBridgeCacheMaxEntries` | `200` | 10–5000 |
|
||||
|
||||
#### Settings schema + migration
|
||||
|
||||
The new `modalityBridge*` keys are Zod-validated in `updateSettingsSchema`
|
||||
(`src/shared/validation/settingsSchemas.ts`): `modalityBridgeVisionEnabled`,
|
||||
`modalityBridgeVisionMode`, `modalityBridgeVisionModel`,
|
||||
`modalityBridgeVisionTaskAware`, `modalityBridgeVisionPrompt`,
|
||||
`modalityBridgeVisionTimeout`, `modalityBridgeVisionMaxImages`, the
|
||||
`modalityBridgeCache*` trio, and the PR-3-reserved `modalityBridgeAudio*`
|
||||
group. Migration `141_modality_bridge_settings.sql` copies existing legacy
|
||||
`visionBridge*` values to the matching new keys (idempotent, never overwrites
|
||||
an operator-set `modalityBridge*` value); the legacy keys stay accepted as a
|
||||
read fallback for one release cycle.
|
||||
|
||||
#### Transparency header + stats
|
||||
|
||||
Describe-transformed responses carry
|
||||
`x-omniroute-modality-bridge: image->text;model=<visionModel>;parts=<n>`
|
||||
(built by `buildModalityBridgeHeader()` in `modalityBridge/bridgeStats.ts`,
|
||||
stamped by `withModalityBridgeHeader()` in `src/sse/handlers/chatHelpers.ts`).
|
||||
Rerouted requests get **no** header — the payload was untouched and the model
|
||||
swap is already visible in the response body's `model` field.
|
||||
|
||||
`GET /api/modality-bridge/stats` (management auth, same tier as
|
||||
`GET /api/settings`) returns the in-memory per-modality counters
|
||||
`{ bridged, cacheHits, failures, lastUsedAt }` for `vision` (and the
|
||||
PR-3-reserved `audio`). Counters reset on process restart by design
|
||||
(telemetry, not accounting).
|
||||
|
||||
**Self-loop admission bypass:** when the describe call routes through OmniRoute's
|
||||
own `/v1` self-loop (non-standard provider model), the sub-request sends
|
||||
@@ -67,8 +143,10 @@ operator-configured `OMNIROUTE_API_KEY` / `ROUTER_API_KEY` env key (#1350) so
|
||||
is only honored for those exact credentials, so external clients cannot use the
|
||||
header to skip admission.
|
||||
|
||||
Defaults live in `src/shared/constants/visionBridgeDefaults.ts`. The guardrail
|
||||
exposes a `deps` constructor option so tests can inject fake `getSettings` and
|
||||
Legacy defaults live in `src/shared/constants/visionBridgeDefaults.ts`; the
|
||||
new mode/task-aware/cache defaults and the settings resolver live in
|
||||
`src/shared/constants/modalityBridgeDefaults.ts`. The guardrail exposes a
|
||||
`deps` constructor option so tests can inject fake `getSettings` and
|
||||
`callVisionModel` implementations.
|
||||
|
||||
### PII Masker (`piiMasker.ts`)
|
||||
|
||||
@@ -18,6 +18,7 @@ import { getHiddenModelsByProvider } from "../../../src/lib/db/models";
|
||||
import { getComboModelString, normalizeComboStep } from "../../../src/lib/combos/steps.ts";
|
||||
import { getProviderByAlias, getProviderById } from "../../../src/shared/constants/providers.ts";
|
||||
import { estimateTokens } from "../contextManager.ts";
|
||||
import { containsMediaKind } from "../../utils/mediaParts.ts";
|
||||
import { getResolvedModelCapabilities } from "../modelCapabilities.ts";
|
||||
import { parseModel, stripContextWindowSuffix } from "../model.ts";
|
||||
import { dedupeTargetsByExecutionKey, isRecord } from "./comboData.ts";
|
||||
@@ -483,21 +484,15 @@ function estimateRequestInputTokens(body: Record<string, unknown>): number {
|
||||
return Object.keys(estimatePayload).length > 0 ? estimateTokens(estimatePayload) : 0;
|
||||
}
|
||||
|
||||
function valueContainsImagePart(value: unknown, depth = 0): boolean {
|
||||
if (depth > 8 || value === null || value === undefined) return false;
|
||||
if (typeof value === "string") return value.startsWith("data:image/");
|
||||
if (Array.isArray(value)) return value.some((entry) => valueContainsImagePart(entry, depth + 1));
|
||||
if (!isRecord(value)) return false;
|
||||
|
||||
const type = typeof value.type === "string" ? value.type.toLowerCase() : null;
|
||||
if (type === "image" || type === "image_url" || type === "input_image") return true;
|
||||
if ("image_url" in value || "input_image" in value) return true;
|
||||
|
||||
const source = isRecord(value.source) ? value.source : null;
|
||||
const mediaType = typeof source?.media_type === "string" ? source.media_type.toLowerCase() : "";
|
||||
if (mediaType.startsWith("image/")) return true;
|
||||
|
||||
return Object.values(value).some((entry) => valueContainsImagePart(entry, depth + 1));
|
||||
function valueContainsImagePart(value: unknown): boolean {
|
||||
// Delegates to the unified media detector (open-sse/utils/mediaParts.ts) —
|
||||
// single source of truth shared with the vision-bridge guardrail. The
|
||||
// detector keeps this filter's legacy permissive matches (image-ish `type`
|
||||
// in any casing, bare `image_url`/`input_image` keys, source.media_type
|
||||
// image/*, bare data:image strings, recursion capped at depth 8) via
|
||||
// "image_indicator" parts. containsMediaKind short-circuits on the first
|
||||
// hit — this runs on every request, so no full-part collection here.
|
||||
return containsMediaKind([{ content: [value] }], "image");
|
||||
}
|
||||
|
||||
export function deriveRequestCompatibilityRequirements(
|
||||
|
||||
250
open-sse/utils/mediaParts.ts
Normal file
250
open-sse/utils/mediaParts.ts
Normal file
@@ -0,0 +1,250 @@
|
||||
/**
|
||||
* Unified media-part detection for request messages.
|
||||
* Single source of truth shared by the vision/audio bridge guardrails (src/)
|
||||
* and the combo compatibility filter (open-sse/) — the two previously kept
|
||||
* divergent copies (guardrail missed input_image; combo saw it).
|
||||
*/
|
||||
export type MediaKind = "image" | "audio";
|
||||
|
||||
export interface MediaPart {
|
||||
kind: MediaKind;
|
||||
/** URL, data URI, or base64 payload reference for the media content. */
|
||||
ref: string;
|
||||
/**
|
||||
* Location of the top-level content part this hit belongs to. For nested
|
||||
* hits (`nested: true`) these indexes point at the CONTAINER part — the
|
||||
* entry of `message.content` under which the media was found — not at the
|
||||
* media object itself.
|
||||
*/
|
||||
messageIndex: number;
|
||||
partIndex: number;
|
||||
/**
|
||||
* True when the media was found below the top level of the content part
|
||||
* (inside another object/array, e.g. an image nested in an audio payload
|
||||
* or a data URI inside a text field). Splice-style consumers can only
|
||||
* replace top-level parts, so they must skip nested hits.
|
||||
*/
|
||||
nested: boolean;
|
||||
/** Original wire shape, for callers that need format-specific handling. */
|
||||
shape:
|
||||
| "image_url"
|
||||
| "image_base64"
|
||||
| "image_source_url"
|
||||
| "input_image"
|
||||
| "data_uri_string"
|
||||
| "input_audio"
|
||||
| "audio_url"
|
||||
/** Audio detected via `source.media_type: audio/*` (no explicit type). */
|
||||
| "audio_source"
|
||||
/**
|
||||
* Combo-parity indicator: the value looks like an image part (image-ish
|
||||
* `type` in any casing, a bare `image_url`/`input_image` key, or a
|
||||
* `source.media_type` of image/*) but carries no extractable ref — `ref`
|
||||
* may be "". Boolean callers (combo compatibility filter) count it;
|
||||
* ref-consuming callers (vision bridge) must skip empty refs.
|
||||
*/
|
||||
| "image_indicator";
|
||||
}
|
||||
|
||||
const MAX_DEPTH = 8;
|
||||
|
||||
interface DetectCtx {
|
||||
out: MediaPart[];
|
||||
messageIndex: number;
|
||||
partIndex: number;
|
||||
/** When set, `found` flips true on the first part of this kind (early exit). */
|
||||
stopAtKind?: MediaKind;
|
||||
found?: boolean;
|
||||
}
|
||||
|
||||
/** Extract a URL from either a bare string or a `{ url }` object. */
|
||||
function urlFrom(raw: unknown): string | undefined {
|
||||
if (typeof raw === "string") return raw;
|
||||
const url = (raw as Record<string, unknown> | undefined)?.url;
|
||||
return typeof url === "string" ? url : undefined;
|
||||
}
|
||||
|
||||
function pushPart(
|
||||
ctx: DetectCtx,
|
||||
kind: MediaKind,
|
||||
ref: string,
|
||||
shape: MediaPart["shape"],
|
||||
depth: number
|
||||
): void {
|
||||
ctx.out.push({
|
||||
kind,
|
||||
ref,
|
||||
messageIndex: ctx.messageIndex,
|
||||
partIndex: ctx.partIndex,
|
||||
nested: depth > 0,
|
||||
shape,
|
||||
});
|
||||
if (ctx.stopAtKind === kind) ctx.found = true;
|
||||
}
|
||||
|
||||
/** Strict image shapes with an extractable ref. Returns true when one was pushed. */
|
||||
function inspectImageShapes(
|
||||
obj: Record<string, unknown>,
|
||||
type: string | undefined,
|
||||
ctx: DetectCtx,
|
||||
depth: number
|
||||
): boolean {
|
||||
if (type === "image_url" || type === "input_image") {
|
||||
const url = urlFrom(obj.image_url);
|
||||
if (url) {
|
||||
pushPart(ctx, "image", url, type === "input_image" ? "input_image" : "image_url", depth);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (type === "image") {
|
||||
const source = obj.source as Record<string, unknown> | undefined;
|
||||
if (source?.type === "base64" && typeof source.data === "string") {
|
||||
const media = typeof source.media_type === "string" ? source.media_type : "image/png";
|
||||
pushPart(ctx, "image", `data:${media};base64,${source.data}`, "image_base64", depth);
|
||||
return true;
|
||||
}
|
||||
// Non-empty url required: an empty `source.url` is not an extractable image
|
||||
// (mirrors the guardrail's historical `if (url)` guard).
|
||||
if (source?.type === "url" && typeof source.url === "string" && source.url) {
|
||||
pushPart(ctx, "image", source.url, "image_source_url", depth);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Audio shapes. Returns true when a part was pushed (at most one per object).
|
||||
* Callers must NOT early-return on audio: the same object can also carry
|
||||
* image indicators or nest image parts inside its payload.
|
||||
*/
|
||||
function inspectAudioShapes(
|
||||
obj: Record<string, unknown>,
|
||||
type: string | undefined,
|
||||
mediaType: unknown,
|
||||
ctx: DetectCtx,
|
||||
depth: number
|
||||
): boolean {
|
||||
if (type === "input_audio") {
|
||||
const audio = obj.input_audio as Record<string, unknown> | undefined;
|
||||
if (typeof audio?.data === "string") {
|
||||
pushPart(ctx, "audio", audio.data, "input_audio", depth);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (type === "audio_url") {
|
||||
const url = urlFrom(obj.audio_url);
|
||||
if (url) {
|
||||
pushPart(ctx, "audio", url, "audio_url", depth);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (typeof mediaType === "string" && mediaType.startsWith("audio/")) {
|
||||
const data = (obj.source as Record<string, unknown>).data;
|
||||
if (typeof data === "string") {
|
||||
pushPart(ctx, "audio", data, "audio_source", depth);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Combo-parity image indicators: the legacy valueContainsImagePart
|
||||
* (comboStructure) matched image-ish `type` names case-insensitively, bare
|
||||
* `image_url`/`input_image` keys, and `source.media_type` image/* — all
|
||||
* without needing an extractable ref. Emit an indicator part (ref
|
||||
* best-effort, possibly "") so boolean callers keep seeing those requests as
|
||||
* vision requests. Returns true when one was pushed.
|
||||
*/
|
||||
function inspectImageIndicators(
|
||||
obj: Record<string, unknown>,
|
||||
type: string | undefined,
|
||||
mediaType: unknown,
|
||||
ctx: DetectCtx,
|
||||
depth: number
|
||||
): boolean {
|
||||
const lowerType = type?.toLowerCase();
|
||||
const looksLikeImage =
|
||||
lowerType === "image" ||
|
||||
lowerType === "image_url" ||
|
||||
lowerType === "input_image" ||
|
||||
"image_url" in obj ||
|
||||
"input_image" in obj;
|
||||
const imageMediaType =
|
||||
typeof mediaType === "string" && mediaType.toLowerCase().startsWith("image/");
|
||||
if (!looksLikeImage && !imageMediaType) return false;
|
||||
pushPart(ctx, "image", urlFrom(obj.image_url ?? obj.input_image) ?? "", "image_indicator", depth);
|
||||
return true;
|
||||
}
|
||||
|
||||
function inspect(value: unknown, ctx: DetectCtx, depth: number): void {
|
||||
if (ctx.found || depth > MAX_DEPTH || value == null) return;
|
||||
if (typeof value === "string") {
|
||||
if (value.startsWith("data:image/")) pushPart(ctx, "image", value, "data_uri_string", depth);
|
||||
return;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
for (const entry of value) {
|
||||
inspect(entry, ctx, depth + 1);
|
||||
if (ctx.found) return;
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (typeof value !== "object") return;
|
||||
const obj = value as Record<string, unknown>;
|
||||
const type = typeof obj.type === "string" ? obj.type : undefined;
|
||||
|
||||
if (inspectImageShapes(obj, type, ctx, depth)) return;
|
||||
|
||||
const mediaType = (obj.source as Record<string, unknown> | undefined)?.media_type;
|
||||
// Audio does not early-return: the same object can also carry image
|
||||
// indicators (bare `image_url`/`input_image` keys the legacy combo filter
|
||||
// matched) or nest image parts inside its payload.
|
||||
inspectAudioShapes(obj, type, mediaType, ctx, depth);
|
||||
if (ctx.found) return;
|
||||
if (inspectImageIndicators(obj, type, mediaType, ctx, depth)) return;
|
||||
for (const nested of Object.values(obj)) {
|
||||
inspect(nested, ctx, depth + 1);
|
||||
if (ctx.found) return;
|
||||
}
|
||||
}
|
||||
|
||||
export function detectMediaParts(
|
||||
messages: ReadonlyArray<{ role?: string; content?: unknown }> | undefined | null
|
||||
): MediaPart[] {
|
||||
const out: MediaPart[] = [];
|
||||
if (!Array.isArray(messages)) return out;
|
||||
for (let messageIndex = 0; messageIndex < messages.length; messageIndex++) {
|
||||
const content = messages[messageIndex]?.content;
|
||||
if (!Array.isArray(content)) continue;
|
||||
for (let partIndex = 0; partIndex < content.length; partIndex++) {
|
||||
inspect(content[partIndex], { out, messageIndex, partIndex }, 0);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Early-exit presence check: returns true as soon as the FIRST part of the
|
||||
* requested kind is found, without collecting the full part list or finishing
|
||||
* the traversal. Prefer this on hot paths (e.g. the combo compatibility
|
||||
* filter runs on every request) over `detectMediaParts(...).some(...)`.
|
||||
*/
|
||||
export function containsMediaKind(
|
||||
messages: ReadonlyArray<{ role?: string; content?: unknown }> | undefined | null,
|
||||
kind: MediaKind
|
||||
): boolean {
|
||||
if (!Array.isArray(messages)) return false;
|
||||
const out: MediaPart[] = [];
|
||||
for (let messageIndex = 0; messageIndex < messages.length; messageIndex++) {
|
||||
const content = messages[messageIndex]?.content;
|
||||
if (!Array.isArray(content)) continue;
|
||||
for (let partIndex = 0; partIndex < content.length; partIndex++) {
|
||||
const ctx: DetectCtx = { out, messageIndex, partIndex, stopAtKind: kind };
|
||||
inspect(content[partIndex], ctx, 0);
|
||||
if (ctx.found) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
21
src/app/api/modality-bridge/stats/route.ts
Normal file
21
src/app/api/modality-bridge/stats/route.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
|
||||
import { getBridgeStats } from "@/lib/guardrails/modalityBridge/bridgeStats";
|
||||
|
||||
/**
|
||||
* GET /api/modality-bridge/stats — read-only, in-memory Modality Bridge
|
||||
* telemetry (per-modality bridged/cacheHits/failures/lastUsedAt counters).
|
||||
* Same MANAGEMENT auth tier as GET /api/settings (routeGuard default —
|
||||
* intentionally NOT local-only: harmless read-only telemetry, no side effects).
|
||||
* Counters reset on process restart; force-dynamic + no-store so the dashboard
|
||||
* always sees live values.
|
||||
*/
|
||||
export const dynamic = "force-dynamic";
|
||||
export const revalidate = 0;
|
||||
|
||||
export async function GET(request: Request) {
|
||||
const authError = await requireManagementAuth(request);
|
||||
if (authError) return authError;
|
||||
|
||||
return NextResponse.json(getBridgeStats(), { headers: { "Cache-Control": "no-store" } });
|
||||
}
|
||||
30
src/lib/db/migrations/141_modality_bridge_settings.sql
Normal file
30
src/lib/db/migrations/141_modality_bridge_settings.sql
Normal file
@@ -0,0 +1,30 @@
|
||||
-- 141_modality_bridge_settings.sql
|
||||
-- Modality Bridge (PR-1): copy legacy visionBridge* settings to the new modalityBridge* keys.
|
||||
-- Legacy keys are left in place for one release cycle (rollback window).
|
||||
-- Idempotent: each INSERT only fires when the new key does not exist yet, so an
|
||||
-- operator-set modalityBridge* value is never overwritten and re-runs are no-ops.
|
||||
|
||||
INSERT INTO key_value (namespace, key, value)
|
||||
SELECT 'settings', 'modalityBridgeVisionEnabled', value FROM key_value
|
||||
WHERE namespace = 'settings' AND key = 'visionBridgeEnabled'
|
||||
AND NOT EXISTS (SELECT 1 FROM key_value WHERE namespace = 'settings' AND key = 'modalityBridgeVisionEnabled');
|
||||
|
||||
INSERT INTO key_value (namespace, key, value)
|
||||
SELECT 'settings', 'modalityBridgeVisionModel', value FROM key_value
|
||||
WHERE namespace = 'settings' AND key = 'visionBridgeModel'
|
||||
AND NOT EXISTS (SELECT 1 FROM key_value WHERE namespace = 'settings' AND key = 'modalityBridgeVisionModel');
|
||||
|
||||
INSERT INTO key_value (namespace, key, value)
|
||||
SELECT 'settings', 'modalityBridgeVisionPrompt', value FROM key_value
|
||||
WHERE namespace = 'settings' AND key = 'visionBridgePrompt'
|
||||
AND NOT EXISTS (SELECT 1 FROM key_value WHERE namespace = 'settings' AND key = 'modalityBridgeVisionPrompt');
|
||||
|
||||
INSERT INTO key_value (namespace, key, value)
|
||||
SELECT 'settings', 'modalityBridgeVisionTimeout', value FROM key_value
|
||||
WHERE namespace = 'settings' AND key = 'visionBridgeTimeout'
|
||||
AND NOT EXISTS (SELECT 1 FROM key_value WHERE namespace = 'settings' AND key = 'modalityBridgeVisionTimeout');
|
||||
|
||||
INSERT INTO key_value (namespace, key, value)
|
||||
SELECT 'settings', 'modalityBridgeVisionMaxImages', value FROM key_value
|
||||
WHERE namespace = 'settings' AND key = 'visionBridgeMaxImages'
|
||||
AND NOT EXISTS (SELECT 1 FROM key_value WHERE namespace = 'settings' AND key = 'modalityBridgeVisionMaxImages');
|
||||
82
src/lib/guardrails/modalityBridge/bridgeCache.ts
Normal file
82
src/lib/guardrails/modalityBridge/bridgeCache.ts
Normal file
@@ -0,0 +1,82 @@
|
||||
/**
|
||||
* Modality Bridge description cache (PR-1).
|
||||
*
|
||||
* In-memory LRU + TTL cache for bridge outputs (image/audio descriptions),
|
||||
* keyed by sha256(contentRef + prompt + model). Avoids re-describing the
|
||||
* same media with the same prompt/model within the configured TTL.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import type { VisionBridgeRuntimeSettings } from "@/shared/constants/modalityBridgeDefaults";
|
||||
|
||||
export function bridgeCacheKey(contentRef: string, prompt: string, model: string): string {
|
||||
// Length-prefix framing: hashing the byte lengths first makes the field
|
||||
// boundaries unambiguous, so ("ab","c") can never collide with ("a","bc").
|
||||
return createHash("sha256")
|
||||
.update(`${Buffer.byteLength(contentRef)}:${Buffer.byteLength(prompt)}:`)
|
||||
.update(contentRef)
|
||||
.update(prompt)
|
||||
.update(model)
|
||||
.digest("hex");
|
||||
}
|
||||
|
||||
export interface BridgeCacheOptions {
|
||||
maxEntries: number;
|
||||
ttlMs: number;
|
||||
/** Injectable clock for tests. */
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
export class BridgeCache {
|
||||
private readonly entries = new Map<string, { value: string; expiresAt: number }>();
|
||||
|
||||
constructor(private readonly opts: BridgeCacheOptions) {}
|
||||
|
||||
get(key: string): string | undefined {
|
||||
const hit = this.entries.get(key);
|
||||
if (!hit) return undefined;
|
||||
const now = (this.opts.now ?? Date.now)();
|
||||
if (hit.expiresAt <= now) {
|
||||
this.entries.delete(key);
|
||||
return undefined;
|
||||
}
|
||||
// Map preserves insertion order — re-insert to mark as most-recently-used.
|
||||
this.entries.delete(key);
|
||||
this.entries.set(key, hit);
|
||||
return hit.value;
|
||||
}
|
||||
|
||||
set(key: string, value: string): void {
|
||||
const now = (this.opts.now ?? Date.now)();
|
||||
this.entries.delete(key);
|
||||
this.entries.set(key, { value, expiresAt: now + this.opts.ttlMs });
|
||||
while (this.entries.size > this.opts.maxEntries) {
|
||||
const oldest = this.entries.keys().next().value;
|
||||
if (oldest === undefined) break;
|
||||
this.entries.delete(oldest);
|
||||
}
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.entries.size;
|
||||
}
|
||||
}
|
||||
|
||||
/** Process-wide singleton used by the bridges; recreated when config changes. */
|
||||
let shared: { cache: BridgeCache; ttlMs: number; maxEntries: number } | null = null;
|
||||
|
||||
export function getSharedBridgeCache(ttlMs: number, maxEntries: number): BridgeCache {
|
||||
if (!shared || shared.ttlMs !== ttlMs || shared.maxEntries !== maxEntries) {
|
||||
shared = { cache: new BridgeCache({ maxEntries, ttlMs }), ttlMs, maxEntries };
|
||||
}
|
||||
return shared.cache;
|
||||
}
|
||||
|
||||
/**
|
||||
* Single conversion point from runtime settings to the shared cache: every
|
||||
* bridge (vision, audio) goes through here so the minutes→ms conversion can
|
||||
* never diverge between callers and thrash the singleton on each request.
|
||||
*/
|
||||
export function getSharedBridgeCacheFor(settings: VisionBridgeRuntimeSettings): BridgeCache {
|
||||
return getSharedBridgeCache(settings.cacheTtlMinutes * 60_000, settings.cacheMaxEntries);
|
||||
}
|
||||
72
src/lib/guardrails/modalityBridge/bridgeStats.ts
Normal file
72
src/lib/guardrails/modalityBridge/bridgeStats.ts
Normal file
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Modality Bridge stats + response transparency header (PR-1 Task 9).
|
||||
*
|
||||
* In-memory, process-global counters for bridge activity ("vision" today,
|
||||
* "audio" reserved for PR-3) plus the builder for the
|
||||
* `x-omniroute-modality-bridge` response header, which tells clients that
|
||||
* their request payload was transparently transformed (image→text describe).
|
||||
* Reroutes do NOT get a header — the payload was untouched, only the model
|
||||
* changed, and that is already visible in the response body's `model` field.
|
||||
*
|
||||
* Counters reset on process restart by design (telemetry, not accounting).
|
||||
*/
|
||||
|
||||
export interface BridgeModalityStats {
|
||||
bridged: number;
|
||||
cacheHits: number;
|
||||
failures: number;
|
||||
lastUsedAt: string | null;
|
||||
}
|
||||
|
||||
const stats: Record<"vision" | "audio", BridgeModalityStats> = {
|
||||
vision: { bridged: 0, cacheHits: 0, failures: 0, lastUsedAt: null },
|
||||
audio: { bridged: 0, cacheHits: 0, failures: 0, lastUsedAt: null },
|
||||
};
|
||||
|
||||
export function recordBridgeUse(
|
||||
kind: "vision" | "audio",
|
||||
opts: { cacheHit?: boolean; failure?: boolean } = {}
|
||||
): void {
|
||||
const s = stats[kind];
|
||||
s.bridged += 1;
|
||||
if (opts.cacheHit) s.cacheHits += 1;
|
||||
if (opts.failure) s.failures += 1;
|
||||
s.lastUsedAt = new Date().toISOString();
|
||||
}
|
||||
|
||||
export function getBridgeStats(): Record<"vision" | "audio", BridgeModalityStats> {
|
||||
return structuredClone(stats);
|
||||
}
|
||||
|
||||
/**
|
||||
* Structural subset of GuardrailExecutionResult (src/lib/guardrails/base.ts) —
|
||||
* only the fields the header builder reads, so callers can pass the registry's
|
||||
* `results` array directly without a type dependency on the guardrail core.
|
||||
*/
|
||||
interface GuardrailMetaEntry {
|
||||
guardrail: string;
|
||||
meta?: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
/** Response header value for a describe-bridged request; null when untouched. */
|
||||
export function buildModalityBridgeHeader(results: GuardrailMetaEntry[]): string | null {
|
||||
const segments: string[] = [];
|
||||
for (const r of results) {
|
||||
const meta = r.meta ?? {};
|
||||
if (
|
||||
r.guardrail === "vision-bridge" &&
|
||||
typeof meta.imagesProcessed === "number" &&
|
||||
!meta.rerouted
|
||||
) {
|
||||
segments.push(
|
||||
`image->text;model=${String(meta.visionModel ?? "unknown")};parts=${meta.imagesProcessed}`
|
||||
);
|
||||
}
|
||||
if (r.guardrail === "audio-bridge" && typeof meta.clipsProcessed === "number") {
|
||||
segments.push(
|
||||
`audio->text;model=${String(meta.sttModel ?? "unknown")};parts=${meta.clipsProcessed}`
|
||||
);
|
||||
}
|
||||
}
|
||||
return segments.length ? segments.join(", ") : null;
|
||||
}
|
||||
@@ -11,14 +11,17 @@ import { getResolvedModelCapabilities } from "@/lib/modelCapabilities";
|
||||
import {
|
||||
extractImageParts,
|
||||
callVisionModel as defaultCallVisionModel,
|
||||
composeVisionPrompt,
|
||||
replaceImageParts,
|
||||
} from "./visionBridgeHelpers";
|
||||
import {
|
||||
VISION_BRIDGE_DEFAULTS,
|
||||
getVisionBridgeConfig,
|
||||
isVisionBridgeForcedModel,
|
||||
} from "@/shared/constants/visionBridgeDefaults";
|
||||
import { resolveVisionBridgeRuntimeSettings } from "@/shared/constants/modalityBridgeDefaults";
|
||||
import { getBestVisionModel } from "./visionBridgeRouter";
|
||||
import { bridgeCacheKey, getSharedBridgeCacheFor } from "./modalityBridge/bridgeCache";
|
||||
import { recordBridgeUse } from "./modalityBridge/bridgeStats";
|
||||
import {
|
||||
isProviderConnectionUsable,
|
||||
hasUsableCredentialsForModel,
|
||||
@@ -94,6 +97,30 @@ async function getComboVisionBridgeDecision(model: string): Promise<ComboVisionB
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the text of the LAST user message that carries any (string content,
|
||||
* or the first `type: "text"` part of an array content). Used as the
|
||||
* task-aware focus hint for the describe path.
|
||||
*/
|
||||
function extractLastUserText(messages: unknown[]): string | undefined {
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
const message = messages[i] as { role?: unknown; content?: unknown } | null | undefined;
|
||||
if (message?.role !== "user") continue;
|
||||
if (typeof message.content === "string" && message.content.trim()) {
|
||||
return message.content;
|
||||
}
|
||||
if (Array.isArray(message.content)) {
|
||||
for (const part of message.content) {
|
||||
const p = part as { type?: unknown; text?: unknown } | null | undefined;
|
||||
if (p?.type === "text" && typeof p.text === "string" && p.text.trim()) {
|
||||
return p.text;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export interface VisionBridgeDependencies {
|
||||
getSettings?: () => Promise<Record<string, unknown>>;
|
||||
callVisionModel?: (
|
||||
@@ -189,13 +216,7 @@ export class VisionBridgeGuardrail extends BaseGuardrail {
|
||||
return { block: false };
|
||||
}
|
||||
|
||||
// 6. Check for images using helper (extractImageParts returns empty if no images)
|
||||
const imageParts = extractImageParts(messages as Parameters<typeof extractImageParts>[0]);
|
||||
if (imageParts.length === 0) {
|
||||
return { block: false };
|
||||
}
|
||||
|
||||
// 7. Get settings (injectable for testing)
|
||||
// 6. Get settings (injectable for testing)
|
||||
const getSettings = this.deps.getSettings ?? defaultGetSettings;
|
||||
let settings: Record<string, unknown> = {};
|
||||
try {
|
||||
@@ -204,9 +225,18 @@ export class VisionBridgeGuardrail extends BaseGuardrail {
|
||||
// If getSettings fails, use defaults
|
||||
}
|
||||
|
||||
// 8. Check if Vision Bridge is enabled in settings
|
||||
const enabled = settings.visionBridgeEnabled ?? VISION_BRIDGE_DEFAULTS.enabled;
|
||||
if (!enabled) {
|
||||
// 7. Resolve runtime settings (new modalityBridge* keys win; legacy
|
||||
// visionBridge* keys stay a one-cycle fallback) and check enabled —
|
||||
// BEFORE any media traversal, so a disabled bridge never pays the
|
||||
// per-request deep scan of every message content part.
|
||||
const runtime = resolveVisionBridgeRuntimeSettings(settings);
|
||||
if (!runtime.enabled) {
|
||||
return { block: false };
|
||||
}
|
||||
|
||||
// 8. Check for images using helper (extractImageParts returns empty if no images)
|
||||
const imageParts = extractImageParts(messages as Parameters<typeof extractImageParts>[0]);
|
||||
if (imageParts.length === 0) {
|
||||
return { block: false };
|
||||
}
|
||||
|
||||
@@ -226,9 +256,16 @@ export class VisionBridgeGuardrail extends BaseGuardrail {
|
||||
// request with model=auto would land on a text-only model (#7871). Keeping
|
||||
// "auto" is never the answer there, so the keep-credentialed-model skip
|
||||
// below does not apply to auto — only the reroute-target credential guard.
|
||||
if ((comboVisionBridgeDecision === "not-combo" || isAuto) && !forceVisionBridge) {
|
||||
const rerouteEligible =
|
||||
(comboVisionBridgeDecision === "not-combo" || isAuto) && !forceVisionBridge;
|
||||
// Forced modes short-circuit BEFORE the auto heuristic (#6640/#7204 untouched):
|
||||
// - "describe" skips the whole reroute block → straight to the describe path.
|
||||
// - "reroute" skips only the keep-credentialed-model guard; the reroute-target
|
||||
// credential guard still applies, and with no usable target it falls through
|
||||
// to describe (raw images must never reach a text-only backend — #8430).
|
||||
if (rerouteEligible && runtime.mode !== "describe") {
|
||||
const checkCreds = this.deps.hasUsableCredentials ?? hasUsableCredentialsForModel;
|
||||
const originalUsable = await checkCreds(model);
|
||||
const originalUsable = runtime.mode === "reroute" ? false : await checkCreds(model);
|
||||
|
||||
if (originalUsable === true && !isAuto) {
|
||||
// Keep the credentialed model; describe images below if needed.
|
||||
@@ -238,14 +275,17 @@ export class VisionBridgeGuardrail extends BaseGuardrail {
|
||||
);
|
||||
} else {
|
||||
// Honor an explicit operator override from the Vision Bridge settings tab
|
||||
// (settings.visionBridgeModel) as the fixed reroute target, for consistency
|
||||
// with the combo/describe path below (step 10) which always honors it via
|
||||
// getVisionBridgeConfig. When unset, auto-select the fastest available
|
||||
// vision-capable model from available providers.
|
||||
const configuredModel =
|
||||
typeof settings.visionBridgeModel === "string" && settings.visionBridgeModel.trim()
|
||||
? settings.visionBridgeModel.trim()
|
||||
: undefined;
|
||||
// as the fixed reroute target, for consistency with the combo/describe
|
||||
// path below (step 10) which always honors it via getVisionBridgeConfig.
|
||||
// New modalityBridgeVisionModel wins over legacy visionBridgeModel (same
|
||||
// precedence as resolveVisionBridgeRuntimeSettings — runtime.model can't
|
||||
// be used here because it backfills the default and this path must
|
||||
// auto-select the fastest available vision-capable model when unset).
|
||||
const rawConfiguredModel = [
|
||||
settings.modalityBridgeVisionModel,
|
||||
settings.visionBridgeModel,
|
||||
].find((value): value is string => typeof value === "string" && value.trim().length > 0);
|
||||
const configuredModel = rawConfiguredModel?.trim();
|
||||
// Propagate the same resolved credential check used by the adjacent
|
||||
// checkCreds() calls above/below (#8430) — without this, the router
|
||||
// falls back to the real DB-backed hasUsableCredentialsForModel and
|
||||
@@ -284,13 +324,15 @@ export class VisionBridgeGuardrail extends BaseGuardrail {
|
||||
// Fall through: describe images as text (or no-op if describe path can't run)
|
||||
}
|
||||
|
||||
// 10. Get configuration
|
||||
// 10. Get configuration — fed from the resolved runtime values so the new
|
||||
// modalityBridge* keys are honored; getVisionBridgeConfig keeps producing
|
||||
// the same VisionModelConfig shape callVisionModel expects.
|
||||
const config = getVisionBridgeConfig({
|
||||
visionBridgeEnabled: settings.visionBridgeEnabled as boolean | undefined,
|
||||
visionBridgeModel: settings.visionBridgeModel as string | undefined,
|
||||
visionBridgePrompt: settings.visionBridgePrompt as string | undefined,
|
||||
visionBridgeTimeout: settings.visionBridgeTimeout as number | undefined,
|
||||
visionBridgeMaxImages: settings.visionBridgeMaxImages as number | undefined,
|
||||
visionBridgeEnabled: runtime.enabled,
|
||||
visionBridgeModel: runtime.model,
|
||||
visionBridgePrompt: runtime.prompt,
|
||||
visionBridgeTimeout: runtime.timeoutMs,
|
||||
visionBridgeMaxImages: runtime.maxImages,
|
||||
});
|
||||
|
||||
// 11. Limit images
|
||||
@@ -301,10 +343,30 @@ export class VisionBridgeGuardrail extends BaseGuardrail {
|
||||
const logger = context.log;
|
||||
const startTime = Date.now();
|
||||
|
||||
// Task-aware focus hint: append the LAST user question so the description
|
||||
// targets what the user actually asked instead of a generic caption.
|
||||
const lastUserText = extractLastUserText(messages);
|
||||
const composedPrompt = composeVisionPrompt(config.prompt, lastUserText, runtime.taskAware);
|
||||
const describeConfig = { ...config, prompt: composedPrompt };
|
||||
|
||||
// Shared describe cache (sha256 of contentRef+prompt+model): the same image
|
||||
// with the same prompt/model is described once per TTL. Failures are never
|
||||
// cached — a throw inside the map happens before the cache write. The model
|
||||
// component is the CONFIGURED bridge model (config.model — the bridge-config
|
||||
// identity), not the model that actually produced the description:
|
||||
// callVisionModel may fall back internally to another vision model, and
|
||||
// keying by attempt would fragment the cache and leak router state into the
|
||||
// key. Intentional and stable — do not "fix" this to key per attempt.
|
||||
const cache = runtime.cacheEnabled ? getSharedBridgeCacheFor(runtime) : null;
|
||||
|
||||
// Process all images in parallel using Promise.allSettled for fail-partial behavior
|
||||
const results = await Promise.allSettled(
|
||||
limitedParts.map(async (imagePart, i) => {
|
||||
const description = await callVision(imagePart.imageUrl, config);
|
||||
const key = cache ? bridgeCacheKey(imagePart.imageUrl, composedPrompt, config.model) : null;
|
||||
const cached = key && cache ? cache.get(key) : undefined;
|
||||
const description = cached ?? (await callVision(imagePart.imageUrl, describeConfig));
|
||||
if (cached === undefined && key && cache) cache.set(key, description);
|
||||
recordBridgeUse("vision", { cacheHit: cached !== undefined });
|
||||
return `[Image ${i + 1}]: ${description}`;
|
||||
})
|
||||
);
|
||||
@@ -320,6 +382,7 @@ export class VisionBridgeGuardrail extends BaseGuardrail {
|
||||
const message =
|
||||
result.reason instanceof Error ? result.reason.message : String(result.reason);
|
||||
logger?.warn?.("VISION-BRIDGE", `Failed to get description for image ${i + 1}: ${message}`);
|
||||
recordBridgeUse("vision", { failure: true });
|
||||
return null;
|
||||
});
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
/**
|
||||
* Vision Bridge helper functions for image processing.
|
||||
*/
|
||||
import { detectMediaParts, type MediaPart } from "@omniroute/open-sse/utils/mediaParts";
|
||||
import { fetchRemoteImage } from "@/shared/network/remoteImageFetch";
|
||||
import { getRuntimePorts } from "@/lib/runtime/ports";
|
||||
import { resolveSelfLoopBearer } from "@/shared/middleware/chatBodyAdmission";
|
||||
@@ -118,53 +119,37 @@ export type RequestContentPart =
|
||||
* vision-bridge guardrail, so the image was silently dropped by a text-only
|
||||
* executor instead of being described.
|
||||
*/
|
||||
/**
|
||||
* Shapes `replaceImageParts` knows how to splice: top-level content parts
|
||||
* whose `type` is `image_url`, `image`, or `input_image`. Everything else the
|
||||
* detector reports (nested hits, `data_uri_string`, `image_indicator`) is
|
||||
* combo-filter material only — extracting it would desync the positional
|
||||
* description consumption in visionBridge (descriptions would shift onto the
|
||||
* wrong images).
|
||||
*/
|
||||
const REPLACEABLE_IMAGE_SHAPES: ReadonlySet<MediaPart["shape"]> = new Set([
|
||||
"image_url",
|
||||
"image_base64",
|
||||
"image_source_url",
|
||||
"input_image",
|
||||
]);
|
||||
|
||||
export function extractImageParts(messages: RequestMessage[]): ImagePart[] {
|
||||
const results: ImagePart[] = [];
|
||||
|
||||
if (!Array.isArray(messages)) {
|
||||
return results;
|
||||
}
|
||||
|
||||
for (let msgIdx = 0; msgIdx < messages.length; msgIdx++) {
|
||||
const message = messages[msgIdx];
|
||||
if (!message || !Array.isArray(message.content)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (let partIdx = 0; partIdx < message.content.length; partIdx++) {
|
||||
const part = message.content[partIdx];
|
||||
|
||||
if (part?.type === "image_url" && part.image_url?.url) {
|
||||
results.push({
|
||||
messageIndex: msgIdx,
|
||||
partIndex: partIdx,
|
||||
imageUrl: part.image_url.url,
|
||||
imageType: "image_url",
|
||||
});
|
||||
} else if (part?.type === "image" && part.source?.type === "base64") {
|
||||
const { media_type, data } = part.source;
|
||||
const dataUri = `data:${media_type};base64,${data}`;
|
||||
results.push({
|
||||
messageIndex: msgIdx,
|
||||
partIndex: partIdx,
|
||||
imageUrl: dataUri,
|
||||
imageType: "image",
|
||||
});
|
||||
} else if (part?.type === "image" && part.source?.type === "url") {
|
||||
const url = part.source.url;
|
||||
if (url) {
|
||||
results.push({
|
||||
messageIndex: msgIdx,
|
||||
partIndex: partIdx,
|
||||
imageUrl: url,
|
||||
imageType: "url",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
// Delegates to the unified detector (open-sse/utils/mediaParts.ts) so the
|
||||
// guardrail and the combo compatibility filter share one source of truth.
|
||||
// Extraction is ALLOWLISTED to top-level (non-nested) parts whose shape
|
||||
// replaceImageParts can splice back — the extract↔replace contract: every
|
||||
// extracted part MUST be replaceable, in the same order, or the positional
|
||||
// descriptions shift onto the wrong images.
|
||||
return detectMediaParts(messages)
|
||||
.filter((p) => p.kind === "image" && !p.nested && REPLACEABLE_IMAGE_SHAPES.has(p.shape))
|
||||
.map((p) => ({
|
||||
messageIndex: p.messageIndex,
|
||||
partIndex: p.partIndex,
|
||||
imageUrl: p.ref,
|
||||
imageType:
|
||||
p.shape === "image_base64" ? "image" : p.shape === "image_source_url" ? "url" : "image_url",
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -223,6 +208,19 @@ export interface VisionModelConfig {
|
||||
maxImages: number;
|
||||
}
|
||||
|
||||
/** Task-aware focus hint (codex-vision-proxy pattern): steer the description
|
||||
* toward what the user actually asked, instead of a generic caption. */
|
||||
export function composeVisionPrompt(
|
||||
basePrompt: string,
|
||||
lastUserText: string | undefined,
|
||||
taskAware: boolean
|
||||
): string {
|
||||
const text = (lastUserText ?? "").trim();
|
||||
if (!taskAware || !text) return basePrompt;
|
||||
const hint = text.length > 500 ? `${text.slice(0, 500)}…` : text;
|
||||
return `${basePrompt}\n\nThe user asked: "${hint}". Focus your description on what is relevant to answering this, and transcribe any text visible in the image.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Call the vision model to get an image description.
|
||||
* Supports both OpenAI-compatible and Anthropic API formats.
|
||||
@@ -703,7 +701,12 @@ export function replaceImageParts(
|
||||
const newContent: RequestContentPart[] = [];
|
||||
|
||||
for (const part of message.content) {
|
||||
if (part?.type === "image_url" || part?.type === "image") {
|
||||
// `input_image` (Responses API) is read through a widened type: it is
|
||||
// not part of the historical RequestContentPart union but MUST be
|
||||
// replaceable — extractImageParts allowlists it, and every extracted
|
||||
// part needs a matching splice here (extract↔replace contract).
|
||||
const partType = (part as { type?: string } | null | undefined)?.type;
|
||||
if (partType === "image_url" || partType === "image" || partType === "input_image") {
|
||||
if (descriptionIndex < descriptions.length) {
|
||||
const description = descriptions[descriptionIndex];
|
||||
descriptionIndex++;
|
||||
|
||||
85
src/shared/constants/modalityBridgeDefaults.ts
Normal file
85
src/shared/constants/modalityBridgeDefaults.ts
Normal file
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* Modality Bridge default configuration values (PR-1).
|
||||
*
|
||||
* New `modalityBridge*` settings keys supersede the legacy `visionBridge*`
|
||||
* keys; the legacy keys stay accepted as a fallback for one release cycle
|
||||
* (rollback window) and are resolved here in a single place.
|
||||
*/
|
||||
import { VISION_BRIDGE_DEFAULTS } from "./visionBridgeDefaults";
|
||||
|
||||
export type VisionBridgeMode = "auto" | "describe" | "reroute";
|
||||
|
||||
export const MODALITY_BRIDGE_DEFAULTS = {
|
||||
visionMode: "auto" as VisionBridgeMode,
|
||||
visionTaskAware: true,
|
||||
cacheEnabled: true,
|
||||
cacheTtlMinutes: 60,
|
||||
cacheMaxEntries: 200,
|
||||
audioEnabled: true,
|
||||
audioModel: "",
|
||||
audioTimeoutMs: 60000,
|
||||
audioMaxClips: 3,
|
||||
} as const;
|
||||
|
||||
export interface VisionBridgeRuntimeSettings {
|
||||
enabled: boolean;
|
||||
mode: VisionBridgeMode;
|
||||
model: string;
|
||||
taskAware: boolean;
|
||||
prompt: string;
|
||||
timeoutMs: number;
|
||||
maxImages: number;
|
||||
cacheEnabled: boolean;
|
||||
cacheTtlMinutes: number;
|
||||
cacheMaxEntries: number;
|
||||
}
|
||||
|
||||
// Typed candidate pickers: a stored value of the wrong type (e.g. the string
|
||||
// "off" in a boolean field) is skipped so the next candidate/default wins.
|
||||
function pickBoolean(...values: unknown[]): boolean | undefined {
|
||||
for (const v of values) if (typeof v === "boolean") return v;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function pickNumber(...values: unknown[]): number | undefined {
|
||||
for (const v of values) if (typeof v === "number" && Number.isFinite(v)) return v;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function pickString(...values: unknown[]): string | undefined {
|
||||
for (const v of values) if (typeof v === "string") return v;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** New modalityBridge* keys win; legacy visionBridge* keys are a one-cycle fallback. */
|
||||
export function resolveVisionBridgeRuntimeSettings(
|
||||
settings: Record<string, unknown> | null | undefined
|
||||
): VisionBridgeRuntimeSettings {
|
||||
const s = settings ?? {};
|
||||
const mode = pickString(s.modalityBridgeVisionMode);
|
||||
return {
|
||||
enabled:
|
||||
pickBoolean(s.modalityBridgeVisionEnabled, s.visionBridgeEnabled) ??
|
||||
VISION_BRIDGE_DEFAULTS.enabled,
|
||||
mode: mode === "describe" || mode === "reroute" ? mode : MODALITY_BRIDGE_DEFAULTS.visionMode,
|
||||
model:
|
||||
pickString(s.modalityBridgeVisionModel, s.visionBridgeModel) ?? VISION_BRIDGE_DEFAULTS.model,
|
||||
taskAware:
|
||||
pickBoolean(s.modalityBridgeVisionTaskAware) ?? MODALITY_BRIDGE_DEFAULTS.visionTaskAware,
|
||||
prompt:
|
||||
pickString(s.modalityBridgeVisionPrompt, s.visionBridgePrompt) ??
|
||||
VISION_BRIDGE_DEFAULTS.prompt,
|
||||
timeoutMs:
|
||||
pickNumber(s.modalityBridgeVisionTimeout, s.visionBridgeTimeout) ??
|
||||
VISION_BRIDGE_DEFAULTS.timeoutMs,
|
||||
maxImages:
|
||||
pickNumber(s.modalityBridgeVisionMaxImages, s.visionBridgeMaxImages) ??
|
||||
VISION_BRIDGE_DEFAULTS.maxImagesPerRequest,
|
||||
cacheEnabled:
|
||||
pickBoolean(s.modalityBridgeCacheEnabled) ?? MODALITY_BRIDGE_DEFAULTS.cacheEnabled,
|
||||
cacheTtlMinutes:
|
||||
pickNumber(s.modalityBridgeCacheTtlMinutes) ?? MODALITY_BRIDGE_DEFAULTS.cacheTtlMinutes,
|
||||
cacheMaxEntries:
|
||||
pickNumber(s.modalityBridgeCacheMaxEntries) ?? MODALITY_BRIDGE_DEFAULTS.cacheMaxEntries,
|
||||
};
|
||||
}
|
||||
@@ -330,6 +330,22 @@ export const updateSettingsSchema = z.object({
|
||||
visionBridgePrompt: z.string().max(5000).optional(),
|
||||
visionBridgeTimeout: z.number().int().min(1000).max(300000).optional(),
|
||||
visionBridgeMaxImages: z.number().int().min(1).max(20).optional(),
|
||||
// Modality Bridge settings (new schema — visionBridge* keys above are the
|
||||
// deprecated legacy aliases, kept accepted for one release cycle)
|
||||
modalityBridgeVisionEnabled: z.boolean().optional(),
|
||||
modalityBridgeVisionMode: z.enum(["auto", "describe", "reroute"]).optional(),
|
||||
modalityBridgeVisionModel: z.string().max(200).optional(),
|
||||
modalityBridgeVisionTaskAware: z.boolean().optional(),
|
||||
modalityBridgeVisionPrompt: z.string().max(5000).optional(),
|
||||
modalityBridgeVisionTimeout: z.number().int().min(1000).max(300000).optional(),
|
||||
modalityBridgeVisionMaxImages: z.number().int().min(1).max(20).optional(),
|
||||
modalityBridgeAudioEnabled: z.boolean().optional(),
|
||||
modalityBridgeAudioModel: z.string().max(200).optional(),
|
||||
modalityBridgeAudioTimeout: z.number().int().min(1000).max(300000).optional(),
|
||||
modalityBridgeAudioMaxClips: z.number().int().min(1).max(10).optional(),
|
||||
modalityBridgeCacheEnabled: z.boolean().optional(),
|
||||
modalityBridgeCacheTtlMinutes: z.number().int().min(1).max(1440).optional(),
|
||||
modalityBridgeCacheMaxEntries: z.number().int().min(10).max(5000).optional(),
|
||||
// Missing settings
|
||||
lkgpEnabled: z.boolean().optional(),
|
||||
// #1311: echo the requested alias/combo name in the response model field (opt-in)
|
||||
|
||||
@@ -77,7 +77,9 @@ import {
|
||||
withSessionHeader,
|
||||
withSelectedConnectionHeader,
|
||||
withCorrelationId,
|
||||
withModalityBridgeHeader,
|
||||
} from "./chatHelpers";
|
||||
import { buildModalityBridgeHeader } from "@/lib/guardrails/modalityBridge/bridgeStats";
|
||||
import {
|
||||
isAntigravityMissingProjectError,
|
||||
PROVIDER_BREAKER_FAILURE_STATUSES,
|
||||
@@ -544,6 +546,10 @@ async function handleChatImplementation(
|
||||
isModelAllowedForKey,
|
||||
log,
|
||||
}));
|
||||
// Modality Bridge transparency (Task 9): non-null only when a pre-call bridge
|
||||
// guardrail transformed the payload (describe path) — stamped on the main
|
||||
// success exits below via withModalityBridgeHeader().
|
||||
const modalityBridgeHeader = buildModalityBridgeHeader(preCallGuardrails.results);
|
||||
telemetry.endPhase();
|
||||
|
||||
// T08: per-key active session limit (0 = unlimited).
|
||||
@@ -907,7 +913,10 @@ async function handleChatImplementation(
|
||||
if (fallbackResponse.ok) {
|
||||
log.info("GLOBAL_FALLBACK", `Global fallback ${fallbackModel} succeeded`);
|
||||
recordTelemetry(telemetry);
|
||||
return withSessionHeader(fallbackResponse, sessionId);
|
||||
return withModalityBridgeHeader(
|
||||
withSessionHeader(fallbackResponse, sessionId),
|
||||
modalityBridgeHeader
|
||||
);
|
||||
}
|
||||
log.warn(
|
||||
"GLOBAL_FALLBACK",
|
||||
@@ -944,7 +953,10 @@ async function handleChatImplementation(
|
||||
});
|
||||
} catch {}
|
||||
}
|
||||
return withCorrelationId(withSessionHeader(response, sessionId), reqId);
|
||||
return withModalityBridgeHeader(
|
||||
withCorrelationId(withSessionHeader(response, sessionId), reqId),
|
||||
modalityBridgeHeader
|
||||
);
|
||||
}
|
||||
telemetry.endPhase();
|
||||
|
||||
@@ -986,7 +998,10 @@ async function handleChatImplementation(
|
||||
false
|
||||
);
|
||||
recordTelemetry(telemetry);
|
||||
return withCorrelationId(withSessionHeader(response, sessionId), reqId);
|
||||
return withModalityBridgeHeader(
|
||||
withCorrelationId(withSessionHeader(response, sessionId), reqId),
|
||||
modalityBridgeHeader
|
||||
);
|
||||
}
|
||||
|
||||
export const handleChat = chatAdmission.withChatAdmission(handleChatImplementation);
|
||||
|
||||
@@ -908,6 +908,31 @@ export function withCorrelationId(response: Response, correlationId: string | nu
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Modality Bridge transparency (PR-1 Task 9): stamp the
|
||||
* `x-omniroute-modality-bridge` header on responses whose request payload was
|
||||
* transparently transformed (e.g. image→text describe). `value` comes from
|
||||
* buildModalityBridgeHeader(); null (untouched/rerouted request) is a no-op.
|
||||
* Same try-set/clone-fallback shape as withSessionHeader — the clone reuses
|
||||
* `response.body`, so SSE streams pass through untouched.
|
||||
*/
|
||||
export function withModalityBridgeHeader(response: Response, value: string | null): Response {
|
||||
if (!response || !value) return response;
|
||||
|
||||
try {
|
||||
response.headers.set("x-omniroute-modality-bridge", value);
|
||||
return response;
|
||||
} catch {
|
||||
const cloned = new Response(response.body, {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
headers: response.headers,
|
||||
});
|
||||
cloned.headers.set("x-omniroute-modality-bridge", value);
|
||||
return cloned;
|
||||
}
|
||||
}
|
||||
|
||||
export function withSelectedConnectionHeader(
|
||||
response: Response,
|
||||
connectionId: string | null | undefined
|
||||
|
||||
79
tests/unit/helpers/decollidedMigrationsDir.ts
Normal file
79
tests/unit/helpers/decollidedMigrationsDir.ts
Normal file
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Test-only workaround for the inherited base-red "Migration version collision
|
||||
* detected" on release/v3.8.50 (originally the `134_ccr_blocks.sql` +
|
||||
* `134_proxy_logs_egress_ip.sql` pair, fixed by #9688; the surviving pair is
|
||||
* `135_connection_runtime_state.sql` + `135_migrate_model_capability_max_token.sql`,
|
||||
* fix #9676 in flight). Any test that exercises a code path opening
|
||||
* the DB (e.g. `VisionBridgeGuardrail.preCall` → `getResolvedModelCapabilities`
|
||||
* → `getDbInstance`) dies at migration-file scan time, BEFORE the code under
|
||||
* test runs — making TDD on those paths impossible until the base is fixed.
|
||||
*
|
||||
* Base-red tracking: issue #9679; remaining fix PR in flight: #9676 (#9688
|
||||
* already landed). Once the base has no duplicate prefixes the copy and
|
||||
* this degrades to a plain pass-through copy — at that point this helper (and
|
||||
* its callsites) can be removed. Grep trigger: 9679 / 9676 / 9688.
|
||||
*
|
||||
* This helper copies the real migrations into a temp dir, renumbering any file
|
||||
* whose numeric prefix duplicates an earlier one to a fresh (max+1) version,
|
||||
* and points `OMNIROUTE_MIGRATIONS_DIR` (supported operator env var — see
|
||||
* `src/lib/db/migrationRunner.ts::resolveMigrationsDir`) at the copy. On the
|
||||
* FRESH per-process test DATA_DIR (tests/_setup/isolateDataDir.ts) the schema
|
||||
* CONTENT applied is byte-identical, but note the renumbering does shift the
|
||||
* displaced duplicate to the END of the migration ORDER (it runs after every
|
||||
* lower-numbered file instead of at its original slot). Harmless for the
|
||||
* current colliding pair — and strictly better than the crash — but not
|
||||
* literally "the same run" as production.
|
||||
*
|
||||
* MUST be called before the first `getDbInstance()` in the process (i.e. at
|
||||
* test-file top level, before any `preCall`). An explicitly configured
|
||||
* `OMNIROUTE_MIGRATIONS_DIR` always wins.
|
||||
*/
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
export function useDecollidedMigrationsDir(): void {
|
||||
if (process.env.OMNIROUTE_MIGRATIONS_DIR) return;
|
||||
|
||||
const realDir = path.resolve("src/lib/db/migrations");
|
||||
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-test-migrations-"));
|
||||
|
||||
const files = fs
|
||||
.readdirSync(realDir, { withFileTypes: true })
|
||||
.filter((entry) => entry.isFile())
|
||||
.map((entry) => entry.name)
|
||||
.sort();
|
||||
|
||||
let maxVersion = 0;
|
||||
for (const file of files) {
|
||||
const match = file.match(/^(\d+)_/);
|
||||
if (match) maxVersion = Math.max(maxVersion, Number.parseInt(match[1], 10));
|
||||
}
|
||||
|
||||
const seenVersions = new Set<number>();
|
||||
for (const file of files) {
|
||||
const match = file.match(/^(\d+)_(.*)$/);
|
||||
let target = file;
|
||||
if (match) {
|
||||
const version = Number.parseInt(match[1], 10);
|
||||
if (seenVersions.has(version)) {
|
||||
// Collision: move the later duplicate to a fresh version slot.
|
||||
maxVersion += 1;
|
||||
target = `${maxVersion}_${match[2]}`;
|
||||
} else {
|
||||
seenVersions.add(version);
|
||||
}
|
||||
}
|
||||
fs.copyFileSync(path.join(realDir, file), path.join(tmp, target));
|
||||
}
|
||||
|
||||
process.env.OMNIROUTE_MIGRATIONS_DIR = tmp;
|
||||
|
||||
process.on("exit", () => {
|
||||
try {
|
||||
fs.rmSync(tmp, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Best-effort cleanup — the OS reaps its temp dir eventually.
|
||||
}
|
||||
});
|
||||
}
|
||||
220
tests/unit/media-parts.test.ts
Normal file
220
tests/unit/media-parts.test.ts
Normal file
@@ -0,0 +1,220 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert";
|
||||
import { containsMediaKind, detectMediaParts } from "../../open-sse/utils/mediaParts";
|
||||
import { extractImageParts, replaceImageParts } from "../../src/lib/guardrails/visionBridgeHelpers";
|
||||
|
||||
const msg = (content: unknown) => [{ role: "user", content }];
|
||||
|
||||
test("detects OpenAI image_url part", () => {
|
||||
const parts = detectMediaParts(
|
||||
msg([{ type: "image_url", image_url: { url: "https://x/i.png" } }])
|
||||
);
|
||||
assert.equal(parts.length, 1);
|
||||
assert.equal(parts[0].kind, "image");
|
||||
assert.equal(parts[0].ref, "https://x/i.png");
|
||||
});
|
||||
|
||||
test("detects Anthropic base64 image source", () => {
|
||||
const parts = detectMediaParts(
|
||||
msg([{ type: "image", source: { type: "base64", media_type: "image/png", data: "AAA" } }])
|
||||
);
|
||||
assert.equal(parts[0].ref, "data:image/png;base64,AAA");
|
||||
});
|
||||
|
||||
test("detects Responses-API input_image", () => {
|
||||
const parts = detectMediaParts(msg([{ type: "input_image", image_url: "https://x/i.png" }]));
|
||||
assert.equal(parts.length, 1);
|
||||
assert.equal(parts[0].kind, "image");
|
||||
});
|
||||
|
||||
test("detects bare data:image string in content array", () => {
|
||||
const parts = detectMediaParts(msg([{ type: "text", text: "data:image/jpeg;base64,QUJD" }]));
|
||||
assert.equal(parts.length, 1);
|
||||
});
|
||||
|
||||
test("detects input_audio and audio_url as audio kind", () => {
|
||||
const parts = detectMediaParts(
|
||||
msg([
|
||||
{ type: "input_audio", input_audio: { data: "QUJD", format: "wav" } },
|
||||
{ type: "audio_url", audio_url: { url: "https://x/a.mp3" } },
|
||||
])
|
||||
);
|
||||
assert.deepEqual(
|
||||
parts.map((p) => p.kind),
|
||||
["audio", "audio"]
|
||||
);
|
||||
});
|
||||
|
||||
test("recursion depth capped at 8", () => {
|
||||
let nested: Record<string, unknown> = { type: "image_url", image_url: { url: "https://x" } };
|
||||
for (let i = 0; i < 10; i++) nested = { wrap: nested };
|
||||
assert.equal(detectMediaParts(msg([nested])).length, 0);
|
||||
});
|
||||
|
||||
test("string content and empty messages yield []", () => {
|
||||
assert.deepEqual(detectMediaParts([{ role: "user", content: "oi" }]), []);
|
||||
assert.deepEqual(detectMediaParts([]), []);
|
||||
});
|
||||
|
||||
test("combo-parity: image indicators without extractable refs are still detected", () => {
|
||||
// Bare image_url key without a `type` (legacy combo matched by key presence).
|
||||
const bareKey = detectMediaParts(msg([{ image_url: { url: "https://x/i.png" } }]));
|
||||
assert.equal(bareKey.length, 1);
|
||||
assert.equal(bareKey[0].kind, "image");
|
||||
assert.equal(bareKey[0].ref, "https://x/i.png");
|
||||
|
||||
// `type: "image"` with no usable source (legacy combo matched by type name).
|
||||
const bareType = detectMediaParts(msg([{ type: "image" }]));
|
||||
assert.equal(bareType.length, 1);
|
||||
assert.equal(bareType[0].shape, "image_indicator");
|
||||
assert.equal(bareType[0].ref, "");
|
||||
|
||||
// source.media_type image/* on an untyped part.
|
||||
const bySourceMedia = detectMediaParts(
|
||||
msg([{ source: { media_type: "image/JPEG", data: "AAA" } }])
|
||||
);
|
||||
assert.equal(bySourceMedia.length, 1);
|
||||
assert.equal(bySourceMedia[0].kind, "image");
|
||||
|
||||
// Case-insensitive type match (legacy combo lowercased `type`).
|
||||
const upperType = detectMediaParts(msg([{ type: "IMAGE_URL", image_url: { url: "https://x" } }]));
|
||||
assert.equal(upperType.length, 1);
|
||||
assert.equal(upperType[0].ref, "https://x");
|
||||
});
|
||||
|
||||
test("audio part does not shadow a bare image_url key on the same object", () => {
|
||||
const parts = detectMediaParts(
|
||||
msg([{ type: "input_audio", input_audio: { data: "QUJD" }, image_url: "https://x/i.png" }])
|
||||
);
|
||||
assert.deepEqual(parts.map((p) => p.kind).sort(), ["audio", "image"]);
|
||||
assert.equal(parts.find((p) => p.kind === "image")?.ref, "https://x/i.png");
|
||||
});
|
||||
|
||||
test("audio_url part does not shadow a bare input_image key on the same object", () => {
|
||||
const parts = detectMediaParts(
|
||||
msg([{ type: "audio_url", audio_url: "https://x/a.mp3", input_image: "https://x/i.png" }])
|
||||
);
|
||||
assert.deepEqual(parts.map((p) => p.kind).sort(), ["audio", "image"]);
|
||||
});
|
||||
|
||||
test("audio media_type part still recurses into sibling values (combo parity)", () => {
|
||||
const parts = detectMediaParts(
|
||||
msg([{ source: { media_type: "audio/mp3", data: "QUJD" }, sibling: { type: "image" } }])
|
||||
);
|
||||
assert.equal(parts.filter((p) => p.kind === "audio").length, 1);
|
||||
assert.equal(parts.filter((p) => p.kind === "image").length, 1);
|
||||
});
|
||||
|
||||
test("image nested inside input_audio payload is still detected", () => {
|
||||
const parts = detectMediaParts(
|
||||
msg([
|
||||
{
|
||||
type: "input_audio",
|
||||
input_audio: {
|
||||
data: "QUJD",
|
||||
cover: { type: "image_url", image_url: { url: "https://x/c.png" } },
|
||||
},
|
||||
},
|
||||
])
|
||||
);
|
||||
assert.equal(parts.filter((p) => p.kind === "audio").length, 1);
|
||||
assert.ok(parts.some((p) => p.kind === "image" && p.ref === "https://x/c.png"));
|
||||
});
|
||||
|
||||
test("bare source.media_type audio/* yields a single audio_source part", () => {
|
||||
const parts = detectMediaParts(msg([{ source: { media_type: "audio/wav", data: "QUJD" } }]));
|
||||
assert.equal(parts.length, 1);
|
||||
assert.equal(parts[0].kind, "audio");
|
||||
assert.equal(parts[0].shape, "audio_source");
|
||||
assert.equal(parts[0].ref, "QUJD");
|
||||
});
|
||||
|
||||
test("containsMediaKind early-exit agrees with detectMediaParts presence", () => {
|
||||
const withImage = msg([
|
||||
{ type: "text", text: "hi" },
|
||||
{ type: "image_url", image_url: { url: "https://x/i.png" } },
|
||||
]);
|
||||
assert.equal(containsMediaKind(withImage, "image"), true);
|
||||
assert.equal(containsMediaKind(withImage, "audio"), false);
|
||||
|
||||
const withAudio = msg([{ type: "input_audio", input_audio: { data: "QUJD", format: "wav" } }]);
|
||||
assert.equal(containsMediaKind(withAudio, "audio"), true);
|
||||
assert.equal(containsMediaKind(withAudio, "image"), false);
|
||||
|
||||
// Nested/indicator hits count for presence (combo parity).
|
||||
assert.equal(containsMediaKind(msg([{ image_url: "https://x" }]), "image"), true);
|
||||
assert.equal(containsMediaKind([{ role: "user", content: "oi" }], "image"), false);
|
||||
assert.equal(containsMediaKind(undefined, "image"), false);
|
||||
});
|
||||
|
||||
test("extractImageParts skips indicator parts without extractable refs", () => {
|
||||
assert.deepEqual(
|
||||
extractImageParts([{ role: "user", content: [{ type: "image" }] } as never]),
|
||||
[]
|
||||
);
|
||||
});
|
||||
|
||||
test("extractImageParts now sees input_image (Responses API)", () => {
|
||||
const parts = extractImageParts([
|
||||
{ role: "user", content: [{ type: "input_image", image_url: "https://x/i.png" }] } as never,
|
||||
]);
|
||||
assert.equal(parts.length, 1);
|
||||
assert.equal(parts[0].imageUrl, "https://x/i.png");
|
||||
});
|
||||
|
||||
test("extract→replace round-trip: input_image does not shift sibling descriptions", () => {
|
||||
const body = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "input_image", image_url: "https://x/a.png" },
|
||||
{ type: "image_url", image_url: { url: "https://x/b.png" } },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
const extracted = extractImageParts(body.messages as never);
|
||||
assert.equal(extracted.length, 2);
|
||||
assert.equal(extracted[0].imageUrl, "https://x/a.png");
|
||||
assert.equal(extracted[1].imageUrl, "https://x/b.png");
|
||||
|
||||
const replaced = replaceImageParts(body as never, ["DA", "DB"]);
|
||||
const content = (replaced.messages as Array<{ content: unknown }>)[0].content;
|
||||
assert.deepEqual(content, [
|
||||
{ type: "text", text: "DA" },
|
||||
{ type: "text", text: "DB" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("extractImageParts does not extract a data URI embedded in a text part", () => {
|
||||
const parts = extractImageParts([
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "data:image/png;base64,QUJD" }],
|
||||
} as never,
|
||||
]);
|
||||
assert.deepEqual(parts, []);
|
||||
});
|
||||
|
||||
test("extractImageParts skips nested and indicator-only detections", () => {
|
||||
const parts = extractImageParts([
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
// Image nested inside an audio payload: detector reports it (combo needs
|
||||
// it) but the replacer cannot splice it — must not be extracted.
|
||||
{
|
||||
type: "input_audio",
|
||||
input_audio: {
|
||||
data: "QUJD",
|
||||
cover: { type: "image_url", image_url: { url: "https://x/c.png" } },
|
||||
},
|
||||
},
|
||||
// Bare image_url key without type: indicator shape, not replaceable.
|
||||
{ image_url: "https://x/bare.png" },
|
||||
],
|
||||
} as never,
|
||||
]);
|
||||
assert.deepEqual(parts, []);
|
||||
});
|
||||
63
tests/unit/modality-bridge-cache.test.ts
Normal file
63
tests/unit/modality-bridge-cache.test.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert";
|
||||
|
||||
import {
|
||||
BridgeCache,
|
||||
bridgeCacheKey,
|
||||
getSharedBridgeCacheFor,
|
||||
} from "../../src/lib/guardrails/modalityBridge/bridgeCache.ts";
|
||||
import { resolveVisionBridgeRuntimeSettings } from "../../src/shared/constants/modalityBridgeDefaults.ts";
|
||||
|
||||
test("key is stable sha256 of content+prompt+model", () => {
|
||||
const a = bridgeCacheKey("data:image/png;base64,AAA", "describe", "gpt-4o-mini");
|
||||
const b = bridgeCacheKey("data:image/png;base64,AAA", "describe", "gpt-4o-mini");
|
||||
assert.equal(a, b);
|
||||
assert.match(a, /^[a-f0-9]{64}$/);
|
||||
assert.notEqual(a, bridgeCacheKey("data:image/png;base64,AAA", "other", "gpt-4o-mini"));
|
||||
});
|
||||
|
||||
test("key framing prevents boundary-shift collisions between fields", () => {
|
||||
// Without length-prefix framing ("ab","c") and ("a","bc") would hash the
|
||||
// same concatenated bytes.
|
||||
assert.notEqual(bridgeCacheKey("ab", "c", "m"), bridgeCacheKey("a", "bc", "m"));
|
||||
assert.notEqual(bridgeCacheKey("x", "yz", "m"), bridgeCacheKey("x", "y", "zm"));
|
||||
});
|
||||
|
||||
test("get/set roundtrip and TTL expiry", () => {
|
||||
let now = 1000;
|
||||
const cache = new BridgeCache({ maxEntries: 10, ttlMs: 500, now: () => now });
|
||||
cache.set("k1", "desc");
|
||||
assert.equal(cache.get("k1"), "desc");
|
||||
now = 1600;
|
||||
assert.equal(cache.get("k1"), undefined);
|
||||
});
|
||||
|
||||
test("LRU evicts oldest when full", () => {
|
||||
const cache = new BridgeCache({ maxEntries: 2, ttlMs: 60000, now: () => 0 });
|
||||
cache.set("a", "1");
|
||||
cache.set("b", "2");
|
||||
cache.get("a");
|
||||
cache.set("c", "3");
|
||||
assert.equal(cache.get("b"), undefined);
|
||||
assert.equal(cache.get("a"), "1");
|
||||
});
|
||||
|
||||
test("getSharedBridgeCacheFor reuses the instance for the same config and recreates on change", () => {
|
||||
const base = resolveVisionBridgeRuntimeSettings({});
|
||||
const a = getSharedBridgeCacheFor(base);
|
||||
const b = getSharedBridgeCacheFor({ ...base });
|
||||
assert.equal(a, b);
|
||||
|
||||
const c = getSharedBridgeCacheFor({ ...base, cacheTtlMinutes: base.cacheTtlMinutes + 5 });
|
||||
assert.notEqual(c, a);
|
||||
|
||||
const d = getSharedBridgeCacheFor({ ...base, cacheTtlMinutes: base.cacheTtlMinutes + 5 });
|
||||
assert.equal(d, c);
|
||||
|
||||
const e = getSharedBridgeCacheFor({
|
||||
...base,
|
||||
cacheTtlMinutes: base.cacheTtlMinutes + 5,
|
||||
cacheMaxEntries: base.cacheMaxEntries + 1,
|
||||
});
|
||||
assert.notEqual(e, d);
|
||||
});
|
||||
116
tests/unit/modality-bridge-header.test.ts
Normal file
116
tests/unit/modality-bridge-header.test.ts
Normal file
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* Modality Bridge stats + transparency header (PR-1 Task 9):
|
||||
* - buildModalityBridgeHeader() derives the `x-omniroute-modality-bridge`
|
||||
* response header value from pre-call guardrail results (describe path only —
|
||||
* reroute and untouched requests get no header).
|
||||
* - recordBridgeUse()/getBridgeStats() keep in-memory per-modality counters.
|
||||
*
|
||||
* Stats and the describe cache are PROCESS-GLOBAL: assertions use >= against
|
||||
* captured before-values and every guardrail case uses a unique image payload
|
||||
* (`auto/` model + mode "describe" keeps the flow DB-free — same recipe as
|
||||
* tests/unit/vision-bridge-describe-cache.test.ts).
|
||||
*/
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert";
|
||||
|
||||
import {
|
||||
buildModalityBridgeHeader,
|
||||
recordBridgeUse,
|
||||
getBridgeStats,
|
||||
} from "../../src/lib/guardrails/modalityBridge/bridgeStats.ts";
|
||||
import { VisionBridgeGuardrail } from "../../src/lib/guardrails/visionBridge.ts";
|
||||
|
||||
test("header built from vision-bridge describe meta", () => {
|
||||
const h = buildModalityBridgeHeader([
|
||||
{ guardrail: "vision-bridge", meta: { imagesProcessed: 2, visionModel: "openai/gpt-4o-mini" } },
|
||||
]);
|
||||
assert.equal(h, "image->text;model=openai/gpt-4o-mini;parts=2");
|
||||
});
|
||||
|
||||
test("no header for reroute or untouched requests", () => {
|
||||
assert.equal(
|
||||
buildModalityBridgeHeader([{ guardrail: "vision-bridge", meta: { rerouted: true } }]),
|
||||
null
|
||||
);
|
||||
assert.equal(buildModalityBridgeHeader([]), null);
|
||||
});
|
||||
|
||||
test("audio-bridge meta produces the audio segment (PR-3 forward-compat)", () => {
|
||||
const h = buildModalityBridgeHeader([
|
||||
{ guardrail: "vision-bridge", meta: { imagesProcessed: 1, visionModel: "m1" } },
|
||||
{ guardrail: "audio-bridge", meta: { clipsProcessed: 3, sttModel: "m2" } },
|
||||
]);
|
||||
assert.equal(h, "image->text;model=m1;parts=1, audio->text;model=m2;parts=3");
|
||||
});
|
||||
|
||||
test("stats counters accumulate", () => {
|
||||
recordBridgeUse("vision", { cacheHit: true });
|
||||
const s = getBridgeStats();
|
||||
assert.ok(s.vision.bridged >= 1);
|
||||
assert.ok(s.vision.cacheHits >= 1);
|
||||
});
|
||||
|
||||
// ── Guardrail-level wiring: describe path bumps the vision counters ─────────
|
||||
|
||||
function statsGuardrail(counter: { calls: number }, behavior?: { failAlways?: boolean }) {
|
||||
return new VisionBridgeGuardrail({
|
||||
deps: {
|
||||
getSettings: async () => ({ modalityBridgeVisionMode: "describe" }),
|
||||
callVisionModel: async () => {
|
||||
counter.calls++;
|
||||
if (behavior?.failAlways) throw new Error("describe indisponível");
|
||||
return "uma descrição da imagem";
|
||||
},
|
||||
hasUsableCredentials: async () => null,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/** Unique per-test payload — the test name lands inside the base64 content. */
|
||||
function bodyWithImage(uniqueRef: string): Record<string, unknown> {
|
||||
return {
|
||||
model: "auto/bridge-stats",
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "o que há na imagem?" },
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: `data:image/png;base64,${Buffer.from(uniqueRef).toString("base64")}`,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const context = { model: "auto/bridge-stats", log: console };
|
||||
|
||||
test("describe path records bridged use; identical repeat counts a cache hit", async () => {
|
||||
const before = getBridgeStats().vision;
|
||||
const counter = { calls: 0 };
|
||||
const guardrail = statsGuardrail(counter);
|
||||
|
||||
await guardrail.preCall(bodyWithImage("bridge-stats-hit-test"), context);
|
||||
const afterFirst = getBridgeStats().vision;
|
||||
assert.ok(afterFirst.bridged >= before.bridged + 1, "successful describe must count as bridged");
|
||||
assert.ok(typeof afterFirst.lastUsedAt === "string", "lastUsedAt must be stamped");
|
||||
|
||||
await guardrail.preCall(bodyWithImage("bridge-stats-hit-test"), context);
|
||||
const afterSecond = getBridgeStats().vision;
|
||||
assert.equal(counter.calls, 1, "second identical request must be served from the cache");
|
||||
assert.ok(afterSecond.bridged >= afterFirst.bridged + 1, "cache-served describe still bridged");
|
||||
assert.ok(afterSecond.cacheHits >= afterFirst.cacheHits + 1, "cache hit must be counted");
|
||||
});
|
||||
|
||||
test("failed describe records a failure", async () => {
|
||||
const before = getBridgeStats().vision;
|
||||
const guardrail = statsGuardrail({ calls: 0 }, { failAlways: true });
|
||||
|
||||
await guardrail.preCall(bodyWithImage("bridge-stats-failure-test"), context);
|
||||
const after = getBridgeStats().vision;
|
||||
assert.ok(after.failures >= before.failures + 1, "failed describe must count as failure");
|
||||
});
|
||||
92
tests/unit/modality-bridge-settings-migration.test.ts
Normal file
92
tests/unit/modality-bridge-settings-migration.test.ts
Normal file
@@ -0,0 +1,92 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import Database from "better-sqlite3";
|
||||
|
||||
const MIGRATION_PATH = path.resolve("src/lib/db/migrations/141_modality_bridge_settings.sql");
|
||||
const INITIAL_SCHEMA_PATH = path.resolve("src/lib/db/migrations/001_initial_schema.sql");
|
||||
|
||||
/** Extract the REAL key_value DDL from the initial schema so the seed can never drift. */
|
||||
function keyValueTableDdl(): string {
|
||||
const schema = fs.readFileSync(INITIAL_SCHEMA_PATH, "utf8");
|
||||
const match = schema.match(/CREATE TABLE IF NOT EXISTS key_value \([\s\S]*?\);/);
|
||||
assert.ok(match, "key_value CREATE TABLE block not found in 001_initial_schema.sql");
|
||||
return match[0];
|
||||
}
|
||||
|
||||
function createSeededDb(): InstanceType<typeof Database> {
|
||||
const db = new Database(":memory:");
|
||||
db.exec(keyValueTableDdl());
|
||||
return db;
|
||||
}
|
||||
|
||||
function getSetting(db: InstanceType<typeof Database>, key: string): string | undefined {
|
||||
const row = db
|
||||
.prepare("SELECT value FROM key_value WHERE namespace = 'settings' AND key = ?")
|
||||
.get(key) as { value: string } | undefined;
|
||||
return row?.value;
|
||||
}
|
||||
|
||||
test("migration 141 copies legacy visionBridge* settings to modalityBridge* keys", () => {
|
||||
const sql = fs.readFileSync(MIGRATION_PATH, "utf8");
|
||||
const db = createSeededDb();
|
||||
try {
|
||||
const seed = db.prepare(
|
||||
"INSERT INTO key_value (namespace, key, value) VALUES ('settings', ?, ?)"
|
||||
);
|
||||
seed.run("visionBridgeEnabled", "true");
|
||||
seed.run("visionBridgeModel", "openai/gpt-4o-mini");
|
||||
seed.run("visionBridgePrompt", "legacy prompt");
|
||||
seed.run("visionBridgeTimeout", "45000");
|
||||
seed.run("visionBridgeMaxImages", "6");
|
||||
|
||||
db.exec(sql);
|
||||
|
||||
assert.equal(getSetting(db, "modalityBridgeVisionEnabled"), "true");
|
||||
assert.equal(getSetting(db, "modalityBridgeVisionModel"), "openai/gpt-4o-mini");
|
||||
assert.equal(getSetting(db, "modalityBridgeVisionPrompt"), "legacy prompt");
|
||||
assert.equal(getSetting(db, "modalityBridgeVisionTimeout"), "45000");
|
||||
assert.equal(getSetting(db, "modalityBridgeVisionMaxImages"), "6");
|
||||
// Legacy keys stay untouched (one-cycle rollback window).
|
||||
assert.equal(getSetting(db, "visionBridgeEnabled"), "true");
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("migration 141 is idempotent and never overwrites an existing new key", () => {
|
||||
const sql = fs.readFileSync(MIGRATION_PATH, "utf8");
|
||||
const db = createSeededDb();
|
||||
try {
|
||||
const seed = db.prepare(
|
||||
"INSERT INTO key_value (namespace, key, value) VALUES ('settings', ?, ?)"
|
||||
);
|
||||
seed.run("visionBridgeModel", "openai/gpt-4o-mini");
|
||||
// Operator already set the new key — the migration must not clobber it.
|
||||
seed.run("modalityBridgeVisionModel", "gemini/gemini-2.5-flash");
|
||||
|
||||
db.exec(sql);
|
||||
db.exec(sql); // re-run: idempotent
|
||||
|
||||
assert.equal(getSetting(db, "modalityBridgeVisionModel"), "gemini/gemini-2.5-flash");
|
||||
const count = db
|
||||
.prepare("SELECT COUNT(*) AS n FROM key_value WHERE namespace = 'settings'")
|
||||
.get() as { n: number };
|
||||
assert.equal(count.n, 2);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("migration 141 is a no-op on a database without legacy settings", () => {
|
||||
const sql = fs.readFileSync(MIGRATION_PATH, "utf8");
|
||||
const db = createSeededDb();
|
||||
try {
|
||||
db.exec(sql);
|
||||
const count = db.prepare("SELECT COUNT(*) AS n FROM key_value").get() as { n: number };
|
||||
assert.equal(count.n, 0);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
96
tests/unit/modality-bridge-settings.test.ts
Normal file
96
tests/unit/modality-bridge-settings.test.ts
Normal file
@@ -0,0 +1,96 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert";
|
||||
|
||||
import {
|
||||
resolveVisionBridgeRuntimeSettings,
|
||||
MODALITY_BRIDGE_DEFAULTS,
|
||||
} from "../../src/shared/constants/modalityBridgeDefaults.ts";
|
||||
import { updateSettingsSchema } from "../../src/shared/validation/settingsSchemas.ts";
|
||||
|
||||
test("new keys win over legacy keys", () => {
|
||||
const s = resolveVisionBridgeRuntimeSettings({
|
||||
modalityBridgeVisionEnabled: false,
|
||||
visionBridgeEnabled: true,
|
||||
modalityBridgeVisionModel: "gemini/gemini-2.5-flash",
|
||||
visionBridgeModel: "openai/gpt-4o-mini",
|
||||
});
|
||||
assert.equal(s.enabled, false);
|
||||
assert.equal(s.model, "gemini/gemini-2.5-flash");
|
||||
});
|
||||
|
||||
test("legacy keys used as fallback when new keys absent (rollback 1 ciclo)", () => {
|
||||
const s = resolveVisionBridgeRuntimeSettings({
|
||||
visionBridgeEnabled: false,
|
||||
visionBridgePrompt: "legacy prompt",
|
||||
});
|
||||
assert.equal(s.enabled, false);
|
||||
assert.equal(s.prompt, "legacy prompt");
|
||||
});
|
||||
|
||||
test("defaults: mode auto, taskAware true, cache on", () => {
|
||||
const s = resolveVisionBridgeRuntimeSettings({});
|
||||
assert.equal(s.mode, "auto");
|
||||
assert.equal(s.taskAware, true);
|
||||
assert.equal(s.cacheEnabled, true);
|
||||
assert.equal(s.cacheTtlMinutes, MODALITY_BRIDGE_DEFAULTS.cacheTtlMinutes);
|
||||
});
|
||||
|
||||
test("wrong-typed stored values are skipped in favor of the next candidate/default", () => {
|
||||
const s = resolveVisionBridgeRuntimeSettings({
|
||||
modalityBridgeVisionEnabled: "off", // string in a boolean field
|
||||
modalityBridgeVisionTimeout: "9000", // string in a number field
|
||||
modalityBridgeVisionModel: 42, // number in a string field
|
||||
});
|
||||
assert.equal(s.enabled, true); // falls through to VISION_BRIDGE_DEFAULTS.enabled
|
||||
assert.equal(s.timeoutMs, 30000);
|
||||
assert.equal(s.model, "openai/gpt-4o-mini");
|
||||
|
||||
// A wrong-typed NEW key must still fall through to a well-typed LEGACY key.
|
||||
const fallback = resolveVisionBridgeRuntimeSettings({
|
||||
modalityBridgeVisionEnabled: "off",
|
||||
visionBridgeEnabled: false,
|
||||
});
|
||||
assert.equal(fallback.enabled, false);
|
||||
});
|
||||
|
||||
test("Modality Bridge settings are accepted by the settings PATCH schema", () => {
|
||||
const validation = updateSettingsSchema.safeParse({
|
||||
modalityBridgeVisionEnabled: true,
|
||||
modalityBridgeVisionMode: "describe",
|
||||
modalityBridgeVisionModel: "gemini/gemini-2.5-flash",
|
||||
modalityBridgeVisionTaskAware: false,
|
||||
modalityBridgeVisionPrompt: "Describe the image contents briefly.",
|
||||
modalityBridgeVisionTimeout: 45000,
|
||||
modalityBridgeVisionMaxImages: 6,
|
||||
modalityBridgeAudioEnabled: true,
|
||||
modalityBridgeAudioModel: "openai/gpt-4o-mini-transcribe",
|
||||
modalityBridgeAudioTimeout: 60000,
|
||||
modalityBridgeAudioMaxClips: 3,
|
||||
modalityBridgeCacheEnabled: true,
|
||||
modalityBridgeCacheTtlMinutes: 60,
|
||||
modalityBridgeCacheMaxEntries: 200,
|
||||
});
|
||||
|
||||
assert.equal(validation.success, true);
|
||||
});
|
||||
|
||||
test("Modality Bridge settings keep numeric bounds enforced (each field individually)", () => {
|
||||
const invalidByField: Record<string, number> = {
|
||||
modalityBridgeVisionTimeout: 999999,
|
||||
modalityBridgeVisionMaxImages: 0,
|
||||
modalityBridgeAudioMaxClips: 11,
|
||||
modalityBridgeCacheTtlMinutes: 0,
|
||||
modalityBridgeCacheMaxEntries: 9,
|
||||
};
|
||||
for (const [field, value] of Object.entries(invalidByField)) {
|
||||
const validation = updateSettingsSchema.safeParse({ [field]: value });
|
||||
assert.equal(validation.success, false, `${field}=${value} should be rejected`);
|
||||
}
|
||||
});
|
||||
|
||||
test("Modality Bridge vision mode rejects values outside the enum", () => {
|
||||
const validation = updateSettingsSchema.safeParse({
|
||||
modalityBridgeVisionMode: "invalid-mode",
|
||||
});
|
||||
assert.equal(validation.success, false);
|
||||
});
|
||||
102
tests/unit/vision-bridge-describe-cache.test.ts
Normal file
102
tests/unit/vision-bridge-describe-cache.test.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* Describe-path cache integration (Modality Bridge PR-1): the describe loop
|
||||
* consults the shared BridgeCache (sha256 of contentRef+prompt+model) so the
|
||||
* same image with the same prompt/model is described once per TTL. Failures
|
||||
* are never cached. Opt-out via `modalityBridgeCacheEnabled: false`.
|
||||
*
|
||||
* The shared cache is PROCESS-WIDE — every test uses a unique image payload so
|
||||
* tests cannot cross-contaminate each other's keys. Guardrail cases use
|
||||
* `model: "auto/..."` + `mode: "describe"` so the flow is DB-free.
|
||||
*/
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { VisionBridgeGuardrail } from "../../src/lib/guardrails/visionBridge.ts";
|
||||
|
||||
function cacheGuardrail(
|
||||
settings: Record<string, unknown>,
|
||||
counter: { calls: number },
|
||||
behavior?: { failFirstCall?: boolean }
|
||||
): InstanceType<typeof VisionBridgeGuardrail> {
|
||||
return new VisionBridgeGuardrail({
|
||||
deps: {
|
||||
getSettings: async () => ({ modalityBridgeVisionMode: "describe", ...settings }),
|
||||
callVisionModel: async () => {
|
||||
counter.calls++;
|
||||
if (behavior?.failFirstCall && counter.calls === 1) {
|
||||
throw new Error("primeiro describe falhou");
|
||||
}
|
||||
return "uma descrição da imagem";
|
||||
},
|
||||
hasUsableCredentials: async () => null,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/** Unique per-test payload — the test name lands inside the base64 content. */
|
||||
function bodyWithImage(uniqueRef: string): Record<string, unknown> {
|
||||
return {
|
||||
model: "auto/describe-cache",
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "o que há na imagem?" },
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: `data:image/png;base64,${Buffer.from(uniqueRef).toString("base64")}`,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const context = { model: "auto/describe-cache", log: console };
|
||||
|
||||
test("same image+prompt+model described twice → single upstream call (cache hit)", async () => {
|
||||
const counter = { calls: 0 };
|
||||
const guardrail = cacheGuardrail({}, counter);
|
||||
|
||||
const first = await guardrail.preCall(bodyWithImage("cache-hit-test"), context);
|
||||
assert.equal((first.meta ?? {}).imagesProcessed, 1);
|
||||
assert.equal(counter.calls, 1);
|
||||
|
||||
const second = await guardrail.preCall(bodyWithImage("cache-hit-test"), context);
|
||||
assert.equal((second.meta ?? {}).imagesProcessed, 1, "cached describe still replaces the image");
|
||||
assert.equal(counter.calls, 1, "second identical request must be served from the cache");
|
||||
|
||||
const descriptions = (second.meta ?? {}).descriptions as string[];
|
||||
assert.ok(
|
||||
descriptions?.[0]?.includes("uma descrição da imagem"),
|
||||
"cached description must be spliced into the payload"
|
||||
);
|
||||
});
|
||||
|
||||
test("modalityBridgeCacheEnabled=false → every request hits the vision model", async () => {
|
||||
const counter = { calls: 0 };
|
||||
const guardrail = cacheGuardrail({ modalityBridgeCacheEnabled: false }, counter);
|
||||
|
||||
await guardrail.preCall(bodyWithImage("cache-disabled-test"), context);
|
||||
await guardrail.preCall(bodyWithImage("cache-disabled-test"), context);
|
||||
assert.equal(counter.calls, 2, "disabled cache must not dedupe describe calls");
|
||||
});
|
||||
|
||||
test("failed describe is NOT cached — the next request retries upstream", async () => {
|
||||
const counter = { calls: 0 };
|
||||
const guardrail = cacheGuardrail({}, counter, { failFirstCall: true });
|
||||
|
||||
await guardrail.preCall(bodyWithImage("failure-not-cached-test"), context);
|
||||
assert.equal(counter.calls, 1);
|
||||
|
||||
const second = await guardrail.preCall(bodyWithImage("failure-not-cached-test"), context);
|
||||
assert.equal(counter.calls, 2, "failure must not be cached; retry must reach upstream");
|
||||
|
||||
const descriptions = (second.meta ?? {}).descriptions as string[];
|
||||
assert.ok(
|
||||
descriptions?.[0]?.includes("uma descrição da imagem"),
|
||||
"successful retry description must be used"
|
||||
);
|
||||
});
|
||||
138
tests/unit/vision-bridge-mode.test.ts
Normal file
138
tests/unit/vision-bridge-mode.test.ts
Normal file
@@ -0,0 +1,138 @@
|
||||
/**
|
||||
* Vision Bridge mode selector (auto | describe | reroute) — Modality Bridge PR-1.
|
||||
*
|
||||
* The forced modes short-circuit BEFORE the auto reroute×describe heuristic, so
|
||||
* the #6640/#7204/#7871/#8430 contracts stay untouched in "auto" (the default):
|
||||
* - "describe": never whole-request-reroutes — straight to the describe path.
|
||||
* - "reroute": skips only the keep-credentialed-model guard; the reroute-target
|
||||
* credential guard still applies, and with no usable target it falls back to
|
||||
* describe (raw images must never reach a text-only backend — #8430).
|
||||
*
|
||||
* Uses dependency injection for settings/vision calls/credentials. The model
|
||||
* capability lookup inside preCall still opens the real (isolated) SQLite DB,
|
||||
* which on the current base dies on the inherited 134 migration collision —
|
||||
* hence the decollided-migrations helper below.
|
||||
*/
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { useDecollidedMigrationsDir } from "./helpers/decollidedMigrationsDir.ts";
|
||||
|
||||
useDecollidedMigrationsDir();
|
||||
|
||||
const { VisionBridgeGuardrail } = await import("../../src/lib/guardrails/visionBridge.ts");
|
||||
|
||||
const TEXT_ONLY_MODEL = "some/text-only-model";
|
||||
|
||||
/**
|
||||
* Unique per-test payload: the describe path caches by image+prompt+model
|
||||
* (Task 8), so reusing the same data URI across tests would turn a later
|
||||
* describe into a cache hit and hide the upstream call being asserted.
|
||||
*/
|
||||
function imageBody(uniqueRef: string): Record<string, unknown> {
|
||||
return {
|
||||
model: TEXT_ONLY_MODEL,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "o que há na imagem?" },
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: `data:image/png;base64,${Buffer.from(uniqueRef).toString("base64")}`,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function metaOf(result: { meta?: Record<string, unknown> | null }): Record<string, unknown> {
|
||||
return result.meta ?? {};
|
||||
}
|
||||
|
||||
test("mode=describe: never reroutes even when a reroute target exists", async () => {
|
||||
const guardrail = new VisionBridgeGuardrail({
|
||||
deps: {
|
||||
getSettings: async () => ({
|
||||
modalityBridgeVisionMode: "describe",
|
||||
// A configured vision model — in auto/reroute this would be a valid
|
||||
// fixed reroute target (credentials indeterminate → fail-open #8430).
|
||||
modalityBridgeVisionModel: "openai/gpt-4o-mini",
|
||||
}),
|
||||
callVisionModel: async () => "uma foto de um gato",
|
||||
hasUsableCredentials: async () => null,
|
||||
},
|
||||
});
|
||||
|
||||
const body = imageBody("mode-describe-test");
|
||||
const result = await guardrail.preCall(body, { model: TEXT_ONLY_MODEL, log: console });
|
||||
|
||||
const meta = metaOf(result);
|
||||
assert.notEqual(meta.rerouted, true, "describe mode must never whole-request-reroute");
|
||||
assert.equal(meta.imagesProcessed, 1, "the image must be described instead");
|
||||
});
|
||||
|
||||
test("mode=reroute: falls back to describe when no reroute target has credentials", async () => {
|
||||
const describeCalls: string[] = [];
|
||||
const guardrail = new VisionBridgeGuardrail({
|
||||
deps: {
|
||||
getSettings: async () => ({ modalityBridgeVisionMode: "reroute" }),
|
||||
callVisionModel: async () => {
|
||||
describeCalls.push("describe");
|
||||
return "desc";
|
||||
},
|
||||
// Every model confirmed unusable — no reroute target can win (#8430).
|
||||
hasUsableCredentials: async () => false,
|
||||
},
|
||||
});
|
||||
|
||||
const body = imageBody("mode-reroute-fallback-test");
|
||||
const result = await guardrail.preCall(body, { model: TEXT_ONLY_MODEL, log: console });
|
||||
|
||||
const meta = metaOf(result);
|
||||
assert.notEqual(meta.rerouted, true, "must not reroute to a target without credentials");
|
||||
assert.ok(describeCalls.length >= 1, "deveria ter caído para o caminho de descrição");
|
||||
});
|
||||
|
||||
test("mode=reroute: forces reroute where auto mode would keep the credentialed model", async () => {
|
||||
const guardrail = new VisionBridgeGuardrail({
|
||||
deps: {
|
||||
getSettings: async () => ({
|
||||
modalityBridgeVisionMode: "reroute",
|
||||
modalityBridgeVisionModel: "openai/gpt-4o-mini",
|
||||
}),
|
||||
callVisionModel: async () => "desc",
|
||||
// Original model IS credentialed (auto mode would keep it, #7204);
|
||||
// reroute target indeterminate → fail-open proceeds (#8430).
|
||||
hasUsableCredentials: async (model: string) => (model === TEXT_ONLY_MODEL ? true : null),
|
||||
},
|
||||
});
|
||||
|
||||
const body = imageBody("mode-reroute-forces-test");
|
||||
const result = await guardrail.preCall(body, { model: TEXT_ONLY_MODEL, log: console });
|
||||
|
||||
const meta = metaOf(result);
|
||||
assert.equal(meta.rerouted, true, "reroute mode must skip the keep-credentialed-model guard");
|
||||
assert.equal(meta.toModel, "openai/gpt-4o-mini");
|
||||
assert.equal(meta.fromModel, TEXT_ONLY_MODEL);
|
||||
});
|
||||
|
||||
test("mode=auto (default): credentialed model is described, not hijacked", async () => {
|
||||
const guardrail = new VisionBridgeGuardrail({
|
||||
deps: {
|
||||
getSettings: async () => ({}),
|
||||
callVisionModel: async () => "desc",
|
||||
hasUsableCredentials: async () => true,
|
||||
},
|
||||
});
|
||||
|
||||
const body = imageBody("mode-auto-default-test");
|
||||
const result = await guardrail.preCall(body, { model: TEXT_ONLY_MODEL, log: console });
|
||||
|
||||
const meta = metaOf(result);
|
||||
assert.notEqual(meta.rerouted, true, "auto mode keeps the credentialed model (#7204)");
|
||||
assert.equal(meta.imagesProcessed, 1, "images are described for the kept model");
|
||||
});
|
||||
115
tests/unit/vision-bridge-task-aware.test.ts
Normal file
115
tests/unit/vision-bridge-task-aware.test.ts
Normal file
@@ -0,0 +1,115 @@
|
||||
/**
|
||||
* Task-aware vision description prompt (codex-vision-proxy pattern) —
|
||||
* Modality Bridge PR-1. The describe path appends the user's last question as
|
||||
* a focus hint so the vision model describes what is relevant to answering it
|
||||
* instead of producing a generic caption. Default ON; disabled via
|
||||
* `modalityBridgeVisionTaskAware: false`.
|
||||
*
|
||||
* Guardrail-level cases use `model: "auto/..."` + `mode: "describe"` so the
|
||||
* whole flow is DB-free (the auto prefix skips the capability/combo lookups
|
||||
* that open SQLite, and the forced describe mode skips the reroute block).
|
||||
*/
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { composeVisionPrompt } from "../../src/lib/guardrails/visionBridgeHelpers.ts";
|
||||
import { VisionBridgeGuardrail } from "../../src/lib/guardrails/visionBridge.ts";
|
||||
import type { VisionModelConfig } from "../../src/lib/guardrails/visionBridgeHelpers.ts";
|
||||
|
||||
// ── composeVisionPrompt (pure) ──────────────────────────────────────────────
|
||||
|
||||
test("appends user focus hint when taskAware", () => {
|
||||
const p = composeVisionPrompt("Describe the image.", "qual o erro no screenshot?", true);
|
||||
assert.ok(p.startsWith("Describe the image."));
|
||||
assert.ok(p.includes("qual o erro no screenshot?"));
|
||||
});
|
||||
|
||||
test("no hint when disabled or no user text", () => {
|
||||
assert.equal(composeVisionPrompt("Base.", "pergunta", false), "Base.");
|
||||
assert.equal(composeVisionPrompt("Base.", undefined, true), "Base.");
|
||||
assert.equal(composeVisionPrompt("Base.", " ", true), "Base.");
|
||||
});
|
||||
|
||||
test("hint truncated to 500 chars", () => {
|
||||
const p = composeVisionPrompt("Base.", "x".repeat(2000), true);
|
||||
assert.ok(p.length < 700, `expected truncated prompt, got length ${p.length}`);
|
||||
assert.ok(p.includes("x".repeat(500)));
|
||||
assert.ok(!p.includes("x".repeat(501)));
|
||||
});
|
||||
|
||||
// ── Guardrail describe path wiring ──────────────────────────────────────────
|
||||
|
||||
function describeGuardrail(
|
||||
settings: Record<string, unknown>,
|
||||
capturedPrompts: string[]
|
||||
): InstanceType<typeof VisionBridgeGuardrail> {
|
||||
return new VisionBridgeGuardrail({
|
||||
deps: {
|
||||
getSettings: async () => ({ modalityBridgeVisionMode: "describe", ...settings }),
|
||||
callVisionModel: async (_imageDataUri: string, config: VisionModelConfig) => {
|
||||
capturedPrompts.push(config.prompt);
|
||||
return "descrição";
|
||||
},
|
||||
hasUsableCredentials: async () => null,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Unique per-test payload: the describe path caches by image+prompt+model
|
||||
* (Task 8), so reusing the same data URI across tests would make a later
|
||||
* describe a cache hit and hide the upstream call whose prompt is asserted.
|
||||
*/
|
||||
function autoImageBody(uniqueRef: string, userText: string): Record<string, unknown> {
|
||||
return {
|
||||
model: "auto/task-aware",
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: userText },
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: `data:image/png;base64,${Buffer.from(uniqueRef).toString("base64")}`,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
test("describe call prompt contains the last user question (taskAware default on)", async () => {
|
||||
const prompts: string[] = [];
|
||||
const guardrail = describeGuardrail({}, prompts);
|
||||
|
||||
const result = await guardrail.preCall(
|
||||
autoImageBody("task-aware-default-on-test", "qual o erro no screenshot?"),
|
||||
{ model: "auto/task-aware", log: console }
|
||||
);
|
||||
|
||||
assert.equal((result.meta ?? {}).imagesProcessed, 1);
|
||||
assert.equal(prompts.length, 1);
|
||||
assert.ok(
|
||||
prompts[0].includes("qual o erro no screenshot?"),
|
||||
`prompt should carry the user question, got: ${prompts[0]}`
|
||||
);
|
||||
});
|
||||
|
||||
test("modalityBridgeVisionTaskAware=false keeps the base prompt untouched", async () => {
|
||||
const prompts: string[] = [];
|
||||
const guardrail = describeGuardrail(
|
||||
{ modalityBridgeVisionTaskAware: false, modalityBridgeVisionPrompt: "Base prompt." },
|
||||
prompts
|
||||
);
|
||||
|
||||
const result = await guardrail.preCall(
|
||||
autoImageBody("task-aware-disabled-test", "pergunta que não deve vazar"),
|
||||
{ model: "auto/task-aware", log: console }
|
||||
);
|
||||
|
||||
assert.equal((result.meta ?? {}).imagesProcessed, 1);
|
||||
assert.equal(prompts.length, 1);
|
||||
assert.equal(prompts[0], "Base prompt.");
|
||||
});
|
||||
Reference in New Issue
Block a user