diff --git a/src/app/(dashboard)/dashboard/settings/components/modalityBridge/ModalityBridgeStatsRow.tsx b/src/app/(dashboard)/dashboard/settings/components/modalityBridge/ModalityBridgeStatsRow.tsx index 15a939edd6..dfe792ed07 100644 --- a/src/app/(dashboard)/dashboard/settings/components/modalityBridge/ModalityBridgeStatsRow.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/modalityBridge/ModalityBridgeStatsRow.tsx @@ -12,6 +12,7 @@ interface BridgeStats { cacheHits: number; failures: number; lastUsedAt: string | null; + latencySamples: number; successes: number; totalLatencyMs: number; } @@ -36,6 +37,19 @@ function parseStats(value: unknown): BridgeStats | null { typeof record.attempts === "number" ? record.attempts : record.bridged + record.failures; const averageLatencyMs = typeof record.averageLatencyMs === "number" ? record.averageLatencyMs : 0; + const totalLatencyMs = + typeof record.totalLatencyMs === "number" ? record.totalLatencyMs : averageLatencyMs * attempts; + // Compatibility with pre-latencySamples servers: positive latency data was + // sampled, while the legacy all-zero shape means timing was never recorded. + const latencySamples = + typeof record.latencySamples === "number" + ? Math.max(0, Math.floor(record.latencySamples)) + : totalLatencyMs > 0 + ? Math.max( + 1, + averageLatencyMs > 0 ? Math.round(totalLatencyMs / averageLatencyMs) : attempts + ) + : 0; return { attempts, averageLatencyMs, @@ -43,11 +57,9 @@ function parseStats(value: unknown): BridgeStats | null { cacheHits: record.cacheHits, failures: record.failures, lastUsedAt: typeof lastUsedAt === "string" ? lastUsedAt : null, + latencySamples, successes: typeof record.successes === "number" ? record.successes : record.bridged, - totalLatencyMs: - typeof record.totalLatencyMs === "number" - ? record.totalLatencyMs - : averageLatencyMs * attempts, + totalLatencyMs, }; } @@ -82,7 +94,7 @@ export default function ModalityBridgeStatsRow({ kind }: ModalityBridgeStatsRowP return (
- {stats.attempts} {tProviderStats("requests").toLowerCase()} + {stats.attempts} {tRoot("requestLogger.attempts").toLowerCase()} {stats.successes} {t("modalityBridgeStatsBridged")} @@ -94,10 +106,12 @@ export default function ModalityBridgeStatsRow({ kind }: ModalityBridgeStatsRowP {stats.failures} {t("modalityBridgeStatsFailures")} - {tRoot("trafficInspector.timingTotalLatency")}: {Math.round(stats.totalLatencyMs)} ms + {tRoot("trafficInspector.timingTotalLatency")}:{" "} + {stats.latencySamples > 0 ? `${Math.round(stats.totalLatencyMs)} ms` : "—"} - {tProviderStats("avgLatency")}: {Math.round(stats.averageLatencyMs)} ms + {tProviderStats("avgLatency")}:{" "} + {stats.latencySamples > 0 ? `${Math.round(stats.averageLatencyMs)} ms` : "—"} {t("modalityBridgeStatsLastUsed")}: {lastUsed} diff --git a/src/app/api/models/route.ts b/src/app/api/models/route.ts index 268d1fa6e8..88d6cf00f4 100644 --- a/src/app/api/models/route.ts +++ b/src/app/api/models/route.ts @@ -5,7 +5,10 @@ import { updateModelAliasSchema } from "@/shared/validation/schemas"; import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; import { hasEligibleConnectionForModel } from "@/domain/connectionModelRules"; import { getSettings } from "@/lib/db/settings"; -import { getResolvedModelCapabilities } from "@/lib/modelCapabilities"; +import { + createModelCapabilityResolutionSnapshot, + getResolvedModelCapabilities, +} from "@/lib/modelCapabilities"; import { isFreeModel, providerHasFreeModels } from "@/shared/utils/freeModels"; // GET /api/models - Get models with aliases (only from active providers by default) @@ -77,18 +80,6 @@ export async function GET(request: Request) { } } - const models = AI_MODELS.map((m: any) => { - const fullModel = `${m.provider}/${m.model}`; - const available = !activeProviders || activeProviders.has(m.provider); - return { - ...m, - fullModel, - alias: modelAliases[fullModel] || m.model, - available, - supportsVision: getResolvedModelCapabilities(fullModel).supportsVision === true, - }; - }).filter((m: any) => showAll || m.available); - // #6328 (follow-up to #6495): REMOVE — not just hide — paid models from the // dashboard model picker when the operator opts into hidePaidModels. Mirrors // the `shouldHidePaid` guard in `src/app/api/v1/models/catalog.ts` (public @@ -98,14 +89,33 @@ export async function GET(request: Request) { const settings = await getSettings(); hidePaid = settings?.hidePaidModels === true; } catch {} - const filtered = hidePaid - ? models.filter( - (m: { provider: string; model: string }) => - providerHasFreeModels(m.provider) && isFreeModel(m.provider, { id: m.model }) - ) - : models; - return NextResponse.json({ models: filtered }); + // Filter before capability resolution so unavailable/paid rows cannot trigger + // needless capability work. One request-local snapshot supplies all persisted + // capability and custom-vision rows to the remaining resolutions. + const candidates = AI_MODELS.filter((model: any) => { + if (!showAll && activeProviders && !activeProviders.has(model.provider)) return false; + return ( + !hidePaid || + (providerHasFreeModels(model.provider) && isFreeModel(model.provider, { id: model.model })) + ); + }); + const capabilitySnapshot = createModelCapabilityResolutionSnapshot(); + const models = candidates.map((m: any) => { + const fullModel = `${m.provider}/${m.model}`; + const available = !activeProviders || activeProviders.has(m.provider); + return { + ...m, + fullModel, + alias: modelAliases[fullModel] || m.model, + available, + supportsVision: + getResolvedModelCapabilities(fullModel, undefined, capabilitySnapshot).supportsVision === + true, + }; + }); + + return NextResponse.json({ models }); } catch (error) { console.log("Error fetching models:", error); return NextResponse.json({ error: "Failed to fetch models" }, { status: 500 }); diff --git a/src/lib/db/models.ts b/src/lib/db/models.ts index 97c6f81653..41af595e7b 100644 --- a/src/lib/db/models.ts +++ b/src/lib/db/models.ts @@ -90,6 +90,74 @@ export async function getAllCustomModels() { return result; } +/** Nested provider → model map of explicit custom-model vision overrides. */ +export type CustomModelVisionOverrideMap = ReadonlyMap>; + +function readVisionOverrideFromModels(value: string | null, modelId: string): boolean | null { + if (!value) return null; + try { + const models = JSON.parse(value) as unknown; + if (!Array.isArray(models)) return null; + const entry = models.find( + (candidate): candidate is { id: string; supportsVision?: boolean } => + candidate !== null && + typeof candidate === "object" && + !Array.isArray(candidate) && + (candidate as { id?: unknown }).id === modelId + ); + return entry && typeof entry.supportsVision === "boolean" ? entry.supportsVision : null; + } catch { + return null; + } +} + +/** + * Resolve one explicit custom-model vision override. A supplied bulk map avoids + * SQLite reads for request/build-local capability resolution. + */ +export function getCustomModelVisionOverride( + providerId: string, + modelId: string, + bulk?: CustomModelVisionOverrideMap | null +): boolean | null { + if (bulk) return bulk.get(providerId)?.get(modelId) ?? null; + const row = getDbInstance() + .prepare("SELECT value FROM key_value WHERE namespace = 'customModels' AND key = ?") + .get(providerId); + return readVisionOverrideFromModels(getKeyValue(row).value, modelId); +} + +/** Bulk-load explicit custom-model vision overrides with one SQLite query. */ +export function listCustomModelVisionOverrides(): CustomModelVisionOverrideMap { + const rows = getDbInstance() + .prepare("SELECT key, value FROM key_value WHERE namespace = 'customModels'") + .all(); + const result = new Map>(); + for (const row of rows) { + const { key, value } = getKeyValue(row); + if (!key || !value) continue; + try { + const models = JSON.parse(value) as unknown; + if (!Array.isArray(models)) continue; + const byModel = new Map(); + for (const candidate of models) { + if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) continue; + const { id, supportsVision } = candidate as { + id?: unknown; + supportsVision?: unknown; + }; + if (typeof id === "string" && typeof supportsVision === "boolean") { + byModel.set(id, supportsVision); + } + } + if (byModel.size > 0) result.set(key, byModel); + } catch { + // Malformed custom-model rows do not participate in capability resolution. + } + } + return result; +} + export async function addCustomModel( providerId: string, modelId: string, diff --git a/src/lib/guardrails/modalityBridge/bridgeCache.ts b/src/lib/guardrails/modalityBridge/bridgeCache.ts index f4efb797fa..2158638a0f 100644 --- a/src/lib/guardrails/modalityBridge/bridgeCache.ts +++ b/src/lib/guardrails/modalityBridge/bridgeCache.ts @@ -27,12 +27,22 @@ export interface BridgeCacheOptions { now?: () => number; } +export interface BridgeCacheEntry { + value: string; + /** Actual successful producer, which may differ from the routing-plan model after fallback. */ + producerModel?: string; +} + export class BridgeCache { - private readonly entries = new Map(); + private readonly entries = new Map(); constructor(private readonly opts: BridgeCacheOptions) {} get(key: string): string | undefined { + return this.getEntry(key)?.value; + } + + getEntry(key: string): BridgeCacheEntry | undefined { const hit = this.entries.get(key); if (!hit) return undefined; const now = (this.opts.now ?? Date.now)(); @@ -43,13 +53,17 @@ export class BridgeCache { // Map preserves insertion order — re-insert to mark as most-recently-used. this.entries.delete(key); this.entries.set(key, hit); - return hit.value; + return hit.entry; } set(key: string, value: string): void { + this.setEntry(key, { value }); + } + + setEntry(key: string, entry: BridgeCacheEntry): void { const now = (this.opts.now ?? Date.now)(); this.entries.delete(key); - this.entries.set(key, { value, expiresAt: now + this.opts.ttlMs }); + this.entries.set(key, { entry, expiresAt: now + this.opts.ttlMs }); while (this.entries.size > this.opts.maxEntries) { const oldest = this.entries.keys().next().value; if (oldest === undefined) break; diff --git a/src/lib/guardrails/modalityBridge/bridgeStats.ts b/src/lib/guardrails/modalityBridge/bridgeStats.ts index 6f07f1e247..c447043860 100644 --- a/src/lib/guardrails/modalityBridge/bridgeStats.ts +++ b/src/lib/guardrails/modalityBridge/bridgeStats.ts @@ -18,6 +18,7 @@ export interface BridgeModalityStats { cacheHits: number; failures: number; lastUsedAt: string | null; + latencySamples: number; successes: number; totalLatencyMs: number; } @@ -38,6 +39,7 @@ function emptyStats(): BridgeModalityStats { cacheHits: 0, failures: 0, lastUsedAt: null, + latencySamples: 0, successes: 0, totalLatencyMs: 0, }; @@ -64,8 +66,9 @@ export function recordBridgeUse( s.cacheHits += cacheHits; if (typeof opts.latencyMs === "number" && Number.isFinite(opts.latencyMs)) { s.totalLatencyMs += Math.max(0, opts.latencyMs); + s.latencySamples += 1; } - s.averageLatencyMs = s.attempts > 0 ? s.totalLatencyMs / s.attempts : 0; + s.averageLatencyMs = s.latencySamples > 0 ? s.totalLatencyMs / s.latencySamples : 0; s.lastUsedAt = new Date().toISOString(); } diff --git a/src/lib/guardrails/videoBridge.ts b/src/lib/guardrails/videoBridge.ts index 6309df0f3a..ed9ecc6747 100644 --- a/src/lib/guardrails/videoBridge.ts +++ b/src/lib/guardrails/videoBridge.ts @@ -30,6 +30,12 @@ type VideoBridgeBody = { [key: string]: unknown; }; +function combineModelIdentities(models: ReadonlySet, fallback: string): string { + if (models.size === 0) return fallback; + if (models.size === 1) return models.values().next().value ?? fallback; + return "mixed"; +} + export interface VideoBridgeDependencies { getSettings?: () => Promise>; getCapabilities?: (model: string) => { supportsVideo: boolean | null }; @@ -83,7 +89,8 @@ export class VideoBridgeGuardrail extends BaseGuardrail { const visionRuntime = resolveVisionBridgeRuntimeSettings(persisted); const configuredModel = runtime.model.trim() || visionRuntime.model.trim(); - let effectiveVideoModel = configuredModel || "auto"; + const routingPlanModel = configuredModel || "auto"; + const successfulModels = new Set(); let selectedModelPromise: Promise | null = null; const selectVideoModel = (): Promise => { if (!selectedModelPromise) { @@ -119,7 +126,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { context.signal ); if (context.signal?.aborted) throw new Error("Video Bridge processing was aborted"); - if (described.modelUsed) effectiveVideoModel = described.modelUsed; + if (described.modelUsed) successfulModels.add(described.modelUsed); const videoCacheHits = described.cacheHits ?? 0; descriptions.push(described.description); totalFramesRequested += described.framesRequested; @@ -181,7 +188,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { framesUsed: totalFramesUsed, processingTimeMs: Date.now() - startedAt, attempts: attemptedParts.length, - videoModel: effectiveVideoModel, + videoModel: combineModelIdentities(successfulModels, routingPlanModel), videosProcessed, videosReplaced, }, @@ -201,6 +208,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { const cache = runtime.cacheEnabled ? getSharedBridgeCacheFor(runtime) : null; const callVisionModel = this.deps.callVisionModel ?? defaultCallVisionModel; let cacheHits = 0; + const successfulModels = new Set(); const described = await defaultDescribeVideoPart( part, { @@ -213,23 +221,33 @@ export class VideoBridgeGuardrail extends BaseGuardrail { const key = cache ? bridgeCacheKey(frameDataUri, `${prompt}@${timestampSeconds.toFixed(3)}`, selectedModel) : null; - const cached = key && cache ? cache.get(key) : undefined; - if (cached !== undefined) { + const cached = key && cache ? cache.getEntry(key) : undefined; + if (cached) { cacheHits += 1; - return cached; + successfulModels.add(cached.producerModel ?? selectedModel); + return cached.value; } + let producerModel = selectedModel; const caption = await callVisionModel(frameDataUri, { maxImages: 1, model: selectedModel, + onModelUsed: (model) => { + producerModel = model; + }, prompt, signal, timeoutMs: runtime.timeoutMs, }); - if (key && cache) cache.set(key, caption); + successfulModels.add(producerModel); + if (key && cache) cache.setEntry(key, { value: caption, producerModel }); return caption; }, { extractFrames: this.deps.extractFrames } ); - return { ...described, cacheHits, modelUsed: selectedModel }; + return { + ...described, + cacheHits, + modelUsed: combineModelIdentities(successfulModels, selectedModel), + }; } } diff --git a/src/lib/guardrails/visionBridgeHelpers.ts b/src/lib/guardrails/visionBridgeHelpers.ts index fbc8e06847..4d2f6e1804 100644 --- a/src/lib/guardrails/visionBridgeHelpers.ts +++ b/src/lib/guardrails/visionBridgeHelpers.ts @@ -350,6 +350,8 @@ export interface VisionModelConfig { signal?: AbortSignal; /** Injectable fetch (tests). Defaults to undici fetch to bypass the runtime's hooked global fetch. */ fetchImpl?: typeof fetch; + /** Receives the actual successful model while the public return value remains a string. */ + onModelUsed?: (model: string) => void; } /** Task-aware focus hint (codex-vision-proxy pattern): steer the description @@ -416,6 +418,11 @@ export async function callVisionModel( apiKey ); recordLatency(currentModel, Date.now() - attemptStart, true); + try { + config.onModelUsed?.(currentModel); + } catch { + // Observability callbacks must never turn a successful caption into a retry. + } return result; } catch (error) { recordLatency(currentModel, Date.now() - attemptStart, false); diff --git a/src/lib/modelCapabilities.ts b/src/lib/modelCapabilities.ts index 38acf56476..d1f87d7ab5 100644 --- a/src/lib/modelCapabilities.ts +++ b/src/lib/modelCapabilities.ts @@ -14,8 +14,7 @@ import { getSyncedCapability } from "@/lib/modelsDevSync"; import { MODELS_DEV_PROVIDER_MAP } from "@/lib/modelsDevSync/transform"; import { getModelContextOverride } from "@/lib/db/modelContextOverrides"; import { getModelCapabilityOverride } from "@/lib/db/modelCapabilityOverrides"; -import { getDbInstance } from "@/lib/db/core"; -import { getKeyValue } from "@/lib/db/models/shared"; +import { getCustomModelVisionOverride } from "@/lib/db/models"; import type { ModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot"; import { resolveAudioCapability, resolveVideoCapability } from "@/lib/modelCapabilityModalities"; @@ -462,31 +461,6 @@ function modalitiesDeclareVision(modalities: readonly string[]): boolean { }); } -/** - * #9195: Read the customModels supportsVision override for a given provider/model - * pair from the database. Returns true/false when an explicit override exists, or - * null if no custom model entry or no explicit flag. Sync read (better-sqlite3). - */ -function getCustomModelVisionOverride(provider: string, model: string): boolean | null { - try { - const db = getDbInstance(); - const row = db - .prepare("SELECT value FROM key_value WHERE namespace = 'customModels' AND key = ?") - .get(provider); - if (!row) return null; - const parsed = getKeyValue(row); - if (!parsed.value) return null; - const models: Array<{ id: string; supportsVision?: boolean }> = JSON.parse(parsed.value); - const entry = models.find((m) => m.id === model); - if (entry && typeof entry.supportsVision === "boolean") { - return entry.supportsVision; - } - return null; - } catch { - return null; - } -} - function resolveVisionCapability( spec: ModelSpec | undefined, registryModel: { supportsVision?: boolean } | null, @@ -782,7 +756,11 @@ export function getResolvedModelCapabilities( // dashboard "Vision capable" toggle affects Combo routing. const customVisionOverride = resolved.provider && resolved.model - ? getCustomModelVisionOverride(resolved.provider, resolved.model) + ? getCustomModelVisionOverride( + resolved.provider, + resolved.model, + snapshot?.customVisionOverrides + ) : null; const supportsVision = resolveVisionCapability( diff --git a/src/lib/modelCapabilityResolutionSnapshot.ts b/src/lib/modelCapabilityResolutionSnapshot.ts index 1ef49e680f..b5724a62bb 100644 --- a/src/lib/modelCapabilityResolutionSnapshot.ts +++ b/src/lib/modelCapabilityResolutionSnapshot.ts @@ -1,7 +1,7 @@ /** * Build-local capability/context/override resolution snapshot (#9199). * - * Catalog preparation bulk-loads the three capability tables once into a + * Catalog preparation bulk-loads the capability and custom-model tables once into a * build-local view for pure in-memory resolution. This must not flip models.dev's * module-global all-row cache, and ordinary runtime callers keep on-demand DB reads. * @@ -10,6 +10,7 @@ */ import { listModelCapabilityOverrides } from "@/lib/db/modelCapabilityOverrides"; import { listModelContextOverrides } from "@/lib/db/modelContextOverrides"; +import { listCustomModelVisionOverrides, type CustomModelVisionOverrideMap } from "@/lib/db/models"; import { loadAllSyncedCapabilitiesUncached, type CapabilitiesByProvider, @@ -23,6 +24,7 @@ export interface ModelCapabilityResolutionSnapshot { readonly maxTokenOverrides: NestedOverrideMap; readonly maxInputTokenOverrides: NestedOverrideMap; readonly contextOverrides: NestedOverrideMap; + readonly customVisionOverrides: CustomModelVisionOverrideMap; } function setNestedOverride( @@ -40,7 +42,7 @@ function setNestedOverride( } /** - * Load all three capability tables in one uninterrupted JS turn. + * Load all capability/custom-model tables in one uninterrupted JS turn. * Callers must not yield between the bulk reads if they need a coherent view; * existing catalog generation guards remain authoritative across later yields. */ @@ -67,5 +69,6 @@ export function createModelCapabilityResolutionSnapshot(): ModelCapabilityResolu maxTokenOverrides, maxInputTokenOverrides, contextOverrides, + customVisionOverrides: listCustomModelVisionOverrides(), }; } diff --git a/tests/unit/api-models-hide-paid-6328.test.ts b/tests/unit/api-models-hide-paid-6328.test.ts index 81cf702458..e9b3becdd1 100644 --- a/tests/unit/api-models-hide-paid-6328.test.ts +++ b/tests/unit/api-models-hide-paid-6328.test.ts @@ -38,15 +38,49 @@ test.after(() => { test("/api/models retains genuine resolved vision capability for the Video Bridge picker", async () => { await settingsDb.updateSettings({ hidePaidModels: false }); - const models = await fetchModels(); + const db = core.getDbInstance(); + db.prepare( + "INSERT OR REPLACE INTO key_value (namespace, key, value) " + "VALUES ('customModels', ?, ?)" + ).run("openai", JSON.stringify([{ id: "gpt-4o", supportsVision: false }])); + const originalPrepare = db.prepare; + const callPrepare = originalPrepare.bind(db); + let customModelReads = 0; + (db as unknown as { prepare: typeof db.prepare }).prepare = ((sql: string) => { + const normalized = String(sql).replace(/\s+/g, " ").trim(); + if (normalized.includes("FROM key_value WHERE namespace = 'customModels'")) { + customModelReads++; + } + return callPrepare(sql); + }) as typeof db.prepare; + + let models: Awaited>; + try { + models = await fetchModels(); + } finally { + (db as unknown as { prepare: typeof db.prepare }).prepare = originalPrepare; + } const vision = models.find( (model) => model.provider === "openai" && model.model === "gpt-4o-mini" ); const textOnly = models.find((model) => model.provider === "deepgram"); + const explicitlyDowngraded = models.find( + (model) => model.provider === "openai" && model.model === "gpt-4o" + ); assert.ok(vision, "known static vision model must be present in the real producer response"); assert.equal(vision.supportsVision, true); if (textOnly) assert.notEqual(textOnly.supportsVision, true); + assert.ok(explicitlyDowngraded, "custom-overridden model must remain in the real producer"); + assert.equal( + explicitlyDowngraded.supportsVision, + false, + "request snapshot must preserve the explicit custom supportsVision override" + ); + assert.equal( + customModelReads, + 1, + "one request-level capability snapshot must bulk-read custom vision overrides once" + ); }); test("#6328 /api/models removes paid models when hidePaidModels is on", async () => { diff --git a/tests/unit/guardrails/videoBridge.test.ts b/tests/unit/guardrails/videoBridge.test.ts index e3329ddab0..4ba5d336f4 100644 --- a/tests/unit/guardrails/videoBridge.test.ts +++ b/tests/unit/guardrails/videoBridge.test.ts @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import test from "node:test"; import { VideoBridgeGuardrail } from "../../../src/lib/guardrails/videoBridge.ts"; +import { callVisionModel } from "../../../src/lib/guardrails/visionBridgeHelpers.ts"; import { buildModalityBridgeHeader, getBridgeStats, @@ -339,6 +340,60 @@ test("real Video Bridge cache hit avoids a second model call and records the hit assert.equal(second.meta?.cacheHits, 1); }); +test("real primary failure reports and caches the successful fallback model identity", async () => { + const primary = "openai/gpt-4o-mini"; + const fallback = "anthropic/claude-fable-5"; + const attemptedModels: string[] = []; + const fetchImpl: typeof fetch = async (_input, init) => { + const body = JSON.parse(String(init?.body)) as { model: string }; + attemptedModels.push(body.model); + if (body.model === "gpt-4o-mini") { + return new Response("primary unavailable", { status: 503 }); + } + return Response.json({ content: [{ type: "text", text: "fallback observation" }] }); + }; + const bridge = new VideoBridgeGuardrail({ + deps: { + getSettings: async () => ({ + modalityBridgeVideoEnabled: true, + modalityBridgeVideoModel: primary, + modalityBridgeVisionPrompt: "fallback identity integration 9760", + modalityBridgeCacheEnabled: true, + modalityBridgeCacheTtlMinutes: 62, + modalityBridgeCacheMaxEntries: 52, + }), + getCapabilities: () => ({ supportsVideo: false }), + selectVisionModel: async () => primary, + extractFrames: async () => ({ + durationSeconds: 1, + frames: [{ timestampSeconds: 0.5, dataUri: "data:image/jpeg;base64,FALLBACK9760" }], + }), + callVisionModel: (image, config) => + callVisionModel( + image, + { ...config, fetchImpl }, + "sk-fallback-test", + { maxFallbackAttempts: 2 }, + { + hasUsableCredentials: async (model) => model === primary || model === fallback, + } + ), + }, + }); + + const first = await bridge.preCall(payload(), {}); + const second = await bridge.preCall(payload(), {}); + + assert.deepEqual(attemptedModels, ["gpt-4o-mini", "claude-fable-5"]); + assert.equal(first.meta?.videoModel, fallback, "meta must name the successful fallback"); + assert.equal(second.meta?.videoModel, fallback, "cache hit must retain the producer identity"); + assert.equal(second.meta?.cacheHits, 1); + assert.equal( + buildModalityBridgeHeader([{ guardrail: "video-bridge", meta: second.meta }]), + `video->text;model=${fallback};parts=1` + ); +}); + test("cache keys miss on timestamp, prompt, and effective model changes; failures are not cached", async () => { let timestamp = 0.25; let prompt = "prompt-a-9760"; diff --git a/tests/unit/ui/modality-bridge-audio-tab.test.tsx b/tests/unit/ui/modality-bridge-audio-tab.test.tsx index 3b6691024b..0cf884f724 100644 --- a/tests/unit/ui/modality-bridge-audio-tab.test.tsx +++ b/tests/unit/ui/modality-bridge-audio-tab.test.tsx @@ -117,6 +117,10 @@ describe("ModalityBridgeAudioTab", () => { expect(optionValues).not.toContain("deepgram/aura"); expect(optionValues).not.toContain("openai/gpt-5.6"); expect(el.textContent).toContain("4 modalityBridgeStatsBridged"); + expect(el.textContent).toContain("5 requestlogger.attempts"); + expect(el.textContent).toContain("trafficInspector.timingTotalLatency: —"); + expect(el.textContent).toContain("avgLatency: —"); + expect(el.textContent).not.toContain("0 ms"); }); it("PATCHes only the new audio setting when toggled", async () => { diff --git a/tests/unit/ui/modality-bridge-vision-tab.test.tsx b/tests/unit/ui/modality-bridge-vision-tab.test.tsx index 3c4c4e920d..3cccb0b14a 100644 --- a/tests/unit/ui/modality-bridge-vision-tab.test.tsx +++ b/tests/unit/ui/modality-bridge-vision-tab.test.tsx @@ -154,6 +154,10 @@ describe("ModalityBridgeVisionTab", () => { await waitFor(() => el.textContent?.includes("3 modalityBridgeStatsBridged") ?? false, "stats"); expect(el.textContent).toContain("1 modalityBridgeStatsCacheHits"); + expect(el.textContent).toContain("3 requestlogger.attempts"); + expect(el.textContent).toContain("trafficInspector.timingTotalLatency: —"); + expect(el.textContent).toContain("avgLatency: —"); + expect(el.textContent).not.toContain("0 ms"); }); it("clamps advanced numeric settings to the schema bounds before PATCHing", async () => { diff --git a/tests/unit/video-bridge-header-stats.test.ts b/tests/unit/video-bridge-header-stats.test.ts index be8507b96a..67c5b902e8 100644 --- a/tests/unit/video-bridge-header-stats.test.ts +++ b/tests/unit/video-bridge-header-stats.test.ts @@ -41,10 +41,22 @@ test("tracks attempts, successes, failures, cache hits, and latency without coun assert.equal(after.cacheHits - before.cacheHits, 2); assert.equal(after.failures - before.failures, 1); assert.equal(after.totalLatencyMs - before.totalLatencyMs, 200); - assert.equal(after.averageLatencyMs, after.totalLatencyMs / after.attempts); + assert.equal(after.latencySamples - before.latencySamples, 2); + assert.equal(after.averageLatencyMs, after.totalLatencyMs / after.latencySamples); assert.match(after.lastUsedAt ?? "", /^\d{4}-\d{2}-\d{2}T/); }); +test("Vision and Audio attempts without timing do not fabricate zero-millisecond samples", () => { + for (const kind of ["vision", "audio"] as const) { + const before = getBridgeStats()[kind]; + recordBridgeUse(kind, { cacheHit: true }); + const after = getBridgeStats()[kind]; + assert.equal(after.attempts - before.attempts, 1); + assert.equal(after.latencySamples, before.latencySamples); + assert.equal(after.totalLatencyMs, before.totalLatencyMs); + } +}); + test("video header is omitted when every attempted video failed", () => { assert.equal( buildModalityBridgeHeader([