From 91ea94fb508487d43072a7eaab9f2754f93f8861 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Tue, 18 Aug 2026 00:18:38 -0300 Subject: [PATCH] feat(video): cache full video-bridge results with metadata --- src/lib/guardrails/videoBridge.ts | 140 +++++++++++++++++++++- tests/unit/guardrails/videoBridge.test.ts | 30 +++-- 2 files changed, 157 insertions(+), 13 deletions(-) diff --git a/src/lib/guardrails/videoBridge.ts b/src/lib/guardrails/videoBridge.ts index 2a6de89191..9eb4b0098f 100644 --- a/src/lib/guardrails/videoBridge.ts +++ b/src/lib/guardrails/videoBridge.ts @@ -38,6 +38,28 @@ function combineModelIdentities(models: ReadonlySet, fallback: string): return "mixed"; } +const VIDEO_BRIDGE_RESULT_CACHE_VERSION = "v2"; +const VIDEO_BRIDGE_RESULT_CACHE_POLICY = "default"; +const VIDEO_BRIDGE_RESULT_CACHE_STRATEGY = "uniform"; +const VIDEO_BRIDGE_RESULT_CACHE_KEY_KIND = "video-result-v2"; + +interface VideoResultCacheMetadata { + cacheVersion: string; + policyVersion: string; + extractorVersion: string; + strategy: string; + model: string; + prompt: string; + frameCount: number; + maxVideos: number; + durationSeconds: number; + framesRequested: number; + framesExtracted: number; + framesUsed: number; + cacheBytes: number; + modelUsed: string; +} + export interface VideoBridgeDependencies { getSettings?: () => Promise>; getCapabilities?: (model: string) => { supportsVideo: boolean | null }; @@ -51,6 +73,27 @@ export interface VideoBridgeDependencies { ) => Promise; } +function isVideoResultCacheMetadata(value: unknown): value is VideoResultCacheMetadata { + if (!value || typeof value !== "object") return false; + const record = value as Record; + return ( + typeof record.cacheVersion === "string" && + typeof record.policyVersion === "string" && + typeof record.extractorVersion === "string" && + typeof record.strategy === "string" && + typeof record.model === "string" && + typeof record.prompt === "string" && + typeof record.frameCount === "number" && + typeof record.maxVideos === "number" && + typeof record.durationSeconds === "number" && + typeof record.framesRequested === "number" && + typeof record.framesExtracted === "number" && + typeof record.framesUsed === "number" && + typeof record.cacheBytes === "number" && + typeof record.modelUsed === "string" + ); +} + export class VideoBridgeGuardrail extends BaseGuardrail { name = "video-bridge"; priority = 7; @@ -92,6 +135,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { const visionRuntime = resolveVisionBridgeRuntimeSettings(persisted); const configuredModel = runtime.model.trim() || visionRuntime.model.trim(); const routingPlanModel = configuredModel || "auto"; + const cache = runtime.cacheEnabled ? getSharedBridgeCacheFor(runtime) : null; const successfulModels = new Set(); let selectedModelPromise: Promise | null = null; const selectVideoModel = (): Promise => { @@ -115,31 +159,115 @@ export class VideoBridgeGuardrail extends BaseGuardrail { const attemptedParts = parts.slice(0, runtime.maxVideos); for (let index = 0; index < attemptedParts.length; index++) { if (context.signal?.aborted) throw new Error("Video Bridge processing was aborted"); - const part = parts[index]; + const part = attemptedParts[index]; const attemptStartedAt = Date.now(); try { + const selectedModel = await selectVideoModel(); + const resultCacheKey = + cache && selectedModel + ? bridgeCacheKey(part.ref, visionRuntime.prompt, selectedModel, { + kind: VIDEO_BRIDGE_RESULT_CACHE_KEY_KIND, + extractorVersion: VIDEO_BRIDGE_RESULT_CACHE_VERSION, + policyVersion: VIDEO_BRIDGE_RESULT_CACHE_POLICY, + strategy: VIDEO_BRIDGE_RESULT_CACHE_STRATEGY, + frameCount: runtime.frameCount, + maxVideos: runtime.maxVideos, + version: VIDEO_BRIDGE_RESULT_CACHE_VERSION, + }) + : null; + const cachedResult = resultCacheKey ? cache.getEntry(resultCacheKey) : null; + if (cachedResult && isVideoResultCacheMetadata(cachedResult.metadata)) { + const meta = cachedResult.metadata; + const matchPolicy = + meta.cacheVersion === VIDEO_BRIDGE_RESULT_CACHE_VERSION && + meta.policyVersion === VIDEO_BRIDGE_RESULT_CACHE_POLICY && + meta.extractorVersion === VIDEO_BRIDGE_RESULT_CACHE_VERSION && + meta.strategy === VIDEO_BRIDGE_RESULT_CACHE_STRATEGY && + meta.frameCount === runtime.frameCount && + meta.maxVideos === runtime.maxVideos && + meta.model === selectedModel && + meta.prompt === visionRuntime.prompt; + if (matchPolicy) { + const elapsed = Date.now() - attemptStartedAt; + descriptions.push(cachedResult.value); + totalFramesRequested += meta.framesRequested; + totalFramesExtracted += meta.framesExtracted; + totalFramesUsed += meta.framesUsed; + totalDurationSeconds += meta.durationSeconds; + if (cachedResult.producerModel) { + successfulModels.add(cachedResult.producerModel); + } + if (meta.modelUsed) { + successfulModels.add(meta.modelUsed); + } + recordBridgeUse("video", { + latencyMs: elapsed, + resultCacheHit: true, + resultCacheBytes: meta.cacheBytes, + resultCacheLatencyMs: elapsed, + }); + continue; + } + cache.delete(resultCacheKey); + } else if (cachedResult) { + cache.delete(resultCacheKey); + } + const cacheStartAt = Date.now(); const described = this.deps.describePart ? await this.deps.describePart(part) : await this.describeWithVisionModel( part, runtime, visionRuntime, - await selectVideoModel(), + selectedModel, context.signal ); if (context.signal?.aborted) throw new Error("Video Bridge processing was aborted"); if (described.modelUsed) successfulModels.add(described.modelUsed); const videoCacheHits = described.cacheHits ?? 0; + const processingLatencyMs = Date.now() - attemptStartedAt; descriptions.push(described.description); totalFramesRequested += described.framesRequested; totalFramesExtracted += described.framesExtracted ?? described.framesUsed; totalFramesUsed += described.framesUsed; totalDurationSeconds += described.durationSeconds; totalCacheHits += videoCacheHits; - recordBridgeUse("video", { - cacheHits: videoCacheHits, - latencyMs: Date.now() - attemptStartedAt, - }); + if (resultCacheKey && selectedModel) { + const resultCacheBytes = Buffer.byteLength(described.description, "utf8"); + const cacheLatencyMs = Date.now() - cacheStartAt; + cache.setEntry(resultCacheKey, { + value: described.description, + producerModel: described.modelUsed ?? selectedModel, + metadata: { + cacheVersion: VIDEO_BRIDGE_RESULT_CACHE_VERSION, + policyVersion: VIDEO_BRIDGE_RESULT_CACHE_POLICY, + extractorVersion: VIDEO_BRIDGE_RESULT_CACHE_VERSION, + strategy: VIDEO_BRIDGE_RESULT_CACHE_STRATEGY, + model: selectedModel, + prompt: visionRuntime.prompt, + frameCount: runtime.frameCount, + maxVideos: runtime.maxVideos, + durationSeconds: described.durationSeconds, + framesRequested: described.framesRequested, + framesExtracted: described.framesExtracted ?? described.framesUsed, + framesUsed: described.framesUsed, + cacheBytes: resultCacheBytes, + modelUsed: described.modelUsed ?? selectedModel, + }, + }); + recordBridgeUse("video", { + cacheHits: videoCacheHits, + latencyMs: processingLatencyMs, + resultCacheBytes, + resultCacheHit: false, + resultCacheLatencyMs: cacheLatencyMs, + }); + } else { + recordBridgeUse("video", { + cacheHits: videoCacheHits, + latencyMs: processingLatencyMs, + }); + } } catch (error) { if (context.signal?.aborted) throw new Error("Video Bridge processing was aborted"); failures += 1; diff --git a/tests/unit/guardrails/videoBridge.test.ts b/tests/unit/guardrails/videoBridge.test.ts index 0fcf2172df..7206e7d86c 100644 --- a/tests/unit/guardrails/videoBridge.test.ts +++ b/tests/unit/guardrails/videoBridge.test.ts @@ -317,6 +317,7 @@ test("client abort between videos stops processing and never stubs or falls back test("real Video Bridge cache hit avoids a second model call and records the hit", async () => { let modelCalls = 0; + const beforeStats = getBridgeStats().video; const bridge = new VideoBridgeGuardrail({ deps: { getSettings: async () => ({ @@ -343,12 +344,22 @@ test("real Video Bridge cache hit avoids a second model call and records the hit const second = await bridge.preCall(payload(), {}); assert.equal(modelCalls, 1); assert.equal(first.meta?.cacheHits, 0); - assert.equal(second.meta?.cacheHits, 1); + assert.equal(second.meta?.cacheHits, 0); + const afterStats = getBridgeStats().video; + const firstTextPart = (first.modifiedPayload as ReturnType).messages[0] + .content[0]; + assert.equal(afterStats.resultCacheHits - beforeStats.resultCacheHits, 1); + assert.equal( + afterStats.resultCacheBytes - beforeStats.resultCacheBytes, + Buffer.byteLength(String((firstTextPart as { text: string }).text), "utf8") + ); + assert.equal(afterStats.resultCacheLatencyMs - beforeStats.resultCacheLatencyMs >= 0, true); }); test("real primary failure reports and caches the successful fallback model identity", async () => { const primary = "openai/gpt-4o-mini"; const fallback = "anthropic/claude-fable-5"; + const beforeStats = getBridgeStats().video; const attemptedModels: string[] = []; const fetchImpl: typeof fetch = async (_input, init) => { const body = JSON.parse(String(init?.body)) as { model: string }; @@ -393,15 +404,16 @@ test("real primary failure reports and caches the successful fallback model iden assert.deepEqual(attemptedModels, [primary, fallback]); assert.equal(first.meta?.videoModel, fallback, "meta must name the successful fallback"); assert.equal(second.meta?.videoModel, fallback, "cache hit must retain the producer identity"); - assert.equal(second.meta?.cacheHits, 1); + assert.equal(second.meta?.cacheHits, 0); + const deltaResultCacheHits = getBridgeStats().video.resultCacheHits - beforeStats.resultCacheHits; + assert.equal(deltaResultCacheHits >= 1, true); assert.equal( buildModalityBridgeHeader([{ guardrail: "video-bridge", meta: second.meta }]), `video->text;model=${fallback};parts=1` ); }); -test("cache keys miss on timestamp, prompt, and effective model changes; failures are not cached", async () => { - let timestamp = 0.25; +test("cache keys miss on prompt and effective model changes; failures are not cached", async () => { let prompt = "prompt-a-9760"; let selectedModel = "openai/gpt-4o-mini"; let modelCalls = 0; @@ -420,7 +432,7 @@ test("cache keys miss on timestamp, prompt, and effective model changes; failure selectVisionModel: async () => selectedModel, extractFrames: async () => ({ durationSeconds: 1, - frames: [{ timestampSeconds: timestamp, dataUri: "data:image/jpeg;base64,MISS9760" }], + frames: [{ timestampSeconds: 0.25, dataUri: "data:image/jpeg;base64,MISS9760" }], }), callVisionModel: async () => { modelCalls += 1; @@ -434,13 +446,17 @@ test("cache keys miss on timestamp, prompt, and effective model changes; failure fail = false; await bridge.preCall(payload(), {}); assert.equal(modelCalls, 2, "failed captions must not be cached"); - timestamp = 0.5; + const hitWithSameSettings = await bridge.preCall(payload(), {}); + assert.equal(modelCalls, 2, "result cache must reuse after a success"); + assert.equal(hitWithSameSettings.meta?.cacheHits, 0); await bridge.preCall(payload(), {}); + assert.equal(modelCalls, 2, "frame extraction options did not change on this path"); prompt = "prompt-b-9760"; await bridge.preCall(payload(), {}); + assert.equal(modelCalls, 3, "prompt changes must invalidate result cache"); selectedModel = "google/gemini-2.5-flash"; await bridge.preCall(payload(), {}); - assert.equal(modelCalls, 5); + assert.equal(modelCalls, 4, "effective model changes must invalidate result cache"); }); test("FFmpeg ENOENT is sanitized and counts only as a failed attempt, never a bridged success", async () => {