From e7858d716573449f9cf63e7485741a0fbd400e27 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Tue, 18 Aug 2026 06:08:55 -0300 Subject: [PATCH] feat(video): cap the drill-down cache with a global byte budget The per-session drill-down cache now tracks decoded bytes per entry and evicts least-recently-used entries until an aggregate maxTotalBytes budget fits (route sets 256 MiB); an entry larger than the whole budget is rejected. Prevents the previous worst case of 64 x 32 MiB (~2 GiB) pinned in memory. --- .../modality-bridge/video/drilldown/route.ts | 2 + src/lib/guardrails/videoBridgeDrilldown.ts | 50 ++++++++++++++++--- .../guardrails/videoBridgeDrilldown.test.ts | 49 ++++++++++++++++++ 3 files changed, 93 insertions(+), 8 deletions(-) diff --git a/src/app/api/modality-bridge/video/drilldown/route.ts b/src/app/api/modality-bridge/video/drilldown/route.ts index b0ec1edaac..ba89c3053d 100644 --- a/src/app/api/modality-bridge/video/drilldown/route.ts +++ b/src/app/api/modality-bridge/video/drilldown/route.ts @@ -16,6 +16,8 @@ export const VIDEO_BRIDGE_DRILLDOWN_PATH = "/api/modality-bridge/video/drilldown const MAX_BODY_BYTES = 34 * 1024 * 1024; const drilldownCache = new VideoDrilldownCache({ maxEntries: 64, + // Global decoded-byte ceiling: without it, 64 entries × 32 MiB could pin ~2 GiB. + maxTotalBytes: 256 * 1024 * 1024, ttlMs: 10 * 60 * 1000, }); diff --git a/src/lib/guardrails/videoBridgeDrilldown.ts b/src/lib/guardrails/videoBridgeDrilldown.ts index f0543e3b5b..330da73a46 100644 --- a/src/lib/guardrails/videoBridgeDrilldown.ts +++ b/src/lib/guardrails/videoBridgeDrilldown.ts @@ -20,11 +20,14 @@ export interface VideoDrilldownResult { export interface VideoDrilldownCacheOptions { maxEntries: number; + /** Aggregate decoded-byte budget across every entry; oldest entries are evicted (LRU) to fit. */ + maxTotalBytes?: number; now?: () => number; ttlMs: number; } interface StoredDrilldown extends VideoDrilldownPutValue { + bytes: number; expiresAt: number; sessionId: string; } @@ -37,7 +40,10 @@ function cacheKey(sessionId: string, videoRef: string): string { return createHash("sha256").update(`${sessionId}\0${videoRef}`).digest("hex"); } -function validateFrames(value: VideoDrilldownPutValue): VideoDrilldownFrame[] { +function validateFrames(value: VideoDrilldownPutValue): { + frames: VideoDrilldownFrame[]; + totalBytes: number; +} { if ( !Number.isFinite(value.durationSeconds) || value.durationSeconds <= 0 || @@ -67,12 +73,16 @@ function validateFrames(value: VideoDrilldownPutValue): VideoDrilldownFrame[] { if (totalBytes > MAX_TOTAL_BYTES) throw new Error("Drill-down response byte limit exceeded"); return { dataUri: frame.dataUri, timestampSeconds: frame.timestampSeconds }; }); - return frames.sort((left, right) => left.timestampSeconds - right.timestampSeconds); + return { + frames: frames.sort((left, right) => left.timestampSeconds - right.timestampSeconds), + totalBytes, + }; } export class VideoDrilldownCache { private readonly entries = new Map(); private readonly now: () => number; + private totalBytes = 0; constructor(private readonly options: VideoDrilldownCacheOptions) { if (!Number.isFinite(options.ttlMs) || options.ttlMs <= 0) { @@ -81,24 +91,47 @@ export class VideoDrilldownCache { if (!Number.isInteger(options.maxEntries) || options.maxEntries < 1) { throw new Error("Drill-down cache entry limit is invalid"); } + if ( + options.maxTotalBytes !== undefined && + (!Number.isInteger(options.maxTotalBytes) || options.maxTotalBytes < 1) + ) { + throw new Error("Drill-down cache byte budget is invalid"); + } this.now = options.now ?? Date.now; } + private drop(key: string): void { + const stored = this.entries.get(key); + if (!stored) return; + this.entries.delete(key); + this.totalBytes -= stored.bytes; + } + put(sessionId: string, videoRef: string, value: VideoDrilldownPutValue): void { if (!sessionId || sessionId.length > 128 || !videoRef || videoRef.length > 4096) { throw new Error("Drill-down cache key is invalid"); } + const { frames, totalBytes } = validateFrames(value); + if (this.options.maxTotalBytes !== undefined && totalBytes > this.options.maxTotalBytes) { + throw new Error("Drill-down entry exceeds the cache byte budget"); + } const key = cacheKey(sessionId, videoRef); - this.entries.delete(key); + this.drop(key); this.entries.set(key, { + bytes: totalBytes, durationSeconds: value.durationSeconds, expiresAt: this.now() + this.options.ttlMs, - frames: validateFrames(value), + frames, sessionId, }); - while (this.entries.size > this.options.maxEntries) { + this.totalBytes += totalBytes; + while ( + this.entries.size > this.options.maxEntries || + (this.options.maxTotalBytes !== undefined && this.totalBytes > this.options.maxTotalBytes) + ) { const oldest = this.entries.keys().next().value; - if (oldest) this.entries.delete(oldest); + if (!oldest || oldest === key) break; + this.drop(oldest); } } @@ -111,7 +144,7 @@ export class VideoDrilldownCache { const stored = this.entries.get(key); if (!stored) return null; if (stored.expiresAt <= this.now()) { - this.entries.delete(key); + this.drop(key); return null; } this.entries.delete(key); @@ -158,7 +191,7 @@ export class VideoDrilldownCache { let removed = 0; for (const [key, entry] of this.entries.entries()) { if (entry.sessionId === sessionId) { - this.entries.delete(key); + this.drop(key); removed += 1; } } @@ -167,5 +200,6 @@ export class VideoDrilldownCache { clearAll(): void { this.entries.clear(); + this.totalBytes = 0; } } diff --git a/tests/unit/guardrails/videoBridgeDrilldown.test.ts b/tests/unit/guardrails/videoBridgeDrilldown.test.ts index 4f19afea5e..054447d9cb 100644 --- a/tests/unit/guardrails/videoBridgeDrilldown.test.ts +++ b/tests/unit/guardrails/videoBridgeDrilldown.test.ts @@ -60,3 +60,52 @@ test("drill-down cache expires entries and evicts the least recently used key", now = 7000; assert.equal(cache.get("session-b", "video"), null); }); + +test("drill-down cache enforces a global byte budget with LRU eviction", () => { + const bigFrame = (fill: string): VideoDrilldownFrame => ({ + dataUri: `data:image/jpeg;base64,${fill.repeat(4000)}`, + timestampSeconds: 1, + }); + // Each entry is ~3000 decoded bytes; the budget fits two entries. + const cache = new VideoDrilldownCache({ + now: () => 1000, + ttlMs: 5000, + maxEntries: 10, + maxTotalBytes: 7000, + }); + cache.put("s", "v1", { durationSeconds: 10, frames: [bigFrame("A")] }); + cache.put("s", "v2", { durationSeconds: 10, frames: [bigFrame("B")] }); + assert.ok(cache.get("s", "v1")); + assert.ok(cache.get("s", "v2")); + cache.put("s", "v3", { durationSeconds: 10, frames: [bigFrame("C")] }); + assert.equal(cache.get("s", "v1"), null, "the least recently used entry must be evicted"); + assert.ok(cache.get("s", "v2")); + assert.ok(cache.get("s", "v3")); + assert.ok(cache.get("s", "v2")); + cache.put("s", "v4", { durationSeconds: 10, frames: [bigFrame("D")] }); + assert.equal(cache.get("s", "v3"), null, "eviction must follow recency, not insertion order"); + assert.ok(cache.get("s", "v2")); + assert.ok(cache.get("s", "v4")); +}); + +test("drill-down cache rejects an entry larger than the whole byte budget", () => { + const cache = new VideoDrilldownCache({ + now: () => 1000, + ttlMs: 5000, + maxEntries: 4, + maxTotalBytes: 1000, + }); + assert.throws( + () => + cache.put("s", "v1", { + durationSeconds: 10, + frames: [{ dataUri: `data:image/jpeg;base64,${"A".repeat(4000)}`, timestampSeconds: 1 }], + }), + /byte budget/i + ); + assert.equal(cache.get("s", "v1"), null); + assert.throws( + () => new VideoDrilldownCache({ now: () => 0, ttlMs: 1, maxEntries: 1, maxTotalBytes: 0 }), + /byte budget/i + ); +});