From bb22eeba8da30072fedfee5a52979b486e4117a5 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Tue, 18 Aug 2026 02:01:33 -0300 Subject: [PATCH] feat(video): add isolated drill-down cache --- docs/openapi.yaml | 77 ++++++++ docs/security/GUARDRAILS.md | 7 + .../modality-bridge/video/drilldown/route.ts | 140 ++++++++++++++ src/lib/guardrails/videoBridgeDrilldown.ts | 171 ++++++++++++++++++ .../guardrails/videoBridgeDrilldown.test.ts | 62 +++++++ .../unit/video-bridge-drilldown-route.test.ts | 66 +++++++ 6 files changed, 523 insertions(+) create mode 100644 src/app/api/modality-bridge/video/drilldown/route.ts create mode 100644 src/lib/guardrails/videoBridgeDrilldown.ts create mode 100644 tests/unit/guardrails/videoBridgeDrilldown.test.ts create mode 100644 tests/unit/video-bridge-drilldown-route.test.ts diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 7b913ae8c9..483cf8e433 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -5562,6 +5562,83 @@ paths: "504": description: Fixed 120-second broker extraction deadline exceeded + /api/modality-bridge/video/drilldown: + get: + x-loopback-only: true + tags: [System] + summary: Read a bounded Video Bridge drill-down slice + description: Internal loopback/token-authenticated lookup into a short-lived per-session frame cache. It never downloads media or starts a subprocess; start/end and frame count only select already materialized frames. + security: [] + parameters: + - in: query + name: sessionId + required: true + schema: { type: string, maxLength: 128 } + - in: query + name: videoRef + required: true + schema: { type: string, maxLength: 4096 } + - in: query + name: start + required: false + schema: { type: number, minimum: 0 } + - in: query + name: end + required: false + schema: { type: number, minimum: 0 } + - in: query + name: frames + required: false + schema: { type: integer, minimum: 1, maximum: 16 } + responses: + "200": { description: Bounded cached frame slice } + "403": { description: Trusted loopback/token identity required } + "404": { description: Drill-down session or media key was not found } + post: + x-loopback-only: true + tags: [System] + summary: Store a bounded Video Bridge drill-down result + description: Internal lifecycle operation for explicitly authorized callers. The short-lived session cache is isolated by session and media reference and does not alter the primary request cost. + security: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [sessionId, videoRef, durationSeconds, frames] + properties: + sessionId: { type: string, maxLength: 128 } + videoRef: { type: string, maxLength: 4096 } + durationSeconds: { type: number, exclusiveMinimum: 0, maximum: 600 } + frames: + type: array + minItems: 1 + maxItems: 16 + items: + type: object + required: [timestampSeconds, dataUri] + properties: + timestampSeconds: { type: number, minimum: 0 } + dataUri: { type: string, pattern: "^data:image/jpeg;base64," } + responses: + "201": { description: Drill-down result stored } + "403": { description: Trusted loopback/token identity required } + "413": { description: Payload exceeds the bounded session budget } + delete: + x-loopback-only: true + tags: [System] + summary: Delete a Video Bridge drill-down session + security: [] + parameters: + - in: query + name: sessionId + required: true + schema: { type: string, maxLength: 128 } + responses: + "200": { description: Session entries removed } + "403": { description: Trusted loopback/token identity required } + /api/cache/stats: get: tags: [System] diff --git a/docs/security/GUARDRAILS.md b/docs/security/GUARDRAILS.md index 0a7c408b71..ba1418312b 100644 --- a/docs/security/GUARDRAILS.md +++ b/docs/security/GUARDRAILS.md @@ -349,6 +349,13 @@ exact duplicates, and reports a partial result when only one side succeeds. The default Video Bridge path does not invoke speech-to-text or download a second media copy; without that explicit track, it remains video-only. +The internal `/api/modality-bridge/video/drilldown` lifecycle is a separate, +loopback/token-authenticated cache. It stores at most 16 JPEG frames per entry, +keeps entries isolated by session and video reference, expires them after ten +minutes, and supports bounded `start`/`end` reads or explicit session deletion. +It only slices materialized frames and cannot increase the cost of the primary +video request. + Frames are captioned sequentially with the configured Video model. An empty Video override inherits the Vision setting; if both are empty, the Vision auto-router selects the effective vision-capable model. Successful captions diff --git a/src/app/api/modality-bridge/video/drilldown/route.ts b/src/app/api/modality-bridge/video/drilldown/route.ts new file mode 100644 index 0000000000..b0ec1edaac --- /dev/null +++ b/src/app/api/modality-bridge/video/drilldown/route.ts @@ -0,0 +1,140 @@ +import { createErrorResponse } from "@/lib/api/errorResponse"; +import { + VIDEO_BRIDGE_BROKER_PATH, + isVideoBridgeBrokerInternalRequest, +} from "@/lib/guardrails/videoBridgeBrokerAuth"; +import { + VideoDrilldownCache, + type VideoDrilldownFrame, +} from "@/lib/guardrails/videoBridgeDrilldown"; +import { resolveModelSyncInternalBaseUrl } from "@/shared/services/modelSyncScheduler"; + +export const dynamic = "force-dynamic"; +export const revalidate = 0; + +export const VIDEO_BRIDGE_DRILLDOWN_PATH = "/api/modality-bridge/video/drilldown"; +const MAX_BODY_BYTES = 34 * 1024 * 1024; +const drilldownCache = new VideoDrilldownCache({ + maxEntries: 64, + ttlMs: 10 * 60 * 1000, +}); + +function expectedPath(): string { + const basePath = new URL(resolveModelSyncInternalBaseUrl()).pathname.replace(/\/$/, ""); + return `${basePath}${VIDEO_BRIDGE_DRILLDOWN_PATH}`; +} + +function invalid(message: string, status = 400): Response { + return createErrorResponse({ status, message, type: "invalid_request" }); +} + +function parseQuery(url: URL): { + endSeconds?: number; + frameCount?: number; + sessionId: string; + startSeconds?: number; + videoRef: string; +} | null { + const allowed = new Set(["end", "frames", "sessionId", "start", "videoRef"]); + if ([...url.searchParams.keys()].some((key) => !allowed.has(key))) return null; + const sessionId = url.searchParams.get("sessionId")?.trim() ?? ""; + const videoRef = url.searchParams.get("videoRef")?.trim() ?? ""; + if (!sessionId || !videoRef) return null; + const parseNumber = (name: string): number | undefined | null => { + const value = url.searchParams.get(name); + if (value === null) return undefined; + const parsed = Number(value); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : null; + }; + const startSeconds = parseNumber("start"); + const endSeconds = parseNumber("end"); + const rawFrameCount = url.searchParams.get("frames"); + const frameCount = + rawFrameCount === null + ? undefined + : /^\d{1,2}$/.test(rawFrameCount) && Number(rawFrameCount) >= 1 && Number(rawFrameCount) <= 16 + ? Number(rawFrameCount) + : null; + if (startSeconds === null || endSeconds === null || frameCount === null) return null; + return { endSeconds, frameCount, sessionId, startSeconds, videoRef }; +} + +interface VideoDrilldownRouteDependencies { + cache?: VideoDrilldownCache; +} + +export async function handleVideoDrilldownRequest( + request: Request, + dependencies: VideoDrilldownRouteDependencies = {} +): Promise { + const url = new URL(request.url); + if (url.pathname !== expectedPath()) return invalid("Invalid Video Bridge drill-down path", 404); + if (!isVideoBridgeBrokerInternalRequest(request, VIDEO_BRIDGE_BROKER_PATH)) { + return invalid("This endpoint requires an authenticated internal loopback request", 403); + } + const cache = dependencies.cache ?? drilldownCache; + if (request.method === "GET") { + const query = parseQuery(url); + if (!query) return invalid("Invalid Video Bridge drill-down query"); + const result = cache.get(query.sessionId, query.videoRef, query); + return result + ? Response.json(result, { headers: { "Cache-Control": "no-store" } }) + : invalid("Video Bridge drill-down result was not found", 404); + } + if (request.method === "DELETE") { + const sessionId = url.searchParams.get("sessionId")?.trim() ?? ""; + if (!sessionId || [...url.searchParams.keys()].some((key) => key !== "sessionId")) { + return invalid("A sessionId is required"); + } + return Response.json({ removed: cache.clearSession(sessionId) }); + } + if (request.method !== "POST") return invalid("Invalid Video Bridge drill-down method", 405); + if (request.headers.get("content-type")?.toLowerCase() !== "application/json") { + return invalid("Video Bridge drill-down requires application/json"); + } + const declaredLength = Number(request.headers.get("content-length")); + if (Number.isFinite(declaredLength) && declaredLength > MAX_BODY_BYTES) { + return invalid("Video Bridge drill-down payload is too large", 413); + } + let body: unknown; + try { + const bytes = await request.arrayBuffer(); + if (bytes.byteLength > MAX_BODY_BYTES) + return invalid("Video Bridge drill-down payload is too large", 413); + body = JSON.parse(Buffer.from(bytes).toString("utf8")); + } catch { + return invalid("Video Bridge drill-down payload is invalid"); + } + if (!body || typeof body !== "object") + return invalid("Video Bridge drill-down payload is invalid"); + const record = body as Record; + if ( + typeof record.sessionId !== "string" || + typeof record.videoRef !== "string" || + typeof record.durationSeconds !== "number" || + !Array.isArray(record.frames) + ) { + return invalid("Video Bridge drill-down payload is invalid"); + } + try { + cache.put(record.sessionId, record.videoRef, { + durationSeconds: record.durationSeconds, + frames: record.frames as VideoDrilldownFrame[], + }); + } catch { + return invalid("Video Bridge drill-down payload is invalid"); + } + return Response.json({ stored: true }, { status: 201, headers: { "Cache-Control": "no-store" } }); +} + +export async function POST(request: Request): Promise { + return handleVideoDrilldownRequest(request); +} + +export async function GET(request: Request): Promise { + return handleVideoDrilldownRequest(request); +} + +export async function DELETE(request: Request): Promise { + return handleVideoDrilldownRequest(request); +} diff --git a/src/lib/guardrails/videoBridgeDrilldown.ts b/src/lib/guardrails/videoBridgeDrilldown.ts new file mode 100644 index 0000000000..f0543e3b5b --- /dev/null +++ b/src/lib/guardrails/videoBridgeDrilldown.ts @@ -0,0 +1,171 @@ +import { createHash } from "node:crypto"; + +import { resolveVideoFocusWindow, type VideoFocusWindow } from "./videoBridgeRuntime"; + +export interface VideoDrilldownFrame { + dataUri: string; + timestampSeconds: number; +} + +export interface VideoDrilldownPutValue { + durationSeconds: number; + frames: readonly VideoDrilldownFrame[]; +} + +export interface VideoDrilldownResult { + durationSeconds: number; + focusWindow?: VideoFocusWindow; + frames: VideoDrilldownFrame[]; +} + +export interface VideoDrilldownCacheOptions { + maxEntries: number; + now?: () => number; + ttlMs: number; +} + +interface StoredDrilldown extends VideoDrilldownPutValue { + expiresAt: number; + sessionId: string; +} + +const MAX_FRAME_BYTES = 4 * 1024 * 1024; +const MAX_TOTAL_BYTES = 32 * 1024 * 1024; +const MAX_DURATION_SECONDS = 600; + +function cacheKey(sessionId: string, videoRef: string): string { + return createHash("sha256").update(`${sessionId}\0${videoRef}`).digest("hex"); +} + +function validateFrames(value: VideoDrilldownPutValue): VideoDrilldownFrame[] { + if ( + !Number.isFinite(value.durationSeconds) || + value.durationSeconds <= 0 || + value.durationSeconds > MAX_DURATION_SECONDS || + !Array.isArray(value.frames) || + value.frames.length < 1 || + value.frames.length > 16 + ) { + throw new Error("Invalid drill-down duration or frame count"); + } + let totalBytes = 0; + const frames = value.frames.map((frame) => { + if ( + !frame || + !Number.isFinite(frame.timestampSeconds) || + frame.timestampSeconds < 0 || + frame.timestampSeconds > value.durationSeconds || + !/^data:image\/jpeg;base64,[A-Za-z0-9+/=]+$/i.test(frame.dataUri) + ) { + throw new Error("Invalid drill-down JPEG frame"); + } + const encoded = frame.dataUri.slice(frame.dataUri.indexOf(",") + 1); + const bytes = Math.floor((encoded.length * 3) / 4); + if (bytes < 1 || bytes > MAX_FRAME_BYTES) + throw new Error("Drill-down frame byte limit exceeded"); + totalBytes += bytes; + if (totalBytes > MAX_TOTAL_BYTES) throw new Error("Drill-down response byte limit exceeded"); + return { dataUri: frame.dataUri, timestampSeconds: frame.timestampSeconds }; + }); + return frames.sort((left, right) => left.timestampSeconds - right.timestampSeconds); +} + +export class VideoDrilldownCache { + private readonly entries = new Map(); + private readonly now: () => number; + + constructor(private readonly options: VideoDrilldownCacheOptions) { + if (!Number.isFinite(options.ttlMs) || options.ttlMs <= 0) { + throw new Error("Drill-down cache TTL must be positive"); + } + if (!Number.isInteger(options.maxEntries) || options.maxEntries < 1) { + throw new Error("Drill-down cache entry limit is invalid"); + } + this.now = options.now ?? Date.now; + } + + put(sessionId: string, videoRef: string, value: VideoDrilldownPutValue): void { + if (!sessionId || sessionId.length > 128 || !videoRef || videoRef.length > 4096) { + throw new Error("Drill-down cache key is invalid"); + } + const key = cacheKey(sessionId, videoRef); + this.entries.delete(key); + this.entries.set(key, { + durationSeconds: value.durationSeconds, + expiresAt: this.now() + this.options.ttlMs, + frames: validateFrames(value), + sessionId, + }); + while (this.entries.size > this.options.maxEntries) { + const oldest = this.entries.keys().next().value; + if (oldest) this.entries.delete(oldest); + } + } + + get( + sessionId: string, + videoRef: string, + options: { endSeconds?: number; frameCount?: number; startSeconds?: number } = {} + ): VideoDrilldownResult | null { + const key = cacheKey(sessionId, videoRef); + const stored = this.entries.get(key); + if (!stored) return null; + if (stored.expiresAt <= this.now()) { + this.entries.delete(key); + return null; + } + this.entries.delete(key); + this.entries.set(key, stored); + const hasFocus = options.startSeconds !== undefined || options.endSeconds !== undefined; + let focusWindow: VideoFocusWindow | null = null; + try { + focusWindow = hasFocus + ? resolveVideoFocusWindow(stored.durationSeconds, { + endSeconds: options.endSeconds, + startSeconds: options.startSeconds, + }) + : null; + } catch { + return null; + } + const frameCount = + options.frameCount === undefined + ? 16 + : Number.isInteger(options.frameCount) && + options.frameCount >= 1 && + options.frameCount <= 16 + ? options.frameCount + : null; + if (frameCount === null) return null; + const frames = stored.frames + .filter( + (frame) => + !focusWindow || + (frame.timestampSeconds >= focusWindow.startSeconds && + frame.timestampSeconds <= focusWindow.endSeconds) + ) + .slice(0, frameCount) + .map((frame) => ({ ...frame })); + if (frames.length === 0) return null; + return { + durationSeconds: stored.durationSeconds, + ...(focusWindow ? { focusWindow } : {}), + frames, + }; + } + + clearSession(sessionId: string): number { + let removed = 0; + for (const [key, entry] of this.entries.entries()) { + if (entry.sessionId === sessionId) { + this.entries.delete(key); + removed += 1; + } + } + return removed; + } + + clearAll(): void { + this.entries.clear(); + } +} diff --git a/tests/unit/guardrails/videoBridgeDrilldown.test.ts b/tests/unit/guardrails/videoBridgeDrilldown.test.ts new file mode 100644 index 0000000000..4f19afea5e --- /dev/null +++ b/tests/unit/guardrails/videoBridgeDrilldown.test.ts @@ -0,0 +1,62 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + VideoDrilldownCache, + type VideoDrilldownFrame, +} from "../../../src/lib/guardrails/videoBridgeDrilldown"; + +const frames: VideoDrilldownFrame[] = [ + { dataUri: "data:image/jpeg;base64,QQ==", timestampSeconds: 1 }, + { dataUri: "data:image/jpeg;base64,Qg==", timestampSeconds: 5 }, + { dataUri: "data:image/jpeg;base64,Qw==", timestampSeconds: 9 }, +]; + +test("drill-down cache isolates sessions and returns bounded focus slices", () => { + const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); + cache.put("session-a", "video-a", { durationSeconds: 10, frames }); + cache.put("session-b", "video-a", { durationSeconds: 10, frames: [frames[0]] }); + + assert.deepEqual( + cache.get("session-a", "video-a", { endSeconds: 6, frameCount: 2 })?.frames, + frames.slice(0, 2) + ); + assert.equal(cache.get("session-a", "video-b"), null); + assert.equal(cache.get("session-b", "video-a")?.frames.length, 1); +}); + +test("drill-down cache clamps a valid focus and preserves timeline metadata", () => { + const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); + cache.put("session", "video", { durationSeconds: 10, frames }); + const result = cache.get("session", "video", { + endSeconds: 100, + startSeconds: -4, + frameCount: 16, + }); + assert.deepEqual(result?.focusWindow, { endSeconds: 10, startSeconds: 0 }); + assert.equal(result?.durationSeconds, 10); + assert.equal(result?.frames.length, 3); +}); + +test("drill-down cache rejects invalid and oversized frame payloads", () => { + const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); + assert.throws(() => cache.put("session", "video", { durationSeconds: 10, frames: [] }), /frame/i); + assert.throws( + () => + cache.put("session", "video", { + durationSeconds: 10, + frames: [{ dataUri: "data:image/png;base64,QQ==", timestampSeconds: 1 }], + }), + /JPEG/i + ); +}); + +test("drill-down cache expires entries and evicts the least recently used key", () => { + let now = 1000; + const cache = new VideoDrilldownCache({ now: () => now, ttlMs: 5000, maxEntries: 1 }); + cache.put("session-a", "video", { durationSeconds: 10, frames }); + cache.put("session-b", "video", { durationSeconds: 10, frames }); + assert.equal(cache.get("session-a", "video"), null); + now = 7000; + assert.equal(cache.get("session-b", "video"), null); +}); diff --git a/tests/unit/video-bridge-drilldown-route.test.ts b/tests/unit/video-bridge-drilldown-route.test.ts new file mode 100644 index 0000000000..6ee9372752 --- /dev/null +++ b/tests/unit/video-bridge-drilldown-route.test.ts @@ -0,0 +1,66 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { handleVideoDrilldownRequest } from "../../src/app/api/modality-bridge/video/drilldown/route"; +import { buildVideoBridgeBrokerHeaders } from "../../src/lib/guardrails/videoBridgeBrokerAuth"; +import { VideoDrilldownCache } from "../../src/lib/guardrails/videoBridgeDrilldown"; +import { AUTHZ_HEADER_PEER_LOCALITY } from "../../src/server/authz/headers"; +import { isLocalOnlyPath } from "../../src/server/authz/routeGuard"; + +function headers(contentType?: string): Headers { + return new Headers({ + ...buildVideoBridgeBrokerHeaders(), + [AUTHZ_HEADER_PEER_LOCALITY]: "loopback", + ...(contentType ? { "Content-Type": contentType } : {}), + }); +} + +test("drill-down route is loopback/token protected and has no public fallback", async () => { + assert.equal(isLocalOnlyPath("/api/modality-bridge/video/drilldown", "GET"), true); + const response = await handleVideoDrilldownRequest( + new Request("http://localhost/api/modality-bridge/video/drilldown?sessionId=s&videoRef=v") + ); + assert.equal(response.status, 403); +}); + +test("drill-down route stores, slices, and deletes an isolated session result", async () => { + const cache = new VideoDrilldownCache({ maxEntries: 4, now: () => 1000, ttlMs: 5000 }); + const post = await handleVideoDrilldownRequest( + new Request("http://localhost/api/modality-bridge/video/drilldown", { + body: JSON.stringify({ + durationSeconds: 10, + frames: [ + { dataUri: "data:image/jpeg;base64,QQ==", timestampSeconds: 1 }, + { dataUri: "data:image/jpeg;base64,Qg==", timestampSeconds: 5 }, + ], + sessionId: "session-a", + videoRef: "video-a", + }), + headers: headers("application/json"), + method: "POST", + }), + { cache } + ); + assert.equal(post.status, 201); + + const get = await handleVideoDrilldownRequest( + new Request( + "http://localhost/api/modality-bridge/video/drilldown?sessionId=session-a&videoRef=video-a&start=2&end=6&frames=1", + { headers: headers() } + ), + { cache } + ); + assert.equal(get.status, 200); + assert.deepEqual((await get.json()).frames, [ + { dataUri: "data:image/jpeg;base64,Qg==", timestampSeconds: 5 }, + ]); + + const deleted = await handleVideoDrilldownRequest( + new Request("http://localhost/api/modality-bridge/video/drilldown?sessionId=session-a", { + headers: headers(), + method: "DELETE", + }), + { cache } + ); + assert.deepEqual(await deleted.json(), { removed: 1 }); +});