diff --git a/docs/openapi.yaml b/docs/openapi.yaml index e9a11fdf5e..7b913ae8c9 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -5514,7 +5514,7 @@ paths: description: Optional deterministic sampling policy. Scene-aware detection falls back to uniform sampling on detector failure. schema: type: string - enum: [uniform, scene_aware] + enum: [uniform, scene_aware, segment_aware] default: uniform - in: query name: start diff --git a/docs/security/GUARDRAILS.md b/docs/security/GUARDRAILS.md index c223b81f5b..0a7c408b71 100644 --- a/docs/security/GUARDRAILS.md +++ b/docs/security/GUARDRAILS.md @@ -309,14 +309,16 @@ fallback. Videos are limited to 600 seconds, 8,192 pixels per dimension, and 33,554,432 source pixels. FFmpeg samples 1–16 midpoint JPEG frames, scales down the long edge to at most 1,024 pixels without upscaling smaller inputs, and never receives a URL. Sampling is `uniform` by default. The optional -`scene_aware` policy performs one additional fixed FFmpeg pass over the already -validated local stream, selects bounded `showinfo` scene timestamps, and falls -back deterministically to the same uniform midpoints on detector failure, -timeout, malformed output, or an empty candidate set. The hard 16-frame cap is +`scene_aware` and experimental `segment_aware` policies perform one additional +fixed FFmpeg pass over the already validated local stream, select bounded +`showinfo` scene timestamps, and fall back deterministically to the same +uniform midpoints on detector failure, timeout, malformed output, or an empty +candidate set. Segment-aware mode allocates midpoint samples proportionally to +the validated scene intervals. The hard 16-frame cap is applied after selection in every policy. A caller may optionally provide a finite focus window (`start`/`end` seconds); bounds are clamped to the media -duration, reversed or non-finite windows are rejected, and uniform/scene-aware -sampling is performed only inside the normalized interval. The resulting +duration, reversed or non-finite windows are rejected, and all sampling +policies are performed only inside the normalized interval. The resulting window is included in sampling metadata and in the untrusted description prefix so downstream models can distinguish a focused excerpt from the full timeline. @@ -370,14 +372,14 @@ to raw media. Runtime settings are DB-backed and Zod-validated: -| Key | Default | Range / behavior | -| ----------------------------------- | ----------- | -------------------------------------------------------------------- | -| `modalityBridgeVideoEnabled` | `false` | Optional runtime, opt-in | -| `modalityBridgeVideoModel` | `""` | Inherit the Vision Bridge model | -| `modalityBridgeVideoFrameCount` | `8` | 1–16 | -| `modalityBridgeVideoSamplingPolicy` | `"uniform"` | `uniform` or `scene_aware`; detector failure falls back to `uniform` | -| `modalityBridgeVideoMaxVideos` | `1` | 1–4 | -| `modalityBridgeVideoTimeout` | `120000` | 1000–120000 ms | +| Key | Default | Range / behavior | +| ----------------------------------- | ----------- | --------------------------------------------------------------------------------------------------- | +| `modalityBridgeVideoEnabled` | `false` | Optional runtime, opt-in | +| `modalityBridgeVideoModel` | `""` | Inherit the Vision Bridge model | +| `modalityBridgeVideoFrameCount` | `8` | 1–16 | +| `modalityBridgeVideoSamplingPolicy` | `"uniform"` | `uniform`, `scene_aware`, or proportional `segment_aware`; detector failure falls back to `uniform` | +| `modalityBridgeVideoMaxVideos` | `1` | 1–4 | +| `modalityBridgeVideoTimeout` | `120000` | 1000–120000 ms | Legacy persisted Video timeout values above 120 seconds are clamped to the broker deadline; new settings writes above that limit are rejected. diff --git a/src/app/(dashboard)/dashboard/settings/components/modalityBridge/ModalityBridgeVideoTab.tsx b/src/app/(dashboard)/dashboard/settings/components/modalityBridge/ModalityBridgeVideoTab.tsx index 535c8b56c5..e12ceb5789 100644 --- a/src/app/(dashboard)/dashboard/settings/components/modalityBridge/ModalityBridgeVideoTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/modalityBridge/ModalityBridgeVideoTab.tsx @@ -304,6 +304,7 @@ export default function ModalityBridgeVideoTab({ > + diff --git a/src/app/api/modality-bridge/video/extract/route.ts b/src/app/api/modality-bridge/video/extract/route.ts index a1453b4b40..27079a0da5 100644 --- a/src/app/api/modality-bridge/video/extract/route.ts +++ b/src/app/api/modality-bridge/video/extract/route.ts @@ -45,7 +45,12 @@ function parseFrameCount(url: URL): number | null { const raw = url.searchParams.get("frames"); if (!raw || !/^\d{1,2}$/.test(raw)) return null; const samplingPolicy = url.searchParams.get("samplingPolicy"); - if (samplingPolicy !== null && samplingPolicy !== "uniform" && samplingPolicy !== "scene_aware") { + if ( + samplingPolicy !== null && + samplingPolicy !== "uniform" && + samplingPolicy !== "scene_aware" && + samplingPolicy !== "segment_aware" + ) { return null; } const value = Number(raw); @@ -73,7 +78,8 @@ function parseFocusWindow(url: URL): VideoFocusBounds | null { } function parseSamplingPolicy(url: URL): VideoSamplingPolicy { - return url.searchParams.get("samplingPolicy") === "scene_aware" ? "scene_aware" : "uniform"; + const value = url.searchParams.get("samplingPolicy"); + return value === "scene_aware" || value === "segment_aware" ? value : "uniform"; } function expectedBrokerPath(): string { diff --git a/src/lib/guardrails/modalityBridge/bridgeStats.ts b/src/lib/guardrails/modalityBridge/bridgeStats.ts index 66c8f690f6..408b39e6ce 100644 --- a/src/lib/guardrails/modalityBridge/bridgeStats.ts +++ b/src/lib/guardrails/modalityBridge/bridgeStats.ts @@ -154,7 +154,8 @@ export function buildModalityBridgeHeader(results: GuardrailMetaEntry[]): string !meta.rerouted ) { const sampling = - meta.samplingPolicyRequested === "scene_aware" + meta.samplingPolicyRequested === "scene_aware" || + meta.samplingPolicyRequested === "segment_aware" ? `;sampling=${headerModelToken(meta.samplingPolicyEffective ?? "uniform")};candidates=${typeof meta.samplingCandidateCount === "number" ? Math.max(0, Math.floor(meta.samplingCandidateCount)) : 0}` : ""; segments.push( diff --git a/src/lib/guardrails/videoBridge.ts b/src/lib/guardrails/videoBridge.ts index 4f61998ed8..d80b2a71e6 100644 --- a/src/lib/guardrails/videoBridge.ts +++ b/src/lib/guardrails/videoBridge.ts @@ -68,8 +68,8 @@ interface VideoResultCacheMetadata { focusStartSeconds?: number; focusEndSeconds?: number; samplingCandidateCount?: number; - samplingPolicyEffective?: "uniform" | "scene_aware"; - samplingPolicyRequested?: "uniform" | "scene_aware"; + samplingPolicyEffective?: "uniform" | "scene_aware" | "segment_aware"; + samplingPolicyRequested?: "uniform" | "scene_aware" | "segment_aware"; transcriptCuesApplied?: number; cacheBytes: number; modelUsed: string; @@ -112,10 +112,12 @@ function isVideoResultCacheMetadata(value: unknown): value is VideoResultCacheMe (typeof record.samplingCandidateCount === "number" && record.samplingCandidateCount >= 0)) && (record.samplingPolicyEffective === undefined || record.samplingPolicyEffective === "uniform" || - record.samplingPolicyEffective === "scene_aware") && + record.samplingPolicyEffective === "scene_aware" || + record.samplingPolicyEffective === "segment_aware") && (record.samplingPolicyRequested === undefined || record.samplingPolicyRequested === "uniform" || - record.samplingPolicyRequested === "scene_aware") && + record.samplingPolicyRequested === "scene_aware" || + record.samplingPolicyRequested === "segment_aware") && (record.transcriptCuesApplied === undefined || (typeof record.transcriptCuesApplied === "number" && record.transcriptCuesApplied >= 0)) ); @@ -185,7 +187,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { let totalDedupDropped = 0; let focusWindowsApplied = 0; let transcriptCuesApplied = 0; - let samplingPolicyEffective: "uniform" | "scene_aware" = "uniform"; + let samplingPolicyEffective: "uniform" | "scene_aware" | "segment_aware" = "uniform"; let failures = 0; const attemptedParts = parts.slice(0, runtime.maxVideos); @@ -238,8 +240,8 @@ export class VideoBridgeGuardrail extends BaseGuardrail { totalDurationSeconds += meta.durationSeconds; totalSamplingCandidateCount += meta.samplingCandidateCount ?? 0; transcriptCuesApplied += meta.transcriptCuesApplied ?? 0; - if (meta.samplingPolicyEffective === "scene_aware") { - samplingPolicyEffective = "scene_aware"; + if (meta.samplingPolicyEffective && meta.samplingPolicyEffective !== "uniform") { + samplingPolicyEffective = meta.samplingPolicyEffective; } if (cachedResult.producerModel) { successfulModels.add(cachedResult.producerModel); @@ -282,8 +284,11 @@ export class VideoBridgeGuardrail extends BaseGuardrail { transcriptCuesApplied += described.transcriptCues?.length ?? 0; totalDurationSeconds += described.durationSeconds; totalSamplingCandidateCount += described.sampling?.candidateCount ?? 0; - if (described.sampling?.policyEffective === "scene_aware") { - samplingPolicyEffective = "scene_aware"; + if ( + described.sampling?.policyEffective && + described.sampling.policyEffective !== "uniform" + ) { + samplingPolicyEffective = described.sampling.policyEffective; } totalCacheHits += videoCacheHits; if (resultCacheKey && selectedModel) { diff --git a/src/lib/guardrails/videoBridgeBrokerClient.ts b/src/lib/guardrails/videoBridgeBrokerClient.ts index 739b467dc9..95aaf5abc8 100644 --- a/src/lib/guardrails/videoBridgeBrokerClient.ts +++ b/src/lib/guardrails/videoBridgeBrokerClient.ts @@ -109,9 +109,15 @@ function parseBrokerResult(value: unknown, frameCount: number): BrokerExtraction ? (record.sampling as Record) : {}; const policyRequested = - samplingRecord.policyRequested === "scene_aware" ? "scene_aware" : "uniform"; + samplingRecord.policyRequested === "scene_aware" || + samplingRecord.policyRequested === "segment_aware" + ? samplingRecord.policyRequested + : "uniform"; const policyEffective = - samplingRecord.policyEffective === "scene_aware" ? "scene_aware" : "uniform"; + samplingRecord.policyEffective === "scene_aware" || + samplingRecord.policyEffective === "segment_aware" + ? samplingRecord.policyEffective + : "uniform"; const candidateCount = Number(samplingRecord.candidateCount ?? 0); return { durationSeconds, @@ -133,7 +139,7 @@ export async function extractVideoFramesViaBroker( const baseUrl = resolveVideoBridgeBrokerBaseUrl(); const url = new URL(`${baseUrl}${VIDEO_BRIDGE_BROKER_PATH}`); url.searchParams.set("frames", String(options.frameCount)); - if (options.samplingPolicy === "scene_aware") { + if (options.samplingPolicy && options.samplingPolicy !== "uniform") { url.searchParams.set("samplingPolicy", options.samplingPolicy); } if (options.focusWindow?.startSeconds !== undefined) { diff --git a/src/lib/guardrails/videoBridgeRuntime.ts b/src/lib/guardrails/videoBridgeRuntime.ts index b2fd05244c..fea9769381 100644 --- a/src/lib/guardrails/videoBridgeRuntime.ts +++ b/src/lib/guardrails/videoBridgeRuntime.ts @@ -29,7 +29,7 @@ export interface VideoFrameFile { timestampSeconds: number; } -export type VideoSamplingPolicy = "uniform" | "scene_aware"; +export type VideoSamplingPolicy = "uniform" | "scene_aware" | "segment_aware"; export interface VideoSamplingMetadata { candidateCount: number; @@ -234,6 +234,60 @@ export function parseSceneChangeTimestamps(output: string, durationSeconds: numb return normalizeSceneCandidates(durationSeconds, candidates); } +/** Allocate midpoint samples proportionally across validated scene segments. */ +export function calculateSegmentAwareTimestamps( + durationSeconds: number, + requestedFrameCount: number, + sceneCandidates: readonly number[], + focusWindow: VideoFocusWindow | null = null +): number[] { + const startSeconds = focusWindow?.startSeconds ?? 0; + const endSeconds = focusWindow?.endSeconds ?? durationSeconds; + const uniform = calculateFrameTimestamps(endSeconds - startSeconds, requestedFrameCount).map( + (timestamp) => timestamp + startSeconds + ); + const boundaries = normalizeSceneCandidates(durationSeconds, sceneCandidates).filter( + (timestamp) => timestamp > startSeconds && timestamp < endSeconds + ); + if (boundaries.length === 0) return uniform; + const segmentStarts = [startSeconds, ...boundaries]; + const segmentEnds = [...boundaries, endSeconds]; + const lengths = segmentStarts.map((segmentStart, index) => segmentEnds[index] - segmentStart); + const segmentCount = lengths.length; + const frameCount = uniform.length; + if (segmentCount > frameCount) { + return [...uniform].map((timestamp, index) => { + const segmentIndex = Math.min( + segmentCount - 1, + Math.floor((index * segmentCount) / frameCount) + ); + const segmentStart = segmentStarts[segmentIndex]; + const segmentEnd = segmentEnds[segmentIndex]; + return segmentStart + (segmentEnd - segmentStart) / 2; + }); + } + const allocation = lengths.map(() => 1); + let remaining = frameCount - segmentCount; + const idealExtra = lengths.map((length) => (length / (endSeconds - startSeconds)) * remaining); + const extras = idealExtra.map((value) => Math.floor(value)); + remaining -= extras.reduce((sum, value) => sum + value, 0); + const remainderOrder = idealExtra + .map((value, index) => ({ index, remainder: value - Math.floor(value) })) + .sort((left, right) => right.remainder - left.remainder || left.index - right.index); + for (let index = 0; index < remaining; index++) extras[remainderOrder[index].index] += 1; + for (let index = 0; index < allocation.length; index++) allocation[index] += extras[index]; + const timestamps: number[] = []; + for (let segmentIndex = 0; segmentIndex < segmentCount; segmentIndex++) { + const count = allocation[segmentIndex]; + const segmentStart = segmentStarts[segmentIndex]; + const segmentLength = lengths[segmentIndex]; + for (let index = 0; index < count; index++) { + timestamps.push(segmentStart + ((index + 0.5) * segmentLength) / count); + } + } + return timestamps; +} + export function calculateSamplingDecision( durationSeconds: number, requestedFrameCount: number, @@ -246,7 +300,7 @@ export function calculateSamplingDecision( const uniform = calculateFrameTimestamps(endSeconds - startSeconds, requestedFrameCount).map( (timestamp) => timestamp + startSeconds ); - if (policy !== "scene_aware") { + if (policy === "uniform") { return { candidateCount: 0, ...(focusWindow ? { focusWindow } : {}), @@ -264,11 +318,26 @@ export function calculateSamplingDecision( candidateCount: 0, ...(focusWindow ? { focusWindow } : {}), policyEffective: "uniform", - policyRequested: "scene_aware", + policyRequested: policy, timestamps: uniform, }; } + if (policy === "segment_aware") { + return { + candidateCount: candidates.length, + ...(focusWindow ? { focusWindow } : {}), + policyEffective: "segment_aware", + policyRequested: "segment_aware", + timestamps: calculateSegmentAwareTimestamps( + durationSeconds, + requestedFrameCount, + candidates, + focusWindow + ), + }; + } + const frameCount = uniform.length; const selected = candidates.length <= frameCount @@ -478,7 +547,7 @@ export async function extractFramesFromLocalVideo( assertLocalPath(outputDirectory); const policy = options.samplingPolicy ?? "uniform"; let sceneCandidates: number[] = []; - if (policy === "scene_aware") { + if (policy !== "uniform") { try { sceneCandidates = await detectSceneChangeTimestamps(inputPath, { durationSeconds: options.durationSeconds, diff --git a/src/shared/constants/modalityBridgeDefaults.ts b/src/shared/constants/modalityBridgeDefaults.ts index 4b9c07ec4e..e50d710117 100644 --- a/src/shared/constants/modalityBridgeDefaults.ts +++ b/src/shared/constants/modalityBridgeDefaults.ts @@ -8,7 +8,7 @@ import { VISION_BRIDGE_DEFAULTS } from "./visionBridgeDefaults"; export type VisionBridgeMode = "auto" | "describe" | "reroute"; -export type VideoSamplingPolicy = "uniform" | "scene_aware"; +export type VideoSamplingPolicy = "uniform" | "scene_aware" | "segment_aware"; export const VIDEO_BRIDGE_TIMEOUT_MIN_MS = 1_000; export const VIDEO_BRIDGE_TIMEOUT_MAX_MS = 120_000; @@ -150,8 +150,9 @@ export function resolveVideoBridgeRuntimeSettings( frameCount: pickNumber(s.modalityBridgeVideoFrameCount) ?? MODALITY_BRIDGE_DEFAULTS.videoFrameCount, samplingPolicy: - pickString(s.modalityBridgeVideoSamplingPolicy) === "scene_aware" - ? "scene_aware" + pickString(s.modalityBridgeVideoSamplingPolicy) === "scene_aware" || + pickString(s.modalityBridgeVideoSamplingPolicy) === "segment_aware" + ? (pickString(s.modalityBridgeVideoSamplingPolicy) as VideoSamplingPolicy) : MODALITY_BRIDGE_DEFAULTS.videoSamplingPolicy, maxVideos: pickNumber(s.modalityBridgeVideoMaxVideos) ?? MODALITY_BRIDGE_DEFAULTS.videoMaxVideos, diff --git a/src/shared/validation/settingsSchemas.ts b/src/shared/validation/settingsSchemas.ts index da745b0f13..c4cb849573 100644 --- a/src/shared/validation/settingsSchemas.ts +++ b/src/shared/validation/settingsSchemas.ts @@ -363,7 +363,7 @@ export const updateSettingsSchema = z.object({ modalityBridgeVideoEnabled: z.boolean().optional(), modalityBridgeVideoModel: z.string().max(200).optional(), modalityBridgeVideoFrameCount: z.number().int().min(1).max(16).optional(), - modalityBridgeVideoSamplingPolicy: z.enum(["uniform", "scene_aware"]).optional(), + modalityBridgeVideoSamplingPolicy: z.enum(["uniform", "scene_aware", "segment_aware"]).optional(), modalityBridgeVideoMaxVideos: z.number().int().min(1).max(4).optional(), modalityBridgeVideoTimeout: z .number() diff --git a/tests/unit/guardrails/videoBridgeSampler.test.ts b/tests/unit/guardrails/videoBridgeSampler.test.ts index 62e1cd4b76..b8d391a1dc 100644 --- a/tests/unit/guardrails/videoBridgeSampler.test.ts +++ b/tests/unit/guardrails/videoBridgeSampler.test.ts @@ -3,6 +3,7 @@ import test from "node:test"; import { calculateSamplingDecision, + calculateSegmentAwareTimestamps, extractFramesFromLocalVideo, parseSceneChangeTimestamps, type VideoCommandRunner, @@ -96,3 +97,19 @@ test("scene detection timeout or runtime failure falls back to uniform sampling" policyRequested: "scene_aware", }); }); + +test("segment-aware sampling allocates frames across long and short scene segments", () => { + const timestamps = calculateSegmentAwareTimestamps(20, 6, [2, 10, 12]); + assert.equal(timestamps.length, 6); + assert.ok(timestamps.some((timestamp) => timestamp < 2)); + assert.ok(timestamps.some((timestamp) => timestamp > 2 && timestamp < 10)); + assert.ok(timestamps.some((timestamp) => timestamp > 12)); + assert.ok(timestamps.every((timestamp) => timestamp > 0 && timestamp < 20)); +}); + +test("segment-aware sampling falls back to uniform when boundaries are unusable", () => { + const decision = calculateSamplingDecision(8, 4, "segment_aware", []); + assert.equal(decision.policyRequested, "segment_aware"); + assert.equal(decision.policyEffective, "uniform"); + assert.deepEqual(decision.timestamps, [1, 3, 5, 7]); +}); diff --git a/tests/unit/video-bridge-settings.test.ts b/tests/unit/video-bridge-settings.test.ts index c0972e11b2..833df63ab6 100644 --- a/tests/unit/video-bridge-settings.test.ts +++ b/tests/unit/video-bridge-settings.test.ts @@ -41,6 +41,10 @@ test("Video Bridge settings default to a bounded disabled runtime and accept val modalityBridgeVideoTimeout: 120_000, }); assert.equal(valid.success, true); + assert.equal( + updateSettingsSchema.safeParse({ modalityBridgeVideoSamplingPolicy: "segment_aware" }).success, + true + ); }); test("Video Bridge settings schema rejects values outside extraction bounds", () => { @@ -70,3 +74,11 @@ test("persisted legacy Video Bridge timeouts clamp to the broker's 120 second de ); } }); + +test("persisted segment-aware policy remains an explicit opt-in", () => { + assert.equal( + resolveVideoBridgeRuntimeSettings({ modalityBridgeVideoSamplingPolicy: "segment_aware" }) + .samplingPolicy, + "segment_aware" + ); +});