pjandro/OmniRoute
1
0
Fork 0
mirror of https://github.com/diegosouzapw/OmniRoute.git synced 2026-08-18 21:22:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat(video): add segment-aware sampling

Browse Source
This commit is contained in:
Xiangzhe
2026-08-18 01:55:58 -03:00
parent bcd58975c5
commit edb3abf323
12 changed files with 158 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/openapi.yaml
View File

@@ -5514,7 +5514,7 @@ paths:
description: Optional deterministic sampling policy. Scene-aware detection falls back to uniform sampling on detector failure.
schema:
type: string
enum: [uniform, scene_aware]
enum: [uniform, scene_aware, segment_aware]
default: uniform
- in: query
name: start

30
docs/security/GUARDRAILS.md
View File

@@ -309,14 +309,16 @@ fallback. Videos are limited to 600 seconds, 8,192 pixels per dimension, and
33,554,432 source pixels. FFmpeg samples 1–16 midpoint JPEG frames, scales down
the long edge to at most 1,024 pixels without upscaling smaller inputs, and
never receives a URL. Sampling is `uniform` by default. The optional
`scene_aware` policy performs one additional fixed FFmpeg pass over the already
validated local stream, selects bounded `showinfo` scene timestamps, and falls
back deterministically to the same uniform midpoints on detector failure,
timeout, malformed output, or an empty candidate set. The hard 16-frame cap is
`scene_aware` and experimental `segment_aware` policies perform one additional
fixed FFmpeg pass over the already validated local stream, select bounded
`showinfo` scene timestamps, and fall back deterministically to the same
uniform midpoints on detector failure, timeout, malformed output, or an empty
candidate set. Segment-aware mode allocates midpoint samples proportionally to
the validated scene intervals. The hard 16-frame cap is
applied after selection in every policy. A caller may optionally provide a
finite focus window (`start`/`end` seconds); bounds are clamped to the media
duration, reversed or non-finite windows are rejected, and uniform/scene-aware
sampling is performed only inside the normalized interval. The resulting
duration, reversed or non-finite windows are rejected, and all sampling
policies are performed only inside the normalized interval. The resulting
window is included in sampling metadata and in the untrusted description
prefix so downstream models can distinguish a focused excerpt from the full
timeline.
@@ -370,14 +372,14 @@ to raw media.
Runtime settings are DB-backed and Zod-validated:
| Key | Default | Range / behavior |
| ----------------------------------- | ----------- | -------------------------------------------------------------------- |
| `modalityBridgeVideoEnabled` | `false` | Optional runtime, opt-in |
| `modalityBridgeVideoModel` | `""` | Inherit the Vision Bridge model |
| `modalityBridgeVideoFrameCount` | `8` | 1–16 |
| `modalityBridgeVideoSamplingPolicy` | `"uniform"` | `uniform` or `scene_aware`; detector failure falls back to `uniform` |
| `modalityBridgeVideoMaxVideos` | `1` | 1–4 |
| `modalityBridgeVideoTimeout` | `120000` | 1000–120000 ms |
| Key | Default | Range / behavior |
| ----------------------------------- | ----------- | --------------------------------------------------------------------------------------------------- |
| `modalityBridgeVideoEnabled` | `false` | Optional runtime, opt-in |
| `modalityBridgeVideoModel` | `""` | Inherit the Vision Bridge model |
| `modalityBridgeVideoFrameCount` | `8` | 1–16 |
| `modalityBridgeVideoSamplingPolicy` | `"uniform"` | `uniform`, `scene_aware`, or proportional `segment_aware`; detector failure falls back to `uniform` |
| `modalityBridgeVideoMaxVideos` | `1` | 1–4 |
| `modalityBridgeVideoTimeout` | `120000` | 1000–120000 ms |
Legacy persisted Video timeout values above 120 seconds are clamped to the
broker deadline; new settings writes above that limit are rejected.

1
src/app/(dashboard)/dashboard/settings/components/modalityBridge/ModalityBridgeVideoTab.tsx
View File

@@ -304,6 +304,7 @@ export default function ModalityBridgeVideoTab({
>
<option value="uniform">uniform</option>
<option value="scene_aware">scene_aware</option>
<option value="segment_aware">segment_aware</option>
</select>
</label>
</div>

10
src/app/api/modality-bridge/video/extract/route.ts
View File

@@ -45,7 +45,12 @@ function parseFrameCount(url: URL): number | null {
const raw = url.searchParams.get("frames");
if (!raw || !/^\d{1,2}$/.test(raw)) return null;
const samplingPolicy = url.searchParams.get("samplingPolicy");
if (samplingPolicy !== null && samplingPolicy !== "uniform" && samplingPolicy !== "scene_aware") {
if (
samplingPolicy !== null &&
samplingPolicy !== "uniform" &&
samplingPolicy !== "scene_aware" &&
samplingPolicy !== "segment_aware"
) {
return null;
}
const value = Number(raw);
@@ -73,7 +78,8 @@ function parseFocusWindow(url: URL): VideoFocusBounds | null {
}
function parseSamplingPolicy(url: URL): VideoSamplingPolicy {
return url.searchParams.get("samplingPolicy") === "scene_aware" ? "scene_aware" : "uniform";
const value = url.searchParams.get("samplingPolicy");
return value === "scene_aware" || value === "segment_aware" ? value : "uniform";
}
function expectedBrokerPath(): string {

3
src/lib/guardrails/modalityBridge/bridgeStats.ts
View File

@@ -154,7 +154,8 @@ export function buildModalityBridgeHeader(results: GuardrailMetaEntry[]): string
!meta.rerouted
) {
const sampling =
meta.samplingPolicyRequested === "scene_aware"
meta.samplingPolicyRequested === "scene_aware" ||
meta.samplingPolicyRequested === "segment_aware"
? `;sampling=${headerModelToken(meta.samplingPolicyEffective ?? "uniform")};candidates=${typeof meta.samplingCandidateCount === "number" ? Math.max(0, Math.floor(meta.samplingCandidateCount)) : 0}`
: "";
segments.push(

23
src/lib/guardrails/videoBridge.ts
View File

@@ -68,8 +68,8 @@ interface VideoResultCacheMetadata {
focusStartSeconds?: number;
focusEndSeconds?: number;
samplingCandidateCount?: number;
samplingPolicyEffective?: "uniform" | "scene_aware";
samplingPolicyRequested?: "uniform" | "scene_aware";
samplingPolicyEffective?: "uniform" | "scene_aware" | "segment_aware";
samplingPolicyRequested?: "uniform" | "scene_aware" | "segment_aware";
transcriptCuesApplied?: number;
cacheBytes: number;
modelUsed: string;
@@ -112,10 +112,12 @@ function isVideoResultCacheMetadata(value: unknown): value is VideoResultCacheMe
(typeof record.samplingCandidateCount === "number" && record.samplingCandidateCount >= 0)) &&
(record.samplingPolicyEffective === undefined ||
record.samplingPolicyEffective === "uniform" ||
record.samplingPolicyEffective === "scene_aware") &&
record.samplingPolicyEffective === "scene_aware" ||
record.samplingPolicyEffective === "segment_aware") &&
(record.samplingPolicyRequested === undefined ||
record.samplingPolicyRequested === "uniform" ||
record.samplingPolicyRequested === "scene_aware") &&
record.samplingPolicyRequested === "scene_aware" ||
record.samplingPolicyRequested === "segment_aware") &&
(record.transcriptCuesApplied === undefined ||
(typeof record.transcriptCuesApplied === "number" && record.transcriptCuesApplied >= 0))
);
@@ -185,7 +187,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
let totalDedupDropped = 0;
let focusWindowsApplied = 0;
let transcriptCuesApplied = 0;
let samplingPolicyEffective: "uniform" | "scene_aware" = "uniform";
let samplingPolicyEffective: "uniform" | "scene_aware" | "segment_aware" = "uniform";
let failures = 0;
const attemptedParts = parts.slice(0, runtime.maxVideos);
@@ -238,8 +240,8 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
totalDurationSeconds += meta.durationSeconds;
totalSamplingCandidateCount += meta.samplingCandidateCount ?? 0;
transcriptCuesApplied += meta.transcriptCuesApplied ?? 0;
if (meta.samplingPolicyEffective === "scene_aware") {
samplingPolicyEffective = "scene_aware";
if (meta.samplingPolicyEffective && meta.samplingPolicyEffective !== "uniform") {
samplingPolicyEffective = meta.samplingPolicyEffective;
}
if (cachedResult.producerModel) {
successfulModels.add(cachedResult.producerModel);
@@ -282,8 +284,11 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
transcriptCuesApplied += described.transcriptCues?.length ?? 0;
totalDurationSeconds += described.durationSeconds;
totalSamplingCandidateCount += described.sampling?.candidateCount ?? 0;
if (described.sampling?.policyEffective === "scene_aware") {
samplingPolicyEffective = "scene_aware";
if (
described.sampling?.policyEffective &&
described.sampling.policyEffective !== "uniform"
) {
samplingPolicyEffective = described.sampling.policyEffective;
}
totalCacheHits += videoCacheHits;
if (resultCacheKey && selectedModel) {

12
src/lib/guardrails/videoBridgeBrokerClient.ts
View File

@@ -109,9 +109,15 @@ function parseBrokerResult(value: unknown, frameCount: number): BrokerExtraction
? (record.sampling as Record<string, unknown>)
: {};
const policyRequested =
samplingRecord.policyRequested === "scene_aware" ? "scene_aware" : "uniform";
samplingRecord.policyRequested === "scene_aware" ||
samplingRecord.policyRequested === "segment_aware"
? samplingRecord.policyRequested
: "uniform";
const policyEffective =
samplingRecord.policyEffective === "scene_aware" ? "scene_aware" : "uniform";
samplingRecord.policyEffective === "scene_aware" ||
samplingRecord.policyEffective === "segment_aware"
? samplingRecord.policyEffective
: "uniform";
const candidateCount = Number(samplingRecord.candidateCount ?? 0);
return {
durationSeconds,
@@ -133,7 +139,7 @@ export async function extractVideoFramesViaBroker(
const baseUrl = resolveVideoBridgeBrokerBaseUrl();
const url = new URL(`${baseUrl}${VIDEO_BRIDGE_BROKER_PATH}`);
url.searchParams.set("frames", String(options.frameCount));
if (options.samplingPolicy === "scene_aware") {
if (options.samplingPolicy && options.samplingPolicy !== "uniform") {
url.searchParams.set("samplingPolicy", options.samplingPolicy);
}
if (options.focusWindow?.startSeconds !== undefined) {

77
src/lib/guardrails/videoBridgeRuntime.ts
View File

@@ -29,7 +29,7 @@ export interface VideoFrameFile {
timestampSeconds: number;
}
export type VideoSamplingPolicy = "uniform" | "scene_aware";
export type VideoSamplingPolicy = "uniform" | "scene_aware" | "segment_aware";
export interface VideoSamplingMetadata {
candidateCount: number;
@@ -234,6 +234,60 @@ export function parseSceneChangeTimestamps(output: string, durationSeconds: numb
return normalizeSceneCandidates(durationSeconds, candidates);
}
/** Allocate midpoint samples proportionally across validated scene segments. */
export function calculateSegmentAwareTimestamps(
durationSeconds: number,
requestedFrameCount: number,
sceneCandidates: readonly number[],
focusWindow: VideoFocusWindow | null = null
): number[] {
const startSeconds = focusWindow?.startSeconds ?? 0;
const endSeconds = focusWindow?.endSeconds ?? durationSeconds;
const uniform = calculateFrameTimestamps(endSeconds - startSeconds, requestedFrameCount).map(
(timestamp) => timestamp + startSeconds
);
const boundaries = normalizeSceneCandidates(durationSeconds, sceneCandidates).filter(
(timestamp) => timestamp > startSeconds && timestamp < endSeconds
);
if (boundaries.length === 0) return uniform;
const segmentStarts = [startSeconds, ...boundaries];
const segmentEnds = [...boundaries, endSeconds];
const lengths = segmentStarts.map((segmentStart, index) => segmentEnds[index] - segmentStart);
const segmentCount = lengths.length;
const frameCount = uniform.length;
if (segmentCount > frameCount) {
return [...uniform].map((timestamp, index) => {
const segmentIndex = Math.min(
segmentCount - 1,
Math.floor((index * segmentCount) / frameCount)
);
const segmentStart = segmentStarts[segmentIndex];
const segmentEnd = segmentEnds[segmentIndex];
return segmentStart + (segmentEnd - segmentStart) / 2;
});
}
const allocation = lengths.map(() => 1);
let remaining = frameCount - segmentCount;
const idealExtra = lengths.map((length) => (length / (endSeconds - startSeconds)) * remaining);
const extras = idealExtra.map((value) => Math.floor(value));
remaining -= extras.reduce((sum, value) => sum + value, 0);
const remainderOrder = idealExtra
.map((value, index) => ({ index, remainder: value - Math.floor(value) }))
.sort((left, right) => right.remainder - left.remainder || left.index - right.index);
for (let index = 0; index < remaining; index++) extras[remainderOrder[index].index] += 1;
for (let index = 0; index < allocation.length; index++) allocation[index] += extras[index];
const timestamps: number[] = [];
for (let segmentIndex = 0; segmentIndex < segmentCount; segmentIndex++) {
const count = allocation[segmentIndex];
const segmentStart = segmentStarts[segmentIndex];
const segmentLength = lengths[segmentIndex];
for (let index = 0; index < count; index++) {
timestamps.push(segmentStart + ((index + 0.5) * segmentLength) / count);
}
}
return timestamps;
}
export function calculateSamplingDecision(
durationSeconds: number,
requestedFrameCount: number,
@@ -246,7 +300,7 @@ export function calculateSamplingDecision(
const uniform = calculateFrameTimestamps(endSeconds - startSeconds, requestedFrameCount).map(
(timestamp) => timestamp + startSeconds
);
if (policy !== "scene_aware") {
if (policy === "uniform") {
return {
candidateCount: 0,
...(focusWindow ? { focusWindow } : {}),
@@ -264,11 +318,26 @@ export function calculateSamplingDecision(
candidateCount: 0,
...(focusWindow ? { focusWindow } : {}),
policyEffective: "uniform",
policyRequested: "scene_aware",
policyRequested: policy,
timestamps: uniform,
};
}
if (policy === "segment_aware") {
return {
candidateCount: candidates.length,
...(focusWindow ? { focusWindow } : {}),
policyEffective: "segment_aware",
policyRequested: "segment_aware",
timestamps: calculateSegmentAwareTimestamps(
durationSeconds,
requestedFrameCount,
candidates,
focusWindow
),
};
}
const frameCount = uniform.length;
const selected =
candidates.length <= frameCount
@@ -478,7 +547,7 @@ export async function extractFramesFromLocalVideo(
assertLocalPath(outputDirectory);
const policy = options.samplingPolicy ?? "uniform";
let sceneCandidates: number[] = [];
if (policy === "scene_aware") {
if (policy !== "uniform") {
try {
sceneCandidates = await detectSceneChangeTimestamps(inputPath, {
durationSeconds: options.durationSeconds,

7
src/shared/constants/modalityBridgeDefaults.ts
View File

@@ -8,7 +8,7 @@
import { VISION_BRIDGE_DEFAULTS } from "./visionBridgeDefaults";
export type VisionBridgeMode = "auto" | "describe" | "reroute";
export type VideoSamplingPolicy = "uniform" | "scene_aware";
export type VideoSamplingPolicy = "uniform" | "scene_aware" | "segment_aware";
export const VIDEO_BRIDGE_TIMEOUT_MIN_MS = 1_000;
export const VIDEO_BRIDGE_TIMEOUT_MAX_MS = 120_000;
@@ -150,8 +150,9 @@ export function resolveVideoBridgeRuntimeSettings(
frameCount:
pickNumber(s.modalityBridgeVideoFrameCount) ?? MODALITY_BRIDGE_DEFAULTS.videoFrameCount,
samplingPolicy:
pickString(s.modalityBridgeVideoSamplingPolicy) === "scene_aware"
? "scene_aware"
pickString(s.modalityBridgeVideoSamplingPolicy) === "scene_aware" ||
pickString(s.modalityBridgeVideoSamplingPolicy) === "segment_aware"
? (pickString(s.modalityBridgeVideoSamplingPolicy) as VideoSamplingPolicy)
: MODALITY_BRIDGE_DEFAULTS.videoSamplingPolicy,
maxVideos:
pickNumber(s.modalityBridgeVideoMaxVideos) ?? MODALITY_BRIDGE_DEFAULTS.videoMaxVideos,

2
src/shared/validation/settingsSchemas.ts
View File

@@ -363,7 +363,7 @@ export const updateSettingsSchema = z.object({
modalityBridgeVideoEnabled: z.boolean().optional(),
modalityBridgeVideoModel: z.string().max(200).optional(),
modalityBridgeVideoFrameCount: z.number().int().min(1).max(16).optional(),
modalityBridgeVideoSamplingPolicy: z.enum(["uniform", "scene_aware"]).optional(),
modalityBridgeVideoSamplingPolicy: z.enum(["uniform", "scene_aware", "segment_aware"]).optional(),
modalityBridgeVideoMaxVideos: z.number().int().min(1).max(4).optional(),
modalityBridgeVideoTimeout: z
.number()

17
tests/unit/guardrails/videoBridgeSampler.test.ts
View File

@@ -3,6 +3,7 @@ import test from "node:test";
import {
calculateSamplingDecision,
calculateSegmentAwareTimestamps,
extractFramesFromLocalVideo,
parseSceneChangeTimestamps,
type VideoCommandRunner,
@@ -96,3 +97,19 @@ test("scene detection timeout or runtime failure falls back to uniform sampling"
policyRequested: "scene_aware",
});
});
test("segment-aware sampling allocates frames across long and short scene segments", () => {
const timestamps = calculateSegmentAwareTimestamps(20, 6, [2, 10, 12]);
assert.equal(timestamps.length, 6);
assert.ok(timestamps.some((timestamp) => timestamp < 2));
assert.ok(timestamps.some((timestamp) => timestamp > 2 && timestamp < 10));
assert.ok(timestamps.some((timestamp) => timestamp > 12));
assert.ok(timestamps.every((timestamp) => timestamp > 0 && timestamp < 20));
});
test("segment-aware sampling falls back to uniform when boundaries are unusable", () => {
const decision = calculateSamplingDecision(8, 4, "segment_aware", []);
assert.equal(decision.policyRequested, "segment_aware");
assert.equal(decision.policyEffective, "uniform");
assert.deepEqual(decision.timestamps, [1, 3, 5, 7]);
});

12
tests/unit/video-bridge-settings.test.ts
View File

@@ -41,6 +41,10 @@ test("Video Bridge settings default to a bounded disabled runtime and accept val
modalityBridgeVideoTimeout: 120_000,
});
assert.equal(valid.success, true);
assert.equal(
updateSettingsSchema.safeParse({ modalityBridgeVideoSamplingPolicy: "segment_aware" }).success,
true
);
});
test("Video Bridge settings schema rejects values outside extraction bounds", () => {
@@ -70,3 +74,11 @@ test("persisted legacy Video Bridge timeouts clamp to the broker's 120 second de
);
}
});
test("persisted segment-aware policy remains an explicit opt-in", () => {
assert.equal(
resolveVideoBridgeRuntimeSettings({ modalityBridgeVideoSamplingPolicy: "segment_aware" })
.samplingPolicy,
"segment_aware"
);
});