pjandro/OmniRoute
1
0
Fork 0
mirror of https://github.com/diegosouzapw/OmniRoute.git synced 2026-08-26 17:12:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(responses-continuation): recover a real id/output for passthrough and translate-mode replies (#11434)

Browse Source 
Retargetado para release/v3.8.51 (release/v3.8.50 está congelada — freeze issue #11439). Validado em lote combinado (batch-0824h2, junto de #11435/#11436/#11437) contra o tip de release/v3.8.51: typecheck:core limpo, gates estáticos OK, 127/127 testes focados passando.

Investigação sólida com repro real via container isolado, três causas independentes identificadas e corrigidas com testes de regressão dedicados para cada uma. Obrigado pela contribuição!
This commit is contained in:
Markus Hartung
2026-08-25 00:57:12 +02:00
committed by GitHub
parent 3192eb88d5
commit 04dba0460e
341 changed files with 24483 additions and 2052 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
scripts/build/better-sqlite3-stub-flag.mjs Normal file
View File

@@ -0,0 +1,36 @@
/**
* Decide whether the Next.js build should alias `better-sqlite3` to the
* build-time stub (src/lib/db/better-sqlite3.stub.js).
*
* History (#11343): the alias was UNCONDITIONAL, added to keep the bundler from
* tracing the native addon into a Next.js build worker, whose thread teardown
* can abort with SIGABRT (assertion in node::RemoveEnvironmentCleanupHook) and
* leave the build without standalone output (#10060).
*
* The premise recorded next to that alias — "runtime still uses the real
* package via serverExternalPackages" — does not hold. A Turbopack
* `resolveAlias` rewrites the request BEFORE the externals check runs, so
* `better-sqlite3` becomes a relative path, no longer matches the
* `serverExternalPackages` entry, and the stub is baked into the bundle. Every
* artifact built from that config answered HTTP 500 on every route: the stub's
* default export is not a constructor, the sync driver chain fell through to
* `node:sqlite` and then sql.js, and the instrumentation hook aborted at boot.
*
* This is the same failure shape as #6344 (the @/mitm/manager stub shipping to
* every npm/Electron/VPS artifact), so it gets the same treatment: the alias is
* opt-in, and a default build gets the real, externalized native package.
*
* Set OMNIROUTE_BETTER_SQLITE3_STUB=1 ONLY on a build host that actually hits
* the SIGABRT worker teardown, and never for an artifact that will be run —
* the resulting bundle cannot open a database.
*/
export function shouldStubBetterSqlite3(env = process.env) {
return env.OMNIROUTE_BETTER_SQLITE3_STUB === "1";
}
/** Turbopack resolveAlias fragment for `better-sqlite3`, derived from the env. */
export function betterSqlite3AliasFor(env = process.env) {
return shouldStubBetterSqlite3(env)
? { "better-sqlite3": "./src/lib/db/better-sqlite3.stub.js" }
: {};
}

27
scripts/build/colocate-standalone.mjs
View File

@@ -33,6 +33,14 @@ const STANDALONE = process.env.OMNIROUTE_STANDALONE_DIR
const CALL_LOG_WORKER_REL = join("src", "lib", "usage", "callLogArtifactWorker.js");
const CALL_LOG_WORKER_SRC = join(ROOT, "src", "lib", "usage", "callLogArtifactWorker.ts");
const COMPRESSION_WORKER_REL = join("open-sse", "services", "compression", "compressionWorker.js");
const COMPRESSION_WORKER_SRC = join(
ROOT,
"open-sse",
"services",
"compression",
"compressionWorker.ts"
);
const WORKER_REL = join(
"open-sse",
"services",
@@ -107,9 +115,26 @@ function main() {
);
console.log("[colocate-standalone] ✅ call-log artifact worker bundled");
const compressionWorkerDest = join(STANDALONE, COMPRESSION_WORKER_REL);
mkdirSync(dirname(compressionWorkerDest), { recursive: true });
runBuildTool(
"esbuild",
"esbuild",
[
COMPRESSION_WORKER_SRC,
"--bundle",
"--platform=node",
"--packages=external",
"--format=esm",
`--outfile=${compressionWorkerDest}`,
],
{ stdio: "inherit" }
);
console.log("[colocate-standalone] ✅ compression worker bundled");
// The call-log worker is always present; scope it to ESM immediately. The
// optional LLMLingua worker dir is added below only when its deps are installed.
const workerDirs = [dirname(callLogWorkerDest)];
const workerDirs = [dirname(callLogWorkerDest), dirname(compressionWorkerDest)];
if (!hasOptionals) {
console.log(

19
scripts/build/pack-artifact-policy.ts
View File

@@ -45,6 +45,7 @@ export const APP_STAGING_ALLOWED_EXACT_PATHS: string[] = [
// LLMLingua ONNX worker — esbuild'd standalone .js spawned via worker_threads
// (the Next.js bundler can't trace the computed Worker path). Kept like the MCP server.
"open-sse/services/compression/engines/llmlingua/onnxWorker.js",
"open-sse/services/compression/compressionWorker.js",
"src/lib/usage/callLogArtifactWorker.js",
"package.json",
"peer-stamp.mjs",
@@ -312,13 +313,27 @@ export const PACK_ARTIFACT_NEVER_ALLOWED_SEGMENTS: string[] = ["node_modules"];
export function findUnexpectedArtifactPaths(
filePaths: string[],
{ exactPaths = [], prefixPaths = [] }: { exactPaths?: string[]; prefixPaths?: string[] } = {}
{
exactPaths = [],
prefixPaths = [],
// #9985: the app-STAGING prune (prepublish Step 10.7) must be able to opt out
// of the node_modules segment ban — the standalone server's runtime deps live
// under dist/node_modules and Turbopack-hashed dirs (.build/next/node_modules/
// sql.js-*/dist/sql-wasm.wasm, transformers ort-wasm). Pruning them 500'd every
// DB-backed route in packaged boots while /api/monitoring/health stayed green.
// The PUBLISH gate (validate-pack-artifact) keeps the strict default.
neverAllowedSegments = PACK_ARTIFACT_NEVER_ALLOWED_SEGMENTS,
}: {
exactPaths?: string[];
prefixPaths?: string[];
neverAllowedSegments?: string[];
} = {}
): string[] {
const normalizedExact = new Set(exactPaths.map(normalizeArtifactPath));
const normalizedPrefixes = prefixPaths.map(normalizeArtifactPath);
const hasForbiddenSegment = (filePath: string): boolean =>
filePath.split("/").some((segment) => PACK_ARTIFACT_NEVER_ALLOWED_SEGMENTS.includes(segment));
filePath.split("/").some((segment) => neverAllowedSegments.includes(segment));
return filePaths
.map(normalizeArtifactPath)

24
scripts/build/prepare-electron-standalone.mjs
View File

@@ -1,12 +1,13 @@
#!/usr/bin/env node
import { existsSync, lstatSync, readdirSync, rmSync } from "node:fs";
import { existsSync, lstatSync, mkdirSync, readdirSync, rmSync } from "node:fs";
import { basename, dirname, join, relative } from "node:path";
import { fileURLToPath } from "node:url";
import { assembleStandalone } from "./assembleStandalone.mjs";
import { assertSqlitePrebuildExists } from "./electronRebuildPlan.mjs";
import { pruneElectronRuntimeDocs } from "./electronRuntimeDocs.mjs";
import { stageOptionalPacks } from "./optionalPackStaging.mjs";
import { runBuildTool } from "./buildToolRunner.mjs";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
@@ -169,6 +170,27 @@ assembleStandalone({
// app they would point at the build machine's absolute paths and break on install.
materializeSymlinks: true,
});
const compressionWorkerDest = join(
ELECTRON_STANDALONE_DIR,
"open-sse",
"services",
"compression",
"compressionWorker.js"
);
mkdirSync(dirname(compressionWorkerDest), { recursive: true });
runBuildTool(
"esbuild",
"esbuild",
[
join(ROOT, "open-sse", "services", "compression", "compressionWorker.ts"),
"--bundle",
"--platform=node",
"--packages=external",
"--format=esm",
`--outfile=${compressionWorkerDest}`,
],
{ stdio: "inherit" }
);
const docsPrune = pruneElectronRuntimeDocs(ELECTRON_STANDALONE_DIR);
if (docsPrune.removedFiles > 0) {

40
scripts/build/prepublish.ts
View File

@@ -407,6 +407,40 @@ if (existsSync(llmWorkerSrc)) {
}
}
// ── Step 8.6b: Bundle synchronous compression worker ──────────────────
const compressionWorkerSrc = join(
ROOT,
"open-sse",
"services",
"compression",
"compressionWorker.ts"
);
const compressionWorkerDest = join(
DIST_DIR,
"open-sse",
"services",
"compression",
"compressionWorker.js"
);
if (!existsSync(compressionWorkerSrc)) {
throw new Error("Required compression worker source is missing");
}
console.log(" 🔨 Bundling compression worker...");
mkdirSync(dirname(compressionWorkerDest), { recursive: true });
runBuildTool(
"esbuild",
"esbuild",
[
"open-sse/services/compression/compressionWorker.ts",
"--bundle",
"--platform=node",
"--packages=external",
"--format=esm",
"--outfile=dist/open-sse/services/compression/compressionWorker.js",
],
{ cwd: ROOT, stdio: "inherit" }
);
// ── Step 8.7: Bundle CLI Entrypoint ──────────────────────────
const cliSrcFile = join(ROOT, "bin", "omniroute.ts");
const cliDestFile = join(ROOT, "bin", "omniroute.mjs");
@@ -639,10 +673,15 @@ for (const relativePath of APP_STAGING_REMOVAL_PATHS) {
}
// ── Step 10.7: Prune any staged dist/ file outside the allowed runtime set ──
// #9985: neverAllowedSegments is EMPTY here on purpose — unlike the publish
// tarball gate, the staged dist/ legitimately contains node_modules (the
// standalone server's runtime deps, including Turbopack-hashed packages whose
// wasm files DB init requires). The allowlist prefixes above are the contract.
const stagedFiles = walkFiles(DIST_DIR);
const unexpectedStagedFiles = findUnexpectedArtifactPaths(stagedFiles, {
exactPaths: APP_STAGING_ALLOWED_EXACT_PATHS,
prefixPaths: APP_STAGING_ALLOWED_PATH_PREFIXES,
neverAllowedSegments: [],
});
if (unexpectedStagedFiles.length > 0) {
@@ -657,6 +696,7 @@ if (unexpectedStagedFiles.length > 0) {
const remainingUnexpectedFiles = findUnexpectedArtifactPaths(walkFiles(DIST_DIR), {
exactPaths: APP_STAGING_ALLOWED_EXACT_PATHS,
prefixPaths: APP_STAGING_ALLOWED_PATH_PREFIXES,
neverAllowedSegments: [],
});
if (remainingUnexpectedFiles.length > 0) {

256
scripts/check/check-changelog-integrity.mjs
View File

@@ -1,8 +1,8 @@
#!/usr/bin/env node
// scripts/check/check-changelog-integrity.mjs
//
// Anti "CHANGELOG-eat" gate: no bullet line that exists in the BASE branch's
// CHANGELOG.md may disappear in the merge result. The chronic failure mode is
// Anti "CHANGELOG-eat" gate: no bullet-line occurrence that exists in the BASE
// branch's CHANGELOG.md may disappear in the merge result. The chronic failure mode is
// git's merge auto-resolve silently dropping sibling bullets (or whole version
// sections) when two branches touch adjacent CHANGELOG lines — incident
// 2026-07-05: PR #6193's merge ate 212 lines (the entire [3.8.45] + [3.8.44]
@@ -16,47 +16,221 @@
// quality.yml runs it blocking for own-origin PRs and report-only for forks.
// The release captain's reconciliation rewrites the CHANGELOG legitimately,
// but that happens on the release PR (PR → main, ci.yml), which does not run
// this gate. Escape hatch for intentional removals (e.g. reverting a reverted
// feature's bullet): ALLOW_CHANGELOG_REMOVALS=1 turns failures into a report.
// this gate. There is no runtime escape hatch: every unexplained removal fails.
// Intentional rewrites require a reviewed record in
// config/release/changelog-reconciliations.json. Each record binds the complete base
// and result files by SHA-256 and lists the exact removed/added bullet-line multiset;
// repeated strings encode repeated occurrences. The gate deliberately protects
// bullet lines, not standalone headings, dates, or prose outside a bullet.
//
// Usage:
// node scripts/check/check-changelog-integrity.mjs
// env GITHUB_BASE_REF PR base branch (CI); local fallback: current release/*
// env CHANGELOG_BASE_REF explicit ref override (e.g. origin/release/v3.8.45)
// env ALLOW_CHANGELOG_REMOVALS=1 report-only (never fails)
import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
import { existsSync, readFileSync, readdirSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
const CHANGELOG = "CHANGELOG.md";
const RECONCILIATIONS = "config/release/changelog-reconciliations.json";
const FRAGMENTS_DIR = "changelog.d";
const FRAGMENT_SECTIONS = ["features", "fixes", "maintenance"];
const FRAGMENT_SKIP = new Set(["README.md", ".gitkeep"]);
const SHA256_PATTERN = /^[0-9a-f]{64}$/;
const RECONCILIATION_KEYS = new Set([
"id",
"reason",
"baseChangelogSha256",
"resultChangelogSha256",
"removedBullets",
"addedBullets",
]);
/** Extract the set of bullet lines (trimmed) from a CHANGELOG text. */
export function extractBullets(text) {
const bullets = new Set();
return new Set(extractBulletOccurrences(text));
}
/** Extract every bullet-line occurrence, preserving order and duplicates. */
export function extractBulletOccurrences(text) {
const bullets = [];
for (const raw of String(text || "").split("\n")) {
const line = raw.trim();
if (line.startsWith("- ") && line.length > 4) bullets.add(line);
if (line.startsWith("- ") && line.length > 4) bullets.push(line);
}
return bullets;
}
function findMissingOccurrences(sourceText, targetText) {
const available = new Map();
for (const bullet of extractBulletOccurrences(targetText)) {
available.set(bullet, (available.get(bullet) || 0) + 1);
}
const missing = [];
for (const bullet of extractBulletOccurrences(sourceText)) {
const count = available.get(bullet) || 0;
if (count > 0) available.set(bullet, count - 1);
else missing.push(bullet);
}
return missing;
}
/**
* Bullet lines present in the base CHANGELOG but absent from the head
* CHANGELOG — the "eaten" set. Pure so it has a unit test.
* Bullet-line occurrences present in the base CHANGELOG but absent from the head
* CHANGELOG — including one lost copy of a repeated line. Pure so it has a unit test.
*/
export function findLostBullets(baseText, headText) {
const headBullets = extractBullets(headText);
const lost = [];
for (const b of extractBullets(baseText)) {
if (!headBullets.has(b)) lost.push(b);
return findMissingOccurrences(baseText, headText);
}
/** Bullet-line occurrences present only in the result CHANGELOG. */
export function findAddedBullets(baseText, headText) {
return findMissingOccurrences(headText, baseText);
}
/** Stable digest tying a reconciliation record to the complete file, not just its bullets. */
export function changelogSha256(text) {
return createHash("sha256")
.update(String(text || ""), "utf8")
.digest("hex");
}
function validateBulletList(value, path, { allowEmpty }) {
if (!Array.isArray(value)) return [`${path} must be an array`];
const errors = [];
if (!allowEmpty && value.length === 0) errors.push(`${path} must not be empty`);
for (let index = 0; index < value.length; index++) {
const bullet = value[index];
if (
typeof bullet !== "string" ||
bullet !== bullet.trim() ||
!bullet.startsWith("- ") ||
bullet.length <= 4
) {
errors.push(`${path}[${index}] must be one exact, trimmed markdown bullet`);
}
}
return lost;
return errors;
}
/** Validate the durable reconciliation ledger without trusting any of its claims. */
export function validateReconciliationLedger(value) {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return ["ledger must be a JSON object"];
}
const errors = [];
const topLevelKeys = Object.keys(value);
for (const key of topLevelKeys) {
if (key !== "schemaVersion" && key !== "reconciliations") {
errors.push(`unknown top-level field: ${key}`);
}
}
if (value.schemaVersion !== 1) errors.push("schemaVersion must be 1");
if (!Array.isArray(value.reconciliations)) {
errors.push("reconciliations must be an array");
return errors;
}
const ids = new Set();
const filePairs = new Set();
for (let index = 0; index < value.reconciliations.length; index++) {
const record = value.reconciliations[index];
const path = `reconciliations[${index}]`;
if (!record || typeof record !== "object" || Array.isArray(record)) {
errors.push(`${path} must be an object`);
continue;
}
for (const key of Object.keys(record)) {
if (!RECONCILIATION_KEYS.has(key)) errors.push(`${path} has unknown field: ${key}`);
}
if (typeof record.id !== "string" || !/^[a-z0-9][a-z0-9._-]{2,79}$/.test(record.id)) {
errors.push(`${path}.id must be a 3-80 character lowercase slug`);
} else if (ids.has(record.id)) {
errors.push(`${path}.id duplicates "${record.id}"`);
} else {
ids.add(record.id);
}
if (typeof record.reason !== "string" || record.reason.trim().length < 20) {
errors.push(`${path}.reason must explain the reconciliation in at least 20 characters`);
}
if (!SHA256_PATTERN.test(record.baseChangelogSha256 || "")) {
errors.push(`${path}.baseChangelogSha256 must be a lowercase SHA-256 digest`);
}
if (!SHA256_PATTERN.test(record.resultChangelogSha256 || "")) {
errors.push(`${path}.resultChangelogSha256 must be a lowercase SHA-256 digest`);
}
if (
SHA256_PATTERN.test(record.baseChangelogSha256 || "") &&
record.baseChangelogSha256 === record.resultChangelogSha256
) {
errors.push(`${path} must describe a changed CHANGELOG.md`);
}
errors.push(
...validateBulletList(record.removedBullets, `${path}.removedBullets`, {
allowEmpty: false,
}),
...validateBulletList(record.addedBullets, `${path}.addedBullets`, { allowEmpty: true })
);
if (Array.isArray(record.removedBullets) && Array.isArray(record.addedBullets)) {
const removed = new Set(record.removedBullets);
for (const bullet of record.addedBullets) {
if (removed.has(bullet)) errors.push(`${path} lists the same bullet as removed and added`);
}
}
const pair = `${record.baseChangelogSha256}:${record.resultChangelogSha256}`;
if (filePairs.has(pair)) errors.push(`${path} duplicates an earlier base/result digest pair`);
filePairs.add(pair);
}
return errors;
}
function sameStringMultiset(left, right) {
if (left.length !== right.length) return false;
const remaining = new Map();
for (const item of right) remaining.set(item, (remaining.get(item) || 0) + 1);
for (const item of left) {
const count = remaining.get(item) || 0;
if (count === 0) return false;
remaining.set(item, count - 1);
}
return true;
}
/** Find the single record that exactly explains this complete base → result transition. */
export function findLedgeredReconciliation(baseText, headText, ledger) {
const baseChangelogSha256 = changelogSha256(baseText);
const resultChangelogSha256 = changelogSha256(headText);
const removedBullets = findLostBullets(baseText, headText);
const addedBullets = findAddedBullets(baseText, headText);
return ledger.reconciliations.find(
(record) =>
record.baseChangelogSha256 === baseChangelogSha256 &&
record.resultChangelogSha256 === resultChangelogSha256 &&
sameStringMultiset(record.removedBullets, removedBullets) &&
sameStringMultiset(record.addedBullets, addedBullets)
);
}
function readReconciliationLedger(root = ROOT) {
const path = join(root, RECONCILIATIONS);
if (!existsSync(path)) {
return { ledger: null, errors: [`${RECONCILIATIONS} is missing`] };
}
let ledger;
try {
ledger = JSON.parse(readFileSync(path, "utf8"));
} catch (error) {
return {
ledger: null,
errors: [`${RECONCILIATIONS} is not valid JSON: ${error.message}`],
};
}
return { ledger, errors: validateReconciliationLedger(ledger) };
}
/**
@@ -111,7 +285,13 @@ function resolveBaseRef() {
if (process.env.GITHUB_BASE_REF) return `origin/${process.env.GITHUB_BASE_REF}`;
// Local fallback: the highest release/v* on origin (the active development base).
try {
const branches = git(["branch", "-r", "--list", "origin/release/v*", "--format=%(refname:short)"])
const branches = git([
"branch",
"-r",
"--list",
"origin/release/v*",
"--format=%(refname:short)",
])
.split("\n")
.map((s) => s.trim())
.filter(Boolean)
@@ -123,16 +303,33 @@ function resolveBaseRef() {
}
function main() {
if (Object.hasOwn(process.env, "ALLOW_CHANGELOG_REMOVALS")) {
console.error(
"[changelog-integrity] ALLOW_CHANGELOG_REMOVALS was removed; delete it from the environment and record intentional transformations in config/release/changelog-reconciliations.json."
);
return 1;
}
// Fragment well-formedness first (changelog.d/ — the fragments pattern makes the
// eat-guard below structurally unnecessary for PRs that stop editing CHANGELOG.md).
const invalidFragments = findInvalidFragments();
if (invalidFragments.length > 0) {
console.error(`[changelog-integrity] ${invalidFragments.length} invalid changelog fragment(s):`);
console.error(
`[changelog-integrity] ${invalidFragments.length} invalid changelog fragment(s):`
);
for (const { file, error } of invalidFragments) console.error(` ✗ ${file}: ${error}`);
console.error("\nSee changelog.d/README.md for the fragment convention.");
return 1;
}
const { ledger, errors: ledgerErrors } = readReconciliationLedger();
if (ledgerErrors.length > 0) {
console.error(`[changelog-integrity] invalid reconciliation ledger (${ledgerErrors.length}):`);
for (const error of ledgerErrors) console.error(` ✗ ${error}`);
return 1;
}
const hasExplicitBaseRef = Boolean(process.env.CHANGELOG_BASE_REF || process.env.GITHUB_BASE_REF);
const baseRef = resolveBaseRef();
if (!baseRef) {
console.log("[changelog-integrity] SKIP — could not resolve a base ref (offline/fresh clone).");
@@ -143,6 +340,12 @@ function main() {
try {
baseText = git(["show", `${baseRef}:${CHANGELOG}`]);
} catch {
if (hasExplicitBaseRef) {
console.error(
`[changelog-integrity] FAIL — ${CHANGELOG} not readable at explicit base ${baseRef}.`
);
return 1;
}
console.log(`[changelog-integrity] SKIP — ${CHANGELOG} not readable at ${baseRef}.`);
return 0;
}
@@ -154,21 +357,30 @@ function main() {
return 0;
}
const reconciliation = findLedgeredReconciliation(baseText, headText, ledger);
if (reconciliation) {
console.log(
`[changelog-integrity] OK — ${lost.length} removed base bullet(s) covered by ledgered reconciliation "${reconciliation.id}" vs ${baseRef}.`
);
return 0;
}
console.error(
`[changelog-integrity] ${lost.length} bullet(s) present in ${baseRef} are MISSING from this tree's ${CHANGELOG}:`
);
for (const b of lost.slice(0, 15)) console.error(` ✗ ${b.slice(0, 160)}`);
if (lost.length > 15) console.error(` … and ${lost.length - 15} more`);
const added = findAddedBullets(baseText, headText);
console.error(
"\nThis is the CHANGELOG-eat pattern (merge auto-resolve dropping sibling bullets)." +
"\nFix: restore the base CHANGELOG (`git checkout <base> -- CHANGELOG.md`), re-insert ONLY" +
"\nyour own bullet, and prove the net diff is additive. Intentional removals (rare):" +
"\nre-run with ALLOW_CHANGELOG_REMOVALS=1 and justify in the PR body."
"\nyour own bullet, and prove the net diff is additive." +
`\nIntentional reconciliation: add one exact, reviewed record to ${RECONCILIATIONS}.` +
`\n baseChangelogSha256: ${changelogSha256(baseText)}` +
`\n resultChangelogSha256: ${changelogSha256(headText)}` +
`\n removedBullets: ${lost.length}; addedBullets: ${added.length}` +
"\nThere is no environment-variable bypass."
);
if (process.env.ALLOW_CHANGELOG_REMOVALS === "1") {
console.error("[changelog-integrity] ALLOW_CHANGELOG_REMOVALS=1 — reporting only, not failing.");
return 0;
}
return 1;
}

14
scripts/check/check-cli-i18n.mjs
View File

@@ -69,6 +69,7 @@ const files = walk(COMMANDS_DIR);
const usedKeys = collectTKeys(files);
const en = loadJson(join(LOCALES_DIR, "en.json"));
const ptBR = loadJson(join(LOCALES_DIR, "pt-BR.json"));
const zhLocales = ["zh-CN", "zh-TW"].map((n) => [n, loadJson(join(LOCALES_DIR, `${n}.json`))]);
const enKeys = flattenKeys(en);
let errors = 0;
@@ -95,6 +96,19 @@ if (missingTopLevel.length > 0) {
console.log(`[cli-i18n] ✓ pt-BR.json has all ${enTopLevel.length} top-level sections`);
}
// Check 3: zh-CN and zh-TW have full key parity with en.json
for (const [name, cat] of zhLocales) {
const catKeys = flattenKeys(cat);
const missingKeys = [...enKeys].filter((k) => !catKeys.has(k));
if (missingKeys.length > 0) {
console.error(`[cli-i18n] Keys in en.json missing from ${name}.json:`);
for (const k of missingKeys) console.error(` ✗ ${k}`);
errors += missingKeys.length;
} else {
console.log(`[cli-i18n] ✓ ${name}.json has full parity (${enKeys.size} keys)`);
}
}
if (errors > 0) {
console.error(`[cli-i18n] FAIL — ${errors} error(s) found`);
process.exit(1);

8
scripts/check/check-fabricated-docs.mjs
View File

@@ -93,6 +93,14 @@ const ENV_VAR_ALLOWLIST = new Set([
"DATA_DIR",
"REQUIRE_API_KEY",
"OMNIROUTE_BUILD_PROFILE", // build-time only
// Docker builder-stage knobs. Both are documented in docs/guides/DOCKER_GUIDE.md
// because they are the two levers for a memory-constrained build host, but
// neither is read through process.env in this repo: OMNIROUTE_BUILD_WORKERS is
// a Dockerfile ARG that only feeds CIRCLE_NODE_TOTAL, and CIRCLE_NODE_TOTAL is
// read by Next itself (node_modules) to size the page-data worker pool. Pinned
// by tests/unit/docker-build-memory-budget.test.ts.
"OMNIROUTE_BUILD_WORKERS",
"CIRCLE_NODE_TOTAL",
"OMNIROUTE_BUILD_SHA",
"OMNIROUTE_URL", // used by ad-hoc tooling, validated elsewhere
"OMNIROUTE_KEY", // ditto

29
scripts/docs/render-diagrams.mjs
View File

@@ -18,11 +18,13 @@
* gate on it.
*/
import { spawnSync } from "node:child_process";
import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { tmpdir } from "node:os";
import { ensureSvgAccessibility, validateSvgFile } from "./validate-svg.mjs";
const __dirname = dirname(fileURLToPath(import.meta.url));
const repoRoot = resolve(__dirname, "..", "..");
const srcDir = resolve(repoRoot, "docs", "diagrams");
@@ -75,6 +77,31 @@ for (const src of sources) {
if (result.status !== 0) {
console.error(` [FAIL] ${src} (exit ${result.status})`);
failures += 1;
continue;
}
const source = readFileSync(input, "utf8");
const title = source.match(/^%%\s*svg-title:\s*(.+)$/im)?.[1]?.trim();
const description = source.match(/^%%\s*svg-description:\s*(.+)$/im)?.[1]?.trim();
if (title && description) {
const svg = readFileSync(output, "utf8");
writeFileSync(
output,
ensureSvgAccessibility(svg, {
title,
description,
idBase: src.replace(/\.mmd$/, ""),
})
);
} else if (title || description) {
console.warn(` [WARN] ${src}: svg-title and svg-description must be provided together`);
}
const validation = validateSvgFile(output);
for (const warning of validation.warnings) console.warn(` [WARN] ${src}: ${warning}`);
if (validation.errors.length > 0) {
for (const error of validation.errors) console.error(` [FAIL] ${src}: ${error}`);
failures += 1;
}
}

167
scripts/docs/validate-svg.mjs Normal file
View File

@@ -0,0 +1,167 @@
#!/usr/bin/env node
import { readFileSync, writeFileSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { XMLParser, XMLValidator } from "fast-xml-parser";
const parser = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: "@_",
preserveOrder: true,
});
function collectIds(value, ids) {
if (Array.isArray(value)) {
for (const entry of value) collectIds(entry, ids);
return;
}
if (!value || typeof value !== "object") return;
const attributes = value[":@"];
if (attributes && typeof attributes === "object" && typeof attributes["@_id"] === "string") {
ids.push(attributes["@_id"]);
}
for (const entry of Object.values(value)) collectIds(entry, ids);
}
function escapeXml(value) {
return value
.replaceAll("&", "&amp;")
.replaceAll("<", "&lt;")
.replaceAll(">", "&gt;")
.replaceAll('"', "&quot;")
.replaceAll("'", "&apos;");
}
function replaceRootAttribute(openingTag, name, value) {
const attribute = new RegExp(`\\s${name}=(?:"[^"]*"|'[^']*')`, "i");
const withoutExisting = openingTag.replace(attribute, "");
return withoutExisting.replace(/>$/, ` ${name}="${escapeXml(value)}">`);
}
function escapeRegExp(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
export function ensureSvgAccessibility(svg, { title, description, idBase }) {
const xmlResult = XMLValidator.validate(svg);
if (xmlResult !== true) throw new Error(`invalid XML: ${xmlResult.err.msg}`);
const titleId = `${idBase}-title`;
const descriptionId = `${idBase}-desc`;
const priorTitle = new RegExp(
`<title\\b[^>]*\\bid=["']${escapeRegExp(titleId)}["'][^>]*>[\\s\\S]*?<\\/title>`,
"i"
);
const priorDescription = new RegExp(
`<desc\\b[^>]*\\bid=["']${escapeRegExp(descriptionId)}["'][^>]*>[\\s\\S]*?<\\/desc>`,
"i"
);
const withoutPriorAccessibleName = svg.replace(priorTitle, "").replace(priorDescription, "");
const match = withoutPriorAccessibleName.match(/<svg\b[^>]*>/i);
if (!match) throw new Error("document root is not an SVG element");
let openingTag = replaceRootAttribute(match[0], "role", "img");
openingTag = replaceRootAttribute(openingTag, "aria-labelledby", `${titleId} ${descriptionId}`);
const accessibleName =
`<title id="${escapeXml(titleId)}">${escapeXml(title)}</title>` +
`<desc id="${escapeXml(descriptionId)}">${escapeXml(description)}</desc>`;
return withoutPriorAccessibleName.replace(match[0], `${openingTag}${accessibleName}`);
}
export function validateSvgText(svg) {
const xmlResult = XMLValidator.validate(svg);
if (xmlResult !== true) {
return { errors: [`invalid XML: ${xmlResult.err.msg}`], warnings: [] };
}
const document = parser.parse(svg);
const ids = [];
collectIds(document, ids);
const duplicates = [...new Set(ids.filter((id, index) => ids.indexOf(id) !== index))].sort();
const openingTag = svg.match(/<svg\b[^>]*>/i)?.[0] ?? "";
const warnings = [];
if (!/\srole=["']img["']/i.test(openingTag)) warnings.push('root role is not "img"');
const hasAccessibleName =
/\saria-(?:label|labelledby)=["'][^"']+["']/i.test(openingTag) ||
/<title\b[^>]*>[^<]+<\/title>/i.test(svg);
if (!hasAccessibleName) {
warnings.push("missing accessible name (title, aria-label, or aria-labelledby)");
}
if (!/<desc\b[^>]*>[^<]+<\/desc>/i.test(svg)) warnings.push("missing desc element");
if (/<foreignObject\b/i.test(svg)) warnings.push("foreignObject present (Mermaid output)");
if (/\s(?:width|height)=["'][^"']+["']/i.test(openingTag)) {
warnings.push("fixed root width or height present (Mermaid output)");
}
return {
errors: duplicates.length > 0 ? [`duplicate IDs: ${duplicates.join(", ")}`] : [],
warnings,
};
}
export function validateSvgFile(file) {
return validateSvgText(readFileSync(file, "utf8"));
}
function isDirectExecution() {
if (!process.argv[1]) return false;
return fileURLToPath(import.meta.url) === path.resolve(process.argv[1]);
}
if (isDirectExecution()) {
const args = process.argv.slice(2);
let fixAccessibility = false;
let title;
let description;
const files = [];
for (let index = 0; index < args.length; index += 1) {
const arg = args[index];
if (arg === "--fix-a11y") {
fixAccessibility = true;
} else if (arg === "--title") {
title = args[++index];
} else if (arg === "--description") {
description = args[++index];
} else {
files.push(arg);
}
}
if (files.length === 0) {
console.error(
"Usage: node scripts/docs/validate-svg.mjs [--fix-a11y --title TEXT --description TEXT] <file.svg> [...]"
);
process.exit(2);
}
if (fixAccessibility && (!title || !description)) {
console.error("--fix-a11y requires both --title and --description");
process.exit(2);
}
let failures = 0;
for (const file of files) {
if (fixAccessibility) {
const idBase = path.basename(file, path.extname(file));
const updated = ensureSvgAccessibility(readFileSync(file, "utf8"), {
title,
description,
idBase,
});
writeFileSync(file, updated);
}
const result = validateSvgFile(file);
for (const warning of result.warnings) console.warn(`WARN ${file}: ${warning}`);
if (result.errors.length === 0) {
console.log(`PASS ${file}`);
continue;
}
failures += 1;
for (const error of result.errors) console.error(`FAIL ${file}: ${error}`);
}
if (failures > 0) process.exit(1);
}

75
scripts/perf/video-bridge-bench.ts
View File

@@ -1,24 +1,80 @@
/**
* Video Bridge benchmarks (VB-FU-07 sampler overhead + VB-FU-09 contact sheet A/B).
* Video Bridge benchmarks (VB-FU-03 dedup comparator, VB-FU-07 sampler overhead,
* and VB-FU-09 contact sheet A/B).
*
* Run: node --import tsx/esm scripts/perf/video-bridge-bench.ts
*
* 1. Sampler: measures the pure timestamp-selection cost of uniform vs
* 1. Dedup: measures bounded CPU and process-memory observations for the
* production 16x16 grayscale comparator over the hard 16-frame candidate cap.
* 2. Sampler: measures the pure timestamp-selection cost of uniform vs
* scene_aware vs segment_aware for growing scene-candidate counts. The
* ffmpeg scene-detection pass is shared by both aware policies and is
* I/O-bound, so the incremental policy cost is exactly this selection step.
* 2. Contact sheet: composes synthetic JPEG frames into the timestamped grid
* and compares payload bytes + model calls against individual frames.
* 3. Contact sheet: composes synthetic JPEG frames into the visually timestamped
* grid and compares payload bytes + structural call counts. This microbenchmark
* does not measure real-model tokens, latency, or quality; use
* video-bridge-contact-sheet-eval.ts before considering promotion.
*/
import { performance } from "node:perf_hooks";
import { buildVideoContactSheet } from "../../src/lib/guardrails/videoBridgeContactSheet";
import {
compareVideoFramesByGrayscale,
VIDEO_DEDUP_POLICY_VERSION,
VIDEO_DEDUP_THRESHOLD,
} from "../../src/lib/guardrails/videoBridgeHelpers";
import {
calculateSamplingDecision,
type VideoSamplingPolicy,
} from "../../src/lib/guardrails/videoBridgeRuntime";
const SAMPLER_ITERATIONS = 2_000;
const DEDUP_FRAME_CAP = 16;
const DEDUP_ITERATIONS = 10;
function mebibytes(bytes: number): string {
return (bytes / (1024 * 1024)).toFixed(2);
}
async function benchDedupComparator(): Promise<void> {
const frames = await Promise.all(
Array.from({ length: DEDUP_FRAME_CAP }, async (_unused, index) => ({
dataUri: await syntheticJpegFrame(index, 1024, 576),
timestampSeconds: index,
}))
);
await compareVideoFramesByGrayscale(frames[0], frames[1]);
const memoryBefore = process.memoryUsage();
const maxRssBefore = process.resourceUsage().maxRSS * 1024;
const cpuBefore = process.cpuUsage();
const wallBefore = performance.now();
let comparisons = 0;
for (let iteration = 0; iteration < DEDUP_ITERATIONS; iteration++) {
for (let index = 1; index < frames.length; index++) {
await compareVideoFramesByGrayscale(frames[index - 1], frames[index]);
comparisons += 1;
}
}
const wallMs = performance.now() - wallBefore;
const cpu = process.cpuUsage(cpuBefore);
const memoryAfter = process.memoryUsage();
const maxRssAfter = process.resourceUsage().maxRSS * 1024;
const cpuMs = (cpu.user + cpu.system) / 1000;
console.log("== Visual dedup comparator (synthetic 1024x576 JPEG, bounded) ==");
console.log(
`policy=${VIDEO_DEDUP_POLICY_VERSION} threshold=${VIDEO_DEDUP_THRESHOLD} frames=${DEDUP_FRAME_CAP} iterations=${DEDUP_ITERATIONS} comparisons=${comparisons}`
);
console.log(
`wall_ms=${wallMs.toFixed(1)} cpu_ms=${cpuMs.toFixed(1)} cpu_ms/comparison=${(cpuMs / comparisons).toFixed(3)}`
);
console.log(
`rss_delta_MiB=${mebibytes(memoryAfter.rss - memoryBefore.rss)} heap_delta_MiB=${mebibytes(memoryAfter.heapUsed - memoryBefore.heapUsed)} max_rss_delta_MiB=${mebibytes(Math.max(0, maxRssAfter - maxRssBefore))}`
);
console.log(
"Scope: comparator decode/resize/delta cost only; this does not measure caption-model quality."
);
}
function benchSampler(): void {
console.log("== Sampler timestamp-selection cost (pure, per call) ==");
@@ -47,12 +103,12 @@ function benchSampler(): void {
}
}
async function syntheticJpegFrame(index: number): Promise<string> {
async function syntheticJpegFrame(index: number, width = 512, height = 288): Promise<string> {
const { default: sharp } = await import("sharp");
const buffer = await sharp({
create: {
width: 512,
height: 288,
width,
height,
channels: 3,
background: { r: (index * 37) % 255, g: (index * 91) % 255, b: (index * 53) % 255 },
},
@@ -64,6 +120,9 @@ async function syntheticJpegFrame(index: number): Promise<string> {
async function benchContactSheet(): Promise<void> {
console.log("\n== Contact sheet vs individual frames (synthetic 512x288 JPEG) ==");
console.log(
"STRUCTURAL ONLY: real-model tokens/latency/quality are unmeasured; promotion remains HOLD."
);
console.log("frames | sheet_ms sheet_KiB individual_KiB model_calls(sheet/individual)");
for (const frameCount of [1, 4, 8, 16]) {
const frames = await Promise.all(
@@ -86,5 +145,7 @@ async function benchContactSheet(): Promise<void> {
}
}
await benchDedupComparator();
console.log("");
benchSampler();
await benchContactSheet();

578
scripts/perf/video-bridge-contact-sheet-eval.ts Normal file
View File

@@ -0,0 +1,578 @@
#!/usr/bin/env node
import { createHash } from "node:crypto";
import { readFile } from "node:fs/promises";
import path from "node:path";
import { performance } from "node:perf_hooks";
import { fileURLToPath } from "node:url";
import { z } from "zod";
import {
buildVideoContactSheet,
type ContactSheetFrame,
} from "../../src/lib/guardrails/videoBridgeContactSheet";
export type VideoContactSheetEvalConfigurationState = "configured-not-executed" | "not-configured";
export interface VideoContactSheetEvalHoldReportInput {
caseCount: number;
configurationState: VideoContactSheetEvalConfigurationState;
missingConfiguration?: string[];
}
export interface VideoContactSheetEvalHoldReport {
caseCount: number;
execution: {
realModel: false;
state: VideoContactSheetEvalConfigurationState;
};
kind: "video-contact-sheet-ab-eval";
missingConfiguration: string[];
promotion: {
reasons: ["REAL_MODEL_CONFIGURATION_MISSING" | "REAL_MODEL_EVAL_NOT_EXECUTED"];
status: "HOLD";
};
results: [];
schemaVersion: 1;
summary: null;
}
export interface VideoContactSheetEvalThresholds {
minLatencyReductionRatio: number;
minQualityRetention: number;
minQualityScore: number;
minTokenReductionRatio: number;
}
export interface VideoContactSheetEvalAggregate {
latencyMs: number;
qualityScore: number;
totalTokens: number | null;
}
export type VideoContactSheetPromotionReason =
| "LATENCY_REDUCTION_BELOW_THRESHOLD"
| "QUALITY_RETENTION_BELOW_THRESHOLD"
| "QUALITY_SCORE_BELOW_THRESHOLD"
| "TOKEN_REDUCTION_BELOW_THRESHOLD"
| "TOKEN_USAGE_UNAVAILABLE";
export interface VideoContactSheetPromotionDecision {
metrics: {
latencyReductionRatio: number;
qualityRetention: number;
tokenReductionRatio: number | null;
};
reasons: VideoContactSheetPromotionReason[];
status: "ELIGIBLE" | "HOLD";
}
const MAX_EVAL_FRAME_BASE64_CHARS = 5_592_408;
const evalThresholdsSchema = z
.object({
minLatencyReductionRatio: z.number().positive().max(1),
minQualityRetention: z.number().min(0).max(1),
minQualityScore: z.number().min(0).max(1),
minTokenReductionRatio: z.number().positive().max(1),
})
.strict();
const evalManifestSchema = z
.object({
cases: z
.array(
z
.object({
expectedFacts: z
.array(
z
.object({
id: z.string().min(1),
requiredTerms: z.array(z.string().min(1)).min(1),
timestampSeconds: z.number().finite().nonnegative(),
})
.strict()
)
.min(1),
frames: z
.array(
z
.object({
dataUri: z
.string()
.max("data:image/jpeg;base64,".length + MAX_EVAL_FRAME_BASE64_CHARS)
.regex(
/^data:image\/jpeg;base64,[A-Za-z0-9+/=]{4,5592408}$/i,
"expected a bounded JPEG data URI"
),
timestampSeconds: z.number().finite().nonnegative(),
})
.strict()
)
.min(1)
.max(16),
id: z.string().min(1),
prompt: z.string().min(1),
})
.strict()
)
.min(1),
id: z.string().min(1),
schemaVersion: z.literal(1),
thresholds: evalThresholdsSchema,
})
.strict();
const chatCompletionSchema = z
.object({
choices: z
.array(
z
.object({
message: z.object({ content: z.string() }).passthrough(),
})
.passthrough()
)
.min(1),
usage: z
.object({
completion_tokens: z.number().nonnegative().optional(),
prompt_tokens: z.number().nonnegative().optional(),
total_tokens: z.number().nonnegative().optional(),
})
.passthrough()
.optional(),
})
.passthrough();
export type VideoContactSheetEvalManifest = z.infer<typeof evalManifestSchema>;
export interface VideoContactSheetEvalConfig {
apiKey: string;
endpoint: string;
model: string;
}
interface EvalFactScore {
matchedFactIds: string[];
qualityScore: number;
}
interface EvalPathResult extends EvalFactScore {
latencyMs: number;
modelCalls: number;
responseDigest: string;
totalTokens: number | null;
}
export interface VideoContactSheetEvalCaseResult {
caseId: string;
individual: EvalPathResult;
sheet: EvalPathResult;
}
export interface VideoContactSheetEvalExecutedReport {
caseCount: number;
execution: {
realModel: true;
state: "executed";
};
generatedAt: string;
kind: "video-contact-sheet-ab-eval";
manifestDigest: string;
manifestId: string;
model: string;
promotion: VideoContactSheetPromotionDecision;
results: VideoContactSheetEvalCaseResult[];
schemaVersion: 1;
summary: {
individual: VideoContactSheetEvalAggregate & { modelCalls: number };
sheet: VideoContactSheetEvalAggregate & { modelCalls: number };
};
thresholds: VideoContactSheetEvalThresholds;
}
type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
export function createVideoContactSheetEvalHoldReport(
input: VideoContactSheetEvalHoldReportInput
): VideoContactSheetEvalHoldReport {
const reason =
input.configurationState === "not-configured"
? "REAL_MODEL_CONFIGURATION_MISSING"
: "REAL_MODEL_EVAL_NOT_EXECUTED";
return {
caseCount: input.caseCount,
execution: {
realModel: false,
state: input.configurationState,
},
kind: "video-contact-sheet-ab-eval",
missingConfiguration: [...(input.missingConfiguration ?? [])],
promotion: {
reasons: [reason],
status: "HOLD",
},
results: [],
schemaVersion: 1,
summary: null,
};
}
function reductionRatio(baseline: number, candidate: number): number {
if (baseline <= 0) return 0;
return (baseline - candidate) / baseline;
}
export function assessVideoContactSheetPromotion(input: {
individual: VideoContactSheetEvalAggregate;
sheet: VideoContactSheetEvalAggregate;
thresholds: VideoContactSheetEvalThresholds;
}): VideoContactSheetPromotionDecision {
const latencyReductionRatio = reductionRatio(input.individual.latencyMs, input.sheet.latencyMs);
const qualityRetention =
input.individual.qualityScore > 0
? input.sheet.qualityScore / input.individual.qualityScore
: 0;
const tokenReductionRatio =
input.individual.totalTokens === null || input.sheet.totalTokens === null
? null
: reductionRatio(input.individual.totalTokens, input.sheet.totalTokens);
const reasons: VideoContactSheetPromotionReason[] = [];
const requiredLatencyReduction = Math.max(
Number.EPSILON,
input.thresholds.minLatencyReductionRatio
);
const requiredTokenReduction = Math.max(Number.EPSILON, input.thresholds.minTokenReductionRatio);
if (latencyReductionRatio < requiredLatencyReduction) {
reasons.push("LATENCY_REDUCTION_BELOW_THRESHOLD");
}
if (input.sheet.qualityScore < input.thresholds.minQualityScore) {
reasons.push("QUALITY_SCORE_BELOW_THRESHOLD");
}
if (qualityRetention < input.thresholds.minQualityRetention) {
reasons.push("QUALITY_RETENTION_BELOW_THRESHOLD");
}
if (tokenReductionRatio === null) {
reasons.push("TOKEN_USAGE_UNAVAILABLE");
} else if (tokenReductionRatio < requiredTokenReduction) {
reasons.push("TOKEN_REDUCTION_BELOW_THRESHOLD");
}
return {
metrics: {
latencyReductionRatio,
qualityRetention,
tokenReductionRatio,
},
reasons,
status: reasons.length === 0 ? "ELIGIBLE" : "HOLD",
};
}
function normalizeEvalText(value: string): string {
return value
.normalize("NFD")
.replace(/[\u0300-\u036f]/g, "")
.toLowerCase();
}
function formatEvalTimestamp(timestampSeconds: number): string {
const totalMilliseconds = Math.max(0, Math.round(timestampSeconds * 1000));
const minutes = Math.floor(totalMilliseconds / 60_000);
const seconds = Math.floor((totalMilliseconds % 60_000) / 1000);
const milliseconds = totalMilliseconds % 1000;
return `${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}.${String(milliseconds).padStart(3, "0")}`;
}
function scoreFacts(
response: string,
expectedFacts: VideoContactSheetEvalManifest["cases"][number]["expectedFacts"]
): EvalFactScore {
const normalizedResponse = normalizeEvalText(response);
const matchedFactIds = expectedFacts
.filter((fact) => {
const timestamp = normalizeEvalText(formatEvalTimestamp(fact.timestampSeconds));
const timestampIndex = normalizedResponse.indexOf(timestamp);
if (timestampIndex < 0) return false;
const factWindow = normalizedResponse.slice(
Math.max(0, timestampIndex - 160),
Math.min(normalizedResponse.length, timestampIndex + timestamp.length + 160)
);
return fact.requiredTerms.every((term) => factWindow.includes(normalizeEvalText(term)));
})
.map((fact) => fact.id);
return {
matchedFactIds,
qualityScore: matchedFactIds.length / expectedFacts.length,
};
}
function digestResponse(response: string): string {
return createHash("sha256").update(response).digest("hex");
}
function sumTokens(values: Array<number | null>): number | null {
if (values.some((value) => value === null)) return null;
return values.reduce<number>((sum, value) => sum + (value ?? 0), 0);
}
async function callVisionModel(input: {
config: VideoContactSheetEvalConfig;
dataUri: string;
fetchImpl: FetchLike;
prompt: string;
}): Promise<{ content: string; totalTokens: number | null }> {
const response = await input.fetchImpl(input.config.endpoint, {
body: JSON.stringify({
messages: [
{
content: [
{ text: input.prompt, type: "text" },
{ image_url: { url: input.dataUri }, type: "image_url" },
],
role: "user",
},
],
model: input.config.model,
temperature: 0,
}),
headers: {
authorization: `Bearer ${input.config.apiKey}`,
"content-type": "application/json",
},
method: "POST",
});
if (!response.ok) {
throw new Error(`Video contact-sheet eval request failed with HTTP ${response.status}`);
}
const parsed = chatCompletionSchema.parse(await response.json());
const usage = parsed.usage;
const totalTokens =
usage?.total_tokens ??
(usage?.prompt_tokens !== undefined && usage.completion_tokens !== undefined
? usage.prompt_tokens + usage.completion_tokens
: null);
return {
content: parsed.choices[0].message.content,
totalTokens,
};
}
async function evaluateIndividualFrames(input: {
evalCase: VideoContactSheetEvalManifest["cases"][number];
config: VideoContactSheetEvalConfig;
fetchImpl: FetchLike;
}): Promise<EvalPathResult> {
const startedAt = performance.now();
const calls: Array<{ content: string; totalTokens: number | null }> = [];
for (const frame of input.evalCase.frames) {
calls.push(
await callVisionModel({
config: input.config,
dataUri: frame.dataUri,
fetchImpl: input.fetchImpl,
prompt: `${input.evalCase.prompt}\nAnalyze only the frame at ${formatEvalTimestamp(frame.timestampSeconds)}. Associate every observation with that exact timestamp label.`,
})
);
}
const content = calls.map((call) => call.content).join("\n");
return {
...scoreFacts(content, input.evalCase.expectedFacts),
latencyMs: performance.now() - startedAt,
modelCalls: calls.length,
responseDigest: digestResponse(content),
totalTokens: sumTokens(calls.map((call) => call.totalTokens)),
};
}
async function evaluateContactSheet(input: {
evalCase: VideoContactSheetEvalManifest["cases"][number];
config: VideoContactSheetEvalConfig;
fetchImpl: FetchLike;
}): Promise<EvalPathResult> {
const startedAt = performance.now();
const sheet = await buildVideoContactSheet(input.evalCase.frames as ContactSheetFrame[], {
columns: 4,
timeoutMs: 30_000,
});
if (!sheet.used || !sheet.dataUri) {
throw new Error("Video contact-sheet eval could not compose the bounded JPEG grid");
}
const call = await callVisionModel({
config: input.config,
dataUri: sheet.dataUri,
fetchImpl: input.fetchImpl,
prompt: `${input.evalCase.prompt}\nAnalyze every cell in the contact sheet. Timestamp labels are burned into each cell. Associate every observation with its visible timestamp.`,
});
return {
...scoreFacts(call.content, input.evalCase.expectedFacts),
latencyMs: performance.now() - startedAt,
modelCalls: 1,
responseDigest: digestResponse(call.content),
totalTokens: call.totalTokens,
};
}
function aggregatePathResults(
results: VideoContactSheetEvalCaseResult[],
path: "individual" | "sheet"
): VideoContactSheetEvalAggregate & { modelCalls: number } {
const pathResults = results.map((result) => result[path]);
return {
latencyMs: pathResults.reduce((sum, result) => sum + result.latencyMs, 0),
modelCalls: pathResults.reduce((sum, result) => sum + result.modelCalls, 0),
qualityScore:
pathResults.reduce((sum, result) => sum + result.qualityScore, 0) / pathResults.length,
totalTokens: sumTokens(pathResults.map((result) => result.totalTokens)),
};
}
export async function runVideoContactSheetEval(input: {
config: VideoContactSheetEvalConfig;
fetchImpl?: FetchLike;
manifest: VideoContactSheetEvalManifest;
}): Promise<VideoContactSheetEvalExecutedReport> {
const manifest = evalManifestSchema.parse(input.manifest);
const endpoint = z.string().url().parse(input.config.endpoint);
const config = {
apiKey: z.string().min(1).parse(input.config.apiKey),
endpoint,
model: z.string().min(1).parse(input.config.model),
};
const fetchImpl = input.fetchImpl ?? fetch;
const results: VideoContactSheetEvalCaseResult[] = [];
for (const evalCase of manifest.cases) {
const individual = await evaluateIndividualFrames({ config, evalCase, fetchImpl });
const sheet = await evaluateContactSheet({ config, evalCase, fetchImpl });
results.push({ caseId: evalCase.id, individual, sheet });
}
const individual = aggregatePathResults(results, "individual");
const sheet = aggregatePathResults(results, "sheet");
const promotion = assessVideoContactSheetPromotion({
individual,
sheet,
thresholds: manifest.thresholds,
});
return {
caseCount: manifest.cases.length,
execution: { realModel: true, state: "executed" },
generatedAt: new Date().toISOString(),
kind: "video-contact-sheet-ab-eval",
manifestDigest: createHash("sha256").update(JSON.stringify(manifest)).digest("hex"),
manifestId: manifest.id,
model: config.model,
promotion,
results,
schemaVersion: 1,
summary: { individual, sheet },
thresholds: manifest.thresholds,
};
}
function readArgument(name: string): string | undefined {
const index = process.argv.indexOf(`--${name}`);
if (index < 0) return undefined;
const value = process.argv[index + 1];
return value && !value.startsWith("--") ? value : undefined;
}
function printUsage(): void {
console.log(
[
"Usage:",
" node --import tsx/esm scripts/perf/video-bridge-contact-sheet-eval.ts --manifest <manifest.json> --model <vision-model>",
" node --import tsx/esm scripts/perf/video-bridge-contact-sheet-eval.ts --manifest <manifest.json> --model <vision-model> --execute-real",
"",
"The default command validates configuration and emits HOLD without calling a model.",
"A real paid/networked run requires --execute-real, --model, and the documented variables:",
" OMNIROUTE_BASE_URL",
" OMNIROUTE_API_KEY",
"",
"Manifest v1: id, thresholds, and 1+ cases. Each case has 1-16 bounded JPEG data URIs,",
"timestamps, a prompt, and expectedFacts with timestampSeconds + requiredTerms.",
].join("\n")
);
}
async function loadManifest(manifestPath: string): Promise<VideoContactSheetEvalManifest> {
const raw = await readFile(path.resolve(manifestPath), "utf8");
return evalManifestSchema.parse(JSON.parse(raw));
}
function resolveChatCompletionsEndpoint(baseUrl: string): string {
const normalized = baseUrl.replace(/\/{1,8}$/u, "");
if (normalized.endsWith("/v1/chat/completions")) return normalized;
if (normalized.endsWith("/v1")) return `${normalized}/chat/completions`;
return `${normalized}/v1/chat/completions`;
}
async function main(): Promise<void> {
if (process.argv.includes("--help") || process.argv.includes("-h")) {
printUsage();
return;
}
const manifestPath = readArgument("manifest");
const model = readArgument("model");
const missingConfiguration: string[] = [];
if (!manifestPath) missingConfiguration.push("--manifest");
if (!model) missingConfiguration.push("--model");
const baseUrl = process.env.OMNIROUTE_BASE_URL;
const apiKey = process.env.OMNIROUTE_API_KEY;
if (!baseUrl) missingConfiguration.push("OMNIROUTE_BASE_URL");
if (!apiKey) missingConfiguration.push("OMNIROUTE_API_KEY");
let manifest: VideoContactSheetEvalManifest | null = null;
if (manifestPath) manifest = await loadManifest(manifestPath);
if (missingConfiguration.length > 0) {
console.log(
JSON.stringify(
createVideoContactSheetEvalHoldReport({
caseCount: manifest?.cases.length ?? 0,
configurationState: "not-configured",
missingConfiguration,
}),
null,
2
)
);
return;
}
if (!process.argv.includes("--execute-real")) {
console.log(
JSON.stringify(
createVideoContactSheetEvalHoldReport({
caseCount: manifest?.cases.length ?? 0,
configurationState: "configured-not-executed",
}),
null,
2
)
);
return;
}
if (!manifest || !baseUrl || !apiKey || !model) {
throw new Error("Video contact-sheet eval configuration was not resolved");
}
console.log(
JSON.stringify(
await runVideoContactSheetEval({
config: { apiKey, endpoint: resolveChatCompletionsEndpoint(baseUrl), model },
manifest,
}),
null,
2
)
);
}
const isMainModule =
typeof process.argv[1] === "string" &&
path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
if (isMainModule) {
main().catch(() => {
console.error("Video contact-sheet eval failed validation or execution.");
process.exitCode = 1;
});
}

493
scripts/perf/video-bridge-fu07-eval.ts Normal file
View File

@@ -0,0 +1,493 @@
/**
* Real-media FU-07 structural-sampling evaluation.
*
* Run: node --import tsx/esm scripts/perf/video-bridge-fu07-eval.ts
* Optional estimate: append --caption-cost-per-call-usd <positive number>.
*
* This evaluates deterministic structural oracles, not semantic model quality.
* Model quality and monetary savings remain HOLD without an external receipt.
*/
import { execFile } from "node:child_process";
import { access, mkdir, mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { performance } from "node:perf_hooks";
import { promisify } from "node:util";
import { deduplicateVideoFrames } from "../../src/lib/guardrails/videoBridgeHelpers";
import {
analyzeVideoStructure,
calculateSamplingDecision,
extractFramesFromLocalVideo,
readBoundedExtractedFrames,
type VideoCommandRunner,
type VideoStructuralAnalysis,
type VideoStructuralSample,
} from "../../src/lib/guardrails/videoBridgeRuntime";
const execFileAsync = promisify(execFile);
const REQUIRED_FILTERS = ["scdet", "freezedetect", "blurdetect", "signalstats", "siti"];
const TIME_MARKER = "__FU07_TIME__";
interface ChildCost {
maxRssKiB: number | null;
systemSeconds: number | null;
userSeconds: number | null;
wallMs: number;
}
interface FixtureResult {
captionCallsAvoided: number;
childCost: ChildCost;
freezeIntervals: number;
name: string;
oracle: Record<string, boolean | number | string>;
passed: boolean;
sceneCandidates: number;
structuralFrames: number;
uniformFrames: number;
}
function average(values: Array<number | null | undefined>): number | null {
const finite = values.filter(
(value): value is number => value !== null && value !== undefined && Number.isFinite(value)
);
return finite.length > 0 ? finite.reduce((sum, value) => sum + value, 0) / finite.length : null;
}
function samplesIn(
analysis: VideoStructuralAnalysis,
startSeconds: number,
endSeconds: number
): VideoStructuralSample[] {
return analysis.samples.filter(
(sample) => sample.timestampSeconds >= startSeconds && sample.timestampSeconds < endSeconds
);
}
async function generateFixture(outputPath: string, args: readonly string[]): Promise<void> {
await execFileAsync(
"ffmpeg",
["-hide_banner", "-loglevel", "error", ...args, "-threads", "1", "-y", outputPath],
{ maxBuffer: 1024 * 1024, timeout: 30_000 }
);
}
async function generateStaticFixture(outputPath: string): Promise<void> {
await generateFixture(outputPath, [
"-f",
"lavfi",
"-i",
"color=c=blue:s=320x180:d=8:r=12",
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-pix_fmt",
"yuv420p",
]);
}
async function generateMixedFixture(outputPath: string): Promise<void> {
await generateFixture(outputPath, [
"-f",
"lavfi",
"-i",
"color=c=black:s=320x180:d=6:r=12",
"-f",
"lavfi",
"-i",
"testsrc2=s=320x180:d=4:r=12",
"-filter_complex",
"[0:v][1:v]concat=n=2:v=1:a=0,format=yuv420p[v]",
"-map",
"[v]",
"-c:v",
"libx264",
"-preset",
"ultrafast",
]);
}
async function generateBlurExposureFixture(outputPath: string): Promise<void> {
await generateFixture(outputPath, [
"-f",
"lavfi",
"-i",
"testsrc2=s=320x180:d=3:r=12",
"-f",
"lavfi",
"-i",
"color=c=black:s=320x180:d=3:r=12",
"-f",
"lavfi",
"-i",
"testsrc2=s=320x180:d=4:r=12",
"-filter_complex",
"[0:v]gblur=sigma=12[blur];[blur][1:v][2:v]concat=n=3:v=1:a=0,format=yuv420p[v]",
"-map",
"[v]",
"-c:v",
"libx264",
"-preset",
"ultrafast",
]);
}
async function generateDenseTailFixture(outputPath: string): Promise<void> {
const args: string[] = [];
for (const source of [
"color=c=black:s=160x90:d=0.5:r=10",
"color=c=white:s=160x90:d=0.5:r=10",
"color=c=black:s=160x90:d=0.5:r=10",
"color=c=white:s=160x90:d=0.5:r=10",
"testsrc2=s=160x90:d=8:r=10",
]) {
args.push("-f", "lavfi", "-i", source);
}
args.push(
"-filter_complex",
"[0:v][1:v][2:v][3:v][4:v]concat=n=5:v=1:a=0,format=yuv420p[v]",
"-map",
"[v]",
"-c:v",
"libx264",
"-preset",
"ultrafast"
);
await generateFixture(outputPath, args);
}
async function generateGradualFadeFixture(outputPath: string): Promise<void> {
await generateFixture(outputPath, [
"-f",
"lavfi",
"-i",
"color=c=white:s=320x180:d=8:r=12",
"-vf",
"fade=t=out:st=0:d=8,format=yuv420p",
"-c:v",
"libx264",
"-preset",
"ultrafast",
]);
}
async function supportsTimeBinary(): Promise<boolean> {
try {
await access("/usr/bin/time");
return true;
} catch {
return false;
}
}
function parseTimeCost(stderr: string, wallMs: number): ChildCost {
const match = new RegExp(`${TIME_MARKER} ([\\d.]+) ([\\d.]+) ([\\d.]+)`).exec(stderr);
return {
maxRssKiB: match ? Number(match[3]) : null,
systemSeconds: match ? Number(match[2]) : null,
userSeconds: match ? Number(match[1]) : null,
wallMs,
};
}
async function timedAnalysis(
inputPath: string,
durationSeconds: number,
useTimeBinary: boolean
): Promise<{ analysis: VideoStructuralAnalysis; cost: ChildCost }> {
let cost: ChildCost = {
maxRssKiB: null,
systemSeconds: null,
userSeconds: null,
wallMs: 0,
};
const runner: VideoCommandRunner = async (executable, args, options) => {
const startedAt = performance.now();
const command = useTimeBinary ? "/usr/bin/time" : executable;
const commandArgs = useTimeBinary
? ["-f", `${TIME_MARKER} %U %S %M`, executable, ...args]
: [...args];
const result = await execFileAsync(command, commandArgs, {
encoding: "utf8",
maxBuffer: 1024 * 1024,
signal: options.signal,
timeout: options.timeoutMs,
});
cost = parseTimeCost(String(result.stderr), performance.now() - startedAt);
return { stderr: String(result.stderr), stdout: String(result.stdout) };
};
const analysis = await analyzeVideoStructure(inputPath, {
durationSeconds,
runner,
streamIndex: 0,
timeoutMs: 30_000,
});
return { analysis, cost };
}
function sampling(
durationSeconds: number,
frameCount: number,
analysis: VideoStructuralAnalysis
): { structural: number[]; uniform: number[] } {
const uniform = calculateSamplingDecision(durationSeconds, frameCount, "uniform").timestamps;
const structural = calculateSamplingDecision(
durationSeconds,
frameCount,
"segment_aware",
analysis.sceneCandidates,
null,
analysis
).timestamps;
return { structural, uniform };
}
async function captionCallsAfterDedup(
inputPath: string,
outputDirectory: string,
samplingPolicy: "segment_aware" | "uniform"
): Promise<number> {
await mkdir(outputDirectory, { mode: 0o700 });
const frames = await extractFramesFromLocalVideo(inputPath, outputDirectory, {
durationSeconds: 8,
frameCount: 8,
samplingPolicy,
streamIndex: 0,
timeoutMs: 30_000,
});
const bytes = await readBoundedExtractedFrames(frames);
const deduplicated = await deduplicateVideoFrames(
frames.map((frame, index) => ({
dataUri: `data:image/jpeg;base64,${bytes[index].toString("base64")}`,
timestampSeconds: frame.timestampSeconds,
}))
);
return deduplicated.frames.length;
}
function result(
name: string,
cost: ChildCost,
analysis: VideoStructuralAnalysis,
uniform: number[],
structural: number[],
oracle: Record<string, boolean | number | string>,
captionCallsAvoided = 0
): FixtureResult {
const booleans = Object.values(oracle).filter(
(value): value is boolean => typeof value === "boolean"
);
return {
captionCallsAvoided,
childCost: cost,
freezeIntervals: analysis.freezeIntervals.length,
name,
oracle,
passed: booleans.every(Boolean),
sceneCandidates: analysis.sceneCandidates.length,
structuralFrames: structural.length,
uniformFrames: uniform.length,
};
}
async function main(): Promise<void> {
const version = await execFileAsync("ffmpeg", ["-version"], { timeout: 5_000 });
const filters = await execFileAsync("ffmpeg", ["-hide_banner", "-filters"], {
maxBuffer: 2 * 1024 * 1024,
timeout: 5_000,
});
const missingFilters = REQUIRED_FILTERS.filter(
(filter) => !new RegExp(`\\b${filter}\\b`).test(String(filters.stdout))
);
if (missingFilters.length > 0)
throw new Error(`Missing required FFmpeg filters: ${missingFilters.join(", ")}`);
const directory = await mkdtemp(join(tmpdir(), "video-fu07-eval-"));
const useTimeBinary = await supportsTimeBinary();
const results: FixtureResult[] = [];
try {
const staticPath = join(directory, "static.mp4");
await generateStaticFixture(staticPath);
const staticRun = await timedAnalysis(staticPath, 8, useTimeBinary);
const staticSampling = sampling(8, 8, staticRun.analysis);
const uniformCaptionCalls = await captionCallsAfterDedup(
staticPath,
join(directory, "static-uniform"),
"uniform"
);
const structuralCaptionCalls = await captionCallsAfterDedup(
staticPath,
join(directory, "static-structural"),
"segment_aware"
);
const staticCaptionCallsAvoided = Math.max(0, uniformCaptionCalls - structuralCaptionCalls);
results.push(
result(
"static-caption-savings",
staticRun.cost,
staticRun.analysis,
staticSampling.uniform,
staticSampling.structural,
{
fullFreezeDetected: staticRun.analysis.freezeIntervals.some(
(interval) => interval.startSeconds <= 1 && interval.endSeconds >= 7
),
oneIncrementalCaptionCallAvoided: staticCaptionCallsAvoided === 1,
structuralCaptionCalls,
uniformCaptionCalls,
},
staticCaptionCallsAvoided
)
);
const mixedPath = join(directory, "mixed.mp4");
await generateMixedFixture(mixedPath);
const mixedRun = await timedAnalysis(mixedPath, 10, useTimeBinary);
const mixedSampling = sampling(10, 4, mixedRun.analysis);
const uniformDense = mixedSampling.uniform.filter((timestamp) => timestamp > 6).length;
const structuralDense = mixedSampling.structural.filter((timestamp) => timestamp > 6).length;
results.push(
result(
"dense-budget-quality-oracle",
mixedRun.cost,
mixedRun.analysis,
mixedSampling.uniform,
mixedSampling.structural,
{
denseFramesStructural: structuralDense,
denseFramesUniform: uniformDense,
denseRegionGetsMoreBudget: structuralDense > uniformDense,
frozenRegionRetainsCoverage: mixedSampling.structural.some((timestamp) => timestamp < 6),
}
)
);
const qualityPath = join(directory, "blur-exposure.mp4");
await generateBlurExposureFixture(qualityPath);
const qualityRun = await timedAnalysis(qualityPath, 10, useTimeBinary);
const qualitySampling = sampling(10, 6, qualityRun.analysis);
const blurred = samplesIn(qualityRun.analysis, 0, 3);
const dark = samplesIn(qualityRun.analysis, 3, 6);
const sharp = samplesIn(qualityRun.analysis, 6, 10);
const blurredBlur = average(blurred.map((sample) => sample.blur));
const blurredSpatial = average(blurred.map((sample) => sample.spatialInformation));
const darkLuma = average(dark.map((sample) => sample.brightness));
const sharpBlur = average(sharp.map((sample) => sample.blur));
const sharpSpatial = average(sharp.map((sample) => sample.spatialInformation));
const sharpTemporal = average(sharp.map((sample) => sample.temporalInformation));
const sharpLuma = average(sharp.map((sample) => sample.brightness));
results.push(
result(
"blur-exposure-spatial-temporal-evidence",
qualityRun.cost,
qualityRun.analysis,
qualitySampling.uniform,
qualitySampling.structural,
{
blurMetricSeparated:
blurredBlur !== null && sharpBlur !== null && Math.abs(blurredBlur - sharpBlur) >= 0.05,
blurredBlur: blurredBlur ?? "missing",
darkLuma: darkLuma ?? "missing",
exposureSeparated: darkLuma !== null && sharpLuma !== null && sharpLuma - darkLuma >= 50,
sharpBlur: sharpBlur ?? "missing",
sharpSpatial: sharpSpatial ?? "missing",
sharpTemporal: sharpTemporal ?? "missing",
spatialDetailSeparated:
blurredSpatial !== null && sharpSpatial !== null && sharpSpatial - blurredSpatial >= 20,
structuralKeepsSharpRegion:
qualitySampling.structural.filter((timestamp) => timestamp >= 6).length >= 2,
temporalChangeDetected: sharpTemporal !== null && sharpTemporal >= 5,
}
)
);
const tailPath = join(directory, "dense-tail.mp4");
await generateDenseTailFixture(tailPath);
const tailRun = await timedAnalysis(tailPath, 10, useTimeBinary);
const tailSampling = sampling(10, 4, tailRun.analysis);
results.push(
result(
"dense-cuts-long-tail-regression",
tailRun.cost,
tailRun.analysis,
tailSampling.uniform,
tailSampling.structural,
{
multipleEarlyCuts: tailRun.analysis.sceneCandidates.length >= 3,
trailingEightSecondsRepresented: tailSampling.structural.some(
(timestamp) => timestamp > 2
),
}
)
);
const fadePath = join(directory, "gradual-fade.mp4");
await generateGradualFadeFixture(fadePath);
const fadeRun = await timedAnalysis(fadePath, 8, useTimeBinary);
const fadeSampling = sampling(8, 4, fadeRun.analysis);
results.push(
result(
"gradual-fade-false-positive",
fadeRun.cost,
fadeRun.analysis,
fadeSampling.uniform,
fadeSampling.structural,
{
hardCutFalsePositives: fadeRun.analysis.sceneCandidates.length,
noHardCutBurst: fadeRun.analysis.sceneCandidates.length <= 1,
noCaptionBudgetPruning: fadeSampling.structural.length === fadeSampling.uniform.length,
}
)
);
} finally {
await rm(directory, { force: true, recursive: true });
}
const callsAvoided = results.reduce((sum, fixture) => sum + fixture.captionCallsAvoided, 0);
const costFlag = process.argv.indexOf("--caption-cost-per-call-usd");
const explicitCost = Number(costFlag >= 0 ? process.argv[costFlag + 1] : Number.NaN);
const report = {
captionCost:
Number.isFinite(explicitCost) && explicitCost > 0
? {
estimatedUsdAvoided: callsAvoided * explicitCost,
source: "explicit environment input",
status: "ESTIMATED_FROM_INPUT",
}
: {
reason: "--caption-cost-per-call-usd was not supplied with a positive number",
status: "HOLD",
},
ffmpegVersion: String(version.stdout).split("\n")[0],
fixtures: results,
modelQuality: {
reason:
"No authorized real caption-model endpoint, credentials, or frozen judge rubric were configured; deterministic structural oracles are not semantic quality.",
status: "HOLD",
},
gainCostComparison: {
reason:
"The real post-dedup caption-call delta is measured, but no authorized caption latency/cost receipt or child CPU/RSS receipt is configured.",
status: "HOLD",
},
resourceCost: useTimeBinary
? { source: "/usr/bin/time", status: "MEASURED" }
: {
reason: "/usr/bin/time is unavailable; wall time is measured but child CPU/RSS are not",
status: "HOLD",
},
summary: {
captionCallsAvoided: callsAvoided,
failed: results.filter((fixture) => !fixture.passed).map((fixture) => fixture.name),
passed: results.filter((fixture) => fixture.passed).length,
total: results.length,
},
timeBinary: useTimeBinary ? "/usr/bin/time" : null,
};
console.log(JSON.stringify(report, null, 2));
if (report.summary.failed.length > 0) process.exitCode = 1;
}
await main();

54
scripts/quality/validate-release-green.mjs
View File

@@ -90,13 +90,30 @@ export function baselineValue(metric, root = ROOT) {
}
}
// A line that is unambiguously a PASS. Test reporters print the file name on BOTH the
// pass and the fail line, so a green line for a file whose NAME contains "fail"
// (fail-fast-*.test.ts, failover-*.test.ts) must never be offered as a failure cause.
const GREEN_LINE_RE = /^[✓✔√]/;
// Markers that are only meaningful at the START of a line: "FAIL" also occurs inside test
// FILE NAMES and inside summary prose ("Test Files 1 failed"), so matching it anywhere —
// and case-insensitively — reports a PASSING file as the cause of the red.
const LINE_START_FAILURE_RE = /^(?:[✖✗×]|FAIL\b|not ok\b|REGRESS)/;
// Markers that are unambiguous ANYWHERE in the line: tsc and Node emit them mid-line
// ("src/x.ts(10,5): error TS2322: ..."), so these stay unanchored. They are matched
// case-SENSITIVELY because that is how the emitting tools actually spell them.
const INLINE_FAILURE_RE = /\berror TS\d+\b|\bAssertionError\b|\bError:|\bREGRESS/;
/** Best-effort "first meaningful failure line" from captured command output. */
export function firstFailureLine(out) {
const lines = String(out || "")
.split("\n")
.map((l) => l.trim())
.filter(Boolean);
const hit = lines.find((l) => /✖|✗|not ok|AssertionError|error TS|FAIL|Error:|REGRESS/i.test(l));
const hit = lines.find(
(l) => !GREEN_LINE_RE.test(l) && (LINE_START_FAILURE_RE.test(l) || INLINE_FAILURE_RE.test(l))
);
return (hit || lines[lines.length - 1] || "failed").slice(0, 200);
}
@@ -232,6 +249,36 @@ export function fullCiTimeoutFor(gateId) {
return FULL_CI_TIMEOUT_OVERRIDES_MS[gateId] ?? FULL_CI_DEFAULT_TIMEOUT_MS;
}
// ci.yml gate scripts whose result the CURATED pass already records under a DIFFERENT id.
// Without this map the --full-ci pass re-records them unconditionally as kind:"hard" while
// the curated pass recorded them as kind:"drift", and the SAME gate is printed in BOTH
// verdict buckets of one report (file-size / compression-budget appeared as a hard failure
// and as drift simultaneously in the #9985 verdict).
export const FULL_CI_CURATED_ALIASES = {
lint: "lint-errors",
"check:workflows": "workflow-lint",
"check:complexity-ratchets": "complexity",
};
/** Curated-pass id equivalent to a ci.yml gate script id ("check:file-size" -> "file-size"). */
export function curatedEquivalentId(scriptId) {
const id = String(scriptId || "");
if (Object.hasOwn(FULL_CI_CURATED_ALIASES, id)) return FULL_CI_CURATED_ALIASES[id];
return id.startsWith("check:") ? id.slice("check:".length) : id;
}
/**
* Bucket a --full-ci gate must be reported under: the classification the curated pass already
* gave the equivalent gate, else "hard" (the --full-ci default for gates the curated list does
* not cover). This only changes WHICH BUCKET a result is printed in — it never changes whether
* a gate runs, nor whether it passed.
*/
export function fullCiKindFor(scriptId, results) {
const equivalent = curatedEquivalentId(scriptId);
const curated = (results || []).find((r) => r.id === scriptId || r.id === equivalent);
return curated?.kind ?? "hard";
}
/**
* Parse a ci.yml text and return the ordered, de-duplicated list of gate commands to run.
* Each entry: { id, job, args:["run", <script>, ...("--" + args)], env }.
@@ -716,7 +763,10 @@ async function main() {
record({
id: g.id,
label: `ci.yml:${g.job} → npm ${g.args.join(" ")}`,
kind: "hard",
// Respect the curated classification when the curated pass already ran an equivalent
// gate under a different id — otherwise the same ratchet is reported as a HARD failure
// here AND as drift above, in one self-contradicting verdict.
kind: fullCiKindFor(g.id, results),
ok: code === 0,
detail: code === 0 ? "pass" : firstFailureLine(out),
});

6
scripts/release/merge-train.sh
View File

@@ -57,12 +57,16 @@ for N in "${PRS[@]}"; do
done
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || true)"
# The train worktree is detached, so the changelog gate cannot infer which release
# branch seeded it. Shell-quote the requested base before it enters the eval-backed
# gate list, then bind that exact ref only for the changelog check.
printf -v CHANGELOG_BASE_REF_Q '%q' "origin/${BASE}"
STATIC_GATES=(
"npm run typecheck:core"
"node scripts/check/check-file-size.mjs"
"node scripts/check/check-complexity.mjs"
"node scripts/check/check-cognitive-complexity.mjs"
"node scripts/check/check-changelog-integrity.mjs"
"env CHANGELOG_BASE_REF=${CHANGELOG_BASE_REF_Q} node scripts/check/check-changelog-integrity.mjs"
)
# Full mode: the box-speed runner (same coverage as the two CI shards combined —
# main + dashboard + serial groups — at local concurrency instead of runner-sized).

17
scripts/release/sweep-stale-fragments.mjs
View File

@@ -111,6 +111,18 @@ export function classifyFragments({ fragments = [], changelog = "" }) {
return { stale, keep };
}
/**
* Count a stale list by how each entry was matched, for the human report line.
* classifyFragments only ever sets matchedBy to "pr-number" (the filename convention) or
* "text" (the normalized-bullet fallback); the summary must bucket under those exact values.
* Pure - the two counts always add up to stale.length and never mislabel a category.
*/
export function summarizeStale(stale) {
const byPrNumber = (stale || []).filter((s) => s.matchedBy === "pr-number").length;
const byText = (stale || []).filter((s) => s.matchedBy === "text").length;
return { byPrNumber, byText };
}
export function readFragments(root) {
const out = [];
for (const sub of FRAGMENT_DIRS) {
@@ -141,9 +153,8 @@ function main(argv) {
return 0;
}
const byRef = stale.filter((s) => s.matchedBy === "ref").length;
const byText = stale.length - byRef;
process.stdout.write(` matched by ref: ${byRef} · by text: ${byText}\n`);
const { byPrNumber, byText } = summarizeStale(stale);
process.stdout.write(` matched by pr-number: ${byPrNumber} · by text: ${byText}\n`);
for (const s of stale) process.stdout.write(` ${apply ? "removed" : "stale"}: ${s.rel} — ${s.reason}\n`);
if (!apply) {