merge(cache): resolve conflicts merging release/v3.8.51 into semantic-cache re-land

Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>

# Conflicts:
#	changelog.d/fixes/12910-semantic-cache-exact-id-finalize.md
#	open-sse/handlers/chatCore/semanticCache.ts
#	open-sse/handlers/chatCore/streamingSemanticCacheStore.ts
#	src/app/(dashboard)/dashboard/providers/[id]/hooks/useModelImportHandlers.ts
#	src/lib/providerModels/modelDiscovery.ts
#	tests/unit/chatcore-semantic-cache.test.ts
#	tests/unit/semantic-cache-no-truncated-writes.test.ts
This commit is contained in:
diegosouzapw
2026-09-18 13:47:33 -03:00
11508 changed files with 3087170 additions and 307355 deletions

View File

@@ -163,6 +163,43 @@ const EXTRA_MODULE_ENTRIES = [
dest: ["node_modules", "pino-pretty"],
},
{ label: "split2", src: ["node_modules", "split2"], dest: ["node_modules", "split2"] },
{
// ioredis is a deliberately LAZY dependency (Redis is optional — see the
// #6559 comment in src/shared/utils/rateLimiter.ts) — reached only via a
// runtime `await import("ioredis")` in rateLimiter.ts,
// warmupScheduler/circuitBreakerFactory.ts and quota/redisQuotaStore.ts,
// never through a static top-level import. The standalone tracer only
// follows statically-analyzable imports, so it never sees these call
// sites and drops ioredis from node_modules/ entirely. Any self-hosted
// deployment that actually sets REDIS_URL crashes the first time it
// reaches one of those call sites with "Cannot find module 'ioredis'" —
// reproduced on a production Docker deployment (REDIS_URL configured,
// v3.8.49) where the standalone image shipped ioredis/package.json but
// none of its own dependencies or built/ output.
label: "ioredis (dynamic import — #6559)",
src: ["node_modules", "ioredis"],
dest: ["node_modules", "ioredis"],
},
{
// bcryptjs IS statically imported by src/lib/auth/managementPassword.ts,
// so the main server bundle is fine — Next's server compiler inlines the
// small pure-JS package directly into the compiled route chunk instead of
// leaving it as an external node_modules dependency. bin/cli/settings-
// store.mjs (the `omniroute reset-password` / bin/reset-password.mjs
// CLI, used to recover a lost dashboard password) is a separate,
// unbundled entrypoint that does a plain runtime `import bcrypt from
// "bcryptjs"` and needs the real package physically present in
// node_modules/ — which nothing else requires as a loose runtime
// dependency, so it is never copied. Reproduced on a production
// deployment: `node bin/reset-password.mjs --password-stdin` failed with
// "Cannot find package 'bcryptjs' imported from
// /app/bin/cli/settings-store.mjs" (ERR_MODULE_NOT_FOUND) even though the
// same container's dashboard login (which also depends on bcryptjs) was
// working normally.
label: "bcryptjs (bin/cli/settings-store.mjs — reset-password CLI)",
src: ["node_modules", "bcryptjs"],
dest: ["node_modules", "bcryptjs"],
},
{ label: "migrations", src: ["src", "lib", "db", "migrations"], dest: ["migrations"] },
{ label: "MITM server", src: ["src", "mitm", "server.cjs"], dest: ["src", "mitm", "server.cjs"] },
{
@@ -229,6 +266,15 @@ const EXTRA_MODULE_ENTRIES = [
src: ["scripts", "dev", "responses-ws-proxy.mjs"],
dest: ["responses-ws-proxy.mjs"],
},
{
// server-ws.mjs imports ./httpClientAbortGuard.mjs. In the repo that path is
// the scripts/dev shim re-exporting the shared implementation, but the
// assembled bundle has no src/ tree, so ship the real self-contained
// implementation (no relative imports of its own) under the same file name.
label: "http client abort guard (server-ws.mjs dependency)",
src: ["src", "shared", "utils", "httpClientAbortGuard.mjs"],
dest: ["httpClientAbortGuard.mjs"],
},
{
label: "ChatGPT Web Codex MCP tunnel entrypoint",
src: ["bin", "chatgpt-web-codex-mcp.mjs"],

View File

@@ -59,7 +59,8 @@ const ERROR_STUB = `${HEADER}"use client";\nexport default function BackendOnlyE
// global-error replaces the root layout on a root error, so it must render <html>/<body>.
const GLOBAL_ERROR_STUB = `${HEADER}"use client";\nexport default function BackendOnlyGlobalErrorStub() {\n return (\n <html>\n <body></body>\n </html>\n );\n}\n`;
const UI_BASENAME_RE = /^(page|layout|template|loading|error|global-error|not-found|default)\.(tsx|jsx|ts|js)$/;
const UI_BASENAME_RE =
/^(page|layout|template|loading|error|global-error|not-found|default)\.(tsx|jsx|ts|js)$/;
const ROUTE_FILE_RE = /[\\/]route\.(ts|js|tsx|jsx)$/;
/**
@@ -103,6 +104,11 @@ export function isContributorBuild(env = process.env) {
return env.OMNIROUTE_BUILD_PROFILE === "contributor";
}
/** True when standalone output should be packaged (default true; skipped for contributor or fast build). */
export function shouldBuildStandalone(env = process.env) {
return !isContributorBuild(env) && env.OMNIROUTE_SKIP_STANDALONE !== "1";
}
/** Replace the build-only instrumentation entrypoint to avoid pulling the startup graph. */
export function stubContributorInstrumentation(rootDir = process.cwd(), log = console) {
const stubbed = [];

View File

@@ -67,6 +67,11 @@ function isNativeSqliteLoadError(error) {
const message = error instanceof Error ? error.message : String(error);
const code = error && typeof error === "object" && "code" in error ? error.code : undefined;
// Deliberately narrower than src/lib/db/sqliteLoadError.ts. There, a
// non-callable export means "fall back to another driver". Here, the only
// consumer treats a match as "no encrypted credentials exist", which lets
// STORAGE_ENCRYPTION_KEY be regenerated over a database that still holds
// enc:v1: rows. A generic TypeError must stay loud on this path.
return (
message.includes("Module did not self-register") ||
message.includes("NODE_MODULE_VERSION") ||
@@ -78,6 +83,11 @@ function isNativeSqliteLoadError(error) {
);
}
function isLikelyBrokenNativeBinding(error) {
const message = error instanceof Error ? error.message : String(error);
return message.includes("is not a function") || message.includes("is not a constructor");
}
function hasEncryptedCredentials(dataDir) {
const dbPath = join(dataDir, "storage.sqlite");
if (!existsSync(dbPath)) return false;
@@ -133,7 +143,10 @@ function hasEncryptedCredentials(dataDir) {
}
const message = error instanceof Error ? error.message : String(error);
throw new Error(`Unable to inspect existing database at ${dbPath}: ${message}`);
const hint = isLikelyBrokenNativeBinding(error)
? " The better-sqlite3 native binding loaded but did not expose a usable constructor; try `npm rebuild better-sqlite3`."
: "";
throw new Error(`Unable to inspect existing database at ${dbPath}: ${message}${hint}`);
}
}

View File

@@ -14,11 +14,14 @@ import {
import {
isBackendOnlyBuild,
isContributorBuild,
shouldBuildStandalone,
stubContributorInstrumentation,
stubDashboardPages,
restoreDashboardPages,
} from "./backendOnlyPages.mjs";
export { shouldBuildStandalone } from "./backendOnlyPages.mjs";
/**
* Layer 1: `app/` has been renamed to `dist/` and the App-Router collision is gone.
* The only transient paths remaining are `.tmp/wine32` (Wine prefix used by some
@@ -310,7 +313,7 @@ export async function main() {
const result = await runNextBuild();
const standaloneDir = path.join(distDir, "standalone");
if (result.code === 0 && (await exists(standaloneDir)) && !isContributorBuild()) {
if (result.code === 0 && (await exists(standaloneDir)) && shouldBuildStandalone()) {
try {
await fs.cp(path.join(projectRoot, "docs"), path.join(standaloneDir, "docs"), {
recursive: true,
@@ -377,9 +380,9 @@ export async function main() {
} catch (assembleErr) {
console.warn("[build-next-isolated] Non-fatal error assembling standalone:", assembleErr);
}
} else if (result.code === 0 && isContributorBuild()) {
} else if (result.code === 0 && !shouldBuildStandalone()) {
console.log(
"[build-next-isolated] Contributor profile: skipped standalone packaging (compile-only validation)"
"[build-next-isolated] Skipped standalone packaging (standalone disabled for fast compile)"
);
}
process.exitCode = result.code;

View File

@@ -95,6 +95,22 @@ function main() {
return;
}
const healthWorkerDest = join(STANDALONE, "src/lib/db/healthCheckWorker.js");
mkdirSync(dirname(healthWorkerDest), { recursive: true });
runBuildTool(
"esbuild",
"esbuild",
[
join(ROOT, "src/lib/db/healthCheckWorker.ts"),
"--bundle",
"--platform=node",
"--packages=external",
"--format=esm",
`--outfile=${healthWorkerDest}`,
],
{ stdio: "inherit" }
);
const callLogWorkerDest = join(STANDALONE, CALL_LOG_WORKER_REL);
mkdirSync(dirname(callLogWorkerDest), { recursive: true });
// Never spawn `node_modules/.bin/esbuild` directly: that extensionless path is
@@ -134,7 +150,11 @@ function main() {
// The call-log worker is always present; scope it to ESM immediately. The
// optional LLMLingua worker dir is added below only when its deps are installed.
const workerDirs = [dirname(callLogWorkerDest), dirname(compressionWorkerDest)];
const workerDirs = [
dirname(healthWorkerDest),
dirname(callLogWorkerDest),
dirname(compressionWorkerDest),
];
if (!hasOptionals) {
console.log(

View File

@@ -47,6 +47,7 @@ export const APP_STAGING_ALLOWED_EXACT_PATHS: string[] = [
"open-sse/services/compression/engines/llmlingua/onnxWorker.js",
"open-sse/services/compression/compressionWorker.js",
"src/lib/usage/callLogArtifactWorker.js",
"src/lib/db/healthCheckWorker.js",
"package.json",
"peer-stamp.mjs",
"main-server-timeouts.mjs",
@@ -190,6 +191,7 @@ export const PACK_ARTIFACT_ROOT_ALLOWED_PATH_PREFIXES: string[] = [
export const PACK_ARTIFACT_REQUIRED_PATHS: string[] = [
"dist/open-sse/services/compression/engines/rtk/filters/generic-output.json",
"dist/src/lib/usage/callLogArtifactWorker.js",
"dist/src/lib/db/healthCheckWorker.js",
"dist/open-sse/vendor/codex-chatgpt-web/adapters/chatgpt-web/mcp-server.js",
"dist/open-sse/services/compression/rules/en/filler.json",
"dist/server.js",

View File

@@ -344,6 +344,22 @@ if (existsSync(chatGptWebCodexMcpSrcFile)) {
}
// ── Step 8.6: Bundle call-log artifact worker ────────────────────────
const healthWorkerDest = join(DIST_DIR, "src/lib/db/healthCheckWorker.js");
mkdirSync(dirname(healthWorkerDest), { recursive: true });
runBuildTool(
"esbuild",
"esbuild",
[
"src/lib/db/healthCheckWorker.ts",
"--bundle",
"--platform=node",
"--packages=external",
"--format=esm",
`--outfile=${healthWorkerDest}`,
],
{ cwd: ROOT, stdio: "inherit" }
);
const callLogWorkerSrc = join(ROOT, "src", "lib", "usage", "callLogArtifactWorker.ts");
const callLogWorkerDest = join(DIST_DIR, "src", "lib", "usage", "callLogArtifactWorker.js");
if (!existsSync(callLogWorkerSrc)) {

View File

@@ -203,6 +203,10 @@ const IGNORE_FROM_CODE = new Set([
// Listener-owned self-fetch transport signal. The HTTP/HTTPS launchers set
// this before application imports; it is not user-configurable product env.
"OMNIROUTE_INTERNAL_SCHEME",
// Runner-owned bind-host signal. scripts/dev/run-next.mjs publishes the
// interface it actually binds so the in-process startup guard can name it
// (#13695); operators configure HOST / HOSTNAME, never this.
"OMNIROUTE_BOUND_HOST",
// Source typo / placeholder.
"OMNIROUT",
// Static config alias path (the canonical var is OMNIROUTE_PAYLOAD_RULES_PATH).
@@ -278,6 +282,10 @@ const DOC_ONLY_ALLOWLIST = new Set([
// SQL keyword mentioned in the new VACUUM scheduler docs (#4437).
// The check's regex picks up the bare word in description text.
"VACUUM",
// Source-code constant (open-sse/services/combo/comboPredicates.ts:35 —
// `export const COMBO_LOOP_SAFETY_TIMEOUT_MS = 10 * 60 * 1000`), cited in the
// comboTimeoutMs narrative added by #13857. Not operator-configurable.
"COMBO_LOOP_SAFETY_TIMEOUT_MS",
]);
// Vars present in .env.example but intentionally absent from ENVIRONMENT.md.

View File

@@ -205,7 +205,26 @@ function changedSymbols(root, base, entries) {
return result;
}
function markdown(result, base) {
// A changed hub module (providerRegistry.ts, providers.ts, …) is imported by thousands of
// consumers, and every consumer multiplies by its candidate tests, so the cross-product reaches
// millions of rows. Rendering all of them made `lines.join("\n")` exceed V8's maximum string
// length; the throw landed in main()'s catch, which exits 1 — so an ADVISORY step turned
// "Fast Quality Gates" red on every PR whose diff touched a hub (#13866 follow-up). The header
// keeps the exact totals; only the enumeration is bounded.
const RENDER_LIMIT = 200;
const JSON_ITEM_LIMIT = 5000;
/** First `limit` items plus a one-line note naming how many were withheld. */
function renderBounded(lines, items, format, limit = RENDER_LIMIT) {
for (const item of items.slice(0, limit)) lines.push(format(item));
if (items.length > limit) {
lines.push(
`- _… and ${items.length - limit} more not listed (report bounded at ${limit} rows per section; the counts above are exact)._`
);
}
}
export function markdown(result, base) {
const lines = [
"## Forgotten sibling tests (advisory)",
"",
@@ -218,12 +237,10 @@ function markdown(result, base) {
];
if (result.findings.length) {
lines.push("### Candidate tests absent from this diff", "");
for (const item of result.findings) {
renderBounded(lines, result.findings, (item) => {
const symbol = item.changedSymbols.length ? ` (${item.changedSymbols.join(", ")})` : "";
lines.push(
`- \`${item.changedModule}\`${symbol} -> \`${item.consumer}\` -> \`${item.candidateTest}\``
);
}
return `- \`${item.changedModule}\`${symbol} -> \`${item.consumer}\` -> \`${item.candidateTest}\``;
});
lines.push("", "> Report-only calibration: these findings do not fail the job.", "");
}
for (const [heading, items] of [
@@ -232,10 +249,12 @@ function markdown(result, base) {
]) {
if (!items.length) continue;
lines.push(`### ${heading}`, "");
for (const item of items)
lines.push(
renderBounded(
lines,
items,
(item) =>
`- \`${item.changedModule}\` -> \`${item.consumer}\`${item.candidateTest ? ` -> \`${item.candidateTest}\`` : ""}: ${item.reason || item.message}`
);
);
lines.push("");
}
return `${lines.join("\n")}\n`;
@@ -271,9 +290,26 @@ function main() {
});
const report = markdown(result, base);
process.stdout.write(report);
// The JSON artifact is bounded for the same reason the markdown is: a hub-module diff
// produces millions of rows and `JSON.stringify` would throw the same "Invalid string
// length". `totals` keeps every count exact, so tooling can still see the real numbers.
const jsonResult = {
...result,
totals: {
findings: result.findings.length,
diagnostics: result.diagnostics.length,
suppressed: result.suppressed.length,
maskingRisks: result.maskingRisks.length,
},
itemLimit: JSON_ITEM_LIMIT,
findings: result.findings.slice(0, JSON_ITEM_LIMIT),
diagnostics: result.diagnostics.slice(0, JSON_ITEM_LIMIT),
suppressed: result.suppressed.slice(0, JSON_ITEM_LIMIT),
maskingRisks: result.maskingRisks.slice(0, JSON_ITEM_LIMIT),
};
for (const [target, contents] of [
[summaryPath, report],
[jsonPath, `${JSON.stringify(result, null, 2)}\n`],
[jsonPath, `${JSON.stringify(jsonResult, null, 2)}\n`],
]) {
if (!target) continue;
fs.mkdirSync(path.dirname(target), { recursive: true });

View File

@@ -11,6 +11,7 @@
export {
isClientAbortError,
isRecoverableUpstreamTimeoutError,
shouldSwallowUncaught,
attachRequestStreamGuards,
installProcessCrashGuard,

View File

@@ -430,6 +430,9 @@ class ResponsesWsSession {
this.firstResponseBody = null;
this.currentRequestBody = null;
this.preparedContext = null;
this.leaseId = null;
this.leaseReleased = false;
this.leaseReleaseInFlight = false;
// #7388: logging must be scoped per logical turn (one `response.create`
// through its terminal event), not once for the lifetime of the WS
// connection — a single boolean here silently dropped every turn after
@@ -640,6 +643,23 @@ class ResponsesWsSession {
toStringOrNull(responseBody.service_tier) || toStringOrNull(responseBody.serviceTier),
};
// A reused WS connection re-runs prepare per logical turn, and each prepare
// acquires a fresh per-account lease. Release the previous turn before
// adopting the new lease so one session cannot hoard account slots.
const previousLeaseId = this.leaseId;
const newLeaseId = toStringOrNull(prepared.json?.leaseId);
if (this.closed) {
this.leaseId = null;
this.releaseLeaseId(newLeaseId);
return prepared;
}
this.leaseId = newLeaseId;
if (previousLeaseId && previousLeaseId !== newLeaseId) {
this.releaseLeaseId(previousLeaseId);
}
this.leaseReleased = false;
this.leaseReleaseInFlight = false;
return prepared;
}
@@ -770,6 +790,35 @@ class ResponsesWsSession {
}
}
releaseLease() {
if (this.leaseReleased || this.leaseReleaseInFlight || !this.leaseId) return;
this.leaseReleaseInFlight = true;
const leaseId = this.leaseId;
void callInternal(this.fetchImpl, this.baseUrl, this.bridgeSecret, "release", { leaseId })
.then((response) => {
if (!response.ok) throw new Error("lease release rejected");
this.leaseReleased = true;
this.leaseId = null;
})
.catch(() => {
this.leaseReleaseInFlight = false;
const retry = setTimeout(() => this.releaseLease(), 1000);
retry.unref?.();
});
}
releaseLeaseId(leaseId) {
if (!leaseId) return;
void callInternal(this.fetchImpl, this.baseUrl, this.bridgeSecret, "release", { leaseId })
.then((response) => {
if (!response.ok) throw new Error("lease release rejected");
})
.catch(() => {
const retry = setTimeout(() => this.releaseLeaseId(leaseId), 1000);
retry.unref?.();
});
}
async persistHistory({
status = 200,
success = true,
@@ -820,6 +869,7 @@ class ResponsesWsSession {
close(code = 1000, reason = "normal_closure") {
if (this.closed) return;
this.closed = true;
this.releaseLease();
clearInterval(this.pingTimer);
this.cleanupBuffers();
@@ -847,6 +897,7 @@ class ResponsesWsSession {
dispose() {
if (this.closed) return;
this.closed = true;
this.releaseLease();
clearInterval(this.pingTimer);
this.cleanupBuffers();
try {

View File

@@ -16,10 +16,7 @@ import { isTurbopackCacheCorruption, purgeAllTurbopackCaches } from "./turbopack
import { randomUUID } from "node:crypto";
import { getMainServerTimeoutConfig } from "./main-server-timeouts.mjs";
import { createSystemdNotifier } from "./systemd-notify.mjs";
import {
attachRequestStreamGuards,
installProcessCrashGuard,
} from "./httpClientAbortGuard.mjs";
import { attachRequestStreamGuards, installProcessCrashGuard } from "./httpClientAbortGuard.mjs";
const { maybeHandleDisallowedMethod } = methodGuard;
const { wrapRequestListenerWithHeadResponseGuard } = headResponseGuard;
@@ -104,6 +101,12 @@ process.env.OMNIROUTE_INTERNAL_SCHEME = "http";
const { dashboardPort } = runtimePorts;
const hostname = process.env.HOST || "0.0.0.0";
// Publish the interface this server actually binds so in-process TypeScript
// (src/lib/startup/nonLoopbackApiKeyGuard.ts) can warn about an exposed
// anonymous /v1 without re-deriving it. The standalone/Docker entrypoint
// (scripts/dev/run-standalone.mjs -> Next's own server.js) uses HOSTNAME
// instead, which the guard falls back to. #13695
process.env.OMNIROUTE_BOUND_HOST = hostname;
// Turbopack by default in dev (matches the Next 16 CLI default and the production
// build default in build-next-isolated.mjs); OMNIROUTE_USE_TURBOPACK=0 is the
// webpack escape hatch. Under Bun, Turbopack native V8 bindings are unavailable,
@@ -232,15 +235,31 @@ async function start() {
process.exit(1);
});
let isShuttingDown = false;
const shutdown = async (signal) => {
if (isShuttingDown) {
// Second Ctrl+C / signal forces immediate exit
process.exit(1);
}
isShuttingDown = true;
// Safety net: force exit after 2s if keep-alive sockets or Next.js app close hangs
const forceExitTimer = setTimeout(() => {
process.exit(0);
}, 2000);
forceExitTimer.unref?.();
systemdNotifier.stopping();
try {
server.closeIdleConnections?.();
server.closeAllConnections?.();
await new Promise((resolve) => server.close(resolve));
await globalThis.__omnirouteRequestShutdown?.(signal);
await nextApp.close();
} catch (error) {
console.error("[SHUTDOWN] Failed during signal:", signal, error);
} finally {
clearTimeout(forceExitTimer);
process.exit(0);
}
};

View File

@@ -9,6 +9,17 @@ import headResponseGuard from "./head-response-guard.cjs";
import { resolveTlsOptions, createServerListener } from "./tls-options.mjs";
import { getMainServerTimeoutConfig } from "./main-server-timeouts.mjs";
import { createSystemdNotifier } from "./systemd-notify.mjs";
import { installProcessCrashGuard } from "./httpClientAbortGuard.mjs";
// Safety net (#12861): this is the actual production entry point (see the
// keepAliveTimeout comment below for why `run-next.mjs`-only fixes don't
// reach real installs). Without this, a client abort OR a recoverable
// upstream-fetch timeout that a retry path already handles (see
// open-sse/utils/directResponseStartTimeout.ts) can surface as an
// unhandledRejection -> uncaughtException and take the whole server down —
// exactly the asymmetry `run-next.mjs` already closed for dev. Benign errors
// are swallowed and logged; genuine bugs still crash loudly.
installProcessCrashGuard();
// systemd sd_notify (Type=notify / WatchdogSec=): this process is the one
// whose event loop can freeze (cold /v1/models rebuild), so it must own the

View File

@@ -0,0 +1,146 @@
#!/usr/bin/env node
/**
* OmniRoute — i18n key COMPLETENESS gate (CI gate, blocking).
*
* Every `src/i18n/messages/<locale>.json` must carry exactly the key set of `en.json`:
* no leaf absent, no leaf the source no longer has. A `__MISSING__:` placeholder counts as
* present (the ratio gate judges its content); an ABSENT key is the defect this gate names.
*
* Why the two sibling gates cannot see it (the incident it encodes, 2026-09-15):
* - `check-ui-keys-coverage.mjs` enforces an 80 % floor per locale — 43 absent keys out of
* ~13,000 still reads 99.7 %.
* - `check-new-key-coverage.mjs` judges only the keys a PR ADDS to en.json. A locale batch
* is generated from the en.json of the moment the branch is cut; while its translation
* runs for days the base keeps adding keys, and the batch PR adds none itself — so the
* nine batch-1 catalogs (#13044) landed 43 keys short and the eight batch-2 catalogs
* (#13660) 10 keys short. The home widget test was the first thing that noticed.
*
* This gate is absolute, not diff-based: it compares the tree as it is.
*
* Usage:
* node scripts/i18n/check-key-completeness.mjs # blocking (dashboard catalogs)
* node scripts/i18n/check-key-completeness.mjs --warn # report only, exit 0
* node scripts/i18n/check-key-completeness.mjs --catalog=cli # same gate over bin/cli/locales
* npm run i18n:check-keys
* npm run i18n:check-keys:cli
*/
import { promises as fs } from "node:fs";
import path from "node:path";
import process from "node:process";
import { fileURLToPath, pathToFileURL } from "node:url";
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.resolve(SCRIPT_DIR, "..", "..");
const CATALOG_DIRS = {
ui: path.join(ROOT, "src", "i18n", "messages"),
cli: path.join(ROOT, "bin", "cli", "locales"),
};
const SOURCE_LOCALE = "en";
/** Absolute directory of a flat-JSON catalog family: the dashboard (`ui`) or the CLI (`cli`). */
export function catalogDir(name = "ui") {
const dir = CATALOG_DIRS[name];
if (!dir) throw new Error(`unknown catalog "${name}" (expected ui or cli)`);
return dir;
}
function isPlainObject(value) {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
/** Dotted leaf paths of a catalog tree (objects recurse, everything else is a leaf). */
export function leafPaths(node, prefix = "", out = new Set()) {
if (!isPlainObject(node)) return out;
for (const [key, value] of Object.entries(node)) {
const dotted = prefix ? `${prefix}.${key}` : key;
if (isPlainObject(value)) leafPaths(value, dotted, out);
else out.add(dotted);
}
return out;
}
/**
* Pure core. `en` is the source catalog, `locales` maps locale code → catalog. Returns one
* entry per locale whose key set differs from the source, sorted by locale, with sorted
* `missing` (in en, absent in the locale) and `extra` (in the locale, gone from en) lists.
* Locales with an identical key set are not listed.
*/
export function findIncompleteLocales({ en, locales }) {
const source = leafPaths(en);
const gaps = [];
for (const locale of Object.keys(locales).sort()) {
const target = leafPaths(locales[locale]);
const missing = [...source].filter((k) => !target.has(k)).sort();
const extra = [...target].filter((k) => !source.has(k)).sort();
if (missing.length || extra.length) gaps.push({ locale, missing, extra });
}
return gaps;
}
async function readCatalogs(dir) {
const files = (await fs.readdir(dir)).filter((f) => f.endsWith(".json")).sort();
const locales = {};
let en = null;
for (const file of files) {
const code = file.slice(0, -".json".length);
const parsed = JSON.parse(await fs.readFile(path.join(dir, file), "utf8"));
if (code === SOURCE_LOCALE) en = parsed;
else locales[code] = parsed;
}
if (!en) throw new Error(`${SOURCE_LOCALE}.json not found in ${dir}`);
return { en, locales };
}
function formatReport(gaps, sample = 5) {
const lines = [];
for (const { locale, missing, extra } of gaps) {
const parts = [];
if (missing.length) {
parts.push(
`${missing.length} missing (${missing.slice(0, sample).join(", ")}${missing.length > sample ? ", …" : ""})`
);
}
if (extra.length) {
parts.push(
`${extra.length} extra (${extra.slice(0, sample).join(", ")}${extra.length > sample ? ", …" : ""})`
);
}
lines.push(` - ${locale}: ${parts.join("; ")}`);
}
return lines.join("\n");
}
async function main() {
const warnOnly = process.argv.includes("--warn");
const catalogArg = process.argv.find((a) => a.startsWith("--catalog="));
const catalog = catalogArg ? catalogArg.slice(10) : "ui";
const dir = catalogDir(catalog);
const tag = `[i18n-keys:${catalog}]`;
const { en, locales } = await readCatalogs(dir);
const gaps = findIncompleteLocales({ en, locales });
const total = leafPaths(en).size;
const count = Object.keys(locales).length;
if (gaps.length === 0) {
console.log(`${tag} OK — ${count} locales carry all ${total} keys of en.json, none extra.`);
return;
}
const missingTotal = gaps.reduce((s, g) => s + g.missing.length, 0);
const extraTotal = gaps.reduce((s, g) => s + g.extra.length, 0);
console.error(
`${tag} ${warnOnly ? "WARN" : "FAIL"}${gaps.length}/${count} locales differ from en.json (${missingTotal} missing, ${extraTotal} extra leaves):`
);
console.error(formatReport(gaps));
console.error(
`${tag} Fix: node scripts/i18n/sync-ui-keys.mjs --catalog=${catalog} --locale=<codes> --translate-markers (adds the missing keys and translates them); extra keys mean the source dropped them — remove them from the locale.`
);
if (!warnOnly) process.exitCode = 1;
}
const isDirectRun = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
if (isDirectRun) {
main().catch((err) => {
console.error(`[i18n-keys] ${err.message}`);
process.exitCode = 1;
});
}

View File

@@ -23,13 +23,22 @@
*
* How this gate works: DIFF-AWARE, like its sibling. It compares the English catalog at the
* merge base against the working tree; every key that is NEW in English must be present and
* non-placeholder in every locale. Pre-existing gaps are deliberately frozen — this gate
* judges only what the current change adds, so it can be turned on without a migration.
* TRANSLATED in every locale. Pre-existing gaps are deliberately frozen — this gate judges
* only what the current change adds, so it can be turned on without a migration.
*
* Escape hatch, same as the sibling: set the value to `__MISSING__:<english>` to make the
* runtime fall back to correct English and queue the key for the translation pipeline.
* NOTE that `vi` bans placeholders (tests/unit/i18n-vi-completeness.test.ts), so `vi` needs
* a real translation.
* A `__MISSING__:<english>` marker does NOT satisfy this gate (since 2026-09-17). It used to:
* the marker was the documented deferral, because the runtime falls back to correct English.
* Then on 2026-09-16 eight feature PRs added 61 keys to en.json and stamped the marker into
* all 65 locales instead of translating; this gate accepted every one of them, nothing blocked
* the PRs, and the real-translation ratio gate (`check-translation-ratio`, blocking) went red
* on the release tip for everybody (pt-BR 3.2 % > 2.5 % + 0.5). A marker is an absent
* translation wearing a runtime-safe coat, and it is judged as absent here. Translate:
*
* node scripts/i18n/sync-ui-keys.mjs --locale=<codes> --translate-markers --batch-size=40
* bash scripts/i18n/translate-new-keys.sh # same thing, all locales in parallel
*
* Keys that must stay English (product/engine/flag names a test pins) go in
* `scripts/i18n/untranslatable-keys.json`, never behind a marker.
*
* Usage:
* node scripts/i18n/check-new-key-coverage.mjs # strict, exit 1
@@ -68,8 +77,8 @@ export function flattenLeaves(node, prefix = "", out = {}) {
/**
* Pure core: which (key, locale) pairs are keys new in English that a locale never got?
*
* A `__MISSING__:` placeholder counts as satisfied — it is the documented, runtime-correct
* way to defer a translation.
* A `__MISSING__:` placeholder counts as ABSENT — it is not a translation, and accepting it
* is what let the 2026-09-16 batch ship 61 untranslated keys into 65 locales.
*
* @param {object} args
* @param {object} args.baseEn en.json at the base ref
@@ -91,7 +100,7 @@ export function findUntranslatedNewKeys({ baseEn, headEn, headLocales }) {
for (const key of newKeys) {
const value = flat[key];
const satisfied =
typeof value === "string" && (value.trim() !== "" || value.startsWith(PLACEHOLDER_PREFIX));
typeof value === "string" && value.trim() !== "" && !value.startsWith(PLACEHOLDER_PREFIX);
if (!satisfied) gaps.push({ key, locale });
}
}
@@ -186,14 +195,20 @@ function main() {
}
const label = opts.warn ? "WARN" : "FAIL";
console.error(
`\n[i18n-new-keys] ${label}${byKey.size} new English key(s) missing from some locales:`
`\n[i18n-new-keys] ${label}${byKey.size} new English key(s) untranslated in some locales:`
);
for (const [key, locales] of byKey) {
console.error(`${key}missing in ${locales.length}: ${locales.join(", ")}`);
console.error(`${key}untranslated in ${locales.length}: ${locales.join(", ")}`);
}
const codes = [...new Set(gaps.map((g) => g.locale))].sort().join(",");
console.error(
"\n Translate them, or set `__MISSING__:<english>` to defer (the runtime then falls back\n" +
" to English). `vi` bans placeholders — it needs a real translation."
"\n A `__MISSING__:<english>` marker does not count — it is an absent translation.\n" +
" Translate the keys (needs OMNIROUTE_TRANSLATION_API_URL/_API_KEY/_MODEL in .env):\n" +
` node scripts/i18n/sync-ui-keys.mjs --locale=${codes} --translate-markers --batch-size=40\n` +
" or, all locales in parallel (detached runner):\n" +
" bash scripts/i18n/translate-new-keys.sh\n" +
" A key that must stay English (a pinned product/engine/flag name) belongs in\n" +
" scripts/i18n/untranslatable-keys.json."
);
if (!opts.warn) process.exit(1);
}

View File

@@ -66,7 +66,20 @@ async function main() {
}
const state = JSON.parse(await fs.readFile(STATE_PATH, "utf8"));
const sources = state.sources || {};
// Scope: only the documentation core set is translated on purpose (PR-0 decision, 22
// sources); state entries for other docs (older per-locale extras) are not a CI concern.
// `--all` restores the full-state behaviour for local inspection.
const { computeDocsCoreSet } = await import("./lib/docs-core-set.mjs");
const config = JSON.parse(await fs.readFile(path.join(ROOT, "config", "i18n.json"), "utf8"));
const coreSet = computeDocsCoreSet({ root: ROOT, config });
const coreList = Array.isArray(coreSet)
? coreSet
: (coreSet.coreSet ?? coreSet.files ?? Object.keys(coreSet));
const core = new Set(coreList);
const scopeAll = process.argv.includes("--all");
const sources = Object.fromEntries(
Object.entries(state.sources || {}).filter(([rel]) => scopeAll || core.has(rel))
);
const driftedSources = [];
const missingTargets = [];

View File

@@ -0,0 +1,213 @@
#!/usr/bin/env node
/**
* OmniRoute — site catalog retranslator (omnirouteSite/lang/<code>.json).
*
* The site catalogs are flat { "dotted.key": "text" } files translated once by
* add-locale; 10 % of their leaves were still verbatim English on 2026-09-16
* (th 25 %). For every locale, the leaves equal to lang/_source.en.json —
* outside untranslatable-site-keys.json — are sent to the translation backend
* in batches and written back in place (key order preserved). Same backend
* env as sync-ui-keys (`OMNIROUTE_TRANSLATION_*`, loaded from the repo-root
* `.env` when present).
*
* A batch whose answer cannot be trusted (see `parseBatchResponse`) is retried
* one string at a time; a leaf that still fails keeps its English value so the
* next run picks it up again. Each catalog is written as soon as its locale is
* done, so an aborted run keeps the locales already finished.
*
* Usage:
* node scripts/i18n/retranslate-site.mjs --site-dir=../omnirouteSite \
* [--locale=th,phi] [--dry-run] [--batch-size=40]
*/
import { existsSync, readFileSync } from "node:fs";
import { promises as fs } from "node:fs";
import path from "node:path";
import process from "node:process";
import { fileURLToPath, pathToFileURL } from "node:url";
import { backendConfig, translateBatch, translateString } from "./lib/translate-backend.mjs";
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.resolve(SCRIPT_DIR, "..", "..");
const ALLOWLIST = path.join(SCRIPT_DIR, "untranslatable-site-keys.json");
const LOG_PREFIX = "[site-retranslate]";
// ----- .env loader --------------------------------------------------------
// Same loader as sync-ui-keys.mjs: variables from the repo-root `.env`
// (gitignored) land in process.env unless the shell already set them.
function loadDotEnv() {
const envPath = path.join(ROOT, ".env");
if (!existsSync(envPath)) return;
try {
const raw = readFileSync(envPath, "utf8");
for (const rawLine of raw.split(/\r?\n/)) {
const line = rawLine.trim();
if (!line || line.startsWith("#")) continue;
const eq = line.indexOf("=");
if (eq <= 0) continue;
const key = line.slice(0, eq).trim();
if (!key || process.env[key] !== undefined) continue;
let value = line.slice(eq + 1);
if (
(value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))
) {
value = value.slice(1, -1);
}
process.env[key] = value;
}
} catch {
/* ignore — backendConfig() reports the missing variables */
}
}
/**
* Keys of `source` whose `target` value is still the verbatim English string,
* outside `allow`. Missing target keys are not "identical copies" (that is a
* sync problem, not a translation one); empty and non-string source leaves are
* skipped. Sorted so the batches are deterministic.
*
* @param {Record<string, string>} source lang/_source.en.json
* @param {Record<string, string>} target lang/<code>.json
* @param {Set<string>} allow keys that must stay English
* @returns {string[]}
*/
export function findSiteIdenticalKeys(source, target, allow) {
return Object.keys(source)
.filter(
(k) =>
k in target &&
typeof source[k] === "string" &&
source[k] !== "" &&
target[k] === source[k] &&
!allow.has(k)
)
.sort();
}
function parseArgs(argv) {
const o = { siteDir: null, locales: null, dryRun: false, batchSize: 40 };
for (const a of argv.slice(2)) {
if (a.startsWith("--site-dir=")) o.siteDir = path.resolve(ROOT, a.slice("--site-dir=".length));
else if (a.startsWith("--locale="))
o.locales = a
.slice("--locale=".length)
.split(",")
.map((s) => s.trim())
.filter(Boolean);
else if (a === "--dry-run") o.dryRun = true;
else if (a.startsWith("--batch-size="))
o.batchSize = Math.max(1, Number(a.slice("--batch-size=".length)) || 40);
else throw new Error(`unknown argument: ${a}`);
}
if (!o.siteDir) throw new Error("--site-dir=<omnirouteSite checkout> is required");
return o;
}
async function readJson(file) {
return JSON.parse(await fs.readFile(file, "utf8"));
}
/**
* Translates `keys` of `source` for one locale, writing into `target` in place.
* Returns { translated, failed } counts.
*/
async function translateLocale(keys, source, target, localeEntry, backend, batchSize) {
let translated = 0;
let failed = 0;
for (let i = 0; i < keys.length; i += batchSize) {
const slice = keys.slice(i, i + batchSize);
try {
const out = await translateBatch(
slice.map((id) => ({ id, text: source[id] })),
localeEntry,
backend
);
for (const id of slice) {
const value = out.get(id);
if (typeof value === "string" && value.trim()) {
target[id] = value.trim();
translated++;
} else {
failed++;
}
}
} catch (err) {
console.warn(
`${LOG_PREFIX} ${localeEntry.code}: batch of ${slice.length} failed (${err.message}) — retrying one by one`
);
for (const id of slice) {
try {
const value = await translateString(source[id], localeEntry, backend);
if (typeof value === "string" && value.trim()) {
target[id] = value.trim();
translated++;
} else {
failed++;
}
} catch (inner) {
failed++;
console.warn(`${LOG_PREFIX} ${localeEntry.code}: ${id} failed (${inner.message})`);
}
}
}
}
return { translated, failed };
}
async function main() {
const o = parseArgs(process.argv);
if (!o.dryRun) loadDotEnv();
const langDir = path.join(o.siteDir, "lang");
const source = await readJson(path.join(langDir, "_source.en.json"));
const allow = new Set((await readJson(ALLOWLIST)).keys ?? []);
const config = await readJson(path.join(ROOT, "config", "i18n.json"));
const codes = (o.locales ?? config.locales.map((l) => l.code)).filter((c) => c !== "en");
const backend = o.dryRun ? null : backendConfig();
let total = 0;
let failedTotal = 0;
for (const code of codes) {
const file = path.join(langDir, `${code}.json`);
const localeEntry = config.locales.find((l) => l.code === code);
if (!localeEntry) {
console.warn(`${LOG_PREFIX} ${code}: not in config/i18n.json, skipped`);
continue;
}
let target;
try {
target = await readJson(file);
} catch {
console.warn(`${LOG_PREFIX} ${code}: no catalog, skipped`);
continue;
}
const keys = findSiteIdenticalKeys(source, target, allow);
console.log(
`${LOG_PREFIX} ${code}: ${keys.length} English leaves${o.dryRun ? " (dry-run)" : ""}`
);
if (o.dryRun || keys.length === 0) continue;
const { translated, failed } = await translateLocale(
keys,
source,
target,
localeEntry,
backend,
o.batchSize
);
total += translated;
failedTotal += failed;
if (translated > 0) await fs.writeFile(file, JSON.stringify(target, null, 2) + "\n", "utf8");
if (failed > 0) console.warn(`${LOG_PREFIX} ${code}: ${failed} leaves still English`);
}
console.log(
`${LOG_PREFIX} done — ${total} leaves rewritten across ${codes.length} locales` +
(failedTotal > 0 ? ` (${failedTotal} failed)` : "")
);
if (failedTotal > 0) process.exitCode = 1;
}
const isDirectRun = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
if (isDirectRun) {
main().catch((e) => {
console.error(`${LOG_PREFIX} ${e.message}`);
process.exitCode = 1;
});
}

View File

@@ -0,0 +1,247 @@
#!/usr/bin/env node
/**
* OmniRoute — locale review pass. Sends the leaves a locale changed since a git ref to the
* translation backend with a *reviewer* prompt (native speaker, dashboard context, keep
* placeholders/ICU/markup) and applies only the corrections it returns. Produces a markdown
* report so the operator can read what was changed.
*
* Usage: node scripts/i18n/review-locale.mjs --locale=pt-BR --since=<ref> [--dry-run] [--batch-size=30]
*/
import { existsSync, readFileSync, promises as fs } from "node:fs";
import path from "node:path";
import process from "node:process";
import { execFileSync } from "node:child_process";
import { fileURLToPath, pathToFileURL } from "node:url";
import { backendConfig, callChat } from "./lib/translate-backend.mjs";
// ----- .env loader (same contract as sync-ui-keys.mjs: repo-root .env, already-set vars win)
(function loadDotEnv() {
const envPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".env");
if (!existsSync(envPath)) return;
try {
const raw = readFileSync(envPath, "utf8");
for (const rawLine of raw.split(/\r?\n/)) {
const line = rawLine.trim();
if (!line || line.startsWith("#")) continue;
const eq = line.indexOf("=");
if (eq <= 0) continue;
const key = line.slice(0, eq).trim();
if (!key || process.env[key] !== undefined) continue;
let value = line.slice(eq + 1);
if (
(value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))
) {
value = value.slice(1, -1);
}
process.env[key] = value;
}
} catch {
// unreadable .env — the backend config will report the missing variables
}
})();
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.resolve(SCRIPT_DIR, "..", "..");
const MESSAGES_DIR = path.join(ROOT, "src", "i18n", "messages");
const flat = (o, p = "", out = {}) => {
for (const [k, v] of Object.entries(o)) {
const d = p ? `${p}.${k}` : k;
if (v && typeof v === "object" && !Array.isArray(v)) flat(v, d, out);
else out[d] = v;
}
return out;
};
// 36 leaf keys of en.json carry a dot in their own name
// (`compliance.eventTypes["apiKey.ban"]`), so the flattened id is ambiguous:
// a naive split-and-descend walked into a missing `apiKey` object and the
// review died with "Cannot set properties of undefined (setting 'ban')" —
// after two hours for Hausa, before anything was written. Walk the object
// preferring the longest key that actually exists at each level.
export function setDeep(o, dotted, value) {
const parts = dotted.split(".");
const walk = (node, from) => {
for (let len = parts.length - from; len >= 1; len--) {
const key = parts.slice(from, from + len).join(".");
if (!Object.prototype.hasOwnProperty.call(node, key)) continue;
if (from + len === parts.length) {
node[key] = value;
return true;
}
if (node[key] && typeof node[key] === "object" && walk(node[key], from + len)) return true;
}
return false;
};
if (!walk(o, 0)) throw new Error(`key not found in catalog: ${dotted}`);
}
export function changedLeaves(before, after) {
const out = {};
for (const [k, v] of Object.entries(after)) {
if (typeof v === "string" && before[k] !== v) out[k] = v;
}
return out;
}
// One upstream hiccup used to abort the whole run through main().catch, and
// the catalog was written only at the end — a 13k-leaf review (2h40 for
// Amharic) lost everything. Each batch is retried with a backoff and, if it
// still fails, skipped and reported instead of killing the run.
export async function withRetries(
fn,
{ attempts = 4, delaysMs = [2000, 10000, 30000], onRetry } = {}
) {
let lastErr;
for (let i = 0; i < attempts; i++) {
try {
return await fn(i);
} catch (err) {
lastErr = err;
if (i + 1 < attempts) {
onRetry?.(err, i + 1);
await new Promise((r) => setTimeout(r, delaysMs[Math.min(i, delaysMs.length - 1)]));
}
}
}
throw lastErr;
}
export function parseReviewResponse(text, ids) {
const m = text.match(/\{[\s\S]*\}/);
if (!m) return {};
let parsed;
try {
parsed = JSON.parse(m[0]);
} catch {
return {};
}
const allowed = new Set(ids);
const out = {};
for (const [id, v] of Object.entries(parsed)) {
if (allowed.has(id) && typeof v === "string" && v.trim() && v.trim() !== "OK") {
out[id] = v.trim();
}
}
return out;
}
const REVIEW_SYSTEM = (english, native) =>
`You are a senior ${english} (${native}) localization reviewer for a developer-facing dashboard (an LLM proxy/router called OmniRoute). You receive a JSON object mapping ids to {en, current}. For every id, answer "OK" when the current translation is correct, natural and complete, or return the corrected ${english} string. Rules: keep every placeholder ({name}, {count, plural, …}), ICU syntax, HTML/markdown, product names (OmniRoute, Claude, Codex, MCP, A2A) and technical identifiers exactly as in the English; do not translate code; prefer the terminology already used in "current" when it is fine. Reply with ONLY a JSON object {id: "OK" | "corrected string"}.`;
function parseArgs(argv) {
const o = { locale: null, since: null, dryRun: false, batchSize: 30 };
for (const a of argv.slice(2)) {
if (a.startsWith("--locale=")) o.locale = a.slice(9);
else if (a.startsWith("--since=")) o.since = a.slice(8);
else if (a === "--dry-run") o.dryRun = true;
else if (a.startsWith("--batch-size=")) o.batchSize = Math.max(1, Number(a.slice(13)) || 30);
}
if (!o.locale || !o.since) throw new Error("--locale=<code> and --since=<git ref> are required");
return o;
}
async function main() {
const o = parseArgs(process.argv);
const file = path.join(MESSAGES_DIR, `${o.locale}.json`);
const rel = path.relative(ROOT, file);
const after = JSON.parse(await fs.readFile(file, "utf8"));
// A catalog that did not exist at --since (a locale created after it) is reviewed in full.
let before = {};
try {
before = JSON.parse(
execFileSync("git", ["show", `${o.since}:${rel}`], {
cwd: ROOT,
encoding: "utf8",
maxBuffer: 1 << 28,
stdio: ["ignore", "pipe", "ignore"],
})
);
} catch {
console.log(`[review] ${rel} does not exist at ${o.since} — reviewing every leaf`);
}
const en = flat(JSON.parse(await fs.readFile(path.join(MESSAGES_DIR, "en.json"), "utf8")));
const changed = changedLeaves(flat(before), flat(after));
const ids = Object.keys(changed);
console.log(`[review] ${o.locale}: ${ids.length} leaves changed since ${o.since}`);
if (o.dryRun) return;
const config = JSON.parse(await fs.readFile(path.join(ROOT, "config", "i18n.json"), "utf8"));
const entry = config.locales.find((l) => l.code === o.locale);
const backend = backendConfig();
const fixes = {};
const skipped = [];
const CHECKPOINT_EVERY = 25;
const writeCatalog = async () => {
for (const [id, v] of Object.entries(fixes)) {
try {
setDeep(after, id, v);
} catch (err) {
console.log(`[review] ${err.message} — correction dropped`);
}
}
await fs.writeFile(file, JSON.stringify(after, null, 2) + "\n", "utf8");
};
let batchNo = 0;
for (let i = 0; i < ids.length; i += o.batchSize) {
const slice = ids.slice(i, i + o.batchSize);
const payload = Object.fromEntries(
slice.map((id) => [id, { en: en[id], current: changed[id] }])
);
try {
const text = await withRetries(
() =>
callChat(
[
{ role: "system", content: REVIEW_SYSTEM(entry.english ?? entry.name, entry.native) },
{ role: "user", content: JSON.stringify(payload) },
],
backend
),
{
onRetry: (err, n) =>
console.log(`[review] batch at ${i} failed (${err.message}) — retry ${n}`),
}
);
Object.assign(fixes, parseReviewResponse(text, slice));
} catch (err) {
skipped.push(...slice);
console.log(`[review] batch at ${i} skipped after retries: ${err.message}`);
}
console.log(
`[review] ${Math.min(i + o.batchSize, ids.length)}/${ids.length} reviewed, ${Object.keys(fixes).length} corrections so far`
);
if (++batchNo % CHECKPOINT_EVERY === 0) await writeCatalog();
}
await writeCatalog();
const reportDir = path.join(ROOT, "_artifacts", "i18n-review");
await fs.mkdir(reportDir, { recursive: true });
if (skipped.length) {
await fs.writeFile(
path.join(reportDir, `${o.locale}.skipped.json`),
JSON.stringify(skipped, null, 2) + "\n",
"utf8"
);
}
const report = [
`# Review ${o.locale} since ${o.since}`,
"",
`${ids.length} leaves reviewed, ${Object.keys(fixes).length} corrected, ${skipped.length} skipped (upstream failures).`,
"",
"| key | en | before | after |",
"| --- | --- | --- | --- |",
...Object.entries(fixes).map(([id, v]) => `| \`${id}\` | ${en[id]} | ${changed[id]} | ${v} |`),
].join("\n");
await fs.writeFile(path.join(reportDir, `${o.locale}.md`), report + "\n", "utf8");
console.log(
`[review] ${Object.keys(fixes).length} corrections applied${skipped.length ? `, ${skipped.length} leaves skipped` : ""}; report: _artifacts/i18n-review/${o.locale}.md`
);
}
const isDirectRun = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
if (isDirectRun) {
main().catch((e) => {
console.error(`[review] ${e.message}`);
process.exitCode = 1;
});
}

328
scripts/i18n/run-translation.mjs Executable file → Normal file
View File

@@ -37,6 +37,7 @@
*/
import { promises as fs, existsSync, readFileSync } from "node:fs";
import { execFileSync } from "node:child_process";
import path from "node:path";
import crypto from "node:crypto";
import process from "node:process";
@@ -282,13 +283,23 @@ function targetPathFor(relSource, locale) {
return path.join(DOCS_I18N_DIR, locale, relSource);
}
function extractTopHeading(markdown) {
const m = markdown.match(/^# (.+)\r?\n/);
// Most docs sources open with a YAML front-matter block (`---` … `---`). The
// H1 sits behind it, so the heading helpers look past the block: otherwise the
// mirror gets the file name as its title and the front matter (plus a second
// heading) is translated into its body.
const FRONT_MATTER = /^---\r?\n[\s\S]*?\r?\n---\r?\n+/;
function stripFrontMatter(markdown) {
return markdown.replace(FRONT_MATTER, "");
}
export function extractTopHeading(markdown) {
const m = stripFrontMatter(markdown).match(/^# (.+)\r?\n/);
return m ? m[1].trim() : null;
}
function stripTopHeading(markdown) {
return markdown.replace(/^# .+\r?\n+/, "");
export function stripTopHeading(markdown) {
return stripFrontMatter(markdown).replace(/^# .+\r?\n+/, "");
}
// ----- Translator backend --------------------------------------------------
@@ -451,7 +462,7 @@ function splitOversizedSection(section, maxChars) {
const chunks = [];
let current = [];
let size = 0;
for (const lines of blocks) {
for (const lines of blocks.flatMap((b) => splitOversizedRun(b, maxChars))) {
const length = lines.join("\n").length + 1;
if (size > 0 && size + length > maxChars) {
chunks.push(current.join("\n"));
@@ -465,6 +476,254 @@ function splitOversizedSection(section, maxChars) {
return chunks;
}
// A markdown table or a long bullet list has no blank line inside it, so the
// paragraph splitter kept PROVIDER_REFERENCE.md's 244-row table (40 KB) and
// FREE_TIERS.md's 71-item list (16 KB) as one block each, and the request for
// a verbose script outlived the upstream socket ("fetch failed" for Greek and
// Amharic on every attempt). An oversized block made only of table rows or
// list items (plus their indented continuation lines) is cut before an item
// line; the table header rows travel with the first group only.
const ITEM_LINE = /^\s*(\||[-*+]\s|\d+[.)]\s)/;
const CONTINUATION_LINE = /^\s+\S/;
function splitOversizedRun(lines, maxChars) {
if (lines.join("\n").length <= maxChars) return [lines];
const content = lines.filter((l) => l.trim() !== "");
if (!content.every((l) => ITEM_LINE.test(l) || CONTINUATION_LINE.test(l))) return [lines];
if (!ITEM_LINE.test(content[0])) return [lines];
const groups = [];
let group = [];
let size = 0;
for (const line of lines) {
if (group.length && ITEM_LINE.test(line) && size + line.length + 1 > maxChars) {
groups.push(group);
group = [];
size = 0;
}
group.push(line);
size += line.length + 1;
}
if (group.length) groups.push(group);
return groups;
}
// Chunks are rejoined with a blank line (they were cut on headings and
// paragraphs) — except at a seam between two table rows or two list items,
// where a blank line would break one table (or one tight list) into two.
// True when the text ends with a table row or a list item (its indented
// continuation lines included), i.e. a following item line belongs to the
// same run.
function endsInsideItemRun(text) {
const lines = text.trimEnd().split("\n");
let i = lines.length - 1;
while (i > 0 && CONTINUATION_LINE.test(lines[i])) i--;
return ITEM_LINE.test(lines[i] ?? "");
}
export function joinTranslatedChunks(parts) {
let out = "";
for (let i = 0; i < parts.length; i++) {
if (i === 0) {
out = parts[i];
continue;
}
const nextFirst = parts[i].trimStart().split("\n")[0] ?? "";
const seam = endsInsideItemRun(out) && ITEM_LINE.test(nextFirst) ? "\n" : "\n\n";
out = out.trimEnd() + seam + parts[i].trimStart();
}
return out;
}
// ----- Section cache --------------------------------------------------------
// A mirror is retranslated section by section: the state remembers a short
// hash of every `## ` section of the source at the time of translation, and a
// later run only sends the sections whose hash changed, splicing the untouched
// translated sections of the mirror back in. Sections are matched by index
// only — an inserted or removed section shifts everything after it and costs
// a retranslation of the tail, which is acceptable.
// Section 0 is the preamble (text before the first `## `); every other section
// starts with its `## ` heading line. Joining with "\n\n" reproduces the body
// modulo trailing whitespace. A `## ` inside a fenced code block never splits.
export function splitSections(markdown) {
const sections = [[]];
let inFence = false;
for (const line of markdown.split("\n")) {
if (FENCE_LINE.test(line)) inFence = !inFence;
if (!inFence && /^## /.test(line) && sections.at(-1).length) sections.push([]);
sections.at(-1).push(line);
}
return sections.map((s) => s.join("\n").replace(/\s+$/, ""));
}
export function sectionHashes(sections) {
return sections.map((s) => sha256(Buffer.from(s.trim(), "utf8")).slice(0, 12));
}
// `null` means "no reuse possible — translate the whole body": no recorded
// hashes, or the mirror on disk does not have the section count the hashes
// were recorded for (a hand edit, or a mirror written by an older pipeline).
export function planSectionReuse({ previousHashes, sections, mirrorSections }) {
if (!Array.isArray(previousHashes) || previousHashes.length !== mirrorSections.length) {
return null;
}
if (previousHashes.length !== sections.length) return null;
const now = sectionHashes(sections);
const reuse = new Map();
const translate = [];
now.forEach((h, i) => {
// A mirror section byte-equal to its source section is an untranslated copy (322 mirrors
// of the 36 pre-expansion locales were adopted as English, 2026-09-16 audit) — never reuse it.
// Section 0 of a mirror that starts with a YAML block is the old extractor's leaked
// frontmatter (24 newer locales, 2026-09-16) — always rebuild it.
const leakedFrontmatter = i === 0 && /^---\s*\n/.test(mirrorSections[i]);
const untranslated = looksUntranslated(mirrorSections[i], sections[i]);
if (h === previousHashes[i] && !untranslated && !leakedFrontmatter)
reuse.set(i, mirrorSections[i]);
else translate.push(i);
});
return { reuse, translate };
}
// The mirror without the prefix this script writes in front of the translated
// body: `# Title (native)`, the `🌐 **Languages:**` bar (older mirrors carry a
// translated label, so only the globe is pinned) and the `---` separator.
/**
* A mirror whose source did not change can still need a rebuild: the pre-2026-09 extractor
* leaked the source's YAML frontmatter into the body, and 322 mirrors of the pre-expansion
* locales were plain English copies adopted as translated. Both are invisible to the hash
* comparison, so the task loop asks this before skipping an up-to-date pair.
*/
/** Long prose lines (> 20 chars, not table/code/list scaffolding) of a markdown fragment. */
function proseLines(text) {
return text
.split("\n")
.map((l) => l.trim())
.filter((l) => l.length > 20 && !/^[`|#\-*\d\s]+$/.test(l));
}
/**
* True when a mirror fragment is still (mostly) the English source: byte-equal, or ≥ 80 % of
* its prose lines appear verbatim in the source. Fragments with < 3 prose lines fall back to
* byte equality (code-only sections are legitimately identical).
*/
export function looksUntranslated(mirrorFragment, sourceFragment) {
if (mirrorFragment.trim() === sourceFragment.trim()) return /[A-Za-z]{3,}/.test(sourceFragment);
const mir = proseLines(mirrorFragment);
if (mir.length < 3) return false;
const src = new Set(proseLines(sourceFragment));
return mir.filter((l) => src.has(l)).length / mir.length >= 0.8;
}
export function mirrorNeedsRebuild(mirrorText, sourceText) {
const body = extractMirrorBody(mirrorText);
if (/^---\s*\n[\s\S]{0,600}?\n---\s*\n/.test(body)) return true; // leaked frontmatter
const sourceBody = stripTopHeading(sourceText.replace(/^---\n[\s\S]*?\n---\n+/, ""));
return looksUntranslated(body, sourceBody); // still English
}
export function extractMirrorBody(mirrorText) {
return mirrorText
.replace(/^# .+\r?\n+/, "")
.replace(/^🌐 .*\r?\n+/, "")
.replace(/^---\r?\n+/, "");
}
// Bootstrap for targets translated before section hashes existed: walks the
// file's git history (newest first, at most 200 commits) and returns the text
// whose sha256 equals `sha` — the source that produced the mirror on disk — or
// `null` when it is not in reach (shallow clone, rewritten history).
export async function findSourceTextByHash(rel, sha, { cwd = ROOT } = {}) {
let commits;
try {
commits = execFileSync("git", ["log", "--format=%H", "-n", "200", "--", rel], {
cwd,
encoding: "utf8",
})
.trim()
.split("\n")
.filter(Boolean);
} catch {
return null;
}
for (const commit of commits) {
let text;
try {
text = execFileSync("git", ["show", `${commit}:${rel}`], {
cwd,
encoding: "utf8",
maxBuffer: 1 << 28,
});
} catch {
continue;
}
if (sha256(Buffer.from(text, "utf8")) === sha) return text;
}
return null;
}
// Decides which sections of a task can be spliced in from the mirror on disk.
// Targets recorded without `section_hashes` are bootstrapped from git history
// by the `source_hash` the state remembers for them.
/**
* Bootstrap fallback: the recorded source_hash often belongs to a working-tree state that was
* never committed as such (add-locale rewrites README/bars before translating), so an exact
* hash lookup fails. The source as of the last commit before the translation's `updated_at`
* is the closest committed ancestor — sections unchanged since then are safe to reuse.
*/
export function findSourceTextBefore(rel, isoDate, { cwd = ROOT } = {}) {
try {
const commit = execFileSync(
"git",
["log", "-1", "--format=%H", `--before=${isoDate}`, "--", rel],
{
cwd,
encoding: "utf8",
}
).trim();
if (!commit) return null;
return execFileSync("git", ["show", `${commit}:${rel}`], {
cwd,
encoding: "utf8",
maxBuffer: 1 << 28,
});
} catch {
return null;
}
}
export function mirrorLastCommitDate(mirrorRel, { cwd = ROOT } = {}) {
try {
const out = execFileSync("git", ["log", "-1", "--format=%cI", "--", mirrorRel], {
cwd,
encoding: "utf8",
}).trim();
return out || null;
} catch {
return null;
}
}
async function resolveSectionPlan({ task, state, opts, sections }) {
if (task.missingTarget || opts.force) return null;
const recorded = state.sources[task.rel]?.locales?.[task.locale];
let previousHashes = recorded?.section_hashes;
if (!previousHashes && recorded?.source_hash) {
let oldText = await findSourceTextByHash(task.rel, recorded.source_hash);
if (!oldText) {
// `updated_at` is bumped by `--adopt`, so prefer the date of the last commit that
// actually wrote the mirror (mirrors are only written by translation runs).
const translatedAt =
mirrorLastCommitDate(path.relative(ROOT, task.targetAbs)) || recorded.updated_at;
if (translatedAt) oldText = findSourceTextBefore(task.rel, translatedAt);
}
if (oldText) previousHashes = sectionHashes(splitSections(stripTopHeading(oldText)));
}
if (!previousHashes) return null;
const mirrorText = await fs.readFile(task.targetAbs, "utf8");
const mirrorSections = splitSections(extractMirrorBody(mirrorText));
return planSectionReuse({ previousHashes, sections, mirrorSections });
}
async function translateBody(body, localeEntry, backend) {
const englishName = localeEntry.english ?? localeEntry.name;
const native = localeEntry.native ?? localeEntry.name;
@@ -489,7 +748,7 @@ async function translateBody(body, localeEntry, backend) {
// reliably converts characters but not vocabulary habits, so zh-TW output
// otherwise keeps mainland renderings (默認 for 預設, 緩存 for 快取) and
// wrong-homophone conversions (上遊 for 上游, 儀錶板 for 儀表板).
return normalizeLocaleText(translated.join("\n\n"), localeEntry.code);
return normalizeLocaleText(joinTranslatedChunks(translated), localeEntry.code);
}
// Simple promise-based semaphore (avoid runtime deps).
@@ -521,6 +780,22 @@ function createLimiter(max) {
// ----- Main ----------------------------------------------------------------
/**
* Merge one run's (source, locale) records into a freshly re-read state. Source-level
* `source_hash` follows the record; untouched entries stay as the other runners left them.
* `fallback` is this run's in-memory state, used only when the file could not be read.
*/
export function mergeStateUpdates(fresh, touched, fallback) {
const base = fresh && fresh.sources ? fresh : fallback || { sources: {} };
for (const { rel, locale, sourceHash, record } of touched) {
const entry =
base.sources[rel] || (base.sources[rel] = { source_hash: sourceHash, locales: {} });
entry.source_hash = sourceHash;
entry.locales[locale] = record;
}
return base;
}
async function main() {
const opts = parseArgs(process.argv);
const config = await loadConfig();
@@ -642,6 +917,7 @@ async function main() {
// Build a flat queue of (source, locale) work units.
const tasks = [];
const touched = [];
for (const rel of sources) {
const { hash: sourceHash } = sourceHashes.get(rel);
const entry =
@@ -654,10 +930,17 @@ async function main() {
const previous = entry.locales[locale];
const sourceChanged = previous?.source_hash !== sourceHash;
const missingTarget = !existsSync(targetAbs);
if (!opts.force && !sourceChanged && !missingTarget) {
const needsRebuild =
!opts.force &&
!sourceChanged &&
!missingTarget &&
mirrorNeedsRebuild(await fs.readFile(targetAbs, "utf8"), sourceHashes.get(rel).text);
if (!opts.force && !sourceChanged && !missingTarget && !needsRebuild) {
stats.skipped++;
continue;
}
if (needsRebuild)
logInfo(`${rel}${locale}: mirror needs a rebuild (English copy or leaked frontmatter)`);
tasks.push({ rel, locale, targetAbs, sourceChanged, missingTarget });
}
}
@@ -685,9 +968,26 @@ async function main() {
const topHeading = extractTopHeading(sourceText);
const body = stripTopHeading(sourceText);
const sections = splitSections(body);
let translatedBody;
try {
translatedBody = await translateBody(body, localeEntry, backend);
const plan = await resolveSectionPlan({ task, state, opts, sections });
if (plan && plan.translate.length < sections.length) {
const out = [...sections];
for (const [i, text] of plan.reuse) out[i] = text;
for (const i of plan.translate) {
// An empty preamble (body opening with `## `) has nothing to send.
out[i] = sections[i].trim()
? await translateBody(sections[i], localeEntry, backend)
: "";
}
translatedBody = out.join("\n\n");
logInfo(
`${task.rel}${task.locale}: ${plan.translate.length}/${sections.length} sections retranslated`
);
} else {
translatedBody = await translateBody(body, localeEntry, backend);
}
} catch (err) {
stats.failed++;
failures.push({ rel: task.rel, locale: task.locale, error: err.message });
@@ -709,11 +1009,14 @@ async function main() {
await fs.writeFile(task.targetAbs, finalContent, "utf8");
const targetHash = sha256(Buffer.from(finalContent, "utf8"));
state.sources[task.rel].locales[task.locale] = {
const record = {
source_hash: sourceHash,
target_hash: targetHash,
section_hashes: sectionHashes(sections),
updated_at: new Date().toISOString(),
};
state.sources[task.rel].locales[task.locale] = record;
touched.push({ rel: task.rel, locale: task.locale, sourceHash, record });
stats.translated++;
logInfo(`${task.rel}${task.locale} (${translatedBody.length} chars)`);
@@ -721,8 +1024,11 @@ async function main() {
)
);
// Save state even on partial failure so future runs only retry what failed.
await saveState(state);
// Save state even on partial failure so future runs only retry what failed. Several
// `--locale=<code>` runs execute in parallel during a batch, so re-read the file and merge
// only this run's entries instead of overwriting the whole state (last writer used to win
// and the other runners' work vanished from the state — 2026-09-16).
await saveState(mergeStateUpdates(await loadState(), touched, state));
const elapsedSec = ((Date.now() - startMs) / 1000).toFixed(1);
logInfo(

View File

@@ -72,7 +72,28 @@ import { backendConfig, translateBatch, translateString } from "./lib/translate-
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.resolve(SCRIPT_DIR, "..", "..");
const CONFIG_PATH = path.join(ROOT, "config", "i18n.json");
const MESSAGES_DIR = path.join(ROOT, "src", "i18n", "messages");
const CATALOGS = {
ui: {
name: "ui",
dir: path.join(ROOT, "src", "i18n", "messages"),
allowlistPath: path.join(SCRIPT_DIR, "untranslatable-keys.json"),
},
cli: {
name: "cli",
dir: path.join(ROOT, "bin", "cli", "locales"),
allowlistPath: path.join(SCRIPT_DIR, "untranslatable-cli-keys.json"),
},
};
/** Which flat-JSON catalog family a run targets: the dashboard (`ui`) or the CLI (`cli`). */
export function resolveCatalog(name = "ui") {
const catalog = CATALOGS[name];
if (!catalog) throw new Error(`unknown catalog "${name}" (expected ui or cli)`);
return catalog;
}
let MESSAGES_DIR = CATALOGS.ui.dir; // reassigned in main() from --catalog
const SOURCE_LOCALE = "en";
const PLACEHOLDER_PREFIX = "__MISSING__:";
@@ -96,11 +117,13 @@ function parseArgs(argv) {
retranslateIdentical: false,
concurrency: null,
batchSize: 1,
catalog: "ui",
};
for (const arg of argv.slice(2)) {
if (arg === "--dry-run" || arg === "--dryrun") opts.dryRun = true;
else if (arg === "--translate-markers") opts.translateMarkers = true;
else if (arg === "--retranslate-identical") opts.retranslateIdentical = true;
else if (arg.startsWith("--catalog=")) opts.catalog = arg.slice(10).trim();
else if (arg.startsWith("--locale=")) {
opts.locales = arg
.slice(9)
@@ -125,6 +148,7 @@ function parseArgs(argv) {
"Usage: node scripts/i18n/sync-ui-keys.mjs [options]",
"",
" --locale=<csv> Target locales (default: all except `en`)",
" --catalog=ui|cli Catalog family (default ui = src/i18n/messages; cli = bin/cli/locales)",
" --retranslate-identical Flag leaves still identical to English (outside",
" untranslatable-keys.json) as __MISSING__ so they get",
" retranslated — only meaningful with --translate-markers",
@@ -391,9 +415,7 @@ async function processLocale(locale, source, config, opts, backend) {
const { merged, addedPaths } = mergeMissing(source, target);
if (opts.retranslateIdentical && locale !== SOURCE_LOCALE) {
const allow = new Set(
(await loadJson(path.join(SCRIPT_DIR, "untranslatable-keys.json"))).keys ?? []
);
const allow = new Set((await loadJson(resolveCatalog(opts.catalog).allowlistPath)).keys ?? []);
const flagged = markIdenticalAsMissing(merged, source, allow);
logInfo(`${locale}: ${flagged} English leaves flagged for retranslation`);
}
@@ -442,6 +464,9 @@ async function processLocale(locale, source, config, opts, backend) {
async function main() {
const opts = parseArgs(process.argv);
const catalog = resolveCatalog(opts.catalog);
MESSAGES_DIR = catalog.dir;
logInfo(`catalog: ${catalog.name} (${path.relative(ROOT, catalog.dir)})`);
const config = await loadConfig();
const sourcePath = path.join(MESSAGES_DIR, `${SOURCE_LOCALE}.json`);

View File

@@ -0,0 +1,259 @@
#!/usr/bin/env bash
# OmniRoute — translate the `__MISSING__:<en>` markers a branch adds, every locale in parallel.
#
# Why this exists: on 2026-09-16 eight feature PRs added 61 keys to `src/i18n/messages/en.json`
# and stamped `__MISSING__:<en>` into all 65 locales instead of translating. The new-key gate
# (`scripts/i18n/check-new-key-coverage.mjs`) now rejects markers, so a branch that adds keys
# has to translate them before its PR opens — and translating 65 locales one after the other
# is what makes people skip it. This runner does it in parallel and is safe to detach.
#
# What it does: N workers pop locale codes from a queue (`flock`-serialized) and each runs
# node scripts/i18n/sync-ui-keys.mjs --catalog=<c> --locale=<code> --translate-markers --batch-size=40
# with up to 3 attempts per locale. A locale is DONE only when the run exits 0 AND its catalog
# carries no `__MISSING__` marker any more. Everything is written under `_artifacts/i18n-new-keys/`
# (gitignored, disposable):
# <code>.log the sync-ui-keys output of the last attempt
# <code>.exit 0 on success, else the last exit code (1 when markers survived a rc=0 run)
# batch.log one line per attempt + START/END markers
# batch.status running | done | failed
# batch.rc the script's final exit code (written last — poll this file)
# batch.pid PID of this script (kill by PID, never `pkill -f`)
#
# Usage:
# bash scripts/i18n/translate-new-keys.sh [--catalog=ui|cli] [--workers=N] [--locales=<csv>]
# npm run i18n:translate-new-keys -- --locales=pt-BR,es
# # detached (the session keeps working; poll _artifacts/i18n-new-keys/batch.rc):
# mkdir -p _artifacts/i18n-new-keys
# nohup setsid bash scripts/i18n/translate-new-keys.sh > _artifacts/i18n-new-keys/runner.out 2>&1 &
#
# --catalog=ui|cli ui = src/i18n/messages (default); cli = bin/cli/locales
# --workers=N parallel locales (default 5)
# --locales=<csv> subset of locale codes (default: every <code>.json in the catalog but en)
#
# Backend: the translation env block must be present — in the shell or in the repo `.env`
# (the script reads only the OMNIROUTE_TRANSLATION_* lines of `.env`; already-set variables
# win). Without it the script refuses to start and names the variables. It never leaves
# markers silently: the exit code is non-zero while any selected locale still carries one.
#
# Exit codes: 0 all selected locales translated · 1 some locale still has markers / failed ·
# 2 usage or environment error (nothing ran).
set -u
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
ROOT=$(cd "$SCRIPT_DIR/../.." && pwd)
ART="$ROOT/_artifacts/i18n-new-keys"
CATALOG=ui
WORKERS=5
LOCALES=""
RETRY_SLEEP=${OMNIROUTE_TRANSLATION_RETRY_SLEEP:-30}
usage() {
sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
}
for arg in "$@"; do
case "$arg" in
--catalog=*) CATALOG="${arg#--catalog=}" ;;
--workers=*) WORKERS="${arg#--workers=}" ;;
--locales=*) LOCALES="${arg#--locales=}" ;;
--locale=*) LOCALES="${arg#--locale=}" ;;
-h | --help)
usage
exit 0
;;
*)
echo "[i18n-new-keys] unknown argument: $arg" >&2
usage >&2
exit 2
;;
esac
done
case "$CATALOG" in
ui) DIR="$ROOT/src/i18n/messages" ;;
cli) DIR="$ROOT/bin/cli/locales" ;;
*)
echo "[i18n-new-keys] --catalog must be ui or cli (got: $CATALOG)" >&2
exit 2
;;
esac
case "$WORKERS" in
'' | *[!0-9]* | 0)
echo "[i18n-new-keys] --workers must be a positive integer (got: $WORKERS)" >&2
exit 2
;;
esac
# ----- translation backend env: shell first, then the OMNIROUTE_TRANSLATION_* lines of .env
load_translation_env() {
local env_file="$ROOT/.env" line key value
[ -f "$env_file" ] || return 0
while IFS= read -r line || [ -n "$line" ]; do
line="${line#"${line%%[![:space:]]*}"}" # ltrim
line="${line#export }"
case "$line" in
OMNIROUTE_TRANSLATION_*=*) ;;
*) continue ;;
esac
key="${line%%=*}"
value="${line#*=}"
value="${value%"${value##*[![:space:]]}"}" # rtrim
case "$value" in
\"*\") value="${value#\"}" value="${value%\"}" ;;
\'*\') value="${value#\'}" value="${value%\'}" ;;
esac
if [ -z "${!key:-}" ]; then
export "$key=$value"
fi
done <"$env_file"
}
load_translation_env
REQUIRED_VARS="OMNIROUTE_TRANSLATION_API_URL OMNIROUTE_TRANSLATION_API_KEY OMNIROUTE_TRANSLATION_MODEL"
OPTIONAL_VARS="OMNIROUTE_TRANSLATION_CONCURRENCY OMNIROUTE_TRANSLATION_TIMEOUT_MS"
missing=""
for v in $REQUIRED_VARS; do
[ -n "${!v:-}" ] || missing="$missing $v"
done
if [ -n "$missing" ]; then
{
echo "[i18n-new-keys] REFUSING TO START — translation backend not configured."
echo " Missing (required):$missing"
echo " The full block (put it in $ROOT/.env or export it in the shell):"
for v in $REQUIRED_VARS $OPTIONAL_VARS; do
if [ -n "${!v:-}" ]; then echo " $v (set)"; else echo " $v (MISSING)"; fi
done
echo " Nothing was translated; the __MISSING__ markers are still in place and the"
echo " new-key gate (npm run i18n:check-new-keys) will reject them."
} >&2
exit 2
fi
# ----- locale selection: on-disk catalogs minus en, or the explicit --locales subset
selected=()
if [ -n "$LOCALES" ]; then
IFS=',' read -r -a requested <<<"$LOCALES"
for code in "${requested[@]}"; do
code="${code//[[:space:]]/}"
[ -n "$code" ] || continue
if [ "$code" = "en" ]; then
echo "[i18n-new-keys] en is the source catalog, skipping it" >&2
continue
fi
if [ ! -f "$DIR/$code.json" ]; then
echo "[i18n-new-keys] unknown locale: $code ($DIR/$code.json does not exist)" >&2
exit 2
fi
selected+=("$code")
done
else
for f in "$DIR"/*.json; do
code=$(basename "$f" .json)
[ "$code" = "en" ] && continue
selected+=("$code")
done
fi
if [ "${#selected[@]}" -eq 0 ]; then
echo "[i18n-new-keys] no locale selected" >&2
exit 2
fi
# ----- artifacts (fresh per run for the selected locales)
mkdir -p "$ART"
QUEUE="$ART/queue.txt"
LOCK="$ART/queue.lock"
: >"$QUEUE"
for code in "${selected[@]}"; do
echo "$code" >>"$QUEUE"
rm -f "$ART/$code.exit"
done
rm -f "$ART/batch.rc"
echo running >"$ART/batch.status"
echo $$ >"$ART/batch.pid"
log() { echo "$*" >>"$ART/batch.log"; }
count_markers() {
# grep -c prints 0 and exits 1 when nothing matches — the count is what we want.
grep -c "__MISSING__" "$DIR/$1.json" 2>/dev/null || true
}
next_locale() {
# pop the first line of the queue atomically
(
flock 9
local l
l=$(head -n 1 "$QUEUE")
[ -n "$l" ] && sed -i '1d' "$QUEUE"
echo "$l"
) 9>"$LOCK"
}
worker() {
local id=$1 code rc left t0 wall attempt
while :; do
code=$(next_locale)
[ -z "$code" ] && break
for attempt in 1 2 3; do
log "[$code] w$id attempt $attempt start $(date -Is)"
t0=$(date +%s)
(
cd "$ROOT" && node scripts/i18n/sync-ui-keys.mjs --catalog="$CATALOG" --locale="$code" \
--translate-markers --batch-size=40
) >"$ART/$code.log" 2>&1
rc=$?
wall=$(($(date +%s) - t0))
left=$(count_markers "$code")
log "[$code] w$id attempt $attempt exit=$rc wall=${wall}s markers_left=$left $(date -Is)"
if [ "$rc" -eq 0 ] && [ "$left" -eq 0 ]; then
echo 0 >"$ART/$code.exit"
log "[$code] DONE"
break
fi
if [ "$attempt" -eq 3 ]; then
[ "$rc" -eq 0 ] && rc=1
echo "$rc" >"$ART/$code.exit"
log "[$code] FAILED exit=$rc markers_left=$left"
else
sleep "$RETRY_SLEEP"
fi
done
done
}
echo "[i18n-new-keys] catalog=$CATALOG locales=${#selected[@]} workers=$WORKERS artifacts=$ART"
log "BATCH START catalog=$CATALOG workers=$WORKERS locales=${selected[*]} $(date -Is)"
pids=()
for i in $(seq 1 "$WORKERS"); do
worker "$i" &
pids+=("$!")
done
wait "${pids[@]}"
# ----- verdict: every selected locale must have exit 0 AND zero markers on disk
failed=""
for code in "${selected[@]}"; do
ex=$(cat "$ART/$code.exit" 2>/dev/null || echo missing)
left=$(count_markers "$code")
if [ "$ex" != "0" ] || [ "$left" -ne 0 ]; then
failed="$failed $code(exit=$ex,markers=$left)"
fi
done
if [ -n "$failed" ]; then
log "BATCH END failed:$failed $(date -Is)"
echo failed >"$ART/batch.status"
echo 1 >"$ART/batch.rc"
echo "[i18n-new-keys] FAILED — markers still present or run failed for:$failed" >&2
echo " logs: $ART/<code>.log — re-run with --locales=<csv> for just those." >&2
exit 1
fi
log "BATCH END ok $(date -Is)"
echo done >"$ART/batch.status"
echo 0 >"$ART/batch.rc"
echo "[i18n-new-keys] DONE — ${#selected[@]} locale(s) translated, no __MISSING__ marker left."
echo " Now: npm run i18n:check-keys && npm run i18n:check-ratio && npm run i18n:check-new-keys"
exit 0

View File

@@ -0,0 +1,4 @@
{
"description": "CLI keys that must stay in English (brand names, command names, protocol identifiers). Same shape as untranslatable-keys.json; consumed by sync-ui-keys --catalog=cli --retranslate-identical and check-translation-ratio --catalog=cli.",
"keys": []
}

View File

@@ -0,0 +1,27 @@
{
"description": "Site catalog keys (omnirouteSite/lang/<code>.json) that must stay identical to lang/_source.en.json — brand and product names, package names, URLs, protocol acronyms and literal config values. retranslate-site.mjs never sends these to the translation backend.",
"keys": [
"combos.mode.auto",
"compare.ops.oauth.lite",
"compare.res.tls.cli",
"compare.res.tls.own",
"cta.community.whatsapp.brazil",
"deploy.arm",
"deploy.docker",
"deploy.npm",
"deploy.opencode.cmd",
"deploy.pwa",
"deploy.termux",
"deploy.vscode",
"footer.github",
"footer.protocols",
"hero.cta.github",
"providers.cat.local.ex",
"providers.cat.oauth",
"viral.footer.github",
"viral.footer.home",
"why.flow.ide",
"why.where.reddit.title",
"why.where.x.title"
]
}

View File

@@ -0,0 +1,996 @@
/**
* JON-562: bounded memory profile for the real Claude `/v1/messages` request boundary.
*
* The default driver runs each context size in a fresh child process. The worker uses a
* synthetic request and a local fetch stub, so it exercises admission, parsing, translation,
* request logging and SSE cleanup without credentials or provider calls. Raw payloads are never
* written. Artifacts are private (umask 077) and contain only measurements plus V8 profiles of
* the synthetic process.
*
* Usage:
* node --import tsx/esm scripts/perf/messages-route-memory-profile.ts \
* --output-dir /tmp/omniroute-JON-562
*
* Defaults: 100k, 300k and 600k token-equivalents; six sequential requests per fresh process;
* concurrency=1; stream=true; cancellation=none. Context size is the only changing factor.
*/
import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import fs from "node:fs";
import inspector from "node:inspector";
import os from "node:os";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import v8 from "node:v8";
export const BYTES_PER_TOKEN_EQUIVALENT = 4;
export const DEFAULT_TOKEN_EQUIVALENTS = [100_000, 300_000, 600_000] as const;
const PROVENANCE_FILES = [
"src/lib/usage/completedRequestDetails.ts",
"src/lib/usage/usageHistory.ts",
"tests/unit/active-request-stream-chunks-lifecycle.test.ts",
"scripts/perf/messages-route-memory-profile.ts",
"tests/unit/messages-route-memory-profile.test.ts",
] as const;
type ClaudePayload = {
model: string;
max_tokens: number;
stream: boolean;
messages: Array<{ role: "user"; content: string }>;
};
export type MemoryRow = {
phase: "baseline" | "after_route" | "after_drain" | "settled" | "final";
elapsedMs: number;
requestIndex?: number;
heapUsedBytes: number;
heapTotalBytes: number;
rssBytes: number;
externalBytes: number;
arrayBuffersBytes: number;
admission: {
activeHeavy: number;
activeHealthyHeadroom: number;
inflightBytes: number;
queuedBytes: number;
waiting: number;
};
};
type GrowthSummary = {
baselineHeapUsedBytes: number;
finalSettledHeapUsedBytes: number;
settledGrowthBytes: number;
settledSlopeBytesPerRequest: number;
growthToWireRatio: number;
};
function markerFor(tokenEquivalent: number): string {
return ["JON", "562", tokenEquivalent, "CONTEXT"].join("-");
}
/**
* Build the credential-free environment inherited by profiler children.
* @param source - Environment to copy allowlisted runtime fields from.
* @returns A new environment containing only allowlisted fields and fixed test settings.
*/
export function buildWorkerEnv(
source: NodeJS.ProcessEnv | Record<string, string | undefined> = process.env
): NodeJS.ProcessEnv {
const env: NodeJS.ProcessEnv = {};
for (const key of ["PATH", "HOME", "TMPDIR", "TEMP", "TMP", "LANG", "LC_ALL", "TZ"] as const) {
const value = source[key];
if (value) env[key] = value;
}
env.NODE_ENV = "test";
env.APP_LOG_LEVEL = "error";
return env;
}
/**
* Build an ASCII-only Claude body with an exact, repeatable serialized size.
* @param tokenEquivalent - Context size expressed at four serialized bytes per token.
* @returns The synthetic body, its marker and exact target wire size.
* @throws {RangeError} If `tokenEquivalent` is invalid or too small for the fixed envelope.
* @throws {Error} If serialization does not match the calculated target size.
*/
export function buildClaudeContextPayload(tokenEquivalent: number): {
body: ClaudePayload;
marker: string;
targetWireBytes: number;
} {
if (!Number.isSafeInteger(tokenEquivalent) || tokenEquivalent < 64) {
throw new RangeError("tokenEquivalent must be an integer >= 64");
}
const targetWireBytes = tokenEquivalent * BYTES_PER_TOKEN_EQUIVALENT;
const marker = markerFor(tokenEquivalent);
const body: ClaudePayload = {
model: "openai/gpt-4.1",
max_tokens: 8,
stream: true,
messages: [{ role: "user", content: "" }],
};
const fixedBytes = Buffer.byteLength(JSON.stringify(body), "utf8");
const contentBytes = targetWireBytes - fixedBytes;
if (contentBytes < marker.length) {
throw new RangeError("tokenEquivalent is too small for the fixed request envelope");
}
body.messages[0].content = marker + "x".repeat(contentBytes - marker.length);
const actualBytes = Buffer.byteLength(JSON.stringify(body), "utf8");
if (actualBytes !== targetWireBytes) {
throw new Error(`payload calibration failed: wanted ${targetWireBytes}, got ${actualBytes}`);
}
return { body, marker, targetWireBytes };
}
/**
* Calculate post-GC growth from baseline and settled samples only.
* @param rows - Ordered memory samples from one isolated workload.
* @param wireBytes - Exact serialized request size used to calculate the growth ratio.
* @returns Baseline, final, slope and wire-ratio measurements.
* @throws {Error} If the samples contain no baseline or settled row.
*/
export function summarizeSettledGrowth(
rows: Array<Pick<MemoryRow, "phase" | "heapUsedBytes" | "requestIndex">>,
wireBytes: number
): GrowthSummary {
const baseline = rows.find((row) => row.phase === "baseline");
const settled = rows.filter((row) => row.phase === "settled");
if (!baseline || settled.length === 0) {
throw new Error("baseline and settled samples are required");
}
const final = settled[settled.length - 1];
const settledGrowthBytes = final.heapUsedBytes - baseline.heapUsedBytes;
const settledSlopeBytesPerRequest =
settled.length < 2
? settledGrowthBytes
: (final.heapUsedBytes - settled[0].heapUsedBytes) / (settled.length - 1);
return {
baselineHeapUsedBytes: baseline.heapUsedBytes,
finalSettledHeapUsedBytes: final.heapUsedBytes,
settledGrowthBytes,
settledSlopeBytesPerRequest,
growthToWireRatio: settledGrowthBytes / wireBytes,
};
}
function argValue(flag: string): string | undefined {
const index = process.argv.indexOf(flag);
return index >= 0 ? process.argv[index + 1] : undefined;
}
function positiveIntArg(flag: string, fallback: number): number {
const raw = argValue(flag);
if (raw === undefined) return fallback;
const value = Number(raw);
if (!Number.isSafeInteger(value) || value <= 0) {
throw new RangeError(`${flag} must be a positive integer`);
}
return value;
}
function privateDirectory(directory: string): void {
fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
fs.chmodSync(directory, 0o700);
}
function writePrivateJson(file: string, value: unknown): void {
fs.writeFileSync(file, JSON.stringify(value, null, 2) + "\n", { mode: 0o600 });
fs.chmodSync(file, 0o600);
}
function appendPrivateJsonLine(file: string, value: unknown): void {
fs.appendFileSync(file, JSON.stringify(value) + "\n", { mode: 0o600 });
}
/**
* Run work while guaranteeing removal of its raw heap-snapshot path.
* @param snapshotFile - Raw snapshot path owned by the operation.
* @param work - Worker/analyzer operation to run before cleanup.
* @returns The fulfilled result from `work`.
* @throws The original work or cleanup error.
*/
export async function withRawSnapshotCleanup<T>(
snapshotFile: string,
work: () => Promise<T>
): Promise<T> {
try {
return await work();
} finally {
fs.rmSync(snapshotFile, { force: true });
}
}
/**
* Remove a worker snapshot unless a successful driver handoff owns it.
* @param snapshotFile - Raw worker snapshot path.
* @param state - Whether the worker completed and the driver accepted ownership.
* @returns Nothing.
*/
export function cleanupWorkerSnapshot(
snapshotFile: string,
state: { workerComplete: boolean; snapshotHandoff: boolean }
): void {
if (!state.workerComplete || !state.snapshotHandoff) {
fs.rmSync(snapshotFile, { force: true });
}
}
function forceGc(): void {
if (typeof globalThis.gc !== "function") {
throw new Error("JON-562 worker requires node --expose-gc");
}
for (let index = 0; index < 4; index += 1) globalThis.gc();
}
function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function inspectorPost<T>(
session: inspector.Session,
method: string,
params: Record<string, unknown> = {}
): Promise<T> {
return new Promise((resolve, reject) => {
session.post(method, params, (error, result) => {
if (error) reject(error);
else resolve(result as T);
});
});
}
async function startAllocationSampling(): Promise<{
stop: () => Promise<Record<string, unknown>>;
disconnect: () => void;
}> {
const session = new inspector.Session();
session.connect();
await inspectorPost(session, "HeapProfiler.enable");
await inspectorPost(session, "HeapProfiler.startSampling", {
samplingInterval: 32 * 1024,
includeObjectsCollectedByMajorGC: true,
includeObjectsCollectedByMinorGC: true,
});
return {
stop: async () => {
const result = await inspectorPost<{ profile: Record<string, unknown> }>(
session,
"HeapProfiler.stopSampling"
);
return result.profile;
},
disconnect: () => session.disconnect(),
};
}
function openAiSseResponse(): Response {
const encoder = new TextEncoder();
const frames = [
`data: ${JSON.stringify({
id: "chatcmpl_JON562",
object: "chat.completion.chunk",
created: 1,
model: "gpt-4.1",
choices: [{ index: 0, delta: { role: "assistant", content: "ok" } }],
})}\n\n`,
`data: ${JSON.stringify({
id: "chatcmpl_JON562",
object: "chat.completion.chunk",
created: 1,
model: "gpt-4.1",
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
})}\n\n`,
"data: [DONE]\n\n",
];
return new Response(
new ReadableStream<Uint8Array>({
start(controller) {
for (const frame of frames) controller.enqueue(encoder.encode(frame));
controller.close();
},
}),
{ status: 200, headers: { "content-type": "text/event-stream" } }
);
}
async function runWorker(): Promise<void> {
process.umask(0o077);
const tokenEquivalent = positiveIntArg("--tokens", 100_000);
const iterations = positiveIntArg("--iterations", 6);
const outputDir = path.resolve(argValue("--output-dir") ?? "");
if (!argValue("--output-dir")) throw new Error("--output-dir is required in worker mode");
privateDirectory(outputDir);
const memoryFile = path.join(outputDir, "memory.jsonl");
fs.writeFileSync(memoryFile, "", { mode: 0o600 });
const startedAt = performance.now();
const rows: MemoryRow[] = [];
let providerCalls = 0;
let harness: Awaited<
ReturnType<
typeof import("../../tests/integration/_chatPipelineHarness.ts").createChatPipelineHarness
>
> | null = null;
let sampler: Awaited<ReturnType<typeof startAllocationSampling>> | null = null;
let heapSnapshotFile: string | null = null;
let workerComplete = false;
const snapshotHandoff = process.argv.includes("--snapshot-handoff");
try {
const { createChatPipelineHarness } =
await import("../../tests/integration/_chatPipelineHarness.ts");
harness = await createChatPipelineHarness(`JON-562-${tokenEquivalent}`);
const messagesRoute = await import("../../src/app/api/v1/messages/route.ts");
const { perConnectionAdmissionController } =
await import("../../src/shared/middleware/chatBodyAdmission.ts");
const { reloadResourcePressureRuntime } =
await import("../../open-sse/utils/resourcePressure.ts");
reloadResourcePressureRuntime({
heapThresholdMb: null,
immediateHeapUsedMb: () => 1,
sample: async () => ({
observedAtMs: Date.now(),
v8: { heapUsedBytes: 1, heapLimitBytes: Number.MAX_SAFE_INTEGER },
process: {
rssBytes: 1,
externalBytes: 0,
arrayBuffersBytes: 0,
availableBytes: null,
constrainedBytes: null,
},
cgroup: {
currentBytes: null,
maxBytes: null,
highBytes: null,
fileBytes: null,
events: null,
},
psi: null,
}),
});
harness.BaseExecutor.RETRY_CONFIG.delayMs = 0;
await harness.resetStorage();
await harness.seedConnection("openai", {
name: "JON-562-local-stub",
apiKey: "synthetic-not-a-credential",
});
globalThis.fetch = async () => {
providerCalls += 1;
return openAiSseResponse();
};
const sample = (phase: MemoryRow["phase"], requestIndex?: number): void => {
const usage = process.memoryUsage();
const admission = perConnectionAdmissionController.snapshot();
const row: MemoryRow = {
phase,
elapsedMs: Math.round((performance.now() - startedAt) * 100) / 100,
requestIndex,
heapUsedBytes: usage.heapUsed,
heapTotalBytes: usage.heapTotal,
rssBytes: usage.rss,
externalBytes: usage.external,
arrayBuffersBytes: usage.arrayBuffers,
admission: {
activeHeavy: admission.activeHeavy,
activeHealthyHeadroom: admission.activeHealthyHeadroom,
inflightBytes: admission.inflightBytes,
queuedBytes: admission.queuedBytes,
waiting: admission.waiting,
},
};
rows.push(row);
appendPrivateJsonLine(memoryFile, row);
};
const runRequest = async (
tokens: number,
requestIndex: number,
measured: boolean
): Promise<number> => {
const { body, targetWireBytes } = buildClaudeContextPayload(tokens);
const serialized = JSON.stringify(body);
const request = new Request("http://omniroute.invalid/v1/messages", {
method: "POST",
headers: {
"content-type": "application/json",
"content-length": String(targetWireBytes),
accept: "text/event-stream",
},
body: serialized,
});
const response = await messagesRoute.POST(request, {});
if (measured) sample("after_route", requestIndex);
const responseBody = await response.arrayBuffer();
if (measured) sample("after_drain", requestIndex);
if (response.status !== 200) {
throw new Error(
`route returned ${response.status} (${responseBody.byteLength} response bytes)`
);
}
return responseBody.byteLength;
};
await runRequest(Math.min(tokenEquivalent, 2_048), 0, false);
await delay(50);
forceGc();
// Allocation sampling is deliberately completed before the retention time series. The
// inspector profiler retains its own sampled stack records; leaving it enabled would make
// post-GC heap growth look linear even when the request payload itself was collectible.
sampler = await startAllocationSampling();
await runRequest(tokenEquivalent, 0, false);
const allocationProfile = await sampler.stop();
const allocationFile = path.join(outputDir, "allocation.heapprofile");
writePrivateJson(allocationFile, allocationProfile);
sampler.disconnect();
sampler = null;
await delay(100);
forceGc();
sample("baseline");
const responseBytes: number[] = [];
for (let requestIndex = 1; requestIndex <= iterations; requestIndex += 1) {
responseBytes.push(await runRequest(tokenEquivalent, requestIndex, true));
await delay(50);
forceGc();
sample("settled", requestIndex);
}
await delay(250);
forceGc();
sample("final");
if (!process.argv.includes("--no-snapshot")) {
forceGc();
heapSnapshotFile = path.join(outputDir, "post-gc.heapsnapshot");
v8.writeHeapSnapshot(heapSnapshotFile);
fs.chmodSync(heapSnapshotFile, 0o600);
}
const { targetWireBytes } = buildClaudeContextPayload(tokenEquivalent);
const growth = summarizeSettledGrowth(rows, targetWireBytes);
const settledRows = rows.filter((row) => row.phase === "settled");
const released = settledRows.every(
(row) =>
row.admission.activeHeavy === 0 &&
row.admission.activeHealthyHeadroom === 0 &&
row.admission.inflightBytes === 0 &&
row.admission.waiting === 0
);
if (!released) throw new Error("admission state remained live after an SSE response drained");
const manifest = {
ticket: "JON-562",
status: "complete",
route: "/v1/messages",
tokenEquivalent,
bytesPerTokenEquivalent: BYTES_PER_TOKEN_EQUIVALENT,
exactWireBytes: targetWireBytes,
iterations,
providerCalls,
conditions: {
concurrency: 1,
cancellation: "none",
stream: true,
messageCount: 1,
toolCount: 0,
provider: "local-fetch-stub",
networkCalls: 0,
},
runtime: {
node: process.version,
platform: process.platform,
arch: process.arch,
heapSizeLimitBytes: v8.getHeapStatistics().heap_size_limit,
},
responseBytes,
allocationSampling: {
intervalBytes: 32 * 1024,
file: path.basename(allocationFile),
requestCount: 1,
measuredWindow: "one post-warmup request before the retention time series",
},
retentionSeries: { allocationSamplingEnabled: false },
heapSnapshot: heapSnapshotFile ? path.basename(heapSnapshotFile) : null,
admissionReleasedAfterEveryRequest: released,
growth,
};
writePrivateJson(path.join(outputDir, "manifest.json"), manifest);
workerComplete = true;
process.stdout.write(JSON.stringify({ outputDir, status: "complete", growth }) + "\n");
} finally {
try {
if (sampler) {
await sampler.stop().catch(() => undefined);
sampler.disconnect();
}
await harness?.cleanup();
} finally {
if (heapSnapshotFile) {
cleanupWorkerSnapshot(heapSnapshotFile, { workerComplete, snapshotHandoff });
}
}
}
}
export type HeapSnapshot = {
snapshot: {
meta: {
node_fields: string[];
node_types: Array<string[] | string>;
edge_fields: string[];
edge_types: Array<string[] | string>;
};
};
nodes: number[];
edges: number[];
strings: string[];
};
function safeLabel(value: string, marker: string): string {
if (value.includes(marker) || value.length > 120) return "<SYNTHETIC_CONTEXT_REDACTED>";
if (path.isAbsolute(value)) return `<ABSOLUTE_PATH_REDACTED>/${path.basename(value)}`;
return value;
}
/**
* Classify marker-bearing V8 string graphs by their physical backing size.
* @param parsed - Parsed V8 heap snapshot.
* @param marker - Synthetic context marker to locate.
* @param minimumSelfSizeBytes - Minimum backing-graph size treated as retained context.
* @returns Candidate counts, physical sizes, verdict and large component roots.
*/
export function classifyContextBackingCandidates(
parsed: HeapSnapshot,
marker: string,
minimumSelfSizeBytes: number
): {
matchingPreviewNodes: number;
maxSelfSizeBytes: number;
maxBackingRetainedSizeBytes: number;
retainedContextSelfSizeBytes: number;
largeBackingRetained: boolean;
largeNodeIndexes: number[];
} {
const nodeFields = parsed.snapshot.meta.node_fields;
const nodeWidth = nodeFields.length;
const nodeTypeIndex = nodeFields.indexOf("type");
const nodeNameIndex = nodeFields.indexOf("name");
const nodeSelfSizeIndex = nodeFields.indexOf("self_size");
const edgeCountIndex = nodeFields.indexOf("edge_count");
const edgeFields = parsed.snapshot.meta.edge_fields;
const edgeWidth = edgeFields.length;
const edgeTypeIndex = edgeFields.indexOf("type");
const edgeTargetIndex = edgeFields.indexOf("to_node");
const nodeTypes = parsed.snapshot.meta.node_types[nodeTypeIndex] as string[];
const edgeTypes = parsed.snapshot.meta.edge_types[edgeTypeIndex] as string[];
const nodeCount = parsed.nodes.length / nodeWidth;
const matching: Array<{ nodeIndex: number; selfSizeBytes: number }> = [];
const stringParents: Array<number[] | undefined> = new Array(nodeCount);
const stringChildren: Array<number[] | undefined> = new Array(nodeCount);
const isStringNode = (nodeIndex: number): boolean => {
const type = nodeTypes[parsed.nodes[nodeIndex * nodeWidth + nodeTypeIndex]];
return type === "string" || type === "concatenated string" || type === "sliced string";
};
let edgeOffset = 0;
for (let from = 0; from < nodeCount; from += 1) {
const edgeCount = parsed.nodes[from * nodeWidth + edgeCountIndex];
for (let local = 0; local < edgeCount; local += 1) {
const type = edgeTypes[parsed.edges[edgeOffset + edgeTypeIndex]];
const to = parsed.edges[edgeOffset + edgeTargetIndex] / nodeWidth;
if (type === "internal" && isStringNode(from) && isStringNode(to)) {
(stringChildren[from] ??= []).push(to);
(stringParents[to] ??= []).push(from);
}
edgeOffset += edgeWidth;
}
}
for (let nodeIndex = 0; nodeIndex < nodeCount; nodeIndex += 1) {
const offset = nodeIndex * nodeWidth;
const type = nodeTypes[parsed.nodes[offset + nodeTypeIndex]];
if (type !== "string" && type !== "concatenated string" && type !== "sliced string") continue;
const name = parsed.strings[parsed.nodes[offset + nodeNameIndex]] ?? "";
if (name.includes(marker)) {
matching.push({ nodeIndex, selfSizeBytes: parsed.nodes[offset + nodeSelfSizeIndex] });
}
}
const componentRoots = new Set<number>();
for (const entry of matching) {
const queue = [entry.nodeIndex];
const seen = new Set<number>();
while (queue.length > 0) {
const node = queue.pop() as number;
if (seen.has(node)) continue;
seen.add(node);
const parents = stringParents[node] ?? [];
if (parents.length === 0) componentRoots.add(node);
else queue.push(...parents);
}
}
const components = [...componentRoots].map((root) => {
const nodes = new Set<number>();
const queue = [root];
let retainedSizeBytes = 0;
while (queue.length > 0) {
const node = queue.pop() as number;
if (nodes.has(node)) continue;
nodes.add(node);
retainedSizeBytes += parsed.nodes[node * nodeWidth + nodeSelfSizeIndex];
queue.push(...(stringChildren[node] ?? []));
}
return { root, nodes, retainedSizeBytes };
});
const large = components.filter(
(component) => component.retainedSizeBytes >= minimumSelfSizeBytes
);
const retainedNodes = new Set<number>();
for (const component of large) {
for (const node of component.nodes) retainedNodes.add(node);
}
return {
matchingPreviewNodes: matching.length,
maxSelfSizeBytes: matching.reduce(
(maximum, entry) => Math.max(maximum, entry.selfSizeBytes),
0
),
maxBackingRetainedSizeBytes: components.reduce(
(maximum, component) => Math.max(maximum, component.retainedSizeBytes),
0
),
retainedContextSelfSizeBytes: [...retainedNodes].reduce(
(total, node) => total + parsed.nodes[node * nodeWidth + nodeSelfSizeIndex],
0
),
largeBackingRetained: large.length > 0,
largeNodeIndexes: large.map((component) => component.root),
};
}
function analyzeSnapshot(snapshotFile: string, marker: string, minimumSelfSizeBytes: number) {
const parsed = JSON.parse(fs.readFileSync(snapshotFile, "utf8")) as HeapSnapshot;
const meta = parsed.snapshot.meta;
const nodeFields = meta.node_fields;
const edgeFields = meta.edge_fields;
const nodeWidth = nodeFields.length;
const edgeWidth = edgeFields.length;
const nodeTypeIndex = nodeFields.indexOf("type");
const nodeNameIndex = nodeFields.indexOf("name");
const edgeCountIndex = nodeFields.indexOf("edge_count");
const edgeTypeIndex = edgeFields.indexOf("type");
const edgeNameIndex = edgeFields.indexOf("name_or_index");
const edgeTargetIndex = edgeFields.indexOf("to_node");
const nodeTypes = meta.node_types[nodeTypeIndex] as string[];
const edgeTypes = meta.edge_types[edgeTypeIndex] as string[];
const nodeCount = parsed.nodes.length / nodeWidth;
const classification = classifyContextBackingCandidates(parsed, marker, minimumSelfSizeBytes);
const candidateNodes = new Set(classification.largeNodeIndexes);
const parents: Array<Array<{ from: number; edgeType: string; edgeName: string }> | undefined> =
new Array(nodeCount);
let edgeOffset = 0;
for (let from = 0; from < nodeCount; from += 1) {
const nodeOffset = from * nodeWidth;
const edgeCount = parsed.nodes[nodeOffset + edgeCountIndex];
for (let local = 0; local < edgeCount; local += 1) {
const type = edgeTypes[parsed.edges[edgeOffset + edgeTypeIndex]];
const rawName = parsed.edges[edgeOffset + edgeNameIndex];
const to = parsed.edges[edgeOffset + edgeTargetIndex] / nodeWidth;
if (type !== "weak") {
const edgeName =
type === "element" || type === "hidden" ? String(rawName) : parsed.strings[rawName];
(parents[to] ??= []).push({ from, edgeType: type, edgeName: edgeName ?? "" });
}
edgeOffset += edgeWidth;
}
}
const describeNode = (index: number) => {
const offset = index * nodeWidth;
return {
type: nodeTypes[parsed.nodes[offset + nodeTypeIndex]],
name: safeLabel(parsed.strings[parsed.nodes[offset + nodeNameIndex]] ?? "", marker),
};
};
const paths: unknown[] = [];
for (const target of [...candidateNodes].slice(0, 5)) {
const queue: Array<{ node: number; path: Array<Record<string, unknown>> }> = [
{ node: target, path: [{ node: describeNode(target) }] },
];
const seen = new Set([target]);
let found: Array<Record<string, unknown>> | null = null;
while (queue.length > 0 && !found) {
const current = queue.shift() as { node: number; path: Array<Record<string, unknown>> };
if (current.node === 0 || current.path.length >= 32) {
found = current.path;
break;
}
for (const parent of parents[current.node] ?? []) {
if (seen.has(parent.from)) continue;
seen.add(parent.from);
const nextPath = [
...current.path,
{
retainedBy: describeNode(parent.from),
edgeType: parent.edgeType,
edgeName: safeLabel(parent.edgeName, marker),
},
];
if (parent.from === 0) {
found = nextPath;
break;
}
queue.push({ node: parent.from, path: nextPath });
}
}
paths.push(found ?? [{ node: describeNode(target) }, { finding: "no root within 32 edges" }]);
}
return {
matchingContextPreviewNodes: classification.matchingPreviewNodes,
maxContextNodeSelfSizeBytes: classification.maxSelfSizeBytes,
maxContextBackingRetainedSizeBytes: classification.maxBackingRetainedSizeBytes,
retainedContextSelfSizeBytes: classification.retainedContextSelfSizeBytes,
minimumLargeBackingSelfSizeBytes: minimumSelfSizeBytes,
largeBackingRetained: classification.largeBackingRetained,
finding:
candidateNodes.size === 0
? "Only detached context previews remained; no context-sized backing string crossed the V8 self_size threshold."
: "A context-sized backing string remained after forced GC; redacted root paths follow.",
paths,
};
}
/**
* Enforce the physical large-backing retention gate.
* @param result - Analyzer verdict to enforce.
* @returns Nothing.
* @throws {Error} If a context-sized backing string survived forced garbage collection.
*/
export function assertNoLargeBacking(result: { largeBackingRetained: boolean }): void {
if (result.largeBackingRetained) {
throw new Error("context-sized backing string remained after forced GC");
}
}
async function runAnalyzer(): Promise<void> {
process.umask(0o077);
const snapshotArg = argValue("--snapshot");
if (!snapshotArg) throw new Error("analyzer requires --snapshot");
const snapshotFile = path.resolve(snapshotArg);
const outputArg = argValue("--output-dir");
const marker = argValue("--marker") ?? "";
try {
if (!outputArg || !marker) {
throw new Error("analyzer requires --output-dir and --marker");
}
const minimumSelfSizeBytes = positiveIntArg("--minimum-self-size-bytes", 1_024);
const outputDir = path.resolve(outputArg);
const result = analyzeSnapshot(snapshotFile, marker, minimumSelfSizeBytes);
const snapshotStat = fs.statSync(snapshotFile);
const snapshotSha256 = await sha256File(snapshotFile);
const redactedResult = {
...result,
snapshot: {
state: "deleted-after-local-analysis",
byteSize: snapshotStat.size,
sha256: snapshotSha256,
},
};
writePrivateJson(path.join(outputDir, "retainers.redacted.json"), redactedResult);
assertNoLargeBacking(result);
process.stdout.write(JSON.stringify(redactedResult) + "\n");
} finally {
fs.rmSync(snapshotFile, { force: true });
}
}
function harnessHash(): string {
return createHash("sha256")
.update(fs.readFileSync(new URL(import.meta.url)))
.digest("hex");
}
function gitOutput(args: string[]): string {
const result = spawnSync("git", args, {
encoding: "utf8",
env: buildWorkerEnv(process.env),
});
if (result.status !== 0) {
throw new Error(`git ${args.join(" ")} failed: ${result.stderr.trim()}`);
}
return result.stdout.trim();
}
function sha256(value: string | Buffer): string {
return createHash("sha256").update(value).digest("hex");
}
function sourceProvenance() {
const testedCommit = gitOutput(["rev-parse", "HEAD"]);
if (!/^[0-9a-f]{40}$/.test(testedCommit)) {
throw new Error(`invalid git HEAD: ${testedCommit || "empty"}`);
}
// `git branch --show-current` is empty on a detached HEAD, which is the normal checkout
// state for a CI PR run and for this fix worktree's own detached `git worktree add` — fall
// back to a descriptive marker instead of treating that as an error.
const branch = gitOutput(["branch", "--show-current"]) || `detached@${testedCommit.slice(0, 12)}`;
const sourceFiles = Object.fromEntries(
PROVENANCE_FILES.map((file) => {
if (!fs.existsSync(file)) throw new Error(`provenance file missing: ${file}`);
return [file, sha256(fs.readFileSync(file))];
})
);
const trackedDiff = gitOutput(["diff", "--binary", "HEAD", "--", ...PROVENANCE_FILES]);
return {
testedCommit,
branch,
statusPorcelain: gitOutput(["status", "--porcelain=v1", "--untracked-files=all"])
.split("\n")
.filter(Boolean),
trackedDiffSha256: sha256(trackedDiff),
sourceSetSha256: sha256(JSON.stringify(sourceFiles)),
sourceFiles,
};
}
function sha256File(file: string): Promise<string> {
return new Promise((resolve, reject) => {
const hash = createHash("sha256");
const input = fs.createReadStream(file);
input.on("error", reject);
input.on("data", (chunk) => hash.update(chunk));
input.on("end", () => resolve(hash.digest("hex")));
});
}
async function runDriver(): Promise<void> {
process.umask(0o077);
const outputDir = path.resolve(
argValue("--output-dir") ??
path.join(os.tmpdir(), `omniroute-JON-562-${new Date().toISOString().replace(/[:.]/g, "-")}`)
);
const iterations = positiveIntArg("--iterations", 6);
const tokenCases = (argValue("--tokens") ?? DEFAULT_TOKEN_EQUIVALENTS.join(","))
.split(",")
.map((raw) => Number(raw));
if (tokenCases.some((value) => !Number.isSafeInteger(value) || value < 64)) {
throw new RangeError("--tokens must be a comma-separated list of integers >= 64");
}
privateDirectory(outputDir);
const cases: unknown[] = [];
const scriptFile = fileURLToPath(import.meta.url);
for (const tokenEquivalent of tokenCases) {
const caseDir = path.join(outputDir, `context-${tokenEquivalent}`);
privateDirectory(caseDir);
const snapshotFile = path.join(caseDir, "post-gc.heapsnapshot");
await withRawSnapshotCleanup(snapshotFile, async () => {
const worker = spawnSync(
process.execPath,
[
"--expose-gc",
"--import",
"tsx/esm",
scriptFile,
"--worker",
"--snapshot-handoff",
"--tokens",
String(tokenEquivalent),
"--iterations",
String(iterations),
"--output-dir",
caseDir,
],
{
cwd: process.cwd(),
encoding: "utf8",
env: buildWorkerEnv(process.env),
// Measured ~230s wall for the 100k-token case alone (tsx/esm boot of the full
// route/handler module graph + real request lifecycle) on an idle box — 180s left
// no margin and made SIGKILL-on-timeout indistinguishable from a real worker crash
// (`worker.status` is `null`, which already fails the `!== 0` check below).
timeout: 300_000,
killSignal: "SIGKILL",
}
);
if (worker.status !== 0) {
throw new Error(`worker ${tokenEquivalent} failed:\n${worker.stdout}\n${worker.stderr}`);
}
const analyzer = spawnSync(
process.execPath,
[
"--max-old-space-size=4096",
"--import",
"tsx/esm",
scriptFile,
"--analyze-snapshot",
"--snapshot",
snapshotFile,
"--output-dir",
caseDir,
"--marker",
markerFor(tokenEquivalent),
"--minimum-self-size-bytes",
String(Math.floor((tokenEquivalent * BYTES_PER_TOKEN_EQUIVALENT) / 2)),
],
{
cwd: process.cwd(),
encoding: "utf8",
env: buildWorkerEnv(process.env),
timeout: 180_000,
killSignal: "SIGKILL",
}
);
if (analyzer.status !== 0) {
throw new Error(
`snapshot analyzer ${tokenEquivalent} failed:\n${analyzer.stdout}\n${analyzer.stderr}`
);
}
const retaining = JSON.parse(
fs.readFileSync(path.join(caseDir, "retainers.redacted.json"), "utf8")
);
assertNoLargeBacking(retaining);
const caseManifestFile = path.join(caseDir, "manifest.json");
const caseManifest = JSON.parse(fs.readFileSync(caseManifestFile, "utf8"));
caseManifest.heapSnapshot = retaining.snapshot;
writePrivateJson(caseManifestFile, caseManifest);
cases.push(caseManifest);
});
}
const provenance = sourceProvenance();
const workloadManifest = {
ticket: "JON-562",
status: "complete",
testedCommit: provenance.testedCommit,
provenance,
harnessSha256: harnessHash(),
baseBranch: "release/v3.8.51",
inheritedBaseRed: "diegosouzapw/OmniRoute#12732",
variedFactor: "serialized context bytes only",
fixedConditions: {
concurrency: 1,
cancellation: "none",
stream: true,
iterations,
messageCount: 1,
toolCount: 0,
provider: "local-fetch-stub",
externalProviderCalls: 0,
},
tokenEquivalentCases: tokenCases,
bytesPerTokenEquivalent: BYTES_PER_TOKEN_EQUIVALENT,
byteBudgetEvidence:
"Unit-level only: real-route responses in this checkpoint are small, so the 256-entry cap binds before the 16 MiB byte cap. The route matrix proves sliced backing detachment, not a 16 MiB runtime plateau.",
cases,
};
writePrivateJson(path.join(outputDir, "workload-manifest.json"), workloadManifest);
process.stdout.write(JSON.stringify({ outputDir, status: "complete" }) + "\n");
}
async function main(): Promise<void> {
if (process.argv.includes("--worker")) return runWorker();
if (process.argv.includes("--analyze-snapshot")) return runAnalyzer();
return runDriver();
}
const invokedPath = process.argv[1] ? pathToFileURL(path.resolve(process.argv[1])).href : "";
if (invokedPath === import.meta.url) {
main().catch((error: unknown) => {
const message = error instanceof Error ? error.message : String(error);
process.stderr.write(`[JON-562] ${message}\n`);
process.exitCode = 1;
});
}

View File

@@ -0,0 +1,30 @@
const CLOSE_RE =
/gh issue close\b|issues\.update\b|state=closed/g;
const KEYWORD_RE = new RegExp(
String.raw`\b(?:fix(?:es|ed)?|close[sd]?|resolve[sd]?)\s+#(\d+)\b`,
"i"
);
export function findTrackerCloses(workflowText) {
const hits = [];
const lines = String(workflowText ?? "").split(/\n/);
for (let i = 0; i < lines.length; i++) {
CLOSE_RE.lastIndex = 0;
if (CLOSE_RE.test(lines[i])) {
hits.push({ line: i + 1, text: lines[i].trim() });
}
CLOSE_RE.lastIndex = 0;
}
return hits;
}
export function closingKeywordInBody(body, tracker = 12732) {
const re = new RegExp(KEYWORD_RE.source, KEYWORD_RE.flags.includes("g") ? KEYWORD_RE.flags : `${KEYWORD_RE.flags}g`);
const text = String(body ?? "");
let m;
while ((m = re.exec(text)) !== null) {
if (Number(m[1]) === Number(tracker)) return true;
}
return false;
}

View File

@@ -0,0 +1,102 @@
import fs from "node:fs";
import path from "node:path";
import {
COLLECTORS,
globToRegExp,
} from "../../check/check-test-discovery.mjs";
const UNIT_CI_GLOBS = new Set([
"tests/unit/*.test.ts",
"tests/unit/{api,auth,authz,build,cli,cli-helper,combo,compression,correctness,cors,db,db-adapters,docs,gamification,guardrails,lib,mcp,memory,runtime,security,services,settings,shared,translator,ui,usage}/**/*.test.ts",
"tests/unit/dashboard/**/*.test.ts",
"tests/unit/serial/**/*.test.ts",
"tests/unit/**/*.test.mjs",
]);
const INTEGRATION_GLOBS = new Set([
"tests/integration/*.test.ts",
"tests/integration/combo-matrix/*.test.ts",
]);
function inScope(collector, scopeSuites) {
const suites = new Set(scopeSuites);
if (suites.has("test:unit:ci") && UNIT_CI_GLOBS.has(collector.glob)) return true;
if (suites.has("test:integration") && INTEGRATION_GLOBS.has(collector.glob)) return true;
if (suites.has("test:vitest") && collector.sources?.includes("vitest.mcp.config.ts")) {
return true;
}
return false;
}
function walkTestFiles(root = process.cwd()) {
const out = [];
function walk(dir) {
if (!fs.existsSync(dir)) return;
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
if (e.name === "node_modules" || e.name === ".git") continue;
const p = path.join(dir, e.name);
if (e.isDirectory()) walk(p);
else if (/\.(test|spec)\.(ts|tsx|mjs|js)$/.test(e.name)) {
out.push(path.relative(root, p).split(path.sep).join("/"));
}
}
}
walk(path.join(root, "tests"));
walk(path.join(root, "open-sse"));
walk(path.join(root, "src"));
return out;
}
export function canonicalSet(scopeSuites, collectors = COLLECTORS, files) {
const scoped = collectors.filter((c) => inScope(c, scopeSuites));
const regexes = scoped.map((c) => globToRegExp(c.glob));
const discovered = files ?? walkTestFiles();
return discovered.filter((f) => regexes.some((re) => re.test(f)));
}
export function knownUnexecuted(scopeSuites, collectors = COLLECTORS, baseline, files) {
const discovered = files ?? walkTestFiles();
const orphans = baseline?.orphans ?? [];
const outOfScope = collectors.filter((c) => !inScope(c, scopeSuites));
const collectorsOut = outOfScope.map((c) => {
const re = globToRegExp(c.glob);
const count = discovered.filter((f) => re.test(f)).length;
return {
glob: c.glob,
count,
reason: "collector runner is not a suite of this scope",
};
});
return {
orphans: { count: orphans.length, paths: orphans },
collectors: collectorsOut,
};
}
export function inventoryErrors(scopeSuites, collectors, baseline, discoveredFiles) {
const errors = [];
const full = COLLECTORS;
const givenGlobs = new Set(collectors.map((c) => c.glob));
for (const c of full) {
if (!givenGlobs.has(c.glob)) {
errors.push({
code: "collector_omitted",
glob: c.glob,
detail: `collector ${c.glob} omitted without known_unexecuted listing`,
});
}
}
const ku = knownUnexecuted(scopeSuites, collectors, baseline, discoveredFiles);
const knownGlobs = new Set(ku.collectors.map((c) => c.glob));
const knownOrphans = new Set(ku.orphans.paths);
const scoped = collectors.filter((c) => inScope(c, scopeSuites));
const regexes = scoped.map((c) => globToRegExp(c.glob));
for (const f of discoveredFiles ?? []) {
const inCanonical = regexes.some((re) => re.test(f));
const inKnown = knownOrphans.has(f) || [...knownGlobs].some((g) => globToRegExp(g).test(f));
if (!inCanonical && !inKnown) {
errors.push({ code: "unmapped_file", path: f, detail: "discovered file belongs to no set" });
}
}
return errors;
}

View File

@@ -0,0 +1,41 @@
const SUBTEST = /^# Subtest:\s+(\S+)/;
const RESULT = /^(ok|not ok)\s+\d+\s+-\s+(\S+)/;
export function fromNodeTestTap(tapText, argvFiles) {
const completed = [];
const failed = [];
const seen = new Set();
const lines = String(tapText ?? "").split(/\r?\n/);
let pending = null;
for (const line of lines) {
const sub = line.match(SUBTEST);
if (sub) {
pending = sub[1];
continue;
}
const res = line.match(RESULT);
if (res) {
const file = pending;
const ok = res[1] === "ok";
if (file) {
seen.add(file);
if (!ok) {
if (!failed.includes(file)) failed.push(file);
const i = completed.indexOf(file);
if (i >= 0) completed.splice(i, 1);
} else if (!failed.includes(file) && !completed.includes(file)) {
completed.push(file);
}
}
}
}
const attempted = [...argvFiles];
const missing = attempted.filter((f) => !seen.has(f));
return {
completed,
attempted,
missing,
failed,
pass: completed.length > 0 && missing.length === 0 && failed.length === 0,
};
}

View File

@@ -0,0 +1,243 @@
import { gateKey, sameKey } from "./types.mjs";
export function classifyDependent(prereqStatus, dependentKey, prereqKey) {
if (prereqStatus === "FAIL") {
return { status: "FAIL", cause: prereqKey };
}
if (prereqStatus === "INFRA_ERROR") {
return { status: "INFRA_ERROR", cause: prereqKey };
}
if (prereqStatus == null) {
return {
status: "INFRA_ERROR",
cause: prereqKey,
evidence_error: {
code: "prerequisite_missing",
gate: dependentKey,
detail: `missing prerequisite ${prereqKey.gate_id}`,
},
};
}
if (prereqStatus === "SKIPPED") {
return { status: "SKIPPED", cause: prereqKey };
}
return { status: "RUN", cause: null };
}
function requiredSet(plan) {
return plan.required_gates ?? [];
}
function optionalSet(plan) {
return plan.optional_gates ?? [];
}
function isRequired(plan, k) {
return requiredSet(plan).some((r) => sameKey(r, k));
}
function copies(gates, k) {
return gates.filter((g) => sameKey(gateKey(g), k));
}
function copiesByGateId(gates, gateId) {
return gates.filter((g) => g.gate_id === gateId);
}
function uniqueKeys(keys) {
const out = [];
for (const k of keys) {
if (!out.some((existing) => sameKey(existing, k))) out.push(k);
}
return out;
}
function keysForGateId(plan, gates, gateId) {
return uniqueKeys([
...copiesByGateId(gates, gateId).map((g) => gateKey(g)),
...requiredSet(plan).filter((k) => k.gate_id === gateId),
...optionalSet(plan).filter((k) => k.gate_id === gateId),
]);
}
function statusOf(gates, k) {
const list = copies(gates, k);
if (list.length === 0) return null;
if (list.some((g) => g.status === "INFRA_ERROR")) return "INFRA_ERROR";
if (list.some((g) => g.status === "FAIL")) return "FAIL";
if (list.some((g) => g.status === "SKIPPED")) return "SKIPPED";
return list[0].status;
}
function statusOfGateId(gates, gateId) {
const list = copiesByGateId(gates, gateId);
if (list.length === 0) return null;
if (list.some((g) => g.status === "INFRA_ERROR")) return "INFRA_ERROR";
if (list.some((g) => g.status === "FAIL")) return "FAIL";
if (list.some((g) => g.status === "SKIPPED")) return "SKIPPED";
return list[0].status;
}
function pushEvidenceError(evidence_errors, err) {
if (!err) return;
const already = evidence_errors.some(
(e) =>
e.code === err.code &&
e.detail === err.detail &&
e.gate?.gate_id === err.gate?.gate_id
);
if (!already) evidence_errors.push(err);
}
function patchDependent(gates, depKey, classified, prereqKey, evidence_errors, identity) {
const matches = copies(gates, depKey);
const reason =
classified.status === "SKIPPED" ? `classified from ${prereqKey.gate_id}` : undefined;
const exit_code = classified.status === "FAIL" ? 1 : 2;
if (matches.length === 0) {
gates.push({
gate_id: depKey.gate_id,
suite_id: depKey.suite_id,
shard_index: depKey.shard_index,
shard_total: depKey.shard_total,
tested_sha: identity.tested_sha || "0".repeat(40),
run_id: identity.run_id ?? "0",
run_attempt: identity.run_attempt ?? 1,
command_id: depKey.gate_id,
gate_type: "artifact",
status: classified.status,
cause: classified.cause,
reason,
exit_code,
duration_ms: 0,
evidence: [],
});
if (classified.evidence_error) pushEvidenceError(evidence_errors, classified.evidence_error);
return true;
}
let changed = false;
for (const existing of matches) {
if (
existing.status === classified.status &&
((existing.cause == null && classified.cause == null) ||
(existing.cause && classified.cause && sameKey(existing.cause, classified.cause)))
) {
continue;
}
existing.status = classified.status;
existing.cause = classified.cause;
existing.exit_code = exit_code;
if (classified.status === "SKIPPED" && !existing.reason) existing.reason = reason;
changed = true;
}
if (changed && classified.evidence_error) {
pushEvidenceError(evidence_errors, classified.evidence_error);
}
return changed;
}
function assertAcyclic(deps) {
const visiting = new Set();
const done = new Set();
function walk(id) {
if (done.has(id)) return;
if (visiting.has(id)) throw new Error("cyclic prerequisite");
visiting.add(id);
if (Object.hasOwn(deps, id)) walk(deps[id]);
visiting.delete(id);
done.add(id);
}
for (const id of Object.keys(deps)) walk(id);
}
export function reduce(plan, records) {
const deps = plan.dependencies ?? {};
assertAcyclic(deps);
for (const [depId, prereqId] of Object.entries(deps)) {
const requiredDep = requiredSet(plan).some((k) => k.gate_id === depId);
const optionalPrereq = optionalSet(plan).some((k) => k.gate_id === prereqId);
if (requiredDep && optionalPrereq) {
throw new Error("optional prerequisite");
}
}
const gates = [];
const evidence_errors = [];
for (const rec of records) {
const k = gateKey(rec);
const copy = { ...rec, cause: rec.cause ?? null };
if (copy.status === "SKIPPED" && isRequired(plan, k) && !copy.reason) {
copy.reason = "required skipped";
}
gates.push(copy);
}
const identity = plan.identity ?? {};
const edges = Object.entries(deps);
let changed = true;
let guard = edges.length + 1;
while (changed && guard-- > 0) {
changed = false;
for (const [depId, prereqId] of edges) {
const prereqKey = { gate_id: prereqId, suite_id: null, shard_index: null, shard_total: null };
let depKeys = keysForGateId(plan, gates, depId);
if (depKeys.length === 0) {
depKeys = [{ gate_id: depId, suite_id: null, shard_index: null, shard_total: null }];
}
const prereqStatus = statusOfGateId(gates, prereqId);
for (const depKey of depKeys) {
const classified = classifyDependent(prereqStatus, depKey, prereqKey);
if (classified.status === "RUN") continue;
if (patchDependent(gates, depKey, classified, prereqKey, evidence_errors, identity)) {
changed = true;
}
}
}
}
for (const k of requiredSet(plan)) {
const rec = gates.find((g) => sameKey(gateKey(g), k));
if (!rec) {
evidence_errors.push({
code: "missing_record",
gate: k,
detail: `required gate ${k.gate_id} has no record`,
});
} else if (rec.status === "SKIPPED") {
evidence_errors.push({
code: "required_skipped",
gate: k,
detail: rec.reason ?? "required gate SKIPPED",
});
}
}
const required = requiredSet(plan);
if (required.length === 0) {
evidence_errors.push({
code: "empty_required_set",
gate: { gate_id: "schema", suite_id: null, shard_index: null, shard_total: null },
detail: "required_gates is empty",
});
}
let verdict = "VERIFIED";
const hasFail = gates.some(
(g) => g.status === "FAIL" && isRequired(plan, gateKey(g)) && statusOf(gates, gateKey(g)) === "FAIL"
);
const hasUnverified =
evidence_errors.length > 0 ||
gates.some(
(g) =>
isRequired(plan, gateKey(g)) &&
(g.status === "SKIPPED" || g.status === "INFRA_ERROR")
);
if (hasFail) verdict = "FAILED";
else if (hasUnverified) verdict = "UNVERIFIED";
else if (required.some((k) => !gates.some((g) => sameKey(gateKey(g), k)))) {
verdict = "UNVERIFIED";
}
return { verdict, evidence_errors, gates };
}

View File

@@ -0,0 +1,50 @@
export function adaptCompiler({ commandId, inputDigest, exitCode, diagnostics }) {
const diags = Array.isArray(diagnostics) ? diagnostics : [];
const digest = typeof inputDigest === "string" ? inputDigest : "";
if (exitCode === 0 && digest.length > 0) {
return {
command_id: commandId,
input_digest: digest,
exit_code: 0,
diagnostics: diags,
status: "PASS",
};
}
if (exitCode === 0 && digest.length === 0) {
return {
command_id: commandId,
input_digest: digest,
exit_code: 0,
diagnostics: diags,
status: "INFRA_ERROR",
};
}
return {
command_id: commandId,
input_digest: digest,
exit_code: exitCode,
diagnostics: diags,
status: "FAIL",
};
}
export function adaptScript({ commandId, inputDigest, exitCode, stdout }) {
const digest = typeof inputDigest === "string" ? inputDigest : "";
let parsed = null;
if (typeof stdout === "string" && stdout.trim()) {
try {
parsed = JSON.parse(stdout);
} catch {
parsed = null;
}
}
const diagnostics = parsed ?? { input_digest: digest, exit_code: exitCode, diagnostics: stdout ?? "" };
const status = exitCode === 0 ? (digest ? "PASS" : "INFRA_ERROR") : "FAIL";
return {
command_id: commandId,
input_digest: digest,
exit_code: exitCode,
diagnostics,
status,
};
}

View File

@@ -0,0 +1,19 @@
export const STATUSES = Object.freeze(["PASS", "FAIL", "INFRA_ERROR", "SKIPPED"]);
export const VERDICTS = Object.freeze(["VERIFIED", "FAILED", "UNVERIFIED"]);
export function gateKey(rec) {
return {
gate_id: rec.gate_id,
suite_id: rec.suite_id ?? null,
shard_index: rec.shard_index ?? null,
shard_total: rec.shard_total ?? null,
};
}
export function keyId(k) {
return `${k.gate_id}\0${k.suite_id ?? ""}\0${k.shard_index ?? ""}\0${k.shard_total ?? ""}`;
}
export function sameKey(a, b) {
return keyId(a) === keyId(b);
}

View File

@@ -0,0 +1,81 @@
#!/usr/bin/env node
import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import Ajv from "ajv";
import { reduce } from "./release-acceptance/reduce.mjs";
const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
function loadJson(p) {
return JSON.parse(readFileSync(p, "utf8"));
}
export function exitFor(verdict) {
if (verdict === "VERIFIED") return 0;
if (verdict === "FAILED") return 1;
return 2;
}
export function reduceManifests(plan, manifests) {
const records = [];
for (const m of manifests) {
if (Array.isArray(m.gates)) records.push(...m.gates);
else records.push(m);
}
return reduce(plan, records);
}
export function validateReport(report, schema) {
const ajv = new Ajv({ allErrors: true, strict: false });
const validate = ajv.compile(schema);
return { ok: validate(report), errors: validate.errors };
}
function parseArgs(argv) {
const out = { plan: null, manifests: null, out: join(ROOT, "release-acceptance-report.json") };
for (let i = 2; i < argv.length; i++) {
if (argv[i] === "--plan") out.plan = argv[++i];
else if (argv[i] === "--manifests") out.manifests = argv[++i];
else if (argv[i] === "--out") out.out = argv[++i];
}
return out;
}
export async function main(argv = process.argv) {
const args = parseArgs(argv);
const plan = loadJson(args.plan);
const schema = loadJson(join(ROOT, "config/quality/release-acceptance.schema.json"));
const files = readdirSync(args.manifests)
.filter((f) => f.endsWith(".json"))
.map((f) => loadJson(join(args.manifests, f)));
const reduced = reduceManifests(plan, files);
const report = {
schema_version: 1,
identity: plan.identity,
required_gates: plan.required_gates ?? [],
gates: reduced.gates,
evidence_errors: reduced.evidence_errors,
verdict: reduced.verdict,
artifact: plan.artifact ?? null,
};
const { ok, errors } = validateReport(report, schema);
if (!ok) {
if (report.verdict !== "FAILED") report.verdict = "UNVERIFIED";
const gate =
Array.isArray(plan.required_gates) && plan.required_gates.length > 0
? plan.required_gates[0]
: { gate_id: "schema", suite_id: null, shard_index: null, shard_total: null };
report.evidence_errors = [
...(report.evidence_errors ?? []),
{ code: "schema_invalid", gate, detail: JSON.stringify(errors) },
];
}
mkdirSync(dirname(args.out), { recursive: true });
writeFileSync(args.out, JSON.stringify(report, null, 2) + "\n");
return exitFor(report.verdict);
}
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
main().then((code) => process.exit(code));
}

View File

@@ -1,284 +1,289 @@
#!/usr/bin/env node
/**
* PII redaction for log shipping.
*
* Streams input (stdin or files) to stdout, replacing sensitive tokens
* with stable redaction markers. Pure Node.js stdlib — no `npm install`.
*
* Recognised patterns (in order, longest match wins per position):
*
* 1. Anthropic API keys (sk-ant-...) → [REDACTED_API_KEY]
* 2. Google API keys (AIza...) → [REDACTED_API_KEY]
* 3. GitHub tokens (ghp_/gho_/ghu_/...) → [REDACTED_API_KEY]
* 4. OpenAI keys (sk-..., sk-proj-...) → [REDACTED_API_KEY]
* 5. AWS access keys (AKIA/ASIA) → [REDACTED_AWS_KEY]
* 6. Bearer tokens → [REDACTED_BEARER]
* 7. Email addresses → [REDACTED_EMAIL]
* 8. Generic api_key=value pairs → [REDACTED_API_KEY]
* 9. IPv4 addresses → [REDACTED_IPV4]
* 10. IPv6 addresses → [REDACTED_IPV6]
*
* Provider-specific patterns are listed BEFORE the generic `sk-` rule so
* that an `sk-ant-...` key counts as `ANTHROPIC_KEY` (not `OPENAI_KEY`)
* for the per-call summary.
*
* Why stable markers: downstream log-search queries reference the
* redaction markers (e.g. "show me all log lines with [REDACTED_IPV4]"),
* which makes the redaction reversible by anyone with the original
* vault lookup, but never by a log-search reader alone.
*
* CLI:
* node scripts/sre/redact-logs.mjs < input.log > output.log
* node scripts/sre/redact-logs.mjs --file access.log --output out.log
* node scripts/sre/redact-logs.mjs --strict # exit non-zero on any match
*
* Library:
* import { redact, redactString, RedactTransform } from "./scripts/sre/redact-logs.mjs";
*
* Salvaged from closed PR #5057 (base-stale; reimplemented on release).
*/
import { createReadStream, createWriteStream } from "node:fs";
import { TransformStream } from "node:stream/web";
import process from "node:process";
// ── Pattern catalogue ─────────────────────────────────────────────────────────
//
// Each pattern is [name, regex, marker]. The regex uses the `g` flag so we can
// iterate with `matchAll`. Order matters: longer / more specific patterns go
// first so an `sk-ant-...` key counts as ANTHROPIC_KEY instead of OPENAI_KEY.
export const REDACT_PATTERNS = Object.freeze([
// 1. Anthropic keys: sk-ant-api03-... / sk-ant-... (must beat the generic sk-)
[
"ANTHROPIC_KEY",
/\bsk-ant-[A-Za-z0-9_\-]{20,}\b/g,
"[REDACTED_API_KEY]",
],
// 2. Google API keys: AIza... (39 chars total)
[
"GOOGLE_KEY",
/\bAIza[A-Za-z0-9_\-]{35}\b/g,
"[REDACTED_API_KEY]",
],
// 3. GitHub tokens (classic + fine-grained + PAT prefixes)
[
"GITHUB_TOKEN",
/\b(?:ghp|gho|ghu|ghs|ghr|github_pat)_[A-Za-z0-9]{30,}\b/g,
"[REDACTED_API_KEY]",
],
// 4. OpenAI keys: sk-..., sk-proj-..., proj-... (after the more specific rules above)
[
"OPENAI_KEY",
/\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b|\bproj-[A-Za-z0-9_\-]{20,}\b/g,
"[REDACTED_API_KEY]",
],
// 5. AWS access keys — AKIA / ASIA prefixes, 20 chars total
[
"AWS_KEY",
/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g,
"[REDACTED_AWS_KEY]",
],
// 6. Bearer tokens — Authorization: Bearer xxxx (16+ chars)
[
"BEARER",
/(?:Bearer|Authorization:\s*Bearer)\s+([A-Za-z0-9._\-+/=]{16,})/g,
"[REDACTED_BEARER]",
],
// 7. Email — RFC 5322-ish; rejects obvious junk but stays compact.
[
"EMAIL",
/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,24}\b/g,
"[REDACTED_EMAIL]",
],
// 8. Generic api_key / apiKey / password= value pairs (12+ char secret)
[
"GENERIC_KEY",
/\b(?:api[_-]?key|apikey|password|passwd|pwd|secret|token|auth)\s*[:=]\s*['"]?([A-Za-z0-9._\-+/=]{12,})['"]?/gi,
"[REDACTED_API_KEY]",
],
// 9. IPv4 (incl. port) — strict octet bounds
[
"IPV4",
/\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d?\d)(?::\d{1,5})?\b/g,
"[REDACTED_IPV4]",
],
// 10. IPv6 — full, compressed, ::1, ::ffff:1.2.3.4
// Three alternative shapes:
// (a) full 8-group form (no `::`),
// (b) compressed form with `::` somewhere,
// (c) `::1` / `::` alone anchored by a non-hex lookbehind so it
// doesn't greedily extend `fe80::` into `fe80::foo`.
[
"IPV6",
new RegExp(
[
// (a) Full 8-group: 1:2:3:4:5:6:7:8
"\\b(?:[A-Fa-f0-9]{1,4}:){7}[A-Fa-f0-9]{1,4}\\b",
// (b) Compressed with `::` somewhere in the middle (left side 1+ groups)
"\\b(?:[A-Fa-f0-9]{1,4}:){1,6}[A-Fa-f0-9]{1,4}::[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6}\\b",
"\\b(?:[A-Fa-f0-9]{1,4}:){1,7}:[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6}\\b",
// (c) Leading `::` (no left side): ::1, ::1:2, ::ffff:1.2.3.4
"(?<![A-Fa-f0-9:])::(?:[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6})?\\b",
].join("|"),
"g"
),
"[REDACTED_IPV6]",
],
]);
// ── Library API ──────────────────────────────────────────────────────────────
/**
* Redact all PII tokens in a string. Returns the redacted string and the
* match counts so the caller can decide whether to fail in strict mode.
*
* @param {string} input
* @returns {{ output: string, counts: Record<string, number> }}
*/
export function redactString(input) {
if (typeof input !== "string" || input.length === 0) {
return { output: input ?? "", counts: {} };
}
const counts = {};
let output = input;
for (const [name, regex, marker] of REDACT_PATTERNS) {
output = output.replace(regex, () => {
counts[name] = (counts[name] ?? 0) + 1;
return marker;
});
}
return { output, counts };
}
/**
* Redact a single line. Convenience wrapper around redactString that does
* not allocate an intermediate object for the counts.
*
* @param {string} line
* @returns {string}
*/
export function redact(line) {
return redactString(line).output;
}
/**
* Redact TransformStream.
*
* Implements the WHATWG TransformStream API so callers can do:
* await src.pipeThrough(new TextDecoderStream()).pipeThrough(new RedactTransform()).pipeTo(sink)
*
* Counts are accumulated on the stream instance (`.counts`).
*/
export class RedactTransform extends TransformStream {
constructor() {
const counts = {};
super({
transform(chunk, controller) {
const text = typeof chunk === "string" ? chunk : new TextDecoder("utf-8").decode(chunk);
const { output, counts: localCounts } = redactString(text);
for (const [name, n] of Object.entries(localCounts)) {
counts[name] = (counts[name] ?? 0) + n;
}
controller.enqueue(new TextEncoder().encode(output));
},
});
// Attach counts as an enumerable own property so tests can read it.
Object.defineProperty(this, "counts", {
value: counts,
writable: false,
enumerable: true,
configurable: false,
});
}
}
// ── CLI ──────────────────────────────────────────────────────────────────────
function parseArgs(argv) {
const args = { files: [], output: null, strict: false, help: false };
for (let i = 2; i < argv.length; i += 1) {
const a = argv[i];
if (a === "--file") {
args.files.push(argv[++i]);
} else if (a === "--output" || a === "-o") {
args.output = argv[++i];
} else if (a === "--strict") {
args.strict = true;
} else if (a === "--help" || a === "-h") {
args.help = true;
} else {
process.stderr.write(`unknown argument: ${a}\n`);
process.exit(2);
}
}
return args;
}
function printHelp() {
process.stdout.write(`Usage: redact-logs.mjs [options]
Options:
--file <path> Read from file (repeatable). Defaults to stdin.
--output, -o <p> Write to file. Defaults to stdout.
--strict Exit non-zero if any PII is detected.
--help, -h Show this help.
Library:
import { redact, redactString, RedactTransform } from "./scripts/sre/redact-logs.mjs";
`);
}
async function main() {
const args = parseArgs(process.argv);
if (args.help) {
printHelp();
return;
}
const transform = new RedactTransform();
// Build a WHATWG ReadableStream from each input source. We open files /
// stdin as a Node Readable and convert it via Readable.toWeb().
const { Readable } = await import("node:stream");
const { Writable: WritableStreamWeb } = await import("node:stream/web");
const sources = args.files.length > 0 ? args.files : ["-"];
for (const source of sources) {
const nodeSrc = source === "-" ? process.stdin : createReadStream(source, "utf8");
const webSrc = Readable.toWeb(nodeSrc);
const webDecoded = webSrc.pipeThrough(new TextDecoderStream("utf-8"));
const webEncoded = webDecoded.pipeThrough(transform);
const sink = args.output
? createWriteStream(args.output, "utf8")
: process.stdout;
const webSink = WritableStreamWeb.toWeb(sink);
try {
await webEncoded.pipeTo(webSink);
} catch (err) {
process.stderr.write(`redact-logs: ${err.message}\n`);
process.exit(1);
}
}
const totals = transform.counts;
if (Object.keys(totals).length > 0) {
const summary = Object.entries(totals)
.map(([k, v]) => `${k}=${v}`)
.join(" ");
process.stderr.write(`redact-logs: redacted ${summary}\n`);
if (args.strict) {
process.exit(3);
}
}
}
// Only run as CLI when this module is the entrypoint (not when imported as a
// library). `import.meta.url === pathToFileURL(process.argv[1]).href` is the
// canonical ESM check.
import { pathToFileURL } from "node:url";
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
main();
}
#!/usr/bin/env node
/**
* PII redaction for log shipping.
*
* Streams input (stdin or files) to stdout, replacing sensitive tokens
* with stable redaction markers. Pure Node.js stdlib — no `npm install`.
*
* Recognised patterns (in order, longest match wins per position):
*
* 1. Anthropic API keys (sk-ant-...) → [REDACTED_API_KEY]
* 2. Google API keys (AIza...) → [REDACTED_API_KEY]
* 3. GitHub tokens (ghp_/gho_/ghu_/...) → [REDACTED_API_KEY]
* 4. OpenAI keys (sk-..., sk-proj-...) → [REDACTED_API_KEY]
* 5. AWS access keys (AKIA/ASIA) → [REDACTED_AWS_KEY]
* 6. Bearer tokens → [REDACTED_BEARER]
* 7. Email addresses → [REDACTED_EMAIL]
* 8. Generic api_key=value pairs → [REDACTED_API_KEY]
* 9. IPv4 addresses → [REDACTED_IPV4]
* 10. IPv6 addresses → [REDACTED_IPV6]
*
* Provider-specific patterns are listed BEFORE the generic `sk-` rule so
* that an `sk-ant-...` key counts as `ANTHROPIC_KEY` (not `OPENAI_KEY`)
* for the per-call summary.
*
* Why stable markers: downstream log-search queries reference the
* redaction markers (e.g. "show me all log lines with [REDACTED_IPV4]"),
* which makes the redaction reversible by anyone with the original
* vault lookup, but never by a log-search reader alone.
*
* CLI:
* node scripts/sre/redact-logs.mjs < input.log > output.log
* node scripts/sre/redact-logs.mjs --file access.log --output out.log
* node scripts/sre/redact-logs.mjs --strict # exit non-zero on any match
*
* Library:
* import { redact, redactString, RedactTransform } from "./scripts/sre/redact-logs.mjs";
*
* Salvaged from closed PR #5057 (base-stale; reimplemented on release).
*/
import { createReadStream, createWriteStream } from "node:fs";
import { TransformStream } from "node:stream/web";
import process from "node:process";
// ── Pattern catalogue ─────────────────────────────────────────────────────────
//
// Each pattern is [name, regex, marker]. The regex uses the `g` flag so we can
// iterate with `matchAll`. Order matters: longer / more specific patterns go
// first so an `sk-ant-...` key counts as ANTHROPIC_KEY instead of OPENAI_KEY.
export const REDACT_PATTERNS = Object.freeze([
// 1. Anthropic keys: sk-ant-api03-... / sk-ant-... (must beat the generic sk-)
["ANTHROPIC_KEY", /\bsk-ant-[A-Za-z0-9_\-]{20,}\b/g, "[REDACTED_API_KEY]"],
// 2. Google API keys: AIza... (39 chars total)
["GOOGLE_KEY", /\bAIza[A-Za-z0-9_\-]{35}\b/g, "[REDACTED_API_KEY]"],
// 3. GitHub tokens (classic + fine-grained + PAT prefixes)
[
"GITHUB_TOKEN",
/\b(?:ghp|gho|ghu|ghs|ghr|github_pat)_[A-Za-z0-9]{30,}\b/g,
"[REDACTED_API_KEY]",
],
// 4. OpenAI keys: sk-..., sk-proj-..., proj-... (after the more specific rules above)
[
"OPENAI_KEY",
/\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b|\bproj-[A-Za-z0-9_\-]{20,}\b/g,
"[REDACTED_API_KEY]",
],
// 5. AWS access keys — AKIA / ASIA prefixes, 20 chars total
["AWS_KEY", /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, "[REDACTED_AWS_KEY]"],
// 6. Bearer tokens — Authorization: Bearer xxxx (16+ chars)
[
"BEARER",
/(?:Bearer|Authorization:\s*Bearer)\s+([A-Za-z0-9._\-+/=]{16,})/g,
"[REDACTED_BEARER]",
],
// 7. Email — RFC 5322-ish; rejects obvious junk but stays compact.
["EMAIL", /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,24}\b/g, "[REDACTED_EMAIL]"],
// 8. Generic api_key / apiKey / password= value pairs (12+ char secret)
[
"GENERIC_KEY",
/\b(?:api[_-]?key|apikey|password|passwd|pwd|secret|token|auth)\s*[:=]\s*['"]?([A-Za-z0-9._\-+/=]{12,})['"]?/gi,
"[REDACTED_API_KEY]",
],
// 9. IPv4 (incl. port) — strict octet bounds
[
"IPV4",
/\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d?\d)(?::\d{1,5})?\b/g,
"[REDACTED_IPV4]",
],
// 10. IPv6 — full, compressed, ::1, ::ffff:1.2.3.4
// Three alternative shapes:
// (a) full 8-group form (no `::`),
// (b) compressed form with `::` somewhere,
// (c) `::1` / `::` alone anchored by a non-hex lookbehind so it
// doesn't greedily extend `fe80::` into `fe80::foo`.
[
"IPV6",
new RegExp(
[
// (a) Full 8-group: 1:2:3:4:5:6:7:8
"\\b(?:[A-Fa-f0-9]{1,4}:){7}[A-Fa-f0-9]{1,4}\\b",
// (b) Compressed with `::` somewhere in the middle (left side 1+ groups)
"\\b(?:[A-Fa-f0-9]{1,4}:){1,6}[A-Fa-f0-9]{1,4}::[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6}\\b",
"\\b(?:[A-Fa-f0-9]{1,4}:){1,7}:[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6}\\b",
// (c) Leading `::` (no left side): ::1, ::1:2, ::ffff:1.2.3.4
"(?<![A-Fa-f0-9:])::(?:[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6})?\\b",
].join("|"),
"g"
),
"[REDACTED_IPV6]",
],
]);
// ── Library API ──────────────────────────────────────────────────────────────
/**
* Redact all PII tokens in a string. Returns the redacted string and the
* match counts so the caller can decide whether to fail in strict mode.
*
* @param {string} input
* @returns {{ output: string, counts: Record<string, number> }}
*/
export function redactString(input) {
if (typeof input !== "string" || input.length === 0) {
return { output: input ?? "", counts: {} };
}
const counts = {};
let output = input;
for (const [name, regex, marker] of REDACT_PATTERNS) {
output = output.replace(regex, () => {
counts[name] = (counts[name] ?? 0) + 1;
return marker;
});
}
return { output, counts };
}
/**
* Redact a single line. Convenience wrapper around redactString that does
* not allocate an intermediate object for the counts.
*
* @param {string} line
* @returns {string}
*/
export function redact(line) {
return redactString(line).output;
}
/**
* Redact TransformStream.
*
* Implements the WHATWG TransformStream API so callers can do:
* await src.pipeThrough(new TextDecoderStream()).pipeThrough(new RedactTransform()).pipeTo(sink)
*
* Counts are accumulated on the stream instance (`.counts`).
*/
export class RedactTransform extends TransformStream {
constructor() {
const counts = {};
const decoder = new TextDecoder("utf-8");
const encoder = new TextEncoder();
let pendingLine = "";
const emitRedacted = (text, controller) => {
if (text.length === 0) return;
const { output, counts: localCounts } = redactString(text);
for (const [name, n] of Object.entries(localCounts)) {
counts[name] = (counts[name] ?? 0) + n;
}
controller.enqueue(encoder.encode(output));
};
super({
transform(chunk, controller) {
const text =
typeof chunk === "string"
? decoder.decode() + chunk
: decoder.decode(chunk, { stream: true });
pendingLine += text;
const lastNewline = pendingLine.lastIndexOf("\n");
if (lastNewline >= 0) {
emitRedacted(pendingLine.slice(0, lastNewline + 1), controller);
pendingLine = pendingLine.slice(lastNewline + 1);
}
},
flush(controller) {
pendingLine += decoder.decode();
emitRedacted(pendingLine, controller);
},
});
// Attach counts as an enumerable own property so tests can read it.
Object.defineProperty(this, "counts", {
value: counts,
writable: false,
enumerable: true,
configurable: false,
});
}
}
// ── CLI ──────────────────────────────────────────────────────────────────────
function parseArgs(argv) {
const args = { files: [], output: null, strict: false, help: false };
for (let i = 2; i < argv.length; i += 1) {
const a = argv[i];
if (a === "--file") {
args.files.push(argv[++i]);
} else if (a === "--output" || a === "-o") {
args.output = argv[++i];
} else if (a === "--strict") {
args.strict = true;
} else if (a === "--help" || a === "-h") {
args.help = true;
} else {
process.stderr.write(`unknown argument: ${a}\n`);
process.exit(2);
}
}
return args;
}
function printHelp() {
process.stdout.write(`Usage: redact-logs.mjs [options]
Options:
--file <path> Read from file (repeatable). Defaults to stdin.
--output, -o <p> Write to file. Defaults to stdout.
--strict Exit non-zero if any PII is detected.
--help, -h Show this help.
Library:
import { redact, redactString, RedactTransform } from "./scripts/sre/redact-logs.mjs";
`);
}
async function main() {
const args = parseArgs(process.argv);
if (args.help) {
printHelp();
return;
}
const transform = new RedactTransform();
// Build a WHATWG ReadableStream from each input source. We open files /
// stdin as a Node Readable and convert it via Readable.toWeb().
const { Readable } = await import("node:stream");
const { Writable: WritableStreamWeb } = await import("node:stream/web");
const sources = args.files.length > 0 ? args.files : ["-"];
for (const source of sources) {
const nodeSrc = source === "-" ? process.stdin : createReadStream(source, "utf8");
const webSrc = Readable.toWeb(nodeSrc);
const webDecoded = webSrc.pipeThrough(new TextDecoderStream("utf-8"));
const webEncoded = webDecoded.pipeThrough(transform);
const sink = args.output ? createWriteStream(args.output, "utf8") : process.stdout;
const webSink = WritableStreamWeb.toWeb(sink);
try {
await webEncoded.pipeTo(webSink);
} catch (err) {
process.stderr.write(`redact-logs: ${err.message}\n`);
process.exit(1);
}
}
const totals = transform.counts;
if (Object.keys(totals).length > 0) {
const summary = Object.entries(totals)
.map(([k, v]) => `${k}=${v}`)
.join(" ");
process.stderr.write(`redact-logs: redacted ${summary}\n`);
if (args.strict) {
process.exit(3);
}
}
}
// Only run as CLI when this module is the entrypoint (not when imported as a
// library). `import.meta.url === pathToFileURL(process.argv[1]).href` is the
// canonical ESM check.
import { pathToFileURL } from "node:url";
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
main();
}