mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-21 14:22:14 +03:00
merge(cache): resolve conflicts merging release/v3.8.51 into semantic-cache re-land
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> # Conflicts: # changelog.d/fixes/12910-semantic-cache-exact-id-finalize.md # open-sse/handlers/chatCore/semanticCache.ts # open-sse/handlers/chatCore/streamingSemanticCacheStore.ts # src/app/(dashboard)/dashboard/providers/[id]/hooks/useModelImportHandlers.ts # src/lib/providerModels/modelDiscovery.ts # tests/unit/chatcore-semantic-cache.test.ts # tests/unit/semantic-cache-no-truncated-writes.test.ts
This commit is contained in:
@@ -163,6 +163,43 @@ const EXTRA_MODULE_ENTRIES = [
|
||||
dest: ["node_modules", "pino-pretty"],
|
||||
},
|
||||
{ label: "split2", src: ["node_modules", "split2"], dest: ["node_modules", "split2"] },
|
||||
{
|
||||
// ioredis is a deliberately LAZY dependency (Redis is optional — see the
|
||||
// #6559 comment in src/shared/utils/rateLimiter.ts) — reached only via a
|
||||
// runtime `await import("ioredis")` in rateLimiter.ts,
|
||||
// warmupScheduler/circuitBreakerFactory.ts and quota/redisQuotaStore.ts,
|
||||
// never through a static top-level import. The standalone tracer only
|
||||
// follows statically-analyzable imports, so it never sees these call
|
||||
// sites and drops ioredis from node_modules/ entirely. Any self-hosted
|
||||
// deployment that actually sets REDIS_URL crashes the first time it
|
||||
// reaches one of those call sites with "Cannot find module 'ioredis'" —
|
||||
// reproduced on a production Docker deployment (REDIS_URL configured,
|
||||
// v3.8.49) where the standalone image shipped ioredis/package.json but
|
||||
// none of its own dependencies or built/ output.
|
||||
label: "ioredis (dynamic import — #6559)",
|
||||
src: ["node_modules", "ioredis"],
|
||||
dest: ["node_modules", "ioredis"],
|
||||
},
|
||||
{
|
||||
// bcryptjs IS statically imported by src/lib/auth/managementPassword.ts,
|
||||
// so the main server bundle is fine — Next's server compiler inlines the
|
||||
// small pure-JS package directly into the compiled route chunk instead of
|
||||
// leaving it as an external node_modules dependency. bin/cli/settings-
|
||||
// store.mjs (the `omniroute reset-password` / bin/reset-password.mjs
|
||||
// CLI, used to recover a lost dashboard password) is a separate,
|
||||
// unbundled entrypoint that does a plain runtime `import bcrypt from
|
||||
// "bcryptjs"` and needs the real package physically present in
|
||||
// node_modules/ — which nothing else requires as a loose runtime
|
||||
// dependency, so it is never copied. Reproduced on a production
|
||||
// deployment: `node bin/reset-password.mjs --password-stdin` failed with
|
||||
// "Cannot find package 'bcryptjs' imported from
|
||||
// /app/bin/cli/settings-store.mjs" (ERR_MODULE_NOT_FOUND) even though the
|
||||
// same container's dashboard login (which also depends on bcryptjs) was
|
||||
// working normally.
|
||||
label: "bcryptjs (bin/cli/settings-store.mjs — reset-password CLI)",
|
||||
src: ["node_modules", "bcryptjs"],
|
||||
dest: ["node_modules", "bcryptjs"],
|
||||
},
|
||||
{ label: "migrations", src: ["src", "lib", "db", "migrations"], dest: ["migrations"] },
|
||||
{ label: "MITM server", src: ["src", "mitm", "server.cjs"], dest: ["src", "mitm", "server.cjs"] },
|
||||
{
|
||||
@@ -229,6 +266,15 @@ const EXTRA_MODULE_ENTRIES = [
|
||||
src: ["scripts", "dev", "responses-ws-proxy.mjs"],
|
||||
dest: ["responses-ws-proxy.mjs"],
|
||||
},
|
||||
{
|
||||
// server-ws.mjs imports ./httpClientAbortGuard.mjs. In the repo that path is
|
||||
// the scripts/dev shim re-exporting the shared implementation, but the
|
||||
// assembled bundle has no src/ tree, so ship the real self-contained
|
||||
// implementation (no relative imports of its own) under the same file name.
|
||||
label: "http client abort guard (server-ws.mjs dependency)",
|
||||
src: ["src", "shared", "utils", "httpClientAbortGuard.mjs"],
|
||||
dest: ["httpClientAbortGuard.mjs"],
|
||||
},
|
||||
{
|
||||
label: "ChatGPT Web Codex MCP tunnel entrypoint",
|
||||
src: ["bin", "chatgpt-web-codex-mcp.mjs"],
|
||||
|
||||
@@ -59,7 +59,8 @@ const ERROR_STUB = `${HEADER}"use client";\nexport default function BackendOnlyE
|
||||
// global-error replaces the root layout on a root error, so it must render <html>/<body>.
|
||||
const GLOBAL_ERROR_STUB = `${HEADER}"use client";\nexport default function BackendOnlyGlobalErrorStub() {\n return (\n <html>\n <body></body>\n </html>\n );\n}\n`;
|
||||
|
||||
const UI_BASENAME_RE = /^(page|layout|template|loading|error|global-error|not-found|default)\.(tsx|jsx|ts|js)$/;
|
||||
const UI_BASENAME_RE =
|
||||
/^(page|layout|template|loading|error|global-error|not-found|default)\.(tsx|jsx|ts|js)$/;
|
||||
const ROUTE_FILE_RE = /[\\/]route\.(ts|js|tsx|jsx)$/;
|
||||
|
||||
/**
|
||||
@@ -103,6 +104,11 @@ export function isContributorBuild(env = process.env) {
|
||||
return env.OMNIROUTE_BUILD_PROFILE === "contributor";
|
||||
}
|
||||
|
||||
/** True when standalone output should be packaged (default true; skipped for contributor or fast build). */
|
||||
export function shouldBuildStandalone(env = process.env) {
|
||||
return !isContributorBuild(env) && env.OMNIROUTE_SKIP_STANDALONE !== "1";
|
||||
}
|
||||
|
||||
/** Replace the build-only instrumentation entrypoint to avoid pulling the startup graph. */
|
||||
export function stubContributorInstrumentation(rootDir = process.cwd(), log = console) {
|
||||
const stubbed = [];
|
||||
|
||||
@@ -67,6 +67,11 @@ function isNativeSqliteLoadError(error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const code = error && typeof error === "object" && "code" in error ? error.code : undefined;
|
||||
|
||||
// Deliberately narrower than src/lib/db/sqliteLoadError.ts. There, a
|
||||
// non-callable export means "fall back to another driver". Here, the only
|
||||
// consumer treats a match as "no encrypted credentials exist", which lets
|
||||
// STORAGE_ENCRYPTION_KEY be regenerated over a database that still holds
|
||||
// enc:v1: rows. A generic TypeError must stay loud on this path.
|
||||
return (
|
||||
message.includes("Module did not self-register") ||
|
||||
message.includes("NODE_MODULE_VERSION") ||
|
||||
@@ -78,6 +83,11 @@ function isNativeSqliteLoadError(error) {
|
||||
);
|
||||
}
|
||||
|
||||
function isLikelyBrokenNativeBinding(error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return message.includes("is not a function") || message.includes("is not a constructor");
|
||||
}
|
||||
|
||||
function hasEncryptedCredentials(dataDir) {
|
||||
const dbPath = join(dataDir, "storage.sqlite");
|
||||
if (!existsSync(dbPath)) return false;
|
||||
@@ -133,7 +143,10 @@ function hasEncryptedCredentials(dataDir) {
|
||||
}
|
||||
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(`Unable to inspect existing database at ${dbPath}: ${message}`);
|
||||
const hint = isLikelyBrokenNativeBinding(error)
|
||||
? " The better-sqlite3 native binding loaded but did not expose a usable constructor; try `npm rebuild better-sqlite3`."
|
||||
: "";
|
||||
throw new Error(`Unable to inspect existing database at ${dbPath}: ${message}${hint}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,11 +14,14 @@ import {
|
||||
import {
|
||||
isBackendOnlyBuild,
|
||||
isContributorBuild,
|
||||
shouldBuildStandalone,
|
||||
stubContributorInstrumentation,
|
||||
stubDashboardPages,
|
||||
restoreDashboardPages,
|
||||
} from "./backendOnlyPages.mjs";
|
||||
|
||||
export { shouldBuildStandalone } from "./backendOnlyPages.mjs";
|
||||
|
||||
/**
|
||||
* Layer 1: `app/` has been renamed to `dist/` and the App-Router collision is gone.
|
||||
* The only transient paths remaining are `.tmp/wine32` (Wine prefix used by some
|
||||
@@ -310,7 +313,7 @@ export async function main() {
|
||||
|
||||
const result = await runNextBuild();
|
||||
const standaloneDir = path.join(distDir, "standalone");
|
||||
if (result.code === 0 && (await exists(standaloneDir)) && !isContributorBuild()) {
|
||||
if (result.code === 0 && (await exists(standaloneDir)) && shouldBuildStandalone()) {
|
||||
try {
|
||||
await fs.cp(path.join(projectRoot, "docs"), path.join(standaloneDir, "docs"), {
|
||||
recursive: true,
|
||||
@@ -377,9 +380,9 @@ export async function main() {
|
||||
} catch (assembleErr) {
|
||||
console.warn("[build-next-isolated] Non-fatal error assembling standalone:", assembleErr);
|
||||
}
|
||||
} else if (result.code === 0 && isContributorBuild()) {
|
||||
} else if (result.code === 0 && !shouldBuildStandalone()) {
|
||||
console.log(
|
||||
"[build-next-isolated] Contributor profile: skipped standalone packaging (compile-only validation)"
|
||||
"[build-next-isolated] Skipped standalone packaging (standalone disabled for fast compile)"
|
||||
);
|
||||
}
|
||||
process.exitCode = result.code;
|
||||
|
||||
@@ -95,6 +95,22 @@ function main() {
|
||||
return;
|
||||
}
|
||||
|
||||
const healthWorkerDest = join(STANDALONE, "src/lib/db/healthCheckWorker.js");
|
||||
mkdirSync(dirname(healthWorkerDest), { recursive: true });
|
||||
runBuildTool(
|
||||
"esbuild",
|
||||
"esbuild",
|
||||
[
|
||||
join(ROOT, "src/lib/db/healthCheckWorker.ts"),
|
||||
"--bundle",
|
||||
"--platform=node",
|
||||
"--packages=external",
|
||||
"--format=esm",
|
||||
`--outfile=${healthWorkerDest}`,
|
||||
],
|
||||
{ stdio: "inherit" }
|
||||
);
|
||||
|
||||
const callLogWorkerDest = join(STANDALONE, CALL_LOG_WORKER_REL);
|
||||
mkdirSync(dirname(callLogWorkerDest), { recursive: true });
|
||||
// Never spawn `node_modules/.bin/esbuild` directly: that extensionless path is
|
||||
@@ -134,7 +150,11 @@ function main() {
|
||||
|
||||
// The call-log worker is always present; scope it to ESM immediately. The
|
||||
// optional LLMLingua worker dir is added below only when its deps are installed.
|
||||
const workerDirs = [dirname(callLogWorkerDest), dirname(compressionWorkerDest)];
|
||||
const workerDirs = [
|
||||
dirname(healthWorkerDest),
|
||||
dirname(callLogWorkerDest),
|
||||
dirname(compressionWorkerDest),
|
||||
];
|
||||
|
||||
if (!hasOptionals) {
|
||||
console.log(
|
||||
|
||||
@@ -47,6 +47,7 @@ export const APP_STAGING_ALLOWED_EXACT_PATHS: string[] = [
|
||||
"open-sse/services/compression/engines/llmlingua/onnxWorker.js",
|
||||
"open-sse/services/compression/compressionWorker.js",
|
||||
"src/lib/usage/callLogArtifactWorker.js",
|
||||
"src/lib/db/healthCheckWorker.js",
|
||||
"package.json",
|
||||
"peer-stamp.mjs",
|
||||
"main-server-timeouts.mjs",
|
||||
@@ -190,6 +191,7 @@ export const PACK_ARTIFACT_ROOT_ALLOWED_PATH_PREFIXES: string[] = [
|
||||
export const PACK_ARTIFACT_REQUIRED_PATHS: string[] = [
|
||||
"dist/open-sse/services/compression/engines/rtk/filters/generic-output.json",
|
||||
"dist/src/lib/usage/callLogArtifactWorker.js",
|
||||
"dist/src/lib/db/healthCheckWorker.js",
|
||||
"dist/open-sse/vendor/codex-chatgpt-web/adapters/chatgpt-web/mcp-server.js",
|
||||
"dist/open-sse/services/compression/rules/en/filler.json",
|
||||
"dist/server.js",
|
||||
|
||||
@@ -344,6 +344,22 @@ if (existsSync(chatGptWebCodexMcpSrcFile)) {
|
||||
}
|
||||
|
||||
// ── Step 8.6: Bundle call-log artifact worker ────────────────────────
|
||||
const healthWorkerDest = join(DIST_DIR, "src/lib/db/healthCheckWorker.js");
|
||||
mkdirSync(dirname(healthWorkerDest), { recursive: true });
|
||||
runBuildTool(
|
||||
"esbuild",
|
||||
"esbuild",
|
||||
[
|
||||
"src/lib/db/healthCheckWorker.ts",
|
||||
"--bundle",
|
||||
"--platform=node",
|
||||
"--packages=external",
|
||||
"--format=esm",
|
||||
`--outfile=${healthWorkerDest}`,
|
||||
],
|
||||
{ cwd: ROOT, stdio: "inherit" }
|
||||
);
|
||||
|
||||
const callLogWorkerSrc = join(ROOT, "src", "lib", "usage", "callLogArtifactWorker.ts");
|
||||
const callLogWorkerDest = join(DIST_DIR, "src", "lib", "usage", "callLogArtifactWorker.js");
|
||||
if (!existsSync(callLogWorkerSrc)) {
|
||||
|
||||
@@ -203,6 +203,10 @@ const IGNORE_FROM_CODE = new Set([
|
||||
// Listener-owned self-fetch transport signal. The HTTP/HTTPS launchers set
|
||||
// this before application imports; it is not user-configurable product env.
|
||||
"OMNIROUTE_INTERNAL_SCHEME",
|
||||
// Runner-owned bind-host signal. scripts/dev/run-next.mjs publishes the
|
||||
// interface it actually binds so the in-process startup guard can name it
|
||||
// (#13695); operators configure HOST / HOSTNAME, never this.
|
||||
"OMNIROUTE_BOUND_HOST",
|
||||
// Source typo / placeholder.
|
||||
"OMNIROUT",
|
||||
// Static config alias path (the canonical var is OMNIROUTE_PAYLOAD_RULES_PATH).
|
||||
@@ -278,6 +282,10 @@ const DOC_ONLY_ALLOWLIST = new Set([
|
||||
// SQL keyword mentioned in the new VACUUM scheduler docs (#4437).
|
||||
// The check's regex picks up the bare word in description text.
|
||||
"VACUUM",
|
||||
// Source-code constant (open-sse/services/combo/comboPredicates.ts:35 —
|
||||
// `export const COMBO_LOOP_SAFETY_TIMEOUT_MS = 10 * 60 * 1000`), cited in the
|
||||
// comboTimeoutMs narrative added by #13857. Not operator-configurable.
|
||||
"COMBO_LOOP_SAFETY_TIMEOUT_MS",
|
||||
]);
|
||||
|
||||
// Vars present in .env.example but intentionally absent from ENVIRONMENT.md.
|
||||
|
||||
@@ -205,7 +205,26 @@ function changedSymbols(root, base, entries) {
|
||||
return result;
|
||||
}
|
||||
|
||||
function markdown(result, base) {
|
||||
// A changed hub module (providerRegistry.ts, providers.ts, …) is imported by thousands of
|
||||
// consumers, and every consumer multiplies by its candidate tests, so the cross-product reaches
|
||||
// millions of rows. Rendering all of them made `lines.join("\n")` exceed V8's maximum string
|
||||
// length; the throw landed in main()'s catch, which exits 1 — so an ADVISORY step turned
|
||||
// "Fast Quality Gates" red on every PR whose diff touched a hub (#13866 follow-up). The header
|
||||
// keeps the exact totals; only the enumeration is bounded.
|
||||
const RENDER_LIMIT = 200;
|
||||
const JSON_ITEM_LIMIT = 5000;
|
||||
|
||||
/** First `limit` items plus a one-line note naming how many were withheld. */
|
||||
function renderBounded(lines, items, format, limit = RENDER_LIMIT) {
|
||||
for (const item of items.slice(0, limit)) lines.push(format(item));
|
||||
if (items.length > limit) {
|
||||
lines.push(
|
||||
`- _… and ${items.length - limit} more not listed (report bounded at ${limit} rows per section; the counts above are exact)._`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function markdown(result, base) {
|
||||
const lines = [
|
||||
"## Forgotten sibling tests (advisory)",
|
||||
"",
|
||||
@@ -218,12 +237,10 @@ function markdown(result, base) {
|
||||
];
|
||||
if (result.findings.length) {
|
||||
lines.push("### Candidate tests absent from this diff", "");
|
||||
for (const item of result.findings) {
|
||||
renderBounded(lines, result.findings, (item) => {
|
||||
const symbol = item.changedSymbols.length ? ` (${item.changedSymbols.join(", ")})` : "";
|
||||
lines.push(
|
||||
`- \`${item.changedModule}\`${symbol} -> \`${item.consumer}\` -> \`${item.candidateTest}\``
|
||||
);
|
||||
}
|
||||
return `- \`${item.changedModule}\`${symbol} -> \`${item.consumer}\` -> \`${item.candidateTest}\``;
|
||||
});
|
||||
lines.push("", "> Report-only calibration: these findings do not fail the job.", "");
|
||||
}
|
||||
for (const [heading, items] of [
|
||||
@@ -232,10 +249,12 @@ function markdown(result, base) {
|
||||
]) {
|
||||
if (!items.length) continue;
|
||||
lines.push(`### ${heading}`, "");
|
||||
for (const item of items)
|
||||
lines.push(
|
||||
renderBounded(
|
||||
lines,
|
||||
items,
|
||||
(item) =>
|
||||
`- \`${item.changedModule}\` -> \`${item.consumer}\`${item.candidateTest ? ` -> \`${item.candidateTest}\`` : ""}: ${item.reason || item.message}`
|
||||
);
|
||||
);
|
||||
lines.push("");
|
||||
}
|
||||
return `${lines.join("\n")}\n`;
|
||||
@@ -271,9 +290,26 @@ function main() {
|
||||
});
|
||||
const report = markdown(result, base);
|
||||
process.stdout.write(report);
|
||||
// The JSON artifact is bounded for the same reason the markdown is: a hub-module diff
|
||||
// produces millions of rows and `JSON.stringify` would throw the same "Invalid string
|
||||
// length". `totals` keeps every count exact, so tooling can still see the real numbers.
|
||||
const jsonResult = {
|
||||
...result,
|
||||
totals: {
|
||||
findings: result.findings.length,
|
||||
diagnostics: result.diagnostics.length,
|
||||
suppressed: result.suppressed.length,
|
||||
maskingRisks: result.maskingRisks.length,
|
||||
},
|
||||
itemLimit: JSON_ITEM_LIMIT,
|
||||
findings: result.findings.slice(0, JSON_ITEM_LIMIT),
|
||||
diagnostics: result.diagnostics.slice(0, JSON_ITEM_LIMIT),
|
||||
suppressed: result.suppressed.slice(0, JSON_ITEM_LIMIT),
|
||||
maskingRisks: result.maskingRisks.slice(0, JSON_ITEM_LIMIT),
|
||||
};
|
||||
for (const [target, contents] of [
|
||||
[summaryPath, report],
|
||||
[jsonPath, `${JSON.stringify(result, null, 2)}\n`],
|
||||
[jsonPath, `${JSON.stringify(jsonResult, null, 2)}\n`],
|
||||
]) {
|
||||
if (!target) continue;
|
||||
fs.mkdirSync(path.dirname(target), { recursive: true });
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
|
||||
export {
|
||||
isClientAbortError,
|
||||
isRecoverableUpstreamTimeoutError,
|
||||
shouldSwallowUncaught,
|
||||
attachRequestStreamGuards,
|
||||
installProcessCrashGuard,
|
||||
|
||||
@@ -430,6 +430,9 @@ class ResponsesWsSession {
|
||||
this.firstResponseBody = null;
|
||||
this.currentRequestBody = null;
|
||||
this.preparedContext = null;
|
||||
this.leaseId = null;
|
||||
this.leaseReleased = false;
|
||||
this.leaseReleaseInFlight = false;
|
||||
// #7388: logging must be scoped per logical turn (one `response.create`
|
||||
// through its terminal event), not once for the lifetime of the WS
|
||||
// connection — a single boolean here silently dropped every turn after
|
||||
@@ -640,6 +643,23 @@ class ResponsesWsSession {
|
||||
toStringOrNull(responseBody.service_tier) || toStringOrNull(responseBody.serviceTier),
|
||||
};
|
||||
|
||||
// A reused WS connection re-runs prepare per logical turn, and each prepare
|
||||
// acquires a fresh per-account lease. Release the previous turn before
|
||||
// adopting the new lease so one session cannot hoard account slots.
|
||||
const previousLeaseId = this.leaseId;
|
||||
const newLeaseId = toStringOrNull(prepared.json?.leaseId);
|
||||
if (this.closed) {
|
||||
this.leaseId = null;
|
||||
this.releaseLeaseId(newLeaseId);
|
||||
return prepared;
|
||||
}
|
||||
this.leaseId = newLeaseId;
|
||||
if (previousLeaseId && previousLeaseId !== newLeaseId) {
|
||||
this.releaseLeaseId(previousLeaseId);
|
||||
}
|
||||
this.leaseReleased = false;
|
||||
this.leaseReleaseInFlight = false;
|
||||
|
||||
return prepared;
|
||||
}
|
||||
|
||||
@@ -770,6 +790,35 @@ class ResponsesWsSession {
|
||||
}
|
||||
}
|
||||
|
||||
releaseLease() {
|
||||
if (this.leaseReleased || this.leaseReleaseInFlight || !this.leaseId) return;
|
||||
this.leaseReleaseInFlight = true;
|
||||
const leaseId = this.leaseId;
|
||||
void callInternal(this.fetchImpl, this.baseUrl, this.bridgeSecret, "release", { leaseId })
|
||||
.then((response) => {
|
||||
if (!response.ok) throw new Error("lease release rejected");
|
||||
this.leaseReleased = true;
|
||||
this.leaseId = null;
|
||||
})
|
||||
.catch(() => {
|
||||
this.leaseReleaseInFlight = false;
|
||||
const retry = setTimeout(() => this.releaseLease(), 1000);
|
||||
retry.unref?.();
|
||||
});
|
||||
}
|
||||
|
||||
releaseLeaseId(leaseId) {
|
||||
if (!leaseId) return;
|
||||
void callInternal(this.fetchImpl, this.baseUrl, this.bridgeSecret, "release", { leaseId })
|
||||
.then((response) => {
|
||||
if (!response.ok) throw new Error("lease release rejected");
|
||||
})
|
||||
.catch(() => {
|
||||
const retry = setTimeout(() => this.releaseLeaseId(leaseId), 1000);
|
||||
retry.unref?.();
|
||||
});
|
||||
}
|
||||
|
||||
async persistHistory({
|
||||
status = 200,
|
||||
success = true,
|
||||
@@ -820,6 +869,7 @@ class ResponsesWsSession {
|
||||
close(code = 1000, reason = "normal_closure") {
|
||||
if (this.closed) return;
|
||||
this.closed = true;
|
||||
this.releaseLease();
|
||||
|
||||
clearInterval(this.pingTimer);
|
||||
this.cleanupBuffers();
|
||||
@@ -847,6 +897,7 @@ class ResponsesWsSession {
|
||||
dispose() {
|
||||
if (this.closed) return;
|
||||
this.closed = true;
|
||||
this.releaseLease();
|
||||
clearInterval(this.pingTimer);
|
||||
this.cleanupBuffers();
|
||||
try {
|
||||
|
||||
@@ -16,10 +16,7 @@ import { isTurbopackCacheCorruption, purgeAllTurbopackCaches } from "./turbopack
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { getMainServerTimeoutConfig } from "./main-server-timeouts.mjs";
|
||||
import { createSystemdNotifier } from "./systemd-notify.mjs";
|
||||
import {
|
||||
attachRequestStreamGuards,
|
||||
installProcessCrashGuard,
|
||||
} from "./httpClientAbortGuard.mjs";
|
||||
import { attachRequestStreamGuards, installProcessCrashGuard } from "./httpClientAbortGuard.mjs";
|
||||
|
||||
const { maybeHandleDisallowedMethod } = methodGuard;
|
||||
const { wrapRequestListenerWithHeadResponseGuard } = headResponseGuard;
|
||||
@@ -104,6 +101,12 @@ process.env.OMNIROUTE_INTERNAL_SCHEME = "http";
|
||||
|
||||
const { dashboardPort } = runtimePorts;
|
||||
const hostname = process.env.HOST || "0.0.0.0";
|
||||
// Publish the interface this server actually binds so in-process TypeScript
|
||||
// (src/lib/startup/nonLoopbackApiKeyGuard.ts) can warn about an exposed
|
||||
// anonymous /v1 without re-deriving it. The standalone/Docker entrypoint
|
||||
// (scripts/dev/run-standalone.mjs -> Next's own server.js) uses HOSTNAME
|
||||
// instead, which the guard falls back to. #13695
|
||||
process.env.OMNIROUTE_BOUND_HOST = hostname;
|
||||
// Turbopack by default in dev (matches the Next 16 CLI default and the production
|
||||
// build default in build-next-isolated.mjs); OMNIROUTE_USE_TURBOPACK=0 is the
|
||||
// webpack escape hatch. Under Bun, Turbopack native V8 bindings are unavailable,
|
||||
@@ -232,15 +235,31 @@ async function start() {
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
let isShuttingDown = false;
|
||||
const shutdown = async (signal) => {
|
||||
if (isShuttingDown) {
|
||||
// Second Ctrl+C / signal forces immediate exit
|
||||
process.exit(1);
|
||||
}
|
||||
isShuttingDown = true;
|
||||
|
||||
// Safety net: force exit after 2s if keep-alive sockets or Next.js app close hangs
|
||||
const forceExitTimer = setTimeout(() => {
|
||||
process.exit(0);
|
||||
}, 2000);
|
||||
forceExitTimer.unref?.();
|
||||
|
||||
systemdNotifier.stopping();
|
||||
try {
|
||||
server.closeIdleConnections?.();
|
||||
server.closeAllConnections?.();
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
await globalThis.__omnirouteRequestShutdown?.(signal);
|
||||
await nextApp.close();
|
||||
} catch (error) {
|
||||
console.error("[SHUTDOWN] Failed during signal:", signal, error);
|
||||
} finally {
|
||||
clearTimeout(forceExitTimer);
|
||||
process.exit(0);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -9,6 +9,17 @@ import headResponseGuard from "./head-response-guard.cjs";
|
||||
import { resolveTlsOptions, createServerListener } from "./tls-options.mjs";
|
||||
import { getMainServerTimeoutConfig } from "./main-server-timeouts.mjs";
|
||||
import { createSystemdNotifier } from "./systemd-notify.mjs";
|
||||
import { installProcessCrashGuard } from "./httpClientAbortGuard.mjs";
|
||||
|
||||
// Safety net (#12861): this is the actual production entry point (see the
|
||||
// keepAliveTimeout comment below for why `run-next.mjs`-only fixes don't
|
||||
// reach real installs). Without this, a client abort OR a recoverable
|
||||
// upstream-fetch timeout that a retry path already handles (see
|
||||
// open-sse/utils/directResponseStartTimeout.ts) can surface as an
|
||||
// unhandledRejection -> uncaughtException and take the whole server down —
|
||||
// exactly the asymmetry `run-next.mjs` already closed for dev. Benign errors
|
||||
// are swallowed and logged; genuine bugs still crash loudly.
|
||||
installProcessCrashGuard();
|
||||
|
||||
// systemd sd_notify (Type=notify / WatchdogSec=): this process is the one
|
||||
// whose event loop can freeze (cold /v1/models rebuild), so it must own the
|
||||
|
||||
146
scripts/i18n/check-key-completeness.mjs
Normal file
146
scripts/i18n/check-key-completeness.mjs
Normal file
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* OmniRoute — i18n key COMPLETENESS gate (CI gate, blocking).
|
||||
*
|
||||
* Every `src/i18n/messages/<locale>.json` must carry exactly the key set of `en.json`:
|
||||
* no leaf absent, no leaf the source no longer has. A `__MISSING__:` placeholder counts as
|
||||
* present (the ratio gate judges its content); an ABSENT key is the defect this gate names.
|
||||
*
|
||||
* Why the two sibling gates cannot see it (the incident it encodes, 2026-09-15):
|
||||
* - `check-ui-keys-coverage.mjs` enforces an 80 % floor per locale — 43 absent keys out of
|
||||
* ~13,000 still reads 99.7 %.
|
||||
* - `check-new-key-coverage.mjs` judges only the keys a PR ADDS to en.json. A locale batch
|
||||
* is generated from the en.json of the moment the branch is cut; while its translation
|
||||
* runs for days the base keeps adding keys, and the batch PR adds none itself — so the
|
||||
* nine batch-1 catalogs (#13044) landed 43 keys short and the eight batch-2 catalogs
|
||||
* (#13660) 10 keys short. The home widget test was the first thing that noticed.
|
||||
*
|
||||
* This gate is absolute, not diff-based: it compares the tree as it is.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/i18n/check-key-completeness.mjs # blocking (dashboard catalogs)
|
||||
* node scripts/i18n/check-key-completeness.mjs --warn # report only, exit 0
|
||||
* node scripts/i18n/check-key-completeness.mjs --catalog=cli # same gate over bin/cli/locales
|
||||
* npm run i18n:check-keys
|
||||
* npm run i18n:check-keys:cli
|
||||
*/
|
||||
|
||||
import { promises as fs } from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(SCRIPT_DIR, "..", "..");
|
||||
const CATALOG_DIRS = {
|
||||
ui: path.join(ROOT, "src", "i18n", "messages"),
|
||||
cli: path.join(ROOT, "bin", "cli", "locales"),
|
||||
};
|
||||
const SOURCE_LOCALE = "en";
|
||||
|
||||
/** Absolute directory of a flat-JSON catalog family: the dashboard (`ui`) or the CLI (`cli`). */
|
||||
export function catalogDir(name = "ui") {
|
||||
const dir = CATALOG_DIRS[name];
|
||||
if (!dir) throw new Error(`unknown catalog "${name}" (expected ui or cli)`);
|
||||
return dir;
|
||||
}
|
||||
|
||||
function isPlainObject(value) {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
/** Dotted leaf paths of a catalog tree (objects recurse, everything else is a leaf). */
|
||||
export function leafPaths(node, prefix = "", out = new Set()) {
|
||||
if (!isPlainObject(node)) return out;
|
||||
for (const [key, value] of Object.entries(node)) {
|
||||
const dotted = prefix ? `${prefix}.${key}` : key;
|
||||
if (isPlainObject(value)) leafPaths(value, dotted, out);
|
||||
else out.add(dotted);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure core. `en` is the source catalog, `locales` maps locale code → catalog. Returns one
|
||||
* entry per locale whose key set differs from the source, sorted by locale, with sorted
|
||||
* `missing` (in en, absent in the locale) and `extra` (in the locale, gone from en) lists.
|
||||
* Locales with an identical key set are not listed.
|
||||
*/
|
||||
export function findIncompleteLocales({ en, locales }) {
|
||||
const source = leafPaths(en);
|
||||
const gaps = [];
|
||||
for (const locale of Object.keys(locales).sort()) {
|
||||
const target = leafPaths(locales[locale]);
|
||||
const missing = [...source].filter((k) => !target.has(k)).sort();
|
||||
const extra = [...target].filter((k) => !source.has(k)).sort();
|
||||
if (missing.length || extra.length) gaps.push({ locale, missing, extra });
|
||||
}
|
||||
return gaps;
|
||||
}
|
||||
|
||||
async function readCatalogs(dir) {
|
||||
const files = (await fs.readdir(dir)).filter((f) => f.endsWith(".json")).sort();
|
||||
const locales = {};
|
||||
let en = null;
|
||||
for (const file of files) {
|
||||
const code = file.slice(0, -".json".length);
|
||||
const parsed = JSON.parse(await fs.readFile(path.join(dir, file), "utf8"));
|
||||
if (code === SOURCE_LOCALE) en = parsed;
|
||||
else locales[code] = parsed;
|
||||
}
|
||||
if (!en) throw new Error(`${SOURCE_LOCALE}.json not found in ${dir}`);
|
||||
return { en, locales };
|
||||
}
|
||||
|
||||
function formatReport(gaps, sample = 5) {
|
||||
const lines = [];
|
||||
for (const { locale, missing, extra } of gaps) {
|
||||
const parts = [];
|
||||
if (missing.length) {
|
||||
parts.push(
|
||||
`${missing.length} missing (${missing.slice(0, sample).join(", ")}${missing.length > sample ? ", …" : ""})`
|
||||
);
|
||||
}
|
||||
if (extra.length) {
|
||||
parts.push(
|
||||
`${extra.length} extra (${extra.slice(0, sample).join(", ")}${extra.length > sample ? ", …" : ""})`
|
||||
);
|
||||
}
|
||||
lines.push(` - ${locale}: ${parts.join("; ")}`);
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const warnOnly = process.argv.includes("--warn");
|
||||
const catalogArg = process.argv.find((a) => a.startsWith("--catalog="));
|
||||
const catalog = catalogArg ? catalogArg.slice(10) : "ui";
|
||||
const dir = catalogDir(catalog);
|
||||
const tag = `[i18n-keys:${catalog}]`;
|
||||
const { en, locales } = await readCatalogs(dir);
|
||||
const gaps = findIncompleteLocales({ en, locales });
|
||||
const total = leafPaths(en).size;
|
||||
const count = Object.keys(locales).length;
|
||||
if (gaps.length === 0) {
|
||||
console.log(`${tag} OK — ${count} locales carry all ${total} keys of en.json, none extra.`);
|
||||
return;
|
||||
}
|
||||
const missingTotal = gaps.reduce((s, g) => s + g.missing.length, 0);
|
||||
const extraTotal = gaps.reduce((s, g) => s + g.extra.length, 0);
|
||||
console.error(
|
||||
`${tag} ${warnOnly ? "WARN" : "FAIL"} — ${gaps.length}/${count} locales differ from en.json (${missingTotal} missing, ${extraTotal} extra leaves):`
|
||||
);
|
||||
console.error(formatReport(gaps));
|
||||
console.error(
|
||||
`${tag} Fix: node scripts/i18n/sync-ui-keys.mjs --catalog=${catalog} --locale=<codes> --translate-markers (adds the missing keys and translates them); extra keys mean the source dropped them — remove them from the locale.`
|
||||
);
|
||||
if (!warnOnly) process.exitCode = 1;
|
||||
}
|
||||
|
||||
const isDirectRun = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
|
||||
if (isDirectRun) {
|
||||
main().catch((err) => {
|
||||
console.error(`[i18n-keys] ${err.message}`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -23,13 +23,22 @@
|
||||
*
|
||||
* How this gate works: DIFF-AWARE, like its sibling. It compares the English catalog at the
|
||||
* merge base against the working tree; every key that is NEW in English must be present and
|
||||
* non-placeholder in every locale. Pre-existing gaps are deliberately frozen — this gate
|
||||
* judges only what the current change adds, so it can be turned on without a migration.
|
||||
* TRANSLATED in every locale. Pre-existing gaps are deliberately frozen — this gate judges
|
||||
* only what the current change adds, so it can be turned on without a migration.
|
||||
*
|
||||
* Escape hatch, same as the sibling: set the value to `__MISSING__:<english>` to make the
|
||||
* runtime fall back to correct English and queue the key for the translation pipeline.
|
||||
* NOTE that `vi` bans placeholders (tests/unit/i18n-vi-completeness.test.ts), so `vi` needs
|
||||
* a real translation.
|
||||
* A `__MISSING__:<english>` marker does NOT satisfy this gate (since 2026-09-17). It used to:
|
||||
* the marker was the documented deferral, because the runtime falls back to correct English.
|
||||
* Then on 2026-09-16 eight feature PRs added 61 keys to en.json and stamped the marker into
|
||||
* all 65 locales instead of translating; this gate accepted every one of them, nothing blocked
|
||||
* the PRs, and the real-translation ratio gate (`check-translation-ratio`, blocking) went red
|
||||
* on the release tip for everybody (pt-BR 3.2 % > 2.5 % + 0.5). A marker is an absent
|
||||
* translation wearing a runtime-safe coat, and it is judged as absent here. Translate:
|
||||
*
|
||||
* node scripts/i18n/sync-ui-keys.mjs --locale=<codes> --translate-markers --batch-size=40
|
||||
* bash scripts/i18n/translate-new-keys.sh # same thing, all locales in parallel
|
||||
*
|
||||
* Keys that must stay English (product/engine/flag names a test pins) go in
|
||||
* `scripts/i18n/untranslatable-keys.json`, never behind a marker.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/i18n/check-new-key-coverage.mjs # strict, exit 1
|
||||
@@ -68,8 +77,8 @@ export function flattenLeaves(node, prefix = "", out = {}) {
|
||||
/**
|
||||
* Pure core: which (key, locale) pairs are keys new in English that a locale never got?
|
||||
*
|
||||
* A `__MISSING__:` placeholder counts as satisfied — it is the documented, runtime-correct
|
||||
* way to defer a translation.
|
||||
* A `__MISSING__:` placeholder counts as ABSENT — it is not a translation, and accepting it
|
||||
* is what let the 2026-09-16 batch ship 61 untranslated keys into 65 locales.
|
||||
*
|
||||
* @param {object} args
|
||||
* @param {object} args.baseEn en.json at the base ref
|
||||
@@ -91,7 +100,7 @@ export function findUntranslatedNewKeys({ baseEn, headEn, headLocales }) {
|
||||
for (const key of newKeys) {
|
||||
const value = flat[key];
|
||||
const satisfied =
|
||||
typeof value === "string" && (value.trim() !== "" || value.startsWith(PLACEHOLDER_PREFIX));
|
||||
typeof value === "string" && value.trim() !== "" && !value.startsWith(PLACEHOLDER_PREFIX);
|
||||
if (!satisfied) gaps.push({ key, locale });
|
||||
}
|
||||
}
|
||||
@@ -186,14 +195,20 @@ function main() {
|
||||
}
|
||||
const label = opts.warn ? "WARN" : "FAIL";
|
||||
console.error(
|
||||
`\n[i18n-new-keys] ${label} — ${byKey.size} new English key(s) missing from some locales:`
|
||||
`\n[i18n-new-keys] ${label} — ${byKey.size} new English key(s) untranslated in some locales:`
|
||||
);
|
||||
for (const [key, locales] of byKey) {
|
||||
console.error(` ✗ ${key} — missing in ${locales.length}: ${locales.join(", ")}`);
|
||||
console.error(` ✗ ${key} — untranslated in ${locales.length}: ${locales.join(", ")}`);
|
||||
}
|
||||
const codes = [...new Set(gaps.map((g) => g.locale))].sort().join(",");
|
||||
console.error(
|
||||
"\n Translate them, or set `__MISSING__:<english>` to defer (the runtime then falls back\n" +
|
||||
" to English). `vi` bans placeholders — it needs a real translation."
|
||||
"\n A `__MISSING__:<english>` marker does not count — it is an absent translation.\n" +
|
||||
" Translate the keys (needs OMNIROUTE_TRANSLATION_API_URL/_API_KEY/_MODEL in .env):\n" +
|
||||
` node scripts/i18n/sync-ui-keys.mjs --locale=${codes} --translate-markers --batch-size=40\n` +
|
||||
" or, all locales in parallel (detached runner):\n" +
|
||||
" bash scripts/i18n/translate-new-keys.sh\n" +
|
||||
" A key that must stay English (a pinned product/engine/flag name) belongs in\n" +
|
||||
" scripts/i18n/untranslatable-keys.json."
|
||||
);
|
||||
if (!opts.warn) process.exit(1);
|
||||
}
|
||||
|
||||
@@ -66,7 +66,20 @@ async function main() {
|
||||
}
|
||||
|
||||
const state = JSON.parse(await fs.readFile(STATE_PATH, "utf8"));
|
||||
const sources = state.sources || {};
|
||||
// Scope: only the documentation core set is translated on purpose (PR-0 decision, 22
|
||||
// sources); state entries for other docs (older per-locale extras) are not a CI concern.
|
||||
// `--all` restores the full-state behaviour for local inspection.
|
||||
const { computeDocsCoreSet } = await import("./lib/docs-core-set.mjs");
|
||||
const config = JSON.parse(await fs.readFile(path.join(ROOT, "config", "i18n.json"), "utf8"));
|
||||
const coreSet = computeDocsCoreSet({ root: ROOT, config });
|
||||
const coreList = Array.isArray(coreSet)
|
||||
? coreSet
|
||||
: (coreSet.coreSet ?? coreSet.files ?? Object.keys(coreSet));
|
||||
const core = new Set(coreList);
|
||||
const scopeAll = process.argv.includes("--all");
|
||||
const sources = Object.fromEntries(
|
||||
Object.entries(state.sources || {}).filter(([rel]) => scopeAll || core.has(rel))
|
||||
);
|
||||
|
||||
const driftedSources = [];
|
||||
const missingTargets = [];
|
||||
|
||||
213
scripts/i18n/retranslate-site.mjs
Normal file
213
scripts/i18n/retranslate-site.mjs
Normal file
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* OmniRoute — site catalog retranslator (omnirouteSite/lang/<code>.json).
|
||||
*
|
||||
* The site catalogs are flat { "dotted.key": "text" } files translated once by
|
||||
* add-locale; 10 % of their leaves were still verbatim English on 2026-09-16
|
||||
* (th 25 %). For every locale, the leaves equal to lang/_source.en.json —
|
||||
* outside untranslatable-site-keys.json — are sent to the translation backend
|
||||
* in batches and written back in place (key order preserved). Same backend
|
||||
* env as sync-ui-keys (`OMNIROUTE_TRANSLATION_*`, loaded from the repo-root
|
||||
* `.env` when present).
|
||||
*
|
||||
* A batch whose answer cannot be trusted (see `parseBatchResponse`) is retried
|
||||
* one string at a time; a leaf that still fails keeps its English value so the
|
||||
* next run picks it up again. Each catalog is written as soon as its locale is
|
||||
* done, so an aborted run keeps the locales already finished.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/i18n/retranslate-site.mjs --site-dir=../omnirouteSite \
|
||||
* [--locale=th,phi] [--dry-run] [--batch-size=40]
|
||||
*/
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { promises as fs } from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { backendConfig, translateBatch, translateString } from "./lib/translate-backend.mjs";
|
||||
|
||||
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(SCRIPT_DIR, "..", "..");
|
||||
const ALLOWLIST = path.join(SCRIPT_DIR, "untranslatable-site-keys.json");
|
||||
const LOG_PREFIX = "[site-retranslate]";
|
||||
|
||||
// ----- .env loader --------------------------------------------------------
|
||||
// Same loader as sync-ui-keys.mjs: variables from the repo-root `.env`
|
||||
// (gitignored) land in process.env unless the shell already set them.
|
||||
function loadDotEnv() {
|
||||
const envPath = path.join(ROOT, ".env");
|
||||
if (!existsSync(envPath)) return;
|
||||
try {
|
||||
const raw = readFileSync(envPath, "utf8");
|
||||
for (const rawLine of raw.split(/\r?\n/)) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith("#")) continue;
|
||||
const eq = line.indexOf("=");
|
||||
if (eq <= 0) continue;
|
||||
const key = line.slice(0, eq).trim();
|
||||
if (!key || process.env[key] !== undefined) continue;
|
||||
let value = line.slice(eq + 1);
|
||||
if (
|
||||
(value.startsWith('"') && value.endsWith('"')) ||
|
||||
(value.startsWith("'") && value.endsWith("'"))
|
||||
) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
process.env[key] = value;
|
||||
}
|
||||
} catch {
|
||||
/* ignore — backendConfig() reports the missing variables */
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Keys of `source` whose `target` value is still the verbatim English string,
|
||||
* outside `allow`. Missing target keys are not "identical copies" (that is a
|
||||
* sync problem, not a translation one); empty and non-string source leaves are
|
||||
* skipped. Sorted so the batches are deterministic.
|
||||
*
|
||||
* @param {Record<string, string>} source lang/_source.en.json
|
||||
* @param {Record<string, string>} target lang/<code>.json
|
||||
* @param {Set<string>} allow keys that must stay English
|
||||
* @returns {string[]}
|
||||
*/
|
||||
export function findSiteIdenticalKeys(source, target, allow) {
|
||||
return Object.keys(source)
|
||||
.filter(
|
||||
(k) =>
|
||||
k in target &&
|
||||
typeof source[k] === "string" &&
|
||||
source[k] !== "" &&
|
||||
target[k] === source[k] &&
|
||||
!allow.has(k)
|
||||
)
|
||||
.sort();
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const o = { siteDir: null, locales: null, dryRun: false, batchSize: 40 };
|
||||
for (const a of argv.slice(2)) {
|
||||
if (a.startsWith("--site-dir=")) o.siteDir = path.resolve(ROOT, a.slice("--site-dir=".length));
|
||||
else if (a.startsWith("--locale="))
|
||||
o.locales = a
|
||||
.slice("--locale=".length)
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
else if (a === "--dry-run") o.dryRun = true;
|
||||
else if (a.startsWith("--batch-size="))
|
||||
o.batchSize = Math.max(1, Number(a.slice("--batch-size=".length)) || 40);
|
||||
else throw new Error(`unknown argument: ${a}`);
|
||||
}
|
||||
if (!o.siteDir) throw new Error("--site-dir=<omnirouteSite checkout> is required");
|
||||
return o;
|
||||
}
|
||||
|
||||
async function readJson(file) {
|
||||
return JSON.parse(await fs.readFile(file, "utf8"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Translates `keys` of `source` for one locale, writing into `target` in place.
|
||||
* Returns { translated, failed } counts.
|
||||
*/
|
||||
async function translateLocale(keys, source, target, localeEntry, backend, batchSize) {
|
||||
let translated = 0;
|
||||
let failed = 0;
|
||||
for (let i = 0; i < keys.length; i += batchSize) {
|
||||
const slice = keys.slice(i, i + batchSize);
|
||||
try {
|
||||
const out = await translateBatch(
|
||||
slice.map((id) => ({ id, text: source[id] })),
|
||||
localeEntry,
|
||||
backend
|
||||
);
|
||||
for (const id of slice) {
|
||||
const value = out.get(id);
|
||||
if (typeof value === "string" && value.trim()) {
|
||||
target[id] = value.trim();
|
||||
translated++;
|
||||
} else {
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
`${LOG_PREFIX} ${localeEntry.code}: batch of ${slice.length} failed (${err.message}) — retrying one by one`
|
||||
);
|
||||
for (const id of slice) {
|
||||
try {
|
||||
const value = await translateString(source[id], localeEntry, backend);
|
||||
if (typeof value === "string" && value.trim()) {
|
||||
target[id] = value.trim();
|
||||
translated++;
|
||||
} else {
|
||||
failed++;
|
||||
}
|
||||
} catch (inner) {
|
||||
failed++;
|
||||
console.warn(`${LOG_PREFIX} ${localeEntry.code}: ${id} failed (${inner.message})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return { translated, failed };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const o = parseArgs(process.argv);
|
||||
if (!o.dryRun) loadDotEnv();
|
||||
const langDir = path.join(o.siteDir, "lang");
|
||||
const source = await readJson(path.join(langDir, "_source.en.json"));
|
||||
const allow = new Set((await readJson(ALLOWLIST)).keys ?? []);
|
||||
const config = await readJson(path.join(ROOT, "config", "i18n.json"));
|
||||
const codes = (o.locales ?? config.locales.map((l) => l.code)).filter((c) => c !== "en");
|
||||
const backend = o.dryRun ? null : backendConfig();
|
||||
let total = 0;
|
||||
let failedTotal = 0;
|
||||
for (const code of codes) {
|
||||
const file = path.join(langDir, `${code}.json`);
|
||||
const localeEntry = config.locales.find((l) => l.code === code);
|
||||
if (!localeEntry) {
|
||||
console.warn(`${LOG_PREFIX} ${code}: not in config/i18n.json, skipped`);
|
||||
continue;
|
||||
}
|
||||
let target;
|
||||
try {
|
||||
target = await readJson(file);
|
||||
} catch {
|
||||
console.warn(`${LOG_PREFIX} ${code}: no catalog, skipped`);
|
||||
continue;
|
||||
}
|
||||
const keys = findSiteIdenticalKeys(source, target, allow);
|
||||
console.log(
|
||||
`${LOG_PREFIX} ${code}: ${keys.length} English leaves${o.dryRun ? " (dry-run)" : ""}`
|
||||
);
|
||||
if (o.dryRun || keys.length === 0) continue;
|
||||
const { translated, failed } = await translateLocale(
|
||||
keys,
|
||||
source,
|
||||
target,
|
||||
localeEntry,
|
||||
backend,
|
||||
o.batchSize
|
||||
);
|
||||
total += translated;
|
||||
failedTotal += failed;
|
||||
if (translated > 0) await fs.writeFile(file, JSON.stringify(target, null, 2) + "\n", "utf8");
|
||||
if (failed > 0) console.warn(`${LOG_PREFIX} ${code}: ${failed} leaves still English`);
|
||||
}
|
||||
console.log(
|
||||
`${LOG_PREFIX} done — ${total} leaves rewritten across ${codes.length} locales` +
|
||||
(failedTotal > 0 ? ` (${failedTotal} failed)` : "")
|
||||
);
|
||||
if (failedTotal > 0) process.exitCode = 1;
|
||||
}
|
||||
|
||||
const isDirectRun = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
|
||||
if (isDirectRun) {
|
||||
main().catch((e) => {
|
||||
console.error(`${LOG_PREFIX} ${e.message}`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
247
scripts/i18n/review-locale.mjs
Normal file
247
scripts/i18n/review-locale.mjs
Normal file
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* OmniRoute — locale review pass. Sends the leaves a locale changed since a git ref to the
|
||||
* translation backend with a *reviewer* prompt (native speaker, dashboard context, keep
|
||||
* placeholders/ICU/markup) and applies only the corrections it returns. Produces a markdown
|
||||
* report so the operator can read what was changed.
|
||||
*
|
||||
* Usage: node scripts/i18n/review-locale.mjs --locale=pt-BR --since=<ref> [--dry-run] [--batch-size=30]
|
||||
*/
|
||||
import { existsSync, readFileSync, promises as fs } from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { backendConfig, callChat } from "./lib/translate-backend.mjs";
|
||||
|
||||
// ----- .env loader (same contract as sync-ui-keys.mjs: repo-root .env, already-set vars win)
|
||||
(function loadDotEnv() {
|
||||
const envPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".env");
|
||||
if (!existsSync(envPath)) return;
|
||||
try {
|
||||
const raw = readFileSync(envPath, "utf8");
|
||||
for (const rawLine of raw.split(/\r?\n/)) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith("#")) continue;
|
||||
const eq = line.indexOf("=");
|
||||
if (eq <= 0) continue;
|
||||
const key = line.slice(0, eq).trim();
|
||||
if (!key || process.env[key] !== undefined) continue;
|
||||
let value = line.slice(eq + 1);
|
||||
if (
|
||||
(value.startsWith('"') && value.endsWith('"')) ||
|
||||
(value.startsWith("'") && value.endsWith("'"))
|
||||
) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
process.env[key] = value;
|
||||
}
|
||||
} catch {
|
||||
// unreadable .env — the backend config will report the missing variables
|
||||
}
|
||||
})();
|
||||
|
||||
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(SCRIPT_DIR, "..", "..");
|
||||
const MESSAGES_DIR = path.join(ROOT, "src", "i18n", "messages");
|
||||
|
||||
const flat = (o, p = "", out = {}) => {
|
||||
for (const [k, v] of Object.entries(o)) {
|
||||
const d = p ? `${p}.${k}` : k;
|
||||
if (v && typeof v === "object" && !Array.isArray(v)) flat(v, d, out);
|
||||
else out[d] = v;
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
// 36 leaf keys of en.json carry a dot in their own name
|
||||
// (`compliance.eventTypes["apiKey.ban"]`), so the flattened id is ambiguous:
|
||||
// a naive split-and-descend walked into a missing `apiKey` object and the
|
||||
// review died with "Cannot set properties of undefined (setting 'ban')" —
|
||||
// after two hours for Hausa, before anything was written. Walk the object
|
||||
// preferring the longest key that actually exists at each level.
|
||||
export function setDeep(o, dotted, value) {
|
||||
const parts = dotted.split(".");
|
||||
const walk = (node, from) => {
|
||||
for (let len = parts.length - from; len >= 1; len--) {
|
||||
const key = parts.slice(from, from + len).join(".");
|
||||
if (!Object.prototype.hasOwnProperty.call(node, key)) continue;
|
||||
if (from + len === parts.length) {
|
||||
node[key] = value;
|
||||
return true;
|
||||
}
|
||||
if (node[key] && typeof node[key] === "object" && walk(node[key], from + len)) return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
if (!walk(o, 0)) throw new Error(`key not found in catalog: ${dotted}`);
|
||||
}
|
||||
|
||||
export function changedLeaves(before, after) {
|
||||
const out = {};
|
||||
for (const [k, v] of Object.entries(after)) {
|
||||
if (typeof v === "string" && before[k] !== v) out[k] = v;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// One upstream hiccup used to abort the whole run through main().catch, and
|
||||
// the catalog was written only at the end — a 13k-leaf review (2h40 for
|
||||
// Amharic) lost everything. Each batch is retried with a backoff and, if it
|
||||
// still fails, skipped and reported instead of killing the run.
|
||||
export async function withRetries(
|
||||
fn,
|
||||
{ attempts = 4, delaysMs = [2000, 10000, 30000], onRetry } = {}
|
||||
) {
|
||||
let lastErr;
|
||||
for (let i = 0; i < attempts; i++) {
|
||||
try {
|
||||
return await fn(i);
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
if (i + 1 < attempts) {
|
||||
onRetry?.(err, i + 1);
|
||||
await new Promise((r) => setTimeout(r, delaysMs[Math.min(i, delaysMs.length - 1)]));
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastErr;
|
||||
}
|
||||
|
||||
export function parseReviewResponse(text, ids) {
|
||||
const m = text.match(/\{[\s\S]*\}/);
|
||||
if (!m) return {};
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(m[0]);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
const allowed = new Set(ids);
|
||||
const out = {};
|
||||
for (const [id, v] of Object.entries(parsed)) {
|
||||
if (allowed.has(id) && typeof v === "string" && v.trim() && v.trim() !== "OK") {
|
||||
out[id] = v.trim();
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const REVIEW_SYSTEM = (english, native) =>
|
||||
`You are a senior ${english} (${native}) localization reviewer for a developer-facing dashboard (an LLM proxy/router called OmniRoute). You receive a JSON object mapping ids to {en, current}. For every id, answer "OK" when the current translation is correct, natural and complete, or return the corrected ${english} string. Rules: keep every placeholder ({name}, {count, plural, …}), ICU syntax, HTML/markdown, product names (OmniRoute, Claude, Codex, MCP, A2A) and technical identifiers exactly as in the English; do not translate code; prefer the terminology already used in "current" when it is fine. Reply with ONLY a JSON object {id: "OK" | "corrected string"}.`;
|
||||
|
||||
function parseArgs(argv) {
|
||||
const o = { locale: null, since: null, dryRun: false, batchSize: 30 };
|
||||
for (const a of argv.slice(2)) {
|
||||
if (a.startsWith("--locale=")) o.locale = a.slice(9);
|
||||
else if (a.startsWith("--since=")) o.since = a.slice(8);
|
||||
else if (a === "--dry-run") o.dryRun = true;
|
||||
else if (a.startsWith("--batch-size=")) o.batchSize = Math.max(1, Number(a.slice(13)) || 30);
|
||||
}
|
||||
if (!o.locale || !o.since) throw new Error("--locale=<code> and --since=<git ref> are required");
|
||||
return o;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const o = parseArgs(process.argv);
|
||||
const file = path.join(MESSAGES_DIR, `${o.locale}.json`);
|
||||
const rel = path.relative(ROOT, file);
|
||||
const after = JSON.parse(await fs.readFile(file, "utf8"));
|
||||
// A catalog that did not exist at --since (a locale created after it) is reviewed in full.
|
||||
let before = {};
|
||||
try {
|
||||
before = JSON.parse(
|
||||
execFileSync("git", ["show", `${o.since}:${rel}`], {
|
||||
cwd: ROOT,
|
||||
encoding: "utf8",
|
||||
maxBuffer: 1 << 28,
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
})
|
||||
);
|
||||
} catch {
|
||||
console.log(`[review] ${rel} does not exist at ${o.since} — reviewing every leaf`);
|
||||
}
|
||||
const en = flat(JSON.parse(await fs.readFile(path.join(MESSAGES_DIR, "en.json"), "utf8")));
|
||||
const changed = changedLeaves(flat(before), flat(after));
|
||||
const ids = Object.keys(changed);
|
||||
console.log(`[review] ${o.locale}: ${ids.length} leaves changed since ${o.since}`);
|
||||
if (o.dryRun) return;
|
||||
const config = JSON.parse(await fs.readFile(path.join(ROOT, "config", "i18n.json"), "utf8"));
|
||||
const entry = config.locales.find((l) => l.code === o.locale);
|
||||
const backend = backendConfig();
|
||||
const fixes = {};
|
||||
const skipped = [];
|
||||
const CHECKPOINT_EVERY = 25;
|
||||
const writeCatalog = async () => {
|
||||
for (const [id, v] of Object.entries(fixes)) {
|
||||
try {
|
||||
setDeep(after, id, v);
|
||||
} catch (err) {
|
||||
console.log(`[review] ${err.message} — correction dropped`);
|
||||
}
|
||||
}
|
||||
await fs.writeFile(file, JSON.stringify(after, null, 2) + "\n", "utf8");
|
||||
};
|
||||
let batchNo = 0;
|
||||
for (let i = 0; i < ids.length; i += o.batchSize) {
|
||||
const slice = ids.slice(i, i + o.batchSize);
|
||||
const payload = Object.fromEntries(
|
||||
slice.map((id) => [id, { en: en[id], current: changed[id] }])
|
||||
);
|
||||
try {
|
||||
const text = await withRetries(
|
||||
() =>
|
||||
callChat(
|
||||
[
|
||||
{ role: "system", content: REVIEW_SYSTEM(entry.english ?? entry.name, entry.native) },
|
||||
{ role: "user", content: JSON.stringify(payload) },
|
||||
],
|
||||
backend
|
||||
),
|
||||
{
|
||||
onRetry: (err, n) =>
|
||||
console.log(`[review] batch at ${i} failed (${err.message}) — retry ${n}`),
|
||||
}
|
||||
);
|
||||
Object.assign(fixes, parseReviewResponse(text, slice));
|
||||
} catch (err) {
|
||||
skipped.push(...slice);
|
||||
console.log(`[review] batch at ${i} skipped after retries: ${err.message}`);
|
||||
}
|
||||
console.log(
|
||||
`[review] ${Math.min(i + o.batchSize, ids.length)}/${ids.length} reviewed, ${Object.keys(fixes).length} corrections so far`
|
||||
);
|
||||
if (++batchNo % CHECKPOINT_EVERY === 0) await writeCatalog();
|
||||
}
|
||||
await writeCatalog();
|
||||
const reportDir = path.join(ROOT, "_artifacts", "i18n-review");
|
||||
await fs.mkdir(reportDir, { recursive: true });
|
||||
if (skipped.length) {
|
||||
await fs.writeFile(
|
||||
path.join(reportDir, `${o.locale}.skipped.json`),
|
||||
JSON.stringify(skipped, null, 2) + "\n",
|
||||
"utf8"
|
||||
);
|
||||
}
|
||||
const report = [
|
||||
`# Review ${o.locale} since ${o.since}`,
|
||||
"",
|
||||
`${ids.length} leaves reviewed, ${Object.keys(fixes).length} corrected, ${skipped.length} skipped (upstream failures).`,
|
||||
"",
|
||||
"| key | en | before | after |",
|
||||
"| --- | --- | --- | --- |",
|
||||
...Object.entries(fixes).map(([id, v]) => `| \`${id}\` | ${en[id]} | ${changed[id]} | ${v} |`),
|
||||
].join("\n");
|
||||
await fs.writeFile(path.join(reportDir, `${o.locale}.md`), report + "\n", "utf8");
|
||||
console.log(
|
||||
`[review] ${Object.keys(fixes).length} corrections applied${skipped.length ? `, ${skipped.length} leaves skipped` : ""}; report: _artifacts/i18n-review/${o.locale}.md`
|
||||
);
|
||||
}
|
||||
|
||||
const isDirectRun = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
|
||||
if (isDirectRun) {
|
||||
main().catch((e) => {
|
||||
console.error(`[review] ${e.message}`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
328
scripts/i18n/run-translation.mjs
Executable file → Normal file
328
scripts/i18n/run-translation.mjs
Executable file → Normal file
@@ -37,6 +37,7 @@
|
||||
*/
|
||||
|
||||
import { promises as fs, existsSync, readFileSync } from "node:fs";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import crypto from "node:crypto";
|
||||
import process from "node:process";
|
||||
@@ -282,13 +283,23 @@ function targetPathFor(relSource, locale) {
|
||||
return path.join(DOCS_I18N_DIR, locale, relSource);
|
||||
}
|
||||
|
||||
function extractTopHeading(markdown) {
|
||||
const m = markdown.match(/^# (.+)\r?\n/);
|
||||
// Most docs sources open with a YAML front-matter block (`---` … `---`). The
|
||||
// H1 sits behind it, so the heading helpers look past the block: otherwise the
|
||||
// mirror gets the file name as its title and the front matter (plus a second
|
||||
// heading) is translated into its body.
|
||||
const FRONT_MATTER = /^---\r?\n[\s\S]*?\r?\n---\r?\n+/;
|
||||
|
||||
function stripFrontMatter(markdown) {
|
||||
return markdown.replace(FRONT_MATTER, "");
|
||||
}
|
||||
|
||||
export function extractTopHeading(markdown) {
|
||||
const m = stripFrontMatter(markdown).match(/^# (.+)\r?\n/);
|
||||
return m ? m[1].trim() : null;
|
||||
}
|
||||
|
||||
function stripTopHeading(markdown) {
|
||||
return markdown.replace(/^# .+\r?\n+/, "");
|
||||
export function stripTopHeading(markdown) {
|
||||
return stripFrontMatter(markdown).replace(/^# .+\r?\n+/, "");
|
||||
}
|
||||
|
||||
// ----- Translator backend --------------------------------------------------
|
||||
@@ -451,7 +462,7 @@ function splitOversizedSection(section, maxChars) {
|
||||
const chunks = [];
|
||||
let current = [];
|
||||
let size = 0;
|
||||
for (const lines of blocks) {
|
||||
for (const lines of blocks.flatMap((b) => splitOversizedRun(b, maxChars))) {
|
||||
const length = lines.join("\n").length + 1;
|
||||
if (size > 0 && size + length > maxChars) {
|
||||
chunks.push(current.join("\n"));
|
||||
@@ -465,6 +476,254 @@ function splitOversizedSection(section, maxChars) {
|
||||
return chunks;
|
||||
}
|
||||
|
||||
// A markdown table or a long bullet list has no blank line inside it, so the
|
||||
// paragraph splitter kept PROVIDER_REFERENCE.md's 244-row table (40 KB) and
|
||||
// FREE_TIERS.md's 71-item list (16 KB) as one block each, and the request for
|
||||
// a verbose script outlived the upstream socket ("fetch failed" for Greek and
|
||||
// Amharic on every attempt). An oversized block made only of table rows or
|
||||
// list items (plus their indented continuation lines) is cut before an item
|
||||
// line; the table header rows travel with the first group only.
|
||||
const ITEM_LINE = /^\s*(\||[-*+]\s|\d+[.)]\s)/;
|
||||
const CONTINUATION_LINE = /^\s+\S/;
|
||||
function splitOversizedRun(lines, maxChars) {
|
||||
if (lines.join("\n").length <= maxChars) return [lines];
|
||||
const content = lines.filter((l) => l.trim() !== "");
|
||||
if (!content.every((l) => ITEM_LINE.test(l) || CONTINUATION_LINE.test(l))) return [lines];
|
||||
if (!ITEM_LINE.test(content[0])) return [lines];
|
||||
const groups = [];
|
||||
let group = [];
|
||||
let size = 0;
|
||||
for (const line of lines) {
|
||||
if (group.length && ITEM_LINE.test(line) && size + line.length + 1 > maxChars) {
|
||||
groups.push(group);
|
||||
group = [];
|
||||
size = 0;
|
||||
}
|
||||
group.push(line);
|
||||
size += line.length + 1;
|
||||
}
|
||||
if (group.length) groups.push(group);
|
||||
return groups;
|
||||
}
|
||||
|
||||
// Chunks are rejoined with a blank line (they were cut on headings and
|
||||
// paragraphs) — except at a seam between two table rows or two list items,
|
||||
// where a blank line would break one table (or one tight list) into two.
|
||||
// True when the text ends with a table row or a list item (its indented
|
||||
// continuation lines included), i.e. a following item line belongs to the
|
||||
// same run.
|
||||
function endsInsideItemRun(text) {
|
||||
const lines = text.trimEnd().split("\n");
|
||||
let i = lines.length - 1;
|
||||
while (i > 0 && CONTINUATION_LINE.test(lines[i])) i--;
|
||||
return ITEM_LINE.test(lines[i] ?? "");
|
||||
}
|
||||
|
||||
export function joinTranslatedChunks(parts) {
|
||||
let out = "";
|
||||
for (let i = 0; i < parts.length; i++) {
|
||||
if (i === 0) {
|
||||
out = parts[i];
|
||||
continue;
|
||||
}
|
||||
const nextFirst = parts[i].trimStart().split("\n")[0] ?? "";
|
||||
const seam = endsInsideItemRun(out) && ITEM_LINE.test(nextFirst) ? "\n" : "\n\n";
|
||||
out = out.trimEnd() + seam + parts[i].trimStart();
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ----- Section cache --------------------------------------------------------
|
||||
// A mirror is retranslated section by section: the state remembers a short
|
||||
// hash of every `## ` section of the source at the time of translation, and a
|
||||
// later run only sends the sections whose hash changed, splicing the untouched
|
||||
// translated sections of the mirror back in. Sections are matched by index
|
||||
// only — an inserted or removed section shifts everything after it and costs
|
||||
// a retranslation of the tail, which is acceptable.
|
||||
|
||||
// Section 0 is the preamble (text before the first `## `); every other section
|
||||
// starts with its `## ` heading line. Joining with "\n\n" reproduces the body
|
||||
// modulo trailing whitespace. A `## ` inside a fenced code block never splits.
|
||||
export function splitSections(markdown) {
|
||||
const sections = [[]];
|
||||
let inFence = false;
|
||||
for (const line of markdown.split("\n")) {
|
||||
if (FENCE_LINE.test(line)) inFence = !inFence;
|
||||
if (!inFence && /^## /.test(line) && sections.at(-1).length) sections.push([]);
|
||||
sections.at(-1).push(line);
|
||||
}
|
||||
return sections.map((s) => s.join("\n").replace(/\s+$/, ""));
|
||||
}
|
||||
|
||||
export function sectionHashes(sections) {
|
||||
return sections.map((s) => sha256(Buffer.from(s.trim(), "utf8")).slice(0, 12));
|
||||
}
|
||||
|
||||
// `null` means "no reuse possible — translate the whole body": no recorded
|
||||
// hashes, or the mirror on disk does not have the section count the hashes
|
||||
// were recorded for (a hand edit, or a mirror written by an older pipeline).
|
||||
export function planSectionReuse({ previousHashes, sections, mirrorSections }) {
|
||||
if (!Array.isArray(previousHashes) || previousHashes.length !== mirrorSections.length) {
|
||||
return null;
|
||||
}
|
||||
if (previousHashes.length !== sections.length) return null;
|
||||
const now = sectionHashes(sections);
|
||||
const reuse = new Map();
|
||||
const translate = [];
|
||||
now.forEach((h, i) => {
|
||||
// A mirror section byte-equal to its source section is an untranslated copy (322 mirrors
|
||||
// of the 36 pre-expansion locales were adopted as English, 2026-09-16 audit) — never reuse it.
|
||||
// Section 0 of a mirror that starts with a YAML block is the old extractor's leaked
|
||||
// frontmatter (24 newer locales, 2026-09-16) — always rebuild it.
|
||||
const leakedFrontmatter = i === 0 && /^---\s*\n/.test(mirrorSections[i]);
|
||||
const untranslated = looksUntranslated(mirrorSections[i], sections[i]);
|
||||
if (h === previousHashes[i] && !untranslated && !leakedFrontmatter)
|
||||
reuse.set(i, mirrorSections[i]);
|
||||
else translate.push(i);
|
||||
});
|
||||
return { reuse, translate };
|
||||
}
|
||||
|
||||
// The mirror without the prefix this script writes in front of the translated
|
||||
// body: `# Title (native)`, the `🌐 **Languages:**` bar (older mirrors carry a
|
||||
// translated label, so only the globe is pinned) and the `---` separator.
|
||||
/**
|
||||
* A mirror whose source did not change can still need a rebuild: the pre-2026-09 extractor
|
||||
* leaked the source's YAML frontmatter into the body, and 322 mirrors of the pre-expansion
|
||||
* locales were plain English copies adopted as translated. Both are invisible to the hash
|
||||
* comparison, so the task loop asks this before skipping an up-to-date pair.
|
||||
*/
|
||||
/** Long prose lines (> 20 chars, not table/code/list scaffolding) of a markdown fragment. */
|
||||
function proseLines(text) {
|
||||
return text
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter((l) => l.length > 20 && !/^[`|#\-*\d\s]+$/.test(l));
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a mirror fragment is still (mostly) the English source: byte-equal, or ≥ 80 % of
|
||||
* its prose lines appear verbatim in the source. Fragments with < 3 prose lines fall back to
|
||||
* byte equality (code-only sections are legitimately identical).
|
||||
*/
|
||||
export function looksUntranslated(mirrorFragment, sourceFragment) {
|
||||
if (mirrorFragment.trim() === sourceFragment.trim()) return /[A-Za-z]{3,}/.test(sourceFragment);
|
||||
const mir = proseLines(mirrorFragment);
|
||||
if (mir.length < 3) return false;
|
||||
const src = new Set(proseLines(sourceFragment));
|
||||
return mir.filter((l) => src.has(l)).length / mir.length >= 0.8;
|
||||
}
|
||||
|
||||
export function mirrorNeedsRebuild(mirrorText, sourceText) {
|
||||
const body = extractMirrorBody(mirrorText);
|
||||
if (/^---\s*\n[\s\S]{0,600}?\n---\s*\n/.test(body)) return true; // leaked frontmatter
|
||||
const sourceBody = stripTopHeading(sourceText.replace(/^---\n[\s\S]*?\n---\n+/, ""));
|
||||
return looksUntranslated(body, sourceBody); // still English
|
||||
}
|
||||
|
||||
export function extractMirrorBody(mirrorText) {
|
||||
return mirrorText
|
||||
.replace(/^# .+\r?\n+/, "")
|
||||
.replace(/^🌐 .*\r?\n+/, "")
|
||||
.replace(/^---\r?\n+/, "");
|
||||
}
|
||||
|
||||
// Bootstrap for targets translated before section hashes existed: walks the
|
||||
// file's git history (newest first, at most 200 commits) and returns the text
|
||||
// whose sha256 equals `sha` — the source that produced the mirror on disk — or
|
||||
// `null` when it is not in reach (shallow clone, rewritten history).
|
||||
export async function findSourceTextByHash(rel, sha, { cwd = ROOT } = {}) {
|
||||
let commits;
|
||||
try {
|
||||
commits = execFileSync("git", ["log", "--format=%H", "-n", "200", "--", rel], {
|
||||
cwd,
|
||||
encoding: "utf8",
|
||||
})
|
||||
.trim()
|
||||
.split("\n")
|
||||
.filter(Boolean);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
for (const commit of commits) {
|
||||
let text;
|
||||
try {
|
||||
text = execFileSync("git", ["show", `${commit}:${rel}`], {
|
||||
cwd,
|
||||
encoding: "utf8",
|
||||
maxBuffer: 1 << 28,
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (sha256(Buffer.from(text, "utf8")) === sha) return text;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Decides which sections of a task can be spliced in from the mirror on disk.
|
||||
// Targets recorded without `section_hashes` are bootstrapped from git history
|
||||
// by the `source_hash` the state remembers for them.
|
||||
/**
|
||||
* Bootstrap fallback: the recorded source_hash often belongs to a working-tree state that was
|
||||
* never committed as such (add-locale rewrites README/bars before translating), so an exact
|
||||
* hash lookup fails. The source as of the last commit before the translation's `updated_at`
|
||||
* is the closest committed ancestor — sections unchanged since then are safe to reuse.
|
||||
*/
|
||||
export function findSourceTextBefore(rel, isoDate, { cwd = ROOT } = {}) {
|
||||
try {
|
||||
const commit = execFileSync(
|
||||
"git",
|
||||
["log", "-1", "--format=%H", `--before=${isoDate}`, "--", rel],
|
||||
{
|
||||
cwd,
|
||||
encoding: "utf8",
|
||||
}
|
||||
).trim();
|
||||
if (!commit) return null;
|
||||
return execFileSync("git", ["show", `${commit}:${rel}`], {
|
||||
cwd,
|
||||
encoding: "utf8",
|
||||
maxBuffer: 1 << 28,
|
||||
});
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function mirrorLastCommitDate(mirrorRel, { cwd = ROOT } = {}) {
|
||||
try {
|
||||
const out = execFileSync("git", ["log", "-1", "--format=%cI", "--", mirrorRel], {
|
||||
cwd,
|
||||
encoding: "utf8",
|
||||
}).trim();
|
||||
return out || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveSectionPlan({ task, state, opts, sections }) {
|
||||
if (task.missingTarget || opts.force) return null;
|
||||
const recorded = state.sources[task.rel]?.locales?.[task.locale];
|
||||
let previousHashes = recorded?.section_hashes;
|
||||
if (!previousHashes && recorded?.source_hash) {
|
||||
let oldText = await findSourceTextByHash(task.rel, recorded.source_hash);
|
||||
if (!oldText) {
|
||||
// `updated_at` is bumped by `--adopt`, so prefer the date of the last commit that
|
||||
// actually wrote the mirror (mirrors are only written by translation runs).
|
||||
const translatedAt =
|
||||
mirrorLastCommitDate(path.relative(ROOT, task.targetAbs)) || recorded.updated_at;
|
||||
if (translatedAt) oldText = findSourceTextBefore(task.rel, translatedAt);
|
||||
}
|
||||
if (oldText) previousHashes = sectionHashes(splitSections(stripTopHeading(oldText)));
|
||||
}
|
||||
if (!previousHashes) return null;
|
||||
const mirrorText = await fs.readFile(task.targetAbs, "utf8");
|
||||
const mirrorSections = splitSections(extractMirrorBody(mirrorText));
|
||||
return planSectionReuse({ previousHashes, sections, mirrorSections });
|
||||
}
|
||||
|
||||
async function translateBody(body, localeEntry, backend) {
|
||||
const englishName = localeEntry.english ?? localeEntry.name;
|
||||
const native = localeEntry.native ?? localeEntry.name;
|
||||
@@ -489,7 +748,7 @@ async function translateBody(body, localeEntry, backend) {
|
||||
// reliably converts characters but not vocabulary habits, so zh-TW output
|
||||
// otherwise keeps mainland renderings (默認 for 預設, 緩存 for 快取) and
|
||||
// wrong-homophone conversions (上遊 for 上游, 儀錶板 for 儀表板).
|
||||
return normalizeLocaleText(translated.join("\n\n"), localeEntry.code);
|
||||
return normalizeLocaleText(joinTranslatedChunks(translated), localeEntry.code);
|
||||
}
|
||||
|
||||
// Simple promise-based semaphore (avoid runtime deps).
|
||||
@@ -521,6 +780,22 @@ function createLimiter(max) {
|
||||
|
||||
// ----- Main ----------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Merge one run's (source, locale) records into a freshly re-read state. Source-level
|
||||
* `source_hash` follows the record; untouched entries stay as the other runners left them.
|
||||
* `fallback` is this run's in-memory state, used only when the file could not be read.
|
||||
*/
|
||||
export function mergeStateUpdates(fresh, touched, fallback) {
|
||||
const base = fresh && fresh.sources ? fresh : fallback || { sources: {} };
|
||||
for (const { rel, locale, sourceHash, record } of touched) {
|
||||
const entry =
|
||||
base.sources[rel] || (base.sources[rel] = { source_hash: sourceHash, locales: {} });
|
||||
entry.source_hash = sourceHash;
|
||||
entry.locales[locale] = record;
|
||||
}
|
||||
return base;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const opts = parseArgs(process.argv);
|
||||
const config = await loadConfig();
|
||||
@@ -642,6 +917,7 @@ async function main() {
|
||||
|
||||
// Build a flat queue of (source, locale) work units.
|
||||
const tasks = [];
|
||||
const touched = [];
|
||||
for (const rel of sources) {
|
||||
const { hash: sourceHash } = sourceHashes.get(rel);
|
||||
const entry =
|
||||
@@ -654,10 +930,17 @@ async function main() {
|
||||
const previous = entry.locales[locale];
|
||||
const sourceChanged = previous?.source_hash !== sourceHash;
|
||||
const missingTarget = !existsSync(targetAbs);
|
||||
if (!opts.force && !sourceChanged && !missingTarget) {
|
||||
const needsRebuild =
|
||||
!opts.force &&
|
||||
!sourceChanged &&
|
||||
!missingTarget &&
|
||||
mirrorNeedsRebuild(await fs.readFile(targetAbs, "utf8"), sourceHashes.get(rel).text);
|
||||
if (!opts.force && !sourceChanged && !missingTarget && !needsRebuild) {
|
||||
stats.skipped++;
|
||||
continue;
|
||||
}
|
||||
if (needsRebuild)
|
||||
logInfo(`${rel} → ${locale}: mirror needs a rebuild (English copy or leaked frontmatter)`);
|
||||
tasks.push({ rel, locale, targetAbs, sourceChanged, missingTarget });
|
||||
}
|
||||
}
|
||||
@@ -685,9 +968,26 @@ async function main() {
|
||||
const topHeading = extractTopHeading(sourceText);
|
||||
const body = stripTopHeading(sourceText);
|
||||
|
||||
const sections = splitSections(body);
|
||||
let translatedBody;
|
||||
try {
|
||||
translatedBody = await translateBody(body, localeEntry, backend);
|
||||
const plan = await resolveSectionPlan({ task, state, opts, sections });
|
||||
if (plan && plan.translate.length < sections.length) {
|
||||
const out = [...sections];
|
||||
for (const [i, text] of plan.reuse) out[i] = text;
|
||||
for (const i of plan.translate) {
|
||||
// An empty preamble (body opening with `## `) has nothing to send.
|
||||
out[i] = sections[i].trim()
|
||||
? await translateBody(sections[i], localeEntry, backend)
|
||||
: "";
|
||||
}
|
||||
translatedBody = out.join("\n\n");
|
||||
logInfo(
|
||||
`${task.rel} → ${task.locale}: ${plan.translate.length}/${sections.length} sections retranslated`
|
||||
);
|
||||
} else {
|
||||
translatedBody = await translateBody(body, localeEntry, backend);
|
||||
}
|
||||
} catch (err) {
|
||||
stats.failed++;
|
||||
failures.push({ rel: task.rel, locale: task.locale, error: err.message });
|
||||
@@ -709,11 +1009,14 @@ async function main() {
|
||||
await fs.writeFile(task.targetAbs, finalContent, "utf8");
|
||||
|
||||
const targetHash = sha256(Buffer.from(finalContent, "utf8"));
|
||||
state.sources[task.rel].locales[task.locale] = {
|
||||
const record = {
|
||||
source_hash: sourceHash,
|
||||
target_hash: targetHash,
|
||||
section_hashes: sectionHashes(sections),
|
||||
updated_at: new Date().toISOString(),
|
||||
};
|
||||
state.sources[task.rel].locales[task.locale] = record;
|
||||
touched.push({ rel: task.rel, locale: task.locale, sourceHash, record });
|
||||
|
||||
stats.translated++;
|
||||
logInfo(`✓ ${task.rel} → ${task.locale} (${translatedBody.length} chars)`);
|
||||
@@ -721,8 +1024,11 @@ async function main() {
|
||||
)
|
||||
);
|
||||
|
||||
// Save state even on partial failure so future runs only retry what failed.
|
||||
await saveState(state);
|
||||
// Save state even on partial failure so future runs only retry what failed. Several
|
||||
// `--locale=<code>` runs execute in parallel during a batch, so re-read the file and merge
|
||||
// only this run's entries instead of overwriting the whole state (last writer used to win
|
||||
// and the other runners' work vanished from the state — 2026-09-16).
|
||||
await saveState(mergeStateUpdates(await loadState(), touched, state));
|
||||
|
||||
const elapsedSec = ((Date.now() - startMs) / 1000).toFixed(1);
|
||||
logInfo(
|
||||
|
||||
@@ -72,7 +72,28 @@ import { backendConfig, translateBatch, translateString } from "./lib/translate-
|
||||
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(SCRIPT_DIR, "..", "..");
|
||||
const CONFIG_PATH = path.join(ROOT, "config", "i18n.json");
|
||||
const MESSAGES_DIR = path.join(ROOT, "src", "i18n", "messages");
|
||||
|
||||
const CATALOGS = {
|
||||
ui: {
|
||||
name: "ui",
|
||||
dir: path.join(ROOT, "src", "i18n", "messages"),
|
||||
allowlistPath: path.join(SCRIPT_DIR, "untranslatable-keys.json"),
|
||||
},
|
||||
cli: {
|
||||
name: "cli",
|
||||
dir: path.join(ROOT, "bin", "cli", "locales"),
|
||||
allowlistPath: path.join(SCRIPT_DIR, "untranslatable-cli-keys.json"),
|
||||
},
|
||||
};
|
||||
|
||||
/** Which flat-JSON catalog family a run targets: the dashboard (`ui`) or the CLI (`cli`). */
|
||||
export function resolveCatalog(name = "ui") {
|
||||
const catalog = CATALOGS[name];
|
||||
if (!catalog) throw new Error(`unknown catalog "${name}" (expected ui or cli)`);
|
||||
return catalog;
|
||||
}
|
||||
|
||||
let MESSAGES_DIR = CATALOGS.ui.dir; // reassigned in main() from --catalog
|
||||
const SOURCE_LOCALE = "en";
|
||||
const PLACEHOLDER_PREFIX = "__MISSING__:";
|
||||
|
||||
@@ -96,11 +117,13 @@ function parseArgs(argv) {
|
||||
retranslateIdentical: false,
|
||||
concurrency: null,
|
||||
batchSize: 1,
|
||||
catalog: "ui",
|
||||
};
|
||||
for (const arg of argv.slice(2)) {
|
||||
if (arg === "--dry-run" || arg === "--dryrun") opts.dryRun = true;
|
||||
else if (arg === "--translate-markers") opts.translateMarkers = true;
|
||||
else if (arg === "--retranslate-identical") opts.retranslateIdentical = true;
|
||||
else if (arg.startsWith("--catalog=")) opts.catalog = arg.slice(10).trim();
|
||||
else if (arg.startsWith("--locale=")) {
|
||||
opts.locales = arg
|
||||
.slice(9)
|
||||
@@ -125,6 +148,7 @@ function parseArgs(argv) {
|
||||
"Usage: node scripts/i18n/sync-ui-keys.mjs [options]",
|
||||
"",
|
||||
" --locale=<csv> Target locales (default: all except `en`)",
|
||||
" --catalog=ui|cli Catalog family (default ui = src/i18n/messages; cli = bin/cli/locales)",
|
||||
" --retranslate-identical Flag leaves still identical to English (outside",
|
||||
" untranslatable-keys.json) as __MISSING__ so they get",
|
||||
" retranslated — only meaningful with --translate-markers",
|
||||
@@ -391,9 +415,7 @@ async function processLocale(locale, source, config, opts, backend) {
|
||||
|
||||
const { merged, addedPaths } = mergeMissing(source, target);
|
||||
if (opts.retranslateIdentical && locale !== SOURCE_LOCALE) {
|
||||
const allow = new Set(
|
||||
(await loadJson(path.join(SCRIPT_DIR, "untranslatable-keys.json"))).keys ?? []
|
||||
);
|
||||
const allow = new Set((await loadJson(resolveCatalog(opts.catalog).allowlistPath)).keys ?? []);
|
||||
const flagged = markIdenticalAsMissing(merged, source, allow);
|
||||
logInfo(`${locale}: ${flagged} English leaves flagged for retranslation`);
|
||||
}
|
||||
@@ -442,6 +464,9 @@ async function processLocale(locale, source, config, opts, backend) {
|
||||
|
||||
async function main() {
|
||||
const opts = parseArgs(process.argv);
|
||||
const catalog = resolveCatalog(opts.catalog);
|
||||
MESSAGES_DIR = catalog.dir;
|
||||
logInfo(`catalog: ${catalog.name} (${path.relative(ROOT, catalog.dir)})`);
|
||||
const config = await loadConfig();
|
||||
|
||||
const sourcePath = path.join(MESSAGES_DIR, `${SOURCE_LOCALE}.json`);
|
||||
|
||||
259
scripts/i18n/translate-new-keys.sh
Executable file
259
scripts/i18n/translate-new-keys.sh
Executable file
@@ -0,0 +1,259 @@
|
||||
#!/usr/bin/env bash
|
||||
# OmniRoute — translate the `__MISSING__:<en>` markers a branch adds, every locale in parallel.
|
||||
#
|
||||
# Why this exists: on 2026-09-16 eight feature PRs added 61 keys to `src/i18n/messages/en.json`
|
||||
# and stamped `__MISSING__:<en>` into all 65 locales instead of translating. The new-key gate
|
||||
# (`scripts/i18n/check-new-key-coverage.mjs`) now rejects markers, so a branch that adds keys
|
||||
# has to translate them before its PR opens — and translating 65 locales one after the other
|
||||
# is what makes people skip it. This runner does it in parallel and is safe to detach.
|
||||
#
|
||||
# What it does: N workers pop locale codes from a queue (`flock`-serialized) and each runs
|
||||
# node scripts/i18n/sync-ui-keys.mjs --catalog=<c> --locale=<code> --translate-markers --batch-size=40
|
||||
# with up to 3 attempts per locale. A locale is DONE only when the run exits 0 AND its catalog
|
||||
# carries no `__MISSING__` marker any more. Everything is written under `_artifacts/i18n-new-keys/`
|
||||
# (gitignored, disposable):
|
||||
# <code>.log the sync-ui-keys output of the last attempt
|
||||
# <code>.exit 0 on success, else the last exit code (1 when markers survived a rc=0 run)
|
||||
# batch.log one line per attempt + START/END markers
|
||||
# batch.status running | done | failed
|
||||
# batch.rc the script's final exit code (written last — poll this file)
|
||||
# batch.pid PID of this script (kill by PID, never `pkill -f`)
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/i18n/translate-new-keys.sh [--catalog=ui|cli] [--workers=N] [--locales=<csv>]
|
||||
# npm run i18n:translate-new-keys -- --locales=pt-BR,es
|
||||
# # detached (the session keeps working; poll _artifacts/i18n-new-keys/batch.rc):
|
||||
# mkdir -p _artifacts/i18n-new-keys
|
||||
# nohup setsid bash scripts/i18n/translate-new-keys.sh > _artifacts/i18n-new-keys/runner.out 2>&1 &
|
||||
#
|
||||
# --catalog=ui|cli ui = src/i18n/messages (default); cli = bin/cli/locales
|
||||
# --workers=N parallel locales (default 5)
|
||||
# --locales=<csv> subset of locale codes (default: every <code>.json in the catalog but en)
|
||||
#
|
||||
# Backend: the translation env block must be present — in the shell or in the repo `.env`
|
||||
# (the script reads only the OMNIROUTE_TRANSLATION_* lines of `.env`; already-set variables
|
||||
# win). Without it the script refuses to start and names the variables. It never leaves
|
||||
# markers silently: the exit code is non-zero while any selected locale still carries one.
|
||||
#
|
||||
# Exit codes: 0 all selected locales translated · 1 some locale still has markers / failed ·
|
||||
# 2 usage or environment error (nothing ran).
|
||||
set -u
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
ROOT=$(cd "$SCRIPT_DIR/../.." && pwd)
|
||||
ART="$ROOT/_artifacts/i18n-new-keys"
|
||||
|
||||
CATALOG=ui
|
||||
WORKERS=5
|
||||
LOCALES=""
|
||||
RETRY_SLEEP=${OMNIROUTE_TRANSLATION_RETRY_SLEEP:-30}
|
||||
|
||||
usage() {
|
||||
sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
||||
}
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--catalog=*) CATALOG="${arg#--catalog=}" ;;
|
||||
--workers=*) WORKERS="${arg#--workers=}" ;;
|
||||
--locales=*) LOCALES="${arg#--locales=}" ;;
|
||||
--locale=*) LOCALES="${arg#--locale=}" ;;
|
||||
-h | --help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "[i18n-new-keys] unknown argument: $arg" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
case "$CATALOG" in
|
||||
ui) DIR="$ROOT/src/i18n/messages" ;;
|
||||
cli) DIR="$ROOT/bin/cli/locales" ;;
|
||||
*)
|
||||
echo "[i18n-new-keys] --catalog must be ui or cli (got: $CATALOG)" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
case "$WORKERS" in
|
||||
'' | *[!0-9]* | 0)
|
||||
echo "[i18n-new-keys] --workers must be a positive integer (got: $WORKERS)" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
# ----- translation backend env: shell first, then the OMNIROUTE_TRANSLATION_* lines of .env
|
||||
load_translation_env() {
|
||||
local env_file="$ROOT/.env" line key value
|
||||
[ -f "$env_file" ] || return 0
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
line="${line#"${line%%[![:space:]]*}"}" # ltrim
|
||||
line="${line#export }"
|
||||
case "$line" in
|
||||
OMNIROUTE_TRANSLATION_*=*) ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
key="${line%%=*}"
|
||||
value="${line#*=}"
|
||||
value="${value%"${value##*[![:space:]]}"}" # rtrim
|
||||
case "$value" in
|
||||
\"*\") value="${value#\"}" value="${value%\"}" ;;
|
||||
\'*\') value="${value#\'}" value="${value%\'}" ;;
|
||||
esac
|
||||
if [ -z "${!key:-}" ]; then
|
||||
export "$key=$value"
|
||||
fi
|
||||
done <"$env_file"
|
||||
}
|
||||
load_translation_env
|
||||
|
||||
REQUIRED_VARS="OMNIROUTE_TRANSLATION_API_URL OMNIROUTE_TRANSLATION_API_KEY OMNIROUTE_TRANSLATION_MODEL"
|
||||
OPTIONAL_VARS="OMNIROUTE_TRANSLATION_CONCURRENCY OMNIROUTE_TRANSLATION_TIMEOUT_MS"
|
||||
missing=""
|
||||
for v in $REQUIRED_VARS; do
|
||||
[ -n "${!v:-}" ] || missing="$missing $v"
|
||||
done
|
||||
if [ -n "$missing" ]; then
|
||||
{
|
||||
echo "[i18n-new-keys] REFUSING TO START — translation backend not configured."
|
||||
echo " Missing (required):$missing"
|
||||
echo " The full block (put it in $ROOT/.env or export it in the shell):"
|
||||
for v in $REQUIRED_VARS $OPTIONAL_VARS; do
|
||||
if [ -n "${!v:-}" ]; then echo " $v (set)"; else echo " $v (MISSING)"; fi
|
||||
done
|
||||
echo " Nothing was translated; the __MISSING__ markers are still in place and the"
|
||||
echo " new-key gate (npm run i18n:check-new-keys) will reject them."
|
||||
} >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# ----- locale selection: on-disk catalogs minus en, or the explicit --locales subset
|
||||
selected=()
|
||||
if [ -n "$LOCALES" ]; then
|
||||
IFS=',' read -r -a requested <<<"$LOCALES"
|
||||
for code in "${requested[@]}"; do
|
||||
code="${code//[[:space:]]/}"
|
||||
[ -n "$code" ] || continue
|
||||
if [ "$code" = "en" ]; then
|
||||
echo "[i18n-new-keys] en is the source catalog, skipping it" >&2
|
||||
continue
|
||||
fi
|
||||
if [ ! -f "$DIR/$code.json" ]; then
|
||||
echo "[i18n-new-keys] unknown locale: $code ($DIR/$code.json does not exist)" >&2
|
||||
exit 2
|
||||
fi
|
||||
selected+=("$code")
|
||||
done
|
||||
else
|
||||
for f in "$DIR"/*.json; do
|
||||
code=$(basename "$f" .json)
|
||||
[ "$code" = "en" ] && continue
|
||||
selected+=("$code")
|
||||
done
|
||||
fi
|
||||
if [ "${#selected[@]}" -eq 0 ]; then
|
||||
echo "[i18n-new-keys] no locale selected" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# ----- artifacts (fresh per run for the selected locales)
|
||||
mkdir -p "$ART"
|
||||
QUEUE="$ART/queue.txt"
|
||||
LOCK="$ART/queue.lock"
|
||||
: >"$QUEUE"
|
||||
for code in "${selected[@]}"; do
|
||||
echo "$code" >>"$QUEUE"
|
||||
rm -f "$ART/$code.exit"
|
||||
done
|
||||
rm -f "$ART/batch.rc"
|
||||
echo running >"$ART/batch.status"
|
||||
echo $$ >"$ART/batch.pid"
|
||||
|
||||
log() { echo "$*" >>"$ART/batch.log"; }
|
||||
|
||||
count_markers() {
|
||||
# grep -c prints 0 and exits 1 when nothing matches — the count is what we want.
|
||||
grep -c "__MISSING__" "$DIR/$1.json" 2>/dev/null || true
|
||||
}
|
||||
|
||||
next_locale() {
|
||||
# pop the first line of the queue atomically
|
||||
(
|
||||
flock 9
|
||||
local l
|
||||
l=$(head -n 1 "$QUEUE")
|
||||
[ -n "$l" ] && sed -i '1d' "$QUEUE"
|
||||
echo "$l"
|
||||
) 9>"$LOCK"
|
||||
}
|
||||
|
||||
worker() {
|
||||
local id=$1 code rc left t0 wall attempt
|
||||
while :; do
|
||||
code=$(next_locale)
|
||||
[ -z "$code" ] && break
|
||||
for attempt in 1 2 3; do
|
||||
log "[$code] w$id attempt $attempt start $(date -Is)"
|
||||
t0=$(date +%s)
|
||||
(
|
||||
cd "$ROOT" && node scripts/i18n/sync-ui-keys.mjs --catalog="$CATALOG" --locale="$code" \
|
||||
--translate-markers --batch-size=40
|
||||
) >"$ART/$code.log" 2>&1
|
||||
rc=$?
|
||||
wall=$(($(date +%s) - t0))
|
||||
left=$(count_markers "$code")
|
||||
log "[$code] w$id attempt $attempt exit=$rc wall=${wall}s markers_left=$left $(date -Is)"
|
||||
if [ "$rc" -eq 0 ] && [ "$left" -eq 0 ]; then
|
||||
echo 0 >"$ART/$code.exit"
|
||||
log "[$code] DONE"
|
||||
break
|
||||
fi
|
||||
if [ "$attempt" -eq 3 ]; then
|
||||
[ "$rc" -eq 0 ] && rc=1
|
||||
echo "$rc" >"$ART/$code.exit"
|
||||
log "[$code] FAILED exit=$rc markers_left=$left"
|
||||
else
|
||||
sleep "$RETRY_SLEEP"
|
||||
fi
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
echo "[i18n-new-keys] catalog=$CATALOG locales=${#selected[@]} workers=$WORKERS artifacts=$ART"
|
||||
log "BATCH START catalog=$CATALOG workers=$WORKERS locales=${selected[*]} $(date -Is)"
|
||||
pids=()
|
||||
for i in $(seq 1 "$WORKERS"); do
|
||||
worker "$i" &
|
||||
pids+=("$!")
|
||||
done
|
||||
wait "${pids[@]}"
|
||||
|
||||
# ----- verdict: every selected locale must have exit 0 AND zero markers on disk
|
||||
failed=""
|
||||
for code in "${selected[@]}"; do
|
||||
ex=$(cat "$ART/$code.exit" 2>/dev/null || echo missing)
|
||||
left=$(count_markers "$code")
|
||||
if [ "$ex" != "0" ] || [ "$left" -ne 0 ]; then
|
||||
failed="$failed $code(exit=$ex,markers=$left)"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$failed" ]; then
|
||||
log "BATCH END failed:$failed $(date -Is)"
|
||||
echo failed >"$ART/batch.status"
|
||||
echo 1 >"$ART/batch.rc"
|
||||
echo "[i18n-new-keys] FAILED — markers still present or run failed for:$failed" >&2
|
||||
echo " logs: $ART/<code>.log — re-run with --locales=<csv> for just those." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "BATCH END ok $(date -Is)"
|
||||
echo done >"$ART/batch.status"
|
||||
echo 0 >"$ART/batch.rc"
|
||||
echo "[i18n-new-keys] DONE — ${#selected[@]} locale(s) translated, no __MISSING__ marker left."
|
||||
echo " Now: npm run i18n:check-keys && npm run i18n:check-ratio && npm run i18n:check-new-keys"
|
||||
exit 0
|
||||
4
scripts/i18n/untranslatable-cli-keys.json
Normal file
4
scripts/i18n/untranslatable-cli-keys.json
Normal file
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"description": "CLI keys that must stay in English (brand names, command names, protocol identifiers). Same shape as untranslatable-keys.json; consumed by sync-ui-keys --catalog=cli --retranslate-identical and check-translation-ratio --catalog=cli.",
|
||||
"keys": []
|
||||
}
|
||||
27
scripts/i18n/untranslatable-site-keys.json
Normal file
27
scripts/i18n/untranslatable-site-keys.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"description": "Site catalog keys (omnirouteSite/lang/<code>.json) that must stay identical to lang/_source.en.json — brand and product names, package names, URLs, protocol acronyms and literal config values. retranslate-site.mjs never sends these to the translation backend.",
|
||||
"keys": [
|
||||
"combos.mode.auto",
|
||||
"compare.ops.oauth.lite",
|
||||
"compare.res.tls.cli",
|
||||
"compare.res.tls.own",
|
||||
"cta.community.whatsapp.brazil",
|
||||
"deploy.arm",
|
||||
"deploy.docker",
|
||||
"deploy.npm",
|
||||
"deploy.opencode.cmd",
|
||||
"deploy.pwa",
|
||||
"deploy.termux",
|
||||
"deploy.vscode",
|
||||
"footer.github",
|
||||
"footer.protocols",
|
||||
"hero.cta.github",
|
||||
"providers.cat.local.ex",
|
||||
"providers.cat.oauth",
|
||||
"viral.footer.github",
|
||||
"viral.footer.home",
|
||||
"why.flow.ide",
|
||||
"why.where.reddit.title",
|
||||
"why.where.x.title"
|
||||
]
|
||||
}
|
||||
996
scripts/perf/messages-route-memory-profile.ts
Normal file
996
scripts/perf/messages-route-memory-profile.ts
Normal file
@@ -0,0 +1,996 @@
|
||||
/**
|
||||
* JON-562: bounded memory profile for the real Claude `/v1/messages` request boundary.
|
||||
*
|
||||
* The default driver runs each context size in a fresh child process. The worker uses a
|
||||
* synthetic request and a local fetch stub, so it exercises admission, parsing, translation,
|
||||
* request logging and SSE cleanup without credentials or provider calls. Raw payloads are never
|
||||
* written. Artifacts are private (umask 077) and contain only measurements plus V8 profiles of
|
||||
* the synthetic process.
|
||||
*
|
||||
* Usage:
|
||||
* node --import tsx/esm scripts/perf/messages-route-memory-profile.ts \
|
||||
* --output-dir /tmp/omniroute-JON-562
|
||||
*
|
||||
* Defaults: 100k, 300k and 600k token-equivalents; six sequential requests per fresh process;
|
||||
* concurrency=1; stream=true; cancellation=none. Context size is the only changing factor.
|
||||
*/
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import inspector from "node:inspector";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import v8 from "node:v8";
|
||||
|
||||
export const BYTES_PER_TOKEN_EQUIVALENT = 4;
|
||||
export const DEFAULT_TOKEN_EQUIVALENTS = [100_000, 300_000, 600_000] as const;
|
||||
const PROVENANCE_FILES = [
|
||||
"src/lib/usage/completedRequestDetails.ts",
|
||||
"src/lib/usage/usageHistory.ts",
|
||||
"tests/unit/active-request-stream-chunks-lifecycle.test.ts",
|
||||
"scripts/perf/messages-route-memory-profile.ts",
|
||||
"tests/unit/messages-route-memory-profile.test.ts",
|
||||
] as const;
|
||||
|
||||
type ClaudePayload = {
|
||||
model: string;
|
||||
max_tokens: number;
|
||||
stream: boolean;
|
||||
messages: Array<{ role: "user"; content: string }>;
|
||||
};
|
||||
|
||||
export type MemoryRow = {
|
||||
phase: "baseline" | "after_route" | "after_drain" | "settled" | "final";
|
||||
elapsedMs: number;
|
||||
requestIndex?: number;
|
||||
heapUsedBytes: number;
|
||||
heapTotalBytes: number;
|
||||
rssBytes: number;
|
||||
externalBytes: number;
|
||||
arrayBuffersBytes: number;
|
||||
admission: {
|
||||
activeHeavy: number;
|
||||
activeHealthyHeadroom: number;
|
||||
inflightBytes: number;
|
||||
queuedBytes: number;
|
||||
waiting: number;
|
||||
};
|
||||
};
|
||||
|
||||
type GrowthSummary = {
|
||||
baselineHeapUsedBytes: number;
|
||||
finalSettledHeapUsedBytes: number;
|
||||
settledGrowthBytes: number;
|
||||
settledSlopeBytesPerRequest: number;
|
||||
growthToWireRatio: number;
|
||||
};
|
||||
|
||||
function markerFor(tokenEquivalent: number): string {
|
||||
return ["JON", "562", tokenEquivalent, "CONTEXT"].join("-");
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the credential-free environment inherited by profiler children.
|
||||
* @param source - Environment to copy allowlisted runtime fields from.
|
||||
* @returns A new environment containing only allowlisted fields and fixed test settings.
|
||||
*/
|
||||
export function buildWorkerEnv(
|
||||
source: NodeJS.ProcessEnv | Record<string, string | undefined> = process.env
|
||||
): NodeJS.ProcessEnv {
|
||||
const env: NodeJS.ProcessEnv = {};
|
||||
for (const key of ["PATH", "HOME", "TMPDIR", "TEMP", "TMP", "LANG", "LC_ALL", "TZ"] as const) {
|
||||
const value = source[key];
|
||||
if (value) env[key] = value;
|
||||
}
|
||||
env.NODE_ENV = "test";
|
||||
env.APP_LOG_LEVEL = "error";
|
||||
return env;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an ASCII-only Claude body with an exact, repeatable serialized size.
|
||||
* @param tokenEquivalent - Context size expressed at four serialized bytes per token.
|
||||
* @returns The synthetic body, its marker and exact target wire size.
|
||||
* @throws {RangeError} If `tokenEquivalent` is invalid or too small for the fixed envelope.
|
||||
* @throws {Error} If serialization does not match the calculated target size.
|
||||
*/
|
||||
export function buildClaudeContextPayload(tokenEquivalent: number): {
|
||||
body: ClaudePayload;
|
||||
marker: string;
|
||||
targetWireBytes: number;
|
||||
} {
|
||||
if (!Number.isSafeInteger(tokenEquivalent) || tokenEquivalent < 64) {
|
||||
throw new RangeError("tokenEquivalent must be an integer >= 64");
|
||||
}
|
||||
const targetWireBytes = tokenEquivalent * BYTES_PER_TOKEN_EQUIVALENT;
|
||||
const marker = markerFor(tokenEquivalent);
|
||||
const body: ClaudePayload = {
|
||||
model: "openai/gpt-4.1",
|
||||
max_tokens: 8,
|
||||
stream: true,
|
||||
messages: [{ role: "user", content: "" }],
|
||||
};
|
||||
const fixedBytes = Buffer.byteLength(JSON.stringify(body), "utf8");
|
||||
const contentBytes = targetWireBytes - fixedBytes;
|
||||
if (contentBytes < marker.length) {
|
||||
throw new RangeError("tokenEquivalent is too small for the fixed request envelope");
|
||||
}
|
||||
body.messages[0].content = marker + "x".repeat(contentBytes - marker.length);
|
||||
const actualBytes = Buffer.byteLength(JSON.stringify(body), "utf8");
|
||||
if (actualBytes !== targetWireBytes) {
|
||||
throw new Error(`payload calibration failed: wanted ${targetWireBytes}, got ${actualBytes}`);
|
||||
}
|
||||
return { body, marker, targetWireBytes };
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate post-GC growth from baseline and settled samples only.
|
||||
* @param rows - Ordered memory samples from one isolated workload.
|
||||
* @param wireBytes - Exact serialized request size used to calculate the growth ratio.
|
||||
* @returns Baseline, final, slope and wire-ratio measurements.
|
||||
* @throws {Error} If the samples contain no baseline or settled row.
|
||||
*/
|
||||
export function summarizeSettledGrowth(
|
||||
rows: Array<Pick<MemoryRow, "phase" | "heapUsedBytes" | "requestIndex">>,
|
||||
wireBytes: number
|
||||
): GrowthSummary {
|
||||
const baseline = rows.find((row) => row.phase === "baseline");
|
||||
const settled = rows.filter((row) => row.phase === "settled");
|
||||
if (!baseline || settled.length === 0) {
|
||||
throw new Error("baseline and settled samples are required");
|
||||
}
|
||||
const final = settled[settled.length - 1];
|
||||
const settledGrowthBytes = final.heapUsedBytes - baseline.heapUsedBytes;
|
||||
const settledSlopeBytesPerRequest =
|
||||
settled.length < 2
|
||||
? settledGrowthBytes
|
||||
: (final.heapUsedBytes - settled[0].heapUsedBytes) / (settled.length - 1);
|
||||
return {
|
||||
baselineHeapUsedBytes: baseline.heapUsedBytes,
|
||||
finalSettledHeapUsedBytes: final.heapUsedBytes,
|
||||
settledGrowthBytes,
|
||||
settledSlopeBytesPerRequest,
|
||||
growthToWireRatio: settledGrowthBytes / wireBytes,
|
||||
};
|
||||
}
|
||||
|
||||
function argValue(flag: string): string | undefined {
|
||||
const index = process.argv.indexOf(flag);
|
||||
return index >= 0 ? process.argv[index + 1] : undefined;
|
||||
}
|
||||
|
||||
function positiveIntArg(flag: string, fallback: number): number {
|
||||
const raw = argValue(flag);
|
||||
if (raw === undefined) return fallback;
|
||||
const value = Number(raw);
|
||||
if (!Number.isSafeInteger(value) || value <= 0) {
|
||||
throw new RangeError(`${flag} must be a positive integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function privateDirectory(directory: string): void {
|
||||
fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
|
||||
fs.chmodSync(directory, 0o700);
|
||||
}
|
||||
|
||||
function writePrivateJson(file: string, value: unknown): void {
|
||||
fs.writeFileSync(file, JSON.stringify(value, null, 2) + "\n", { mode: 0o600 });
|
||||
fs.chmodSync(file, 0o600);
|
||||
}
|
||||
|
||||
function appendPrivateJsonLine(file: string, value: unknown): void {
|
||||
fs.appendFileSync(file, JSON.stringify(value) + "\n", { mode: 0o600 });
|
||||
}
|
||||
|
||||
/**
|
||||
* Run work while guaranteeing removal of its raw heap-snapshot path.
|
||||
* @param snapshotFile - Raw snapshot path owned by the operation.
|
||||
* @param work - Worker/analyzer operation to run before cleanup.
|
||||
* @returns The fulfilled result from `work`.
|
||||
* @throws The original work or cleanup error.
|
||||
*/
|
||||
export async function withRawSnapshotCleanup<T>(
|
||||
snapshotFile: string,
|
||||
work: () => Promise<T>
|
||||
): Promise<T> {
|
||||
try {
|
||||
return await work();
|
||||
} finally {
|
||||
fs.rmSync(snapshotFile, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a worker snapshot unless a successful driver handoff owns it.
|
||||
* @param snapshotFile - Raw worker snapshot path.
|
||||
* @param state - Whether the worker completed and the driver accepted ownership.
|
||||
* @returns Nothing.
|
||||
*/
|
||||
export function cleanupWorkerSnapshot(
|
||||
snapshotFile: string,
|
||||
state: { workerComplete: boolean; snapshotHandoff: boolean }
|
||||
): void {
|
||||
if (!state.workerComplete || !state.snapshotHandoff) {
|
||||
fs.rmSync(snapshotFile, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function forceGc(): void {
|
||||
if (typeof globalThis.gc !== "function") {
|
||||
throw new Error("JON-562 worker requires node --expose-gc");
|
||||
}
|
||||
for (let index = 0; index < 4; index += 1) globalThis.gc();
|
||||
}
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function inspectorPost<T>(
|
||||
session: inspector.Session,
|
||||
method: string,
|
||||
params: Record<string, unknown> = {}
|
||||
): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
session.post(method, params, (error, result) => {
|
||||
if (error) reject(error);
|
||||
else resolve(result as T);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function startAllocationSampling(): Promise<{
|
||||
stop: () => Promise<Record<string, unknown>>;
|
||||
disconnect: () => void;
|
||||
}> {
|
||||
const session = new inspector.Session();
|
||||
session.connect();
|
||||
await inspectorPost(session, "HeapProfiler.enable");
|
||||
await inspectorPost(session, "HeapProfiler.startSampling", {
|
||||
samplingInterval: 32 * 1024,
|
||||
includeObjectsCollectedByMajorGC: true,
|
||||
includeObjectsCollectedByMinorGC: true,
|
||||
});
|
||||
return {
|
||||
stop: async () => {
|
||||
const result = await inspectorPost<{ profile: Record<string, unknown> }>(
|
||||
session,
|
||||
"HeapProfiler.stopSampling"
|
||||
);
|
||||
return result.profile;
|
||||
},
|
||||
disconnect: () => session.disconnect(),
|
||||
};
|
||||
}
|
||||
|
||||
function openAiSseResponse(): Response {
|
||||
const encoder = new TextEncoder();
|
||||
const frames = [
|
||||
`data: ${JSON.stringify({
|
||||
id: "chatcmpl_JON562",
|
||||
object: "chat.completion.chunk",
|
||||
created: 1,
|
||||
model: "gpt-4.1",
|
||||
choices: [{ index: 0, delta: { role: "assistant", content: "ok" } }],
|
||||
})}\n\n`,
|
||||
`data: ${JSON.stringify({
|
||||
id: "chatcmpl_JON562",
|
||||
object: "chat.completion.chunk",
|
||||
created: 1,
|
||||
model: "gpt-4.1",
|
||||
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
|
||||
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
|
||||
})}\n\n`,
|
||||
"data: [DONE]\n\n",
|
||||
];
|
||||
return new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
for (const frame of frames) controller.enqueue(encoder.encode(frame));
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "text/event-stream" } }
|
||||
);
|
||||
}
|
||||
|
||||
async function runWorker(): Promise<void> {
|
||||
process.umask(0o077);
|
||||
const tokenEquivalent = positiveIntArg("--tokens", 100_000);
|
||||
const iterations = positiveIntArg("--iterations", 6);
|
||||
const outputDir = path.resolve(argValue("--output-dir") ?? "");
|
||||
if (!argValue("--output-dir")) throw new Error("--output-dir is required in worker mode");
|
||||
privateDirectory(outputDir);
|
||||
|
||||
const memoryFile = path.join(outputDir, "memory.jsonl");
|
||||
fs.writeFileSync(memoryFile, "", { mode: 0o600 });
|
||||
const startedAt = performance.now();
|
||||
const rows: MemoryRow[] = [];
|
||||
let providerCalls = 0;
|
||||
let harness: Awaited<
|
||||
ReturnType<
|
||||
typeof import("../../tests/integration/_chatPipelineHarness.ts").createChatPipelineHarness
|
||||
>
|
||||
> | null = null;
|
||||
let sampler: Awaited<ReturnType<typeof startAllocationSampling>> | null = null;
|
||||
let heapSnapshotFile: string | null = null;
|
||||
let workerComplete = false;
|
||||
const snapshotHandoff = process.argv.includes("--snapshot-handoff");
|
||||
|
||||
try {
|
||||
const { createChatPipelineHarness } =
|
||||
await import("../../tests/integration/_chatPipelineHarness.ts");
|
||||
harness = await createChatPipelineHarness(`JON-562-${tokenEquivalent}`);
|
||||
const messagesRoute = await import("../../src/app/api/v1/messages/route.ts");
|
||||
const { perConnectionAdmissionController } =
|
||||
await import("../../src/shared/middleware/chatBodyAdmission.ts");
|
||||
const { reloadResourcePressureRuntime } =
|
||||
await import("../../open-sse/utils/resourcePressure.ts");
|
||||
|
||||
reloadResourcePressureRuntime({
|
||||
heapThresholdMb: null,
|
||||
immediateHeapUsedMb: () => 1,
|
||||
sample: async () => ({
|
||||
observedAtMs: Date.now(),
|
||||
v8: { heapUsedBytes: 1, heapLimitBytes: Number.MAX_SAFE_INTEGER },
|
||||
process: {
|
||||
rssBytes: 1,
|
||||
externalBytes: 0,
|
||||
arrayBuffersBytes: 0,
|
||||
availableBytes: null,
|
||||
constrainedBytes: null,
|
||||
},
|
||||
cgroup: {
|
||||
currentBytes: null,
|
||||
maxBytes: null,
|
||||
highBytes: null,
|
||||
fileBytes: null,
|
||||
events: null,
|
||||
},
|
||||
psi: null,
|
||||
}),
|
||||
});
|
||||
harness.BaseExecutor.RETRY_CONFIG.delayMs = 0;
|
||||
await harness.resetStorage();
|
||||
await harness.seedConnection("openai", {
|
||||
name: "JON-562-local-stub",
|
||||
apiKey: "synthetic-not-a-credential",
|
||||
});
|
||||
globalThis.fetch = async () => {
|
||||
providerCalls += 1;
|
||||
return openAiSseResponse();
|
||||
};
|
||||
|
||||
const sample = (phase: MemoryRow["phase"], requestIndex?: number): void => {
|
||||
const usage = process.memoryUsage();
|
||||
const admission = perConnectionAdmissionController.snapshot();
|
||||
const row: MemoryRow = {
|
||||
phase,
|
||||
elapsedMs: Math.round((performance.now() - startedAt) * 100) / 100,
|
||||
requestIndex,
|
||||
heapUsedBytes: usage.heapUsed,
|
||||
heapTotalBytes: usage.heapTotal,
|
||||
rssBytes: usage.rss,
|
||||
externalBytes: usage.external,
|
||||
arrayBuffersBytes: usage.arrayBuffers,
|
||||
admission: {
|
||||
activeHeavy: admission.activeHeavy,
|
||||
activeHealthyHeadroom: admission.activeHealthyHeadroom,
|
||||
inflightBytes: admission.inflightBytes,
|
||||
queuedBytes: admission.queuedBytes,
|
||||
waiting: admission.waiting,
|
||||
},
|
||||
};
|
||||
rows.push(row);
|
||||
appendPrivateJsonLine(memoryFile, row);
|
||||
};
|
||||
|
||||
const runRequest = async (
|
||||
tokens: number,
|
||||
requestIndex: number,
|
||||
measured: boolean
|
||||
): Promise<number> => {
|
||||
const { body, targetWireBytes } = buildClaudeContextPayload(tokens);
|
||||
const serialized = JSON.stringify(body);
|
||||
const request = new Request("http://omniroute.invalid/v1/messages", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"content-length": String(targetWireBytes),
|
||||
accept: "text/event-stream",
|
||||
},
|
||||
body: serialized,
|
||||
});
|
||||
const response = await messagesRoute.POST(request, {});
|
||||
if (measured) sample("after_route", requestIndex);
|
||||
const responseBody = await response.arrayBuffer();
|
||||
if (measured) sample("after_drain", requestIndex);
|
||||
if (response.status !== 200) {
|
||||
throw new Error(
|
||||
`route returned ${response.status} (${responseBody.byteLength} response bytes)`
|
||||
);
|
||||
}
|
||||
return responseBody.byteLength;
|
||||
};
|
||||
|
||||
await runRequest(Math.min(tokenEquivalent, 2_048), 0, false);
|
||||
await delay(50);
|
||||
forceGc();
|
||||
|
||||
// Allocation sampling is deliberately completed before the retention time series. The
|
||||
// inspector profiler retains its own sampled stack records; leaving it enabled would make
|
||||
// post-GC heap growth look linear even when the request payload itself was collectible.
|
||||
sampler = await startAllocationSampling();
|
||||
await runRequest(tokenEquivalent, 0, false);
|
||||
const allocationProfile = await sampler.stop();
|
||||
const allocationFile = path.join(outputDir, "allocation.heapprofile");
|
||||
writePrivateJson(allocationFile, allocationProfile);
|
||||
sampler.disconnect();
|
||||
sampler = null;
|
||||
|
||||
await delay(100);
|
||||
forceGc();
|
||||
sample("baseline");
|
||||
|
||||
const responseBytes: number[] = [];
|
||||
for (let requestIndex = 1; requestIndex <= iterations; requestIndex += 1) {
|
||||
responseBytes.push(await runRequest(tokenEquivalent, requestIndex, true));
|
||||
await delay(50);
|
||||
forceGc();
|
||||
sample("settled", requestIndex);
|
||||
}
|
||||
|
||||
await delay(250);
|
||||
forceGc();
|
||||
sample("final");
|
||||
|
||||
if (!process.argv.includes("--no-snapshot")) {
|
||||
forceGc();
|
||||
heapSnapshotFile = path.join(outputDir, "post-gc.heapsnapshot");
|
||||
v8.writeHeapSnapshot(heapSnapshotFile);
|
||||
fs.chmodSync(heapSnapshotFile, 0o600);
|
||||
}
|
||||
|
||||
const { targetWireBytes } = buildClaudeContextPayload(tokenEquivalent);
|
||||
const growth = summarizeSettledGrowth(rows, targetWireBytes);
|
||||
const settledRows = rows.filter((row) => row.phase === "settled");
|
||||
const released = settledRows.every(
|
||||
(row) =>
|
||||
row.admission.activeHeavy === 0 &&
|
||||
row.admission.activeHealthyHeadroom === 0 &&
|
||||
row.admission.inflightBytes === 0 &&
|
||||
row.admission.waiting === 0
|
||||
);
|
||||
if (!released) throw new Error("admission state remained live after an SSE response drained");
|
||||
|
||||
const manifest = {
|
||||
ticket: "JON-562",
|
||||
status: "complete",
|
||||
route: "/v1/messages",
|
||||
tokenEquivalent,
|
||||
bytesPerTokenEquivalent: BYTES_PER_TOKEN_EQUIVALENT,
|
||||
exactWireBytes: targetWireBytes,
|
||||
iterations,
|
||||
providerCalls,
|
||||
conditions: {
|
||||
concurrency: 1,
|
||||
cancellation: "none",
|
||||
stream: true,
|
||||
messageCount: 1,
|
||||
toolCount: 0,
|
||||
provider: "local-fetch-stub",
|
||||
networkCalls: 0,
|
||||
},
|
||||
runtime: {
|
||||
node: process.version,
|
||||
platform: process.platform,
|
||||
arch: process.arch,
|
||||
heapSizeLimitBytes: v8.getHeapStatistics().heap_size_limit,
|
||||
},
|
||||
responseBytes,
|
||||
allocationSampling: {
|
||||
intervalBytes: 32 * 1024,
|
||||
file: path.basename(allocationFile),
|
||||
requestCount: 1,
|
||||
measuredWindow: "one post-warmup request before the retention time series",
|
||||
},
|
||||
retentionSeries: { allocationSamplingEnabled: false },
|
||||
heapSnapshot: heapSnapshotFile ? path.basename(heapSnapshotFile) : null,
|
||||
admissionReleasedAfterEveryRequest: released,
|
||||
growth,
|
||||
};
|
||||
writePrivateJson(path.join(outputDir, "manifest.json"), manifest);
|
||||
workerComplete = true;
|
||||
process.stdout.write(JSON.stringify({ outputDir, status: "complete", growth }) + "\n");
|
||||
} finally {
|
||||
try {
|
||||
if (sampler) {
|
||||
await sampler.stop().catch(() => undefined);
|
||||
sampler.disconnect();
|
||||
}
|
||||
await harness?.cleanup();
|
||||
} finally {
|
||||
if (heapSnapshotFile) {
|
||||
cleanupWorkerSnapshot(heapSnapshotFile, { workerComplete, snapshotHandoff });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export type HeapSnapshot = {
|
||||
snapshot: {
|
||||
meta: {
|
||||
node_fields: string[];
|
||||
node_types: Array<string[] | string>;
|
||||
edge_fields: string[];
|
||||
edge_types: Array<string[] | string>;
|
||||
};
|
||||
};
|
||||
nodes: number[];
|
||||
edges: number[];
|
||||
strings: string[];
|
||||
};
|
||||
|
||||
function safeLabel(value: string, marker: string): string {
|
||||
if (value.includes(marker) || value.length > 120) return "<SYNTHETIC_CONTEXT_REDACTED>";
|
||||
if (path.isAbsolute(value)) return `<ABSOLUTE_PATH_REDACTED>/${path.basename(value)}`;
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify marker-bearing V8 string graphs by their physical backing size.
|
||||
* @param parsed - Parsed V8 heap snapshot.
|
||||
* @param marker - Synthetic context marker to locate.
|
||||
* @param minimumSelfSizeBytes - Minimum backing-graph size treated as retained context.
|
||||
* @returns Candidate counts, physical sizes, verdict and large component roots.
|
||||
*/
|
||||
export function classifyContextBackingCandidates(
|
||||
parsed: HeapSnapshot,
|
||||
marker: string,
|
||||
minimumSelfSizeBytes: number
|
||||
): {
|
||||
matchingPreviewNodes: number;
|
||||
maxSelfSizeBytes: number;
|
||||
maxBackingRetainedSizeBytes: number;
|
||||
retainedContextSelfSizeBytes: number;
|
||||
largeBackingRetained: boolean;
|
||||
largeNodeIndexes: number[];
|
||||
} {
|
||||
const nodeFields = parsed.snapshot.meta.node_fields;
|
||||
const nodeWidth = nodeFields.length;
|
||||
const nodeTypeIndex = nodeFields.indexOf("type");
|
||||
const nodeNameIndex = nodeFields.indexOf("name");
|
||||
const nodeSelfSizeIndex = nodeFields.indexOf("self_size");
|
||||
const edgeCountIndex = nodeFields.indexOf("edge_count");
|
||||
const edgeFields = parsed.snapshot.meta.edge_fields;
|
||||
const edgeWidth = edgeFields.length;
|
||||
const edgeTypeIndex = edgeFields.indexOf("type");
|
||||
const edgeTargetIndex = edgeFields.indexOf("to_node");
|
||||
const nodeTypes = parsed.snapshot.meta.node_types[nodeTypeIndex] as string[];
|
||||
const edgeTypes = parsed.snapshot.meta.edge_types[edgeTypeIndex] as string[];
|
||||
const nodeCount = parsed.nodes.length / nodeWidth;
|
||||
const matching: Array<{ nodeIndex: number; selfSizeBytes: number }> = [];
|
||||
const stringParents: Array<number[] | undefined> = new Array(nodeCount);
|
||||
const stringChildren: Array<number[] | undefined> = new Array(nodeCount);
|
||||
const isStringNode = (nodeIndex: number): boolean => {
|
||||
const type = nodeTypes[parsed.nodes[nodeIndex * nodeWidth + nodeTypeIndex]];
|
||||
return type === "string" || type === "concatenated string" || type === "sliced string";
|
||||
};
|
||||
|
||||
let edgeOffset = 0;
|
||||
for (let from = 0; from < nodeCount; from += 1) {
|
||||
const edgeCount = parsed.nodes[from * nodeWidth + edgeCountIndex];
|
||||
for (let local = 0; local < edgeCount; local += 1) {
|
||||
const type = edgeTypes[parsed.edges[edgeOffset + edgeTypeIndex]];
|
||||
const to = parsed.edges[edgeOffset + edgeTargetIndex] / nodeWidth;
|
||||
if (type === "internal" && isStringNode(from) && isStringNode(to)) {
|
||||
(stringChildren[from] ??= []).push(to);
|
||||
(stringParents[to] ??= []).push(from);
|
||||
}
|
||||
edgeOffset += edgeWidth;
|
||||
}
|
||||
}
|
||||
|
||||
for (let nodeIndex = 0; nodeIndex < nodeCount; nodeIndex += 1) {
|
||||
const offset = nodeIndex * nodeWidth;
|
||||
const type = nodeTypes[parsed.nodes[offset + nodeTypeIndex]];
|
||||
if (type !== "string" && type !== "concatenated string" && type !== "sliced string") continue;
|
||||
const name = parsed.strings[parsed.nodes[offset + nodeNameIndex]] ?? "";
|
||||
if (name.includes(marker)) {
|
||||
matching.push({ nodeIndex, selfSizeBytes: parsed.nodes[offset + nodeSelfSizeIndex] });
|
||||
}
|
||||
}
|
||||
|
||||
const componentRoots = new Set<number>();
|
||||
for (const entry of matching) {
|
||||
const queue = [entry.nodeIndex];
|
||||
const seen = new Set<number>();
|
||||
while (queue.length > 0) {
|
||||
const node = queue.pop() as number;
|
||||
if (seen.has(node)) continue;
|
||||
seen.add(node);
|
||||
const parents = stringParents[node] ?? [];
|
||||
if (parents.length === 0) componentRoots.add(node);
|
||||
else queue.push(...parents);
|
||||
}
|
||||
}
|
||||
|
||||
const components = [...componentRoots].map((root) => {
|
||||
const nodes = new Set<number>();
|
||||
const queue = [root];
|
||||
let retainedSizeBytes = 0;
|
||||
while (queue.length > 0) {
|
||||
const node = queue.pop() as number;
|
||||
if (nodes.has(node)) continue;
|
||||
nodes.add(node);
|
||||
retainedSizeBytes += parsed.nodes[node * nodeWidth + nodeSelfSizeIndex];
|
||||
queue.push(...(stringChildren[node] ?? []));
|
||||
}
|
||||
return { root, nodes, retainedSizeBytes };
|
||||
});
|
||||
const large = components.filter(
|
||||
(component) => component.retainedSizeBytes >= minimumSelfSizeBytes
|
||||
);
|
||||
const retainedNodes = new Set<number>();
|
||||
for (const component of large) {
|
||||
for (const node of component.nodes) retainedNodes.add(node);
|
||||
}
|
||||
return {
|
||||
matchingPreviewNodes: matching.length,
|
||||
maxSelfSizeBytes: matching.reduce(
|
||||
(maximum, entry) => Math.max(maximum, entry.selfSizeBytes),
|
||||
0
|
||||
),
|
||||
maxBackingRetainedSizeBytes: components.reduce(
|
||||
(maximum, component) => Math.max(maximum, component.retainedSizeBytes),
|
||||
0
|
||||
),
|
||||
retainedContextSelfSizeBytes: [...retainedNodes].reduce(
|
||||
(total, node) => total + parsed.nodes[node * nodeWidth + nodeSelfSizeIndex],
|
||||
0
|
||||
),
|
||||
largeBackingRetained: large.length > 0,
|
||||
largeNodeIndexes: large.map((component) => component.root),
|
||||
};
|
||||
}
|
||||
|
||||
function analyzeSnapshot(snapshotFile: string, marker: string, minimumSelfSizeBytes: number) {
|
||||
const parsed = JSON.parse(fs.readFileSync(snapshotFile, "utf8")) as HeapSnapshot;
|
||||
const meta = parsed.snapshot.meta;
|
||||
const nodeFields = meta.node_fields;
|
||||
const edgeFields = meta.edge_fields;
|
||||
const nodeWidth = nodeFields.length;
|
||||
const edgeWidth = edgeFields.length;
|
||||
const nodeTypeIndex = nodeFields.indexOf("type");
|
||||
const nodeNameIndex = nodeFields.indexOf("name");
|
||||
const edgeCountIndex = nodeFields.indexOf("edge_count");
|
||||
const edgeTypeIndex = edgeFields.indexOf("type");
|
||||
const edgeNameIndex = edgeFields.indexOf("name_or_index");
|
||||
const edgeTargetIndex = edgeFields.indexOf("to_node");
|
||||
const nodeTypes = meta.node_types[nodeTypeIndex] as string[];
|
||||
const edgeTypes = meta.edge_types[edgeTypeIndex] as string[];
|
||||
const nodeCount = parsed.nodes.length / nodeWidth;
|
||||
const classification = classifyContextBackingCandidates(parsed, marker, minimumSelfSizeBytes);
|
||||
const candidateNodes = new Set(classification.largeNodeIndexes);
|
||||
|
||||
const parents: Array<Array<{ from: number; edgeType: string; edgeName: string }> | undefined> =
|
||||
new Array(nodeCount);
|
||||
let edgeOffset = 0;
|
||||
for (let from = 0; from < nodeCount; from += 1) {
|
||||
const nodeOffset = from * nodeWidth;
|
||||
const edgeCount = parsed.nodes[nodeOffset + edgeCountIndex];
|
||||
for (let local = 0; local < edgeCount; local += 1) {
|
||||
const type = edgeTypes[parsed.edges[edgeOffset + edgeTypeIndex]];
|
||||
const rawName = parsed.edges[edgeOffset + edgeNameIndex];
|
||||
const to = parsed.edges[edgeOffset + edgeTargetIndex] / nodeWidth;
|
||||
if (type !== "weak") {
|
||||
const edgeName =
|
||||
type === "element" || type === "hidden" ? String(rawName) : parsed.strings[rawName];
|
||||
(parents[to] ??= []).push({ from, edgeType: type, edgeName: edgeName ?? "" });
|
||||
}
|
||||
edgeOffset += edgeWidth;
|
||||
}
|
||||
}
|
||||
|
||||
const describeNode = (index: number) => {
|
||||
const offset = index * nodeWidth;
|
||||
return {
|
||||
type: nodeTypes[parsed.nodes[offset + nodeTypeIndex]],
|
||||
name: safeLabel(parsed.strings[parsed.nodes[offset + nodeNameIndex]] ?? "", marker),
|
||||
};
|
||||
};
|
||||
|
||||
const paths: unknown[] = [];
|
||||
for (const target of [...candidateNodes].slice(0, 5)) {
|
||||
const queue: Array<{ node: number; path: Array<Record<string, unknown>> }> = [
|
||||
{ node: target, path: [{ node: describeNode(target) }] },
|
||||
];
|
||||
const seen = new Set([target]);
|
||||
let found: Array<Record<string, unknown>> | null = null;
|
||||
while (queue.length > 0 && !found) {
|
||||
const current = queue.shift() as { node: number; path: Array<Record<string, unknown>> };
|
||||
if (current.node === 0 || current.path.length >= 32) {
|
||||
found = current.path;
|
||||
break;
|
||||
}
|
||||
for (const parent of parents[current.node] ?? []) {
|
||||
if (seen.has(parent.from)) continue;
|
||||
seen.add(parent.from);
|
||||
const nextPath = [
|
||||
...current.path,
|
||||
{
|
||||
retainedBy: describeNode(parent.from),
|
||||
edgeType: parent.edgeType,
|
||||
edgeName: safeLabel(parent.edgeName, marker),
|
||||
},
|
||||
];
|
||||
if (parent.from === 0) {
|
||||
found = nextPath;
|
||||
break;
|
||||
}
|
||||
queue.push({ node: parent.from, path: nextPath });
|
||||
}
|
||||
}
|
||||
paths.push(found ?? [{ node: describeNode(target) }, { finding: "no root within 32 edges" }]);
|
||||
}
|
||||
|
||||
return {
|
||||
matchingContextPreviewNodes: classification.matchingPreviewNodes,
|
||||
maxContextNodeSelfSizeBytes: classification.maxSelfSizeBytes,
|
||||
maxContextBackingRetainedSizeBytes: classification.maxBackingRetainedSizeBytes,
|
||||
retainedContextSelfSizeBytes: classification.retainedContextSelfSizeBytes,
|
||||
minimumLargeBackingSelfSizeBytes: minimumSelfSizeBytes,
|
||||
largeBackingRetained: classification.largeBackingRetained,
|
||||
finding:
|
||||
candidateNodes.size === 0
|
||||
? "Only detached context previews remained; no context-sized backing string crossed the V8 self_size threshold."
|
||||
: "A context-sized backing string remained after forced GC; redacted root paths follow.",
|
||||
paths,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Enforce the physical large-backing retention gate.
|
||||
* @param result - Analyzer verdict to enforce.
|
||||
* @returns Nothing.
|
||||
* @throws {Error} If a context-sized backing string survived forced garbage collection.
|
||||
*/
|
||||
export function assertNoLargeBacking(result: { largeBackingRetained: boolean }): void {
|
||||
if (result.largeBackingRetained) {
|
||||
throw new Error("context-sized backing string remained after forced GC");
|
||||
}
|
||||
}
|
||||
|
||||
async function runAnalyzer(): Promise<void> {
|
||||
process.umask(0o077);
|
||||
const snapshotArg = argValue("--snapshot");
|
||||
if (!snapshotArg) throw new Error("analyzer requires --snapshot");
|
||||
const snapshotFile = path.resolve(snapshotArg);
|
||||
const outputArg = argValue("--output-dir");
|
||||
const marker = argValue("--marker") ?? "";
|
||||
try {
|
||||
if (!outputArg || !marker) {
|
||||
throw new Error("analyzer requires --output-dir and --marker");
|
||||
}
|
||||
const minimumSelfSizeBytes = positiveIntArg("--minimum-self-size-bytes", 1_024);
|
||||
const outputDir = path.resolve(outputArg);
|
||||
const result = analyzeSnapshot(snapshotFile, marker, minimumSelfSizeBytes);
|
||||
const snapshotStat = fs.statSync(snapshotFile);
|
||||
const snapshotSha256 = await sha256File(snapshotFile);
|
||||
const redactedResult = {
|
||||
...result,
|
||||
snapshot: {
|
||||
state: "deleted-after-local-analysis",
|
||||
byteSize: snapshotStat.size,
|
||||
sha256: snapshotSha256,
|
||||
},
|
||||
};
|
||||
writePrivateJson(path.join(outputDir, "retainers.redacted.json"), redactedResult);
|
||||
assertNoLargeBacking(result);
|
||||
process.stdout.write(JSON.stringify(redactedResult) + "\n");
|
||||
} finally {
|
||||
fs.rmSync(snapshotFile, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function harnessHash(): string {
|
||||
return createHash("sha256")
|
||||
.update(fs.readFileSync(new URL(import.meta.url)))
|
||||
.digest("hex");
|
||||
}
|
||||
|
||||
function gitOutput(args: string[]): string {
|
||||
const result = spawnSync("git", args, {
|
||||
encoding: "utf8",
|
||||
env: buildWorkerEnv(process.env),
|
||||
});
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`git ${args.join(" ")} failed: ${result.stderr.trim()}`);
|
||||
}
|
||||
return result.stdout.trim();
|
||||
}
|
||||
|
||||
function sha256(value: string | Buffer): string {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
function sourceProvenance() {
|
||||
const testedCommit = gitOutput(["rev-parse", "HEAD"]);
|
||||
if (!/^[0-9a-f]{40}$/.test(testedCommit)) {
|
||||
throw new Error(`invalid git HEAD: ${testedCommit || "empty"}`);
|
||||
}
|
||||
// `git branch --show-current` is empty on a detached HEAD, which is the normal checkout
|
||||
// state for a CI PR run and for this fix worktree's own detached `git worktree add` — fall
|
||||
// back to a descriptive marker instead of treating that as an error.
|
||||
const branch = gitOutput(["branch", "--show-current"]) || `detached@${testedCommit.slice(0, 12)}`;
|
||||
const sourceFiles = Object.fromEntries(
|
||||
PROVENANCE_FILES.map((file) => {
|
||||
if (!fs.existsSync(file)) throw new Error(`provenance file missing: ${file}`);
|
||||
return [file, sha256(fs.readFileSync(file))];
|
||||
})
|
||||
);
|
||||
const trackedDiff = gitOutput(["diff", "--binary", "HEAD", "--", ...PROVENANCE_FILES]);
|
||||
return {
|
||||
testedCommit,
|
||||
branch,
|
||||
statusPorcelain: gitOutput(["status", "--porcelain=v1", "--untracked-files=all"])
|
||||
.split("\n")
|
||||
.filter(Boolean),
|
||||
trackedDiffSha256: sha256(trackedDiff),
|
||||
sourceSetSha256: sha256(JSON.stringify(sourceFiles)),
|
||||
sourceFiles,
|
||||
};
|
||||
}
|
||||
|
||||
function sha256File(file: string): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const hash = createHash("sha256");
|
||||
const input = fs.createReadStream(file);
|
||||
input.on("error", reject);
|
||||
input.on("data", (chunk) => hash.update(chunk));
|
||||
input.on("end", () => resolve(hash.digest("hex")));
|
||||
});
|
||||
}
|
||||
|
||||
async function runDriver(): Promise<void> {
|
||||
process.umask(0o077);
|
||||
const outputDir = path.resolve(
|
||||
argValue("--output-dir") ??
|
||||
path.join(os.tmpdir(), `omniroute-JON-562-${new Date().toISOString().replace(/[:.]/g, "-")}`)
|
||||
);
|
||||
const iterations = positiveIntArg("--iterations", 6);
|
||||
const tokenCases = (argValue("--tokens") ?? DEFAULT_TOKEN_EQUIVALENTS.join(","))
|
||||
.split(",")
|
||||
.map((raw) => Number(raw));
|
||||
if (tokenCases.some((value) => !Number.isSafeInteger(value) || value < 64)) {
|
||||
throw new RangeError("--tokens must be a comma-separated list of integers >= 64");
|
||||
}
|
||||
privateDirectory(outputDir);
|
||||
|
||||
const cases: unknown[] = [];
|
||||
const scriptFile = fileURLToPath(import.meta.url);
|
||||
for (const tokenEquivalent of tokenCases) {
|
||||
const caseDir = path.join(outputDir, `context-${tokenEquivalent}`);
|
||||
privateDirectory(caseDir);
|
||||
const snapshotFile = path.join(caseDir, "post-gc.heapsnapshot");
|
||||
await withRawSnapshotCleanup(snapshotFile, async () => {
|
||||
const worker = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--expose-gc",
|
||||
"--import",
|
||||
"tsx/esm",
|
||||
scriptFile,
|
||||
"--worker",
|
||||
"--snapshot-handoff",
|
||||
"--tokens",
|
||||
String(tokenEquivalent),
|
||||
"--iterations",
|
||||
String(iterations),
|
||||
"--output-dir",
|
||||
caseDir,
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
env: buildWorkerEnv(process.env),
|
||||
// Measured ~230s wall for the 100k-token case alone (tsx/esm boot of the full
|
||||
// route/handler module graph + real request lifecycle) on an idle box — 180s left
|
||||
// no margin and made SIGKILL-on-timeout indistinguishable from a real worker crash
|
||||
// (`worker.status` is `null`, which already fails the `!== 0` check below).
|
||||
timeout: 300_000,
|
||||
killSignal: "SIGKILL",
|
||||
}
|
||||
);
|
||||
if (worker.status !== 0) {
|
||||
throw new Error(`worker ${tokenEquivalent} failed:\n${worker.stdout}\n${worker.stderr}`);
|
||||
}
|
||||
|
||||
const analyzer = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--max-old-space-size=4096",
|
||||
"--import",
|
||||
"tsx/esm",
|
||||
scriptFile,
|
||||
"--analyze-snapshot",
|
||||
"--snapshot",
|
||||
snapshotFile,
|
||||
"--output-dir",
|
||||
caseDir,
|
||||
"--marker",
|
||||
markerFor(tokenEquivalent),
|
||||
"--minimum-self-size-bytes",
|
||||
String(Math.floor((tokenEquivalent * BYTES_PER_TOKEN_EQUIVALENT) / 2)),
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
env: buildWorkerEnv(process.env),
|
||||
timeout: 180_000,
|
||||
killSignal: "SIGKILL",
|
||||
}
|
||||
);
|
||||
if (analyzer.status !== 0) {
|
||||
throw new Error(
|
||||
`snapshot analyzer ${tokenEquivalent} failed:\n${analyzer.stdout}\n${analyzer.stderr}`
|
||||
);
|
||||
}
|
||||
const retaining = JSON.parse(
|
||||
fs.readFileSync(path.join(caseDir, "retainers.redacted.json"), "utf8")
|
||||
);
|
||||
assertNoLargeBacking(retaining);
|
||||
const caseManifestFile = path.join(caseDir, "manifest.json");
|
||||
const caseManifest = JSON.parse(fs.readFileSync(caseManifestFile, "utf8"));
|
||||
caseManifest.heapSnapshot = retaining.snapshot;
|
||||
writePrivateJson(caseManifestFile, caseManifest);
|
||||
cases.push(caseManifest);
|
||||
});
|
||||
}
|
||||
|
||||
const provenance = sourceProvenance();
|
||||
const workloadManifest = {
|
||||
ticket: "JON-562",
|
||||
status: "complete",
|
||||
testedCommit: provenance.testedCommit,
|
||||
provenance,
|
||||
harnessSha256: harnessHash(),
|
||||
baseBranch: "release/v3.8.51",
|
||||
inheritedBaseRed: "diegosouzapw/OmniRoute#12732",
|
||||
variedFactor: "serialized context bytes only",
|
||||
fixedConditions: {
|
||||
concurrency: 1,
|
||||
cancellation: "none",
|
||||
stream: true,
|
||||
iterations,
|
||||
messageCount: 1,
|
||||
toolCount: 0,
|
||||
provider: "local-fetch-stub",
|
||||
externalProviderCalls: 0,
|
||||
},
|
||||
tokenEquivalentCases: tokenCases,
|
||||
bytesPerTokenEquivalent: BYTES_PER_TOKEN_EQUIVALENT,
|
||||
byteBudgetEvidence:
|
||||
"Unit-level only: real-route responses in this checkpoint are small, so the 256-entry cap binds before the 16 MiB byte cap. The route matrix proves sliced backing detachment, not a 16 MiB runtime plateau.",
|
||||
cases,
|
||||
};
|
||||
writePrivateJson(path.join(outputDir, "workload-manifest.json"), workloadManifest);
|
||||
process.stdout.write(JSON.stringify({ outputDir, status: "complete" }) + "\n");
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
if (process.argv.includes("--worker")) return runWorker();
|
||||
if (process.argv.includes("--analyze-snapshot")) return runAnalyzer();
|
||||
return runDriver();
|
||||
}
|
||||
|
||||
const invokedPath = process.argv[1] ? pathToFileURL(path.resolve(process.argv[1])).href : "";
|
||||
if (invokedPath === import.meta.url) {
|
||||
main().catch((error: unknown) => {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
process.stderr.write(`[JON-562] ${message}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
30
scripts/quality/release-acceptance/closeOracle.mjs
Normal file
30
scripts/quality/release-acceptance/closeOracle.mjs
Normal file
@@ -0,0 +1,30 @@
|
||||
const CLOSE_RE =
|
||||
/gh issue close\b|issues\.update\b|state=closed/g;
|
||||
|
||||
const KEYWORD_RE = new RegExp(
|
||||
String.raw`\b(?:fix(?:es|ed)?|close[sd]?|resolve[sd]?)\s+#(\d+)\b`,
|
||||
"i"
|
||||
);
|
||||
|
||||
export function findTrackerCloses(workflowText) {
|
||||
const hits = [];
|
||||
const lines = String(workflowText ?? "").split(/\n/);
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
CLOSE_RE.lastIndex = 0;
|
||||
if (CLOSE_RE.test(lines[i])) {
|
||||
hits.push({ line: i + 1, text: lines[i].trim() });
|
||||
}
|
||||
CLOSE_RE.lastIndex = 0;
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
export function closingKeywordInBody(body, tracker = 12732) {
|
||||
const re = new RegExp(KEYWORD_RE.source, KEYWORD_RE.flags.includes("g") ? KEYWORD_RE.flags : `${KEYWORD_RE.flags}g`);
|
||||
const text = String(body ?? "");
|
||||
let m;
|
||||
while ((m = re.exec(text)) !== null) {
|
||||
if (Number(m[1]) === Number(tracker)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
102
scripts/quality/release-acceptance/inventory.mjs
Normal file
102
scripts/quality/release-acceptance/inventory.mjs
Normal file
@@ -0,0 +1,102 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import {
|
||||
COLLECTORS,
|
||||
globToRegExp,
|
||||
} from "../../check/check-test-discovery.mjs";
|
||||
|
||||
const UNIT_CI_GLOBS = new Set([
|
||||
"tests/unit/*.test.ts",
|
||||
"tests/unit/{api,auth,authz,build,cli,cli-helper,combo,compression,correctness,cors,db,db-adapters,docs,gamification,guardrails,lib,mcp,memory,runtime,security,services,settings,shared,translator,ui,usage}/**/*.test.ts",
|
||||
"tests/unit/dashboard/**/*.test.ts",
|
||||
"tests/unit/serial/**/*.test.ts",
|
||||
"tests/unit/**/*.test.mjs",
|
||||
]);
|
||||
|
||||
const INTEGRATION_GLOBS = new Set([
|
||||
"tests/integration/*.test.ts",
|
||||
"tests/integration/combo-matrix/*.test.ts",
|
||||
]);
|
||||
|
||||
function inScope(collector, scopeSuites) {
|
||||
const suites = new Set(scopeSuites);
|
||||
if (suites.has("test:unit:ci") && UNIT_CI_GLOBS.has(collector.glob)) return true;
|
||||
if (suites.has("test:integration") && INTEGRATION_GLOBS.has(collector.glob)) return true;
|
||||
if (suites.has("test:vitest") && collector.sources?.includes("vitest.mcp.config.ts")) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function walkTestFiles(root = process.cwd()) {
|
||||
const out = [];
|
||||
function walk(dir) {
|
||||
if (!fs.existsSync(dir)) return;
|
||||
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
if (e.name === "node_modules" || e.name === ".git") continue;
|
||||
const p = path.join(dir, e.name);
|
||||
if (e.isDirectory()) walk(p);
|
||||
else if (/\.(test|spec)\.(ts|tsx|mjs|js)$/.test(e.name)) {
|
||||
out.push(path.relative(root, p).split(path.sep).join("/"));
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(path.join(root, "tests"));
|
||||
walk(path.join(root, "open-sse"));
|
||||
walk(path.join(root, "src"));
|
||||
return out;
|
||||
}
|
||||
|
||||
export function canonicalSet(scopeSuites, collectors = COLLECTORS, files) {
|
||||
const scoped = collectors.filter((c) => inScope(c, scopeSuites));
|
||||
const regexes = scoped.map((c) => globToRegExp(c.glob));
|
||||
const discovered = files ?? walkTestFiles();
|
||||
return discovered.filter((f) => regexes.some((re) => re.test(f)));
|
||||
}
|
||||
|
||||
export function knownUnexecuted(scopeSuites, collectors = COLLECTORS, baseline, files) {
|
||||
const discovered = files ?? walkTestFiles();
|
||||
const orphans = baseline?.orphans ?? [];
|
||||
const outOfScope = collectors.filter((c) => !inScope(c, scopeSuites));
|
||||
const collectorsOut = outOfScope.map((c) => {
|
||||
const re = globToRegExp(c.glob);
|
||||
const count = discovered.filter((f) => re.test(f)).length;
|
||||
return {
|
||||
glob: c.glob,
|
||||
count,
|
||||
reason: "collector runner is not a suite of this scope",
|
||||
};
|
||||
});
|
||||
return {
|
||||
orphans: { count: orphans.length, paths: orphans },
|
||||
collectors: collectorsOut,
|
||||
};
|
||||
}
|
||||
|
||||
export function inventoryErrors(scopeSuites, collectors, baseline, discoveredFiles) {
|
||||
const errors = [];
|
||||
const full = COLLECTORS;
|
||||
const givenGlobs = new Set(collectors.map((c) => c.glob));
|
||||
for (const c of full) {
|
||||
if (!givenGlobs.has(c.glob)) {
|
||||
errors.push({
|
||||
code: "collector_omitted",
|
||||
glob: c.glob,
|
||||
detail: `collector ${c.glob} omitted without known_unexecuted listing`,
|
||||
});
|
||||
}
|
||||
}
|
||||
const ku = knownUnexecuted(scopeSuites, collectors, baseline, discoveredFiles);
|
||||
const knownGlobs = new Set(ku.collectors.map((c) => c.glob));
|
||||
const knownOrphans = new Set(ku.orphans.paths);
|
||||
const scoped = collectors.filter((c) => inScope(c, scopeSuites));
|
||||
const regexes = scoped.map((c) => globToRegExp(c.glob));
|
||||
for (const f of discoveredFiles ?? []) {
|
||||
const inCanonical = regexes.some((re) => re.test(f));
|
||||
const inKnown = knownOrphans.has(f) || [...knownGlobs].some((g) => globToRegExp(g).test(f));
|
||||
if (!inCanonical && !inKnown) {
|
||||
errors.push({ code: "unmapped_file", path: f, detail: "discovered file belongs to no set" });
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
41
scripts/quality/release-acceptance/nodeReporter.mjs
Normal file
41
scripts/quality/release-acceptance/nodeReporter.mjs
Normal file
@@ -0,0 +1,41 @@
|
||||
const SUBTEST = /^# Subtest:\s+(\S+)/;
|
||||
const RESULT = /^(ok|not ok)\s+\d+\s+-\s+(\S+)/;
|
||||
|
||||
export function fromNodeTestTap(tapText, argvFiles) {
|
||||
const completed = [];
|
||||
const failed = [];
|
||||
const seen = new Set();
|
||||
const lines = String(tapText ?? "").split(/\r?\n/);
|
||||
let pending = null;
|
||||
for (const line of lines) {
|
||||
const sub = line.match(SUBTEST);
|
||||
if (sub) {
|
||||
pending = sub[1];
|
||||
continue;
|
||||
}
|
||||
const res = line.match(RESULT);
|
||||
if (res) {
|
||||
const file = pending;
|
||||
const ok = res[1] === "ok";
|
||||
if (file) {
|
||||
seen.add(file);
|
||||
if (!ok) {
|
||||
if (!failed.includes(file)) failed.push(file);
|
||||
const i = completed.indexOf(file);
|
||||
if (i >= 0) completed.splice(i, 1);
|
||||
} else if (!failed.includes(file) && !completed.includes(file)) {
|
||||
completed.push(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const attempted = [...argvFiles];
|
||||
const missing = attempted.filter((f) => !seen.has(f));
|
||||
return {
|
||||
completed,
|
||||
attempted,
|
||||
missing,
|
||||
failed,
|
||||
pass: completed.length > 0 && missing.length === 0 && failed.length === 0,
|
||||
};
|
||||
}
|
||||
243
scripts/quality/release-acceptance/reduce.mjs
Normal file
243
scripts/quality/release-acceptance/reduce.mjs
Normal file
@@ -0,0 +1,243 @@
|
||||
import { gateKey, sameKey } from "./types.mjs";
|
||||
|
||||
export function classifyDependent(prereqStatus, dependentKey, prereqKey) {
|
||||
if (prereqStatus === "FAIL") {
|
||||
return { status: "FAIL", cause: prereqKey };
|
||||
}
|
||||
if (prereqStatus === "INFRA_ERROR") {
|
||||
return { status: "INFRA_ERROR", cause: prereqKey };
|
||||
}
|
||||
if (prereqStatus == null) {
|
||||
return {
|
||||
status: "INFRA_ERROR",
|
||||
cause: prereqKey,
|
||||
evidence_error: {
|
||||
code: "prerequisite_missing",
|
||||
gate: dependentKey,
|
||||
detail: `missing prerequisite ${prereqKey.gate_id}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
if (prereqStatus === "SKIPPED") {
|
||||
return { status: "SKIPPED", cause: prereqKey };
|
||||
}
|
||||
return { status: "RUN", cause: null };
|
||||
}
|
||||
|
||||
function requiredSet(plan) {
|
||||
return plan.required_gates ?? [];
|
||||
}
|
||||
|
||||
function optionalSet(plan) {
|
||||
return plan.optional_gates ?? [];
|
||||
}
|
||||
|
||||
function isRequired(plan, k) {
|
||||
return requiredSet(plan).some((r) => sameKey(r, k));
|
||||
}
|
||||
|
||||
function copies(gates, k) {
|
||||
return gates.filter((g) => sameKey(gateKey(g), k));
|
||||
}
|
||||
|
||||
function copiesByGateId(gates, gateId) {
|
||||
return gates.filter((g) => g.gate_id === gateId);
|
||||
}
|
||||
|
||||
function uniqueKeys(keys) {
|
||||
const out = [];
|
||||
for (const k of keys) {
|
||||
if (!out.some((existing) => sameKey(existing, k))) out.push(k);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function keysForGateId(plan, gates, gateId) {
|
||||
return uniqueKeys([
|
||||
...copiesByGateId(gates, gateId).map((g) => gateKey(g)),
|
||||
...requiredSet(plan).filter((k) => k.gate_id === gateId),
|
||||
...optionalSet(plan).filter((k) => k.gate_id === gateId),
|
||||
]);
|
||||
}
|
||||
|
||||
function statusOf(gates, k) {
|
||||
const list = copies(gates, k);
|
||||
if (list.length === 0) return null;
|
||||
if (list.some((g) => g.status === "INFRA_ERROR")) return "INFRA_ERROR";
|
||||
if (list.some((g) => g.status === "FAIL")) return "FAIL";
|
||||
if (list.some((g) => g.status === "SKIPPED")) return "SKIPPED";
|
||||
return list[0].status;
|
||||
}
|
||||
|
||||
function statusOfGateId(gates, gateId) {
|
||||
const list = copiesByGateId(gates, gateId);
|
||||
if (list.length === 0) return null;
|
||||
if (list.some((g) => g.status === "INFRA_ERROR")) return "INFRA_ERROR";
|
||||
if (list.some((g) => g.status === "FAIL")) return "FAIL";
|
||||
if (list.some((g) => g.status === "SKIPPED")) return "SKIPPED";
|
||||
return list[0].status;
|
||||
}
|
||||
|
||||
function pushEvidenceError(evidence_errors, err) {
|
||||
if (!err) return;
|
||||
const already = evidence_errors.some(
|
||||
(e) =>
|
||||
e.code === err.code &&
|
||||
e.detail === err.detail &&
|
||||
e.gate?.gate_id === err.gate?.gate_id
|
||||
);
|
||||
if (!already) evidence_errors.push(err);
|
||||
}
|
||||
|
||||
function patchDependent(gates, depKey, classified, prereqKey, evidence_errors, identity) {
|
||||
const matches = copies(gates, depKey);
|
||||
const reason =
|
||||
classified.status === "SKIPPED" ? `classified from ${prereqKey.gate_id}` : undefined;
|
||||
const exit_code = classified.status === "FAIL" ? 1 : 2;
|
||||
if (matches.length === 0) {
|
||||
gates.push({
|
||||
gate_id: depKey.gate_id,
|
||||
suite_id: depKey.suite_id,
|
||||
shard_index: depKey.shard_index,
|
||||
shard_total: depKey.shard_total,
|
||||
tested_sha: identity.tested_sha || "0".repeat(40),
|
||||
run_id: identity.run_id ?? "0",
|
||||
run_attempt: identity.run_attempt ?? 1,
|
||||
command_id: depKey.gate_id,
|
||||
gate_type: "artifact",
|
||||
status: classified.status,
|
||||
cause: classified.cause,
|
||||
reason,
|
||||
exit_code,
|
||||
duration_ms: 0,
|
||||
evidence: [],
|
||||
});
|
||||
if (classified.evidence_error) pushEvidenceError(evidence_errors, classified.evidence_error);
|
||||
return true;
|
||||
}
|
||||
let changed = false;
|
||||
for (const existing of matches) {
|
||||
if (
|
||||
existing.status === classified.status &&
|
||||
((existing.cause == null && classified.cause == null) ||
|
||||
(existing.cause && classified.cause && sameKey(existing.cause, classified.cause)))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
existing.status = classified.status;
|
||||
existing.cause = classified.cause;
|
||||
existing.exit_code = exit_code;
|
||||
if (classified.status === "SKIPPED" && !existing.reason) existing.reason = reason;
|
||||
changed = true;
|
||||
}
|
||||
if (changed && classified.evidence_error) {
|
||||
pushEvidenceError(evidence_errors, classified.evidence_error);
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
function assertAcyclic(deps) {
|
||||
const visiting = new Set();
|
||||
const done = new Set();
|
||||
function walk(id) {
|
||||
if (done.has(id)) return;
|
||||
if (visiting.has(id)) throw new Error("cyclic prerequisite");
|
||||
visiting.add(id);
|
||||
if (Object.hasOwn(deps, id)) walk(deps[id]);
|
||||
visiting.delete(id);
|
||||
done.add(id);
|
||||
}
|
||||
for (const id of Object.keys(deps)) walk(id);
|
||||
}
|
||||
|
||||
export function reduce(plan, records) {
|
||||
const deps = plan.dependencies ?? {};
|
||||
assertAcyclic(deps);
|
||||
for (const [depId, prereqId] of Object.entries(deps)) {
|
||||
const requiredDep = requiredSet(plan).some((k) => k.gate_id === depId);
|
||||
const optionalPrereq = optionalSet(plan).some((k) => k.gate_id === prereqId);
|
||||
if (requiredDep && optionalPrereq) {
|
||||
throw new Error("optional prerequisite");
|
||||
}
|
||||
}
|
||||
|
||||
const gates = [];
|
||||
const evidence_errors = [];
|
||||
|
||||
for (const rec of records) {
|
||||
const k = gateKey(rec);
|
||||
const copy = { ...rec, cause: rec.cause ?? null };
|
||||
if (copy.status === "SKIPPED" && isRequired(plan, k) && !copy.reason) {
|
||||
copy.reason = "required skipped";
|
||||
}
|
||||
gates.push(copy);
|
||||
}
|
||||
|
||||
const identity = plan.identity ?? {};
|
||||
const edges = Object.entries(deps);
|
||||
let changed = true;
|
||||
let guard = edges.length + 1;
|
||||
while (changed && guard-- > 0) {
|
||||
changed = false;
|
||||
for (const [depId, prereqId] of edges) {
|
||||
const prereqKey = { gate_id: prereqId, suite_id: null, shard_index: null, shard_total: null };
|
||||
let depKeys = keysForGateId(plan, gates, depId);
|
||||
if (depKeys.length === 0) {
|
||||
depKeys = [{ gate_id: depId, suite_id: null, shard_index: null, shard_total: null }];
|
||||
}
|
||||
const prereqStatus = statusOfGateId(gates, prereqId);
|
||||
for (const depKey of depKeys) {
|
||||
const classified = classifyDependent(prereqStatus, depKey, prereqKey);
|
||||
if (classified.status === "RUN") continue;
|
||||
if (patchDependent(gates, depKey, classified, prereqKey, evidence_errors, identity)) {
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const k of requiredSet(plan)) {
|
||||
const rec = gates.find((g) => sameKey(gateKey(g), k));
|
||||
if (!rec) {
|
||||
evidence_errors.push({
|
||||
code: "missing_record",
|
||||
gate: k,
|
||||
detail: `required gate ${k.gate_id} has no record`,
|
||||
});
|
||||
} else if (rec.status === "SKIPPED") {
|
||||
evidence_errors.push({
|
||||
code: "required_skipped",
|
||||
gate: k,
|
||||
detail: rec.reason ?? "required gate SKIPPED",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const required = requiredSet(plan);
|
||||
if (required.length === 0) {
|
||||
evidence_errors.push({
|
||||
code: "empty_required_set",
|
||||
gate: { gate_id: "schema", suite_id: null, shard_index: null, shard_total: null },
|
||||
detail: "required_gates is empty",
|
||||
});
|
||||
}
|
||||
|
||||
let verdict = "VERIFIED";
|
||||
const hasFail = gates.some(
|
||||
(g) => g.status === "FAIL" && isRequired(plan, gateKey(g)) && statusOf(gates, gateKey(g)) === "FAIL"
|
||||
);
|
||||
const hasUnverified =
|
||||
evidence_errors.length > 0 ||
|
||||
gates.some(
|
||||
(g) =>
|
||||
isRequired(plan, gateKey(g)) &&
|
||||
(g.status === "SKIPPED" || g.status === "INFRA_ERROR")
|
||||
);
|
||||
if (hasFail) verdict = "FAILED";
|
||||
else if (hasUnverified) verdict = "UNVERIFIED";
|
||||
else if (required.some((k) => !gates.some((g) => sameKey(gateKey(g), k)))) {
|
||||
verdict = "UNVERIFIED";
|
||||
}
|
||||
|
||||
return { verdict, evidence_errors, gates };
|
||||
}
|
||||
50
scripts/quality/release-acceptance/staticAdapter.mjs
Normal file
50
scripts/quality/release-acceptance/staticAdapter.mjs
Normal file
@@ -0,0 +1,50 @@
|
||||
export function adaptCompiler({ commandId, inputDigest, exitCode, diagnostics }) {
|
||||
const diags = Array.isArray(diagnostics) ? diagnostics : [];
|
||||
const digest = typeof inputDigest === "string" ? inputDigest : "";
|
||||
if (exitCode === 0 && digest.length > 0) {
|
||||
return {
|
||||
command_id: commandId,
|
||||
input_digest: digest,
|
||||
exit_code: 0,
|
||||
diagnostics: diags,
|
||||
status: "PASS",
|
||||
};
|
||||
}
|
||||
if (exitCode === 0 && digest.length === 0) {
|
||||
return {
|
||||
command_id: commandId,
|
||||
input_digest: digest,
|
||||
exit_code: 0,
|
||||
diagnostics: diags,
|
||||
status: "INFRA_ERROR",
|
||||
};
|
||||
}
|
||||
return {
|
||||
command_id: commandId,
|
||||
input_digest: digest,
|
||||
exit_code: exitCode,
|
||||
diagnostics: diags,
|
||||
status: "FAIL",
|
||||
};
|
||||
}
|
||||
|
||||
export function adaptScript({ commandId, inputDigest, exitCode, stdout }) {
|
||||
const digest = typeof inputDigest === "string" ? inputDigest : "";
|
||||
let parsed = null;
|
||||
if (typeof stdout === "string" && stdout.trim()) {
|
||||
try {
|
||||
parsed = JSON.parse(stdout);
|
||||
} catch {
|
||||
parsed = null;
|
||||
}
|
||||
}
|
||||
const diagnostics = parsed ?? { input_digest: digest, exit_code: exitCode, diagnostics: stdout ?? "" };
|
||||
const status = exitCode === 0 ? (digest ? "PASS" : "INFRA_ERROR") : "FAIL";
|
||||
return {
|
||||
command_id: commandId,
|
||||
input_digest: digest,
|
||||
exit_code: exitCode,
|
||||
diagnostics,
|
||||
status,
|
||||
};
|
||||
}
|
||||
19
scripts/quality/release-acceptance/types.mjs
Normal file
19
scripts/quality/release-acceptance/types.mjs
Normal file
@@ -0,0 +1,19 @@
|
||||
export const STATUSES = Object.freeze(["PASS", "FAIL", "INFRA_ERROR", "SKIPPED"]);
|
||||
export const VERDICTS = Object.freeze(["VERIFIED", "FAILED", "UNVERIFIED"]);
|
||||
|
||||
export function gateKey(rec) {
|
||||
return {
|
||||
gate_id: rec.gate_id,
|
||||
suite_id: rec.suite_id ?? null,
|
||||
shard_index: rec.shard_index ?? null,
|
||||
shard_total: rec.shard_total ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export function keyId(k) {
|
||||
return `${k.gate_id}\0${k.suite_id ?? ""}\0${k.shard_index ?? ""}\0${k.shard_total ?? ""}`;
|
||||
}
|
||||
|
||||
export function sameKey(a, b) {
|
||||
return keyId(a) === keyId(b);
|
||||
}
|
||||
81
scripts/quality/validate-release-acceptance.mjs
Normal file
81
scripts/quality/validate-release-acceptance.mjs
Normal file
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env node
|
||||
import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import Ajv from "ajv";
|
||||
import { reduce } from "./release-acceptance/reduce.mjs";
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
||||
|
||||
function loadJson(p) {
|
||||
return JSON.parse(readFileSync(p, "utf8"));
|
||||
}
|
||||
|
||||
export function exitFor(verdict) {
|
||||
if (verdict === "VERIFIED") return 0;
|
||||
if (verdict === "FAILED") return 1;
|
||||
return 2;
|
||||
}
|
||||
|
||||
export function reduceManifests(plan, manifests) {
|
||||
const records = [];
|
||||
for (const m of manifests) {
|
||||
if (Array.isArray(m.gates)) records.push(...m.gates);
|
||||
else records.push(m);
|
||||
}
|
||||
return reduce(plan, records);
|
||||
}
|
||||
|
||||
export function validateReport(report, schema) {
|
||||
const ajv = new Ajv({ allErrors: true, strict: false });
|
||||
const validate = ajv.compile(schema);
|
||||
return { ok: validate(report), errors: validate.errors };
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = { plan: null, manifests: null, out: join(ROOT, "release-acceptance-report.json") };
|
||||
for (let i = 2; i < argv.length; i++) {
|
||||
if (argv[i] === "--plan") out.plan = argv[++i];
|
||||
else if (argv[i] === "--manifests") out.manifests = argv[++i];
|
||||
else if (argv[i] === "--out") out.out = argv[++i];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv) {
|
||||
const args = parseArgs(argv);
|
||||
const plan = loadJson(args.plan);
|
||||
const schema = loadJson(join(ROOT, "config/quality/release-acceptance.schema.json"));
|
||||
const files = readdirSync(args.manifests)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.map((f) => loadJson(join(args.manifests, f)));
|
||||
const reduced = reduceManifests(plan, files);
|
||||
const report = {
|
||||
schema_version: 1,
|
||||
identity: plan.identity,
|
||||
required_gates: plan.required_gates ?? [],
|
||||
gates: reduced.gates,
|
||||
evidence_errors: reduced.evidence_errors,
|
||||
verdict: reduced.verdict,
|
||||
artifact: plan.artifact ?? null,
|
||||
};
|
||||
const { ok, errors } = validateReport(report, schema);
|
||||
if (!ok) {
|
||||
if (report.verdict !== "FAILED") report.verdict = "UNVERIFIED";
|
||||
const gate =
|
||||
Array.isArray(plan.required_gates) && plan.required_gates.length > 0
|
||||
? plan.required_gates[0]
|
||||
: { gate_id: "schema", suite_id: null, shard_index: null, shard_total: null };
|
||||
report.evidence_errors = [
|
||||
...(report.evidence_errors ?? []),
|
||||
{ code: "schema_invalid", gate, detail: JSON.stringify(errors) },
|
||||
];
|
||||
}
|
||||
mkdirSync(dirname(args.out), { recursive: true });
|
||||
writeFileSync(args.out, JSON.stringify(report, null, 2) + "\n");
|
||||
return exitFor(report.verdict);
|
||||
}
|
||||
|
||||
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
main().then((code) => process.exit(code));
|
||||
}
|
||||
@@ -1,284 +1,289 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* PII redaction for log shipping.
|
||||
*
|
||||
* Streams input (stdin or files) to stdout, replacing sensitive tokens
|
||||
* with stable redaction markers. Pure Node.js stdlib — no `npm install`.
|
||||
*
|
||||
* Recognised patterns (in order, longest match wins per position):
|
||||
*
|
||||
* 1. Anthropic API keys (sk-ant-...) → [REDACTED_API_KEY]
|
||||
* 2. Google API keys (AIza...) → [REDACTED_API_KEY]
|
||||
* 3. GitHub tokens (ghp_/gho_/ghu_/...) → [REDACTED_API_KEY]
|
||||
* 4. OpenAI keys (sk-..., sk-proj-...) → [REDACTED_API_KEY]
|
||||
* 5. AWS access keys (AKIA/ASIA) → [REDACTED_AWS_KEY]
|
||||
* 6. Bearer tokens → [REDACTED_BEARER]
|
||||
* 7. Email addresses → [REDACTED_EMAIL]
|
||||
* 8. Generic api_key=value pairs → [REDACTED_API_KEY]
|
||||
* 9. IPv4 addresses → [REDACTED_IPV4]
|
||||
* 10. IPv6 addresses → [REDACTED_IPV6]
|
||||
*
|
||||
* Provider-specific patterns are listed BEFORE the generic `sk-` rule so
|
||||
* that an `sk-ant-...` key counts as `ANTHROPIC_KEY` (not `OPENAI_KEY`)
|
||||
* for the per-call summary.
|
||||
*
|
||||
* Why stable markers: downstream log-search queries reference the
|
||||
* redaction markers (e.g. "show me all log lines with [REDACTED_IPV4]"),
|
||||
* which makes the redaction reversible by anyone with the original
|
||||
* vault lookup, but never by a log-search reader alone.
|
||||
*
|
||||
* CLI:
|
||||
* node scripts/sre/redact-logs.mjs < input.log > output.log
|
||||
* node scripts/sre/redact-logs.mjs --file access.log --output out.log
|
||||
* node scripts/sre/redact-logs.mjs --strict # exit non-zero on any match
|
||||
*
|
||||
* Library:
|
||||
* import { redact, redactString, RedactTransform } from "./scripts/sre/redact-logs.mjs";
|
||||
*
|
||||
* Salvaged from closed PR #5057 (base-stale; reimplemented on release).
|
||||
*/
|
||||
|
||||
import { createReadStream, createWriteStream } from "node:fs";
|
||||
import { TransformStream } from "node:stream/web";
|
||||
import process from "node:process";
|
||||
|
||||
// ── Pattern catalogue ─────────────────────────────────────────────────────────
|
||||
//
|
||||
// Each pattern is [name, regex, marker]. The regex uses the `g` flag so we can
|
||||
// iterate with `matchAll`. Order matters: longer / more specific patterns go
|
||||
// first so an `sk-ant-...` key counts as ANTHROPIC_KEY instead of OPENAI_KEY.
|
||||
|
||||
export const REDACT_PATTERNS = Object.freeze([
|
||||
// 1. Anthropic keys: sk-ant-api03-... / sk-ant-... (must beat the generic sk-)
|
||||
[
|
||||
"ANTHROPIC_KEY",
|
||||
/\bsk-ant-[A-Za-z0-9_\-]{20,}\b/g,
|
||||
"[REDACTED_API_KEY]",
|
||||
],
|
||||
// 2. Google API keys: AIza... (39 chars total)
|
||||
[
|
||||
"GOOGLE_KEY",
|
||||
/\bAIza[A-Za-z0-9_\-]{35}\b/g,
|
||||
"[REDACTED_API_KEY]",
|
||||
],
|
||||
// 3. GitHub tokens (classic + fine-grained + PAT prefixes)
|
||||
[
|
||||
"GITHUB_TOKEN",
|
||||
/\b(?:ghp|gho|ghu|ghs|ghr|github_pat)_[A-Za-z0-9]{30,}\b/g,
|
||||
"[REDACTED_API_KEY]",
|
||||
],
|
||||
// 4. OpenAI keys: sk-..., sk-proj-..., proj-... (after the more specific rules above)
|
||||
[
|
||||
"OPENAI_KEY",
|
||||
/\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b|\bproj-[A-Za-z0-9_\-]{20,}\b/g,
|
||||
"[REDACTED_API_KEY]",
|
||||
],
|
||||
// 5. AWS access keys — AKIA / ASIA prefixes, 20 chars total
|
||||
[
|
||||
"AWS_KEY",
|
||||
/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g,
|
||||
"[REDACTED_AWS_KEY]",
|
||||
],
|
||||
// 6. Bearer tokens — Authorization: Bearer xxxx (16+ chars)
|
||||
[
|
||||
"BEARER",
|
||||
/(?:Bearer|Authorization:\s*Bearer)\s+([A-Za-z0-9._\-+/=]{16,})/g,
|
||||
"[REDACTED_BEARER]",
|
||||
],
|
||||
// 7. Email — RFC 5322-ish; rejects obvious junk but stays compact.
|
||||
[
|
||||
"EMAIL",
|
||||
/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,24}\b/g,
|
||||
"[REDACTED_EMAIL]",
|
||||
],
|
||||
// 8. Generic api_key / apiKey / password= value pairs (12+ char secret)
|
||||
[
|
||||
"GENERIC_KEY",
|
||||
/\b(?:api[_-]?key|apikey|password|passwd|pwd|secret|token|auth)\s*[:=]\s*['"]?([A-Za-z0-9._\-+/=]{12,})['"]?/gi,
|
||||
"[REDACTED_API_KEY]",
|
||||
],
|
||||
// 9. IPv4 (incl. port) — strict octet bounds
|
||||
[
|
||||
"IPV4",
|
||||
/\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d?\d)(?::\d{1,5})?\b/g,
|
||||
"[REDACTED_IPV4]",
|
||||
],
|
||||
// 10. IPv6 — full, compressed, ::1, ::ffff:1.2.3.4
|
||||
// Three alternative shapes:
|
||||
// (a) full 8-group form (no `::`),
|
||||
// (b) compressed form with `::` somewhere,
|
||||
// (c) `::1` / `::` alone anchored by a non-hex lookbehind so it
|
||||
// doesn't greedily extend `fe80::` into `fe80::foo`.
|
||||
[
|
||||
"IPV6",
|
||||
new RegExp(
|
||||
[
|
||||
// (a) Full 8-group: 1:2:3:4:5:6:7:8
|
||||
"\\b(?:[A-Fa-f0-9]{1,4}:){7}[A-Fa-f0-9]{1,4}\\b",
|
||||
// (b) Compressed with `::` somewhere in the middle (left side 1+ groups)
|
||||
"\\b(?:[A-Fa-f0-9]{1,4}:){1,6}[A-Fa-f0-9]{1,4}::[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6}\\b",
|
||||
"\\b(?:[A-Fa-f0-9]{1,4}:){1,7}:[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6}\\b",
|
||||
// (c) Leading `::` (no left side): ::1, ::1:2, ::ffff:1.2.3.4
|
||||
"(?<![A-Fa-f0-9:])::(?:[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6})?\\b",
|
||||
].join("|"),
|
||||
"g"
|
||||
),
|
||||
"[REDACTED_IPV6]",
|
||||
],
|
||||
]);
|
||||
|
||||
// ── Library API ──────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Redact all PII tokens in a string. Returns the redacted string and the
|
||||
* match counts so the caller can decide whether to fail in strict mode.
|
||||
*
|
||||
* @param {string} input
|
||||
* @returns {{ output: string, counts: Record<string, number> }}
|
||||
*/
|
||||
export function redactString(input) {
|
||||
if (typeof input !== "string" || input.length === 0) {
|
||||
return { output: input ?? "", counts: {} };
|
||||
}
|
||||
const counts = {};
|
||||
let output = input;
|
||||
for (const [name, regex, marker] of REDACT_PATTERNS) {
|
||||
output = output.replace(regex, () => {
|
||||
counts[name] = (counts[name] ?? 0) + 1;
|
||||
return marker;
|
||||
});
|
||||
}
|
||||
return { output, counts };
|
||||
}
|
||||
|
||||
/**
|
||||
* Redact a single line. Convenience wrapper around redactString that does
|
||||
* not allocate an intermediate object for the counts.
|
||||
*
|
||||
* @param {string} line
|
||||
* @returns {string}
|
||||
*/
|
||||
export function redact(line) {
|
||||
return redactString(line).output;
|
||||
}
|
||||
|
||||
/**
|
||||
* Redact TransformStream.
|
||||
*
|
||||
* Implements the WHATWG TransformStream API so callers can do:
|
||||
* await src.pipeThrough(new TextDecoderStream()).pipeThrough(new RedactTransform()).pipeTo(sink)
|
||||
*
|
||||
* Counts are accumulated on the stream instance (`.counts`).
|
||||
*/
|
||||
export class RedactTransform extends TransformStream {
|
||||
constructor() {
|
||||
const counts = {};
|
||||
super({
|
||||
transform(chunk, controller) {
|
||||
const text = typeof chunk === "string" ? chunk : new TextDecoder("utf-8").decode(chunk);
|
||||
const { output, counts: localCounts } = redactString(text);
|
||||
for (const [name, n] of Object.entries(localCounts)) {
|
||||
counts[name] = (counts[name] ?? 0) + n;
|
||||
}
|
||||
controller.enqueue(new TextEncoder().encode(output));
|
||||
},
|
||||
});
|
||||
// Attach counts as an enumerable own property so tests can read it.
|
||||
Object.defineProperty(this, "counts", {
|
||||
value: counts,
|
||||
writable: false,
|
||||
enumerable: true,
|
||||
configurable: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ── CLI ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { files: [], output: null, strict: false, help: false };
|
||||
for (let i = 2; i < argv.length; i += 1) {
|
||||
const a = argv[i];
|
||||
if (a === "--file") {
|
||||
args.files.push(argv[++i]);
|
||||
} else if (a === "--output" || a === "-o") {
|
||||
args.output = argv[++i];
|
||||
} else if (a === "--strict") {
|
||||
args.strict = true;
|
||||
} else if (a === "--help" || a === "-h") {
|
||||
args.help = true;
|
||||
} else {
|
||||
process.stderr.write(`unknown argument: ${a}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
process.stdout.write(`Usage: redact-logs.mjs [options]
|
||||
|
||||
Options:
|
||||
--file <path> Read from file (repeatable). Defaults to stdin.
|
||||
--output, -o <p> Write to file. Defaults to stdout.
|
||||
--strict Exit non-zero if any PII is detected.
|
||||
--help, -h Show this help.
|
||||
|
||||
Library:
|
||||
import { redact, redactString, RedactTransform } from "./scripts/sre/redact-logs.mjs";
|
||||
`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv);
|
||||
if (args.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const transform = new RedactTransform();
|
||||
|
||||
// Build a WHATWG ReadableStream from each input source. We open files /
|
||||
// stdin as a Node Readable and convert it via Readable.toWeb().
|
||||
const { Readable } = await import("node:stream");
|
||||
const { Writable: WritableStreamWeb } = await import("node:stream/web");
|
||||
const sources = args.files.length > 0 ? args.files : ["-"];
|
||||
|
||||
for (const source of sources) {
|
||||
const nodeSrc = source === "-" ? process.stdin : createReadStream(source, "utf8");
|
||||
const webSrc = Readable.toWeb(nodeSrc);
|
||||
const webDecoded = webSrc.pipeThrough(new TextDecoderStream("utf-8"));
|
||||
const webEncoded = webDecoded.pipeThrough(transform);
|
||||
|
||||
const sink = args.output
|
||||
? createWriteStream(args.output, "utf8")
|
||||
: process.stdout;
|
||||
const webSink = WritableStreamWeb.toWeb(sink);
|
||||
|
||||
try {
|
||||
await webEncoded.pipeTo(webSink);
|
||||
} catch (err) {
|
||||
process.stderr.write(`redact-logs: ${err.message}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
const totals = transform.counts;
|
||||
if (Object.keys(totals).length > 0) {
|
||||
const summary = Object.entries(totals)
|
||||
.map(([k, v]) => `${k}=${v}`)
|
||||
.join(" ");
|
||||
process.stderr.write(`redact-logs: redacted ${summary}\n`);
|
||||
if (args.strict) {
|
||||
process.exit(3);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only run as CLI when this module is the entrypoint (not when imported as a
|
||||
// library). `import.meta.url === pathToFileURL(process.argv[1]).href` is the
|
||||
// canonical ESM check.
|
||||
import { pathToFileURL } from "node:url";
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main();
|
||||
}
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* PII redaction for log shipping.
|
||||
*
|
||||
* Streams input (stdin or files) to stdout, replacing sensitive tokens
|
||||
* with stable redaction markers. Pure Node.js stdlib — no `npm install`.
|
||||
*
|
||||
* Recognised patterns (in order, longest match wins per position):
|
||||
*
|
||||
* 1. Anthropic API keys (sk-ant-...) → [REDACTED_API_KEY]
|
||||
* 2. Google API keys (AIza...) → [REDACTED_API_KEY]
|
||||
* 3. GitHub tokens (ghp_/gho_/ghu_/...) → [REDACTED_API_KEY]
|
||||
* 4. OpenAI keys (sk-..., sk-proj-...) → [REDACTED_API_KEY]
|
||||
* 5. AWS access keys (AKIA/ASIA) → [REDACTED_AWS_KEY]
|
||||
* 6. Bearer tokens → [REDACTED_BEARER]
|
||||
* 7. Email addresses → [REDACTED_EMAIL]
|
||||
* 8. Generic api_key=value pairs → [REDACTED_API_KEY]
|
||||
* 9. IPv4 addresses → [REDACTED_IPV4]
|
||||
* 10. IPv6 addresses → [REDACTED_IPV6]
|
||||
*
|
||||
* Provider-specific patterns are listed BEFORE the generic `sk-` rule so
|
||||
* that an `sk-ant-...` key counts as `ANTHROPIC_KEY` (not `OPENAI_KEY`)
|
||||
* for the per-call summary.
|
||||
*
|
||||
* Why stable markers: downstream log-search queries reference the
|
||||
* redaction markers (e.g. "show me all log lines with [REDACTED_IPV4]"),
|
||||
* which makes the redaction reversible by anyone with the original
|
||||
* vault lookup, but never by a log-search reader alone.
|
||||
*
|
||||
* CLI:
|
||||
* node scripts/sre/redact-logs.mjs < input.log > output.log
|
||||
* node scripts/sre/redact-logs.mjs --file access.log --output out.log
|
||||
* node scripts/sre/redact-logs.mjs --strict # exit non-zero on any match
|
||||
*
|
||||
* Library:
|
||||
* import { redact, redactString, RedactTransform } from "./scripts/sre/redact-logs.mjs";
|
||||
*
|
||||
* Salvaged from closed PR #5057 (base-stale; reimplemented on release).
|
||||
*/
|
||||
|
||||
import { createReadStream, createWriteStream } from "node:fs";
|
||||
import { TransformStream } from "node:stream/web";
|
||||
import process from "node:process";
|
||||
|
||||
// ── Pattern catalogue ─────────────────────────────────────────────────────────
|
||||
//
|
||||
// Each pattern is [name, regex, marker]. The regex uses the `g` flag so we can
|
||||
// iterate with `matchAll`. Order matters: longer / more specific patterns go
|
||||
// first so an `sk-ant-...` key counts as ANTHROPIC_KEY instead of OPENAI_KEY.
|
||||
|
||||
export const REDACT_PATTERNS = Object.freeze([
|
||||
// 1. Anthropic keys: sk-ant-api03-... / sk-ant-... (must beat the generic sk-)
|
||||
["ANTHROPIC_KEY", /\bsk-ant-[A-Za-z0-9_\-]{20,}\b/g, "[REDACTED_API_KEY]"],
|
||||
// 2. Google API keys: AIza... (39 chars total)
|
||||
["GOOGLE_KEY", /\bAIza[A-Za-z0-9_\-]{35}\b/g, "[REDACTED_API_KEY]"],
|
||||
// 3. GitHub tokens (classic + fine-grained + PAT prefixes)
|
||||
[
|
||||
"GITHUB_TOKEN",
|
||||
/\b(?:ghp|gho|ghu|ghs|ghr|github_pat)_[A-Za-z0-9]{30,}\b/g,
|
||||
"[REDACTED_API_KEY]",
|
||||
],
|
||||
// 4. OpenAI keys: sk-..., sk-proj-..., proj-... (after the more specific rules above)
|
||||
[
|
||||
"OPENAI_KEY",
|
||||
/\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b|\bproj-[A-Za-z0-9_\-]{20,}\b/g,
|
||||
"[REDACTED_API_KEY]",
|
||||
],
|
||||
// 5. AWS access keys — AKIA / ASIA prefixes, 20 chars total
|
||||
["AWS_KEY", /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, "[REDACTED_AWS_KEY]"],
|
||||
// 6. Bearer tokens — Authorization: Bearer xxxx (16+ chars)
|
||||
[
|
||||
"BEARER",
|
||||
/(?:Bearer|Authorization:\s*Bearer)\s+([A-Za-z0-9._\-+/=]{16,})/g,
|
||||
"[REDACTED_BEARER]",
|
||||
],
|
||||
// 7. Email — RFC 5322-ish; rejects obvious junk but stays compact.
|
||||
["EMAIL", /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,24}\b/g, "[REDACTED_EMAIL]"],
|
||||
// 8. Generic api_key / apiKey / password= value pairs (12+ char secret)
|
||||
[
|
||||
"GENERIC_KEY",
|
||||
/\b(?:api[_-]?key|apikey|password|passwd|pwd|secret|token|auth)\s*[:=]\s*['"]?([A-Za-z0-9._\-+/=]{12,})['"]?/gi,
|
||||
"[REDACTED_API_KEY]",
|
||||
],
|
||||
// 9. IPv4 (incl. port) — strict octet bounds
|
||||
[
|
||||
"IPV4",
|
||||
/\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d?\d)(?::\d{1,5})?\b/g,
|
||||
"[REDACTED_IPV4]",
|
||||
],
|
||||
// 10. IPv6 — full, compressed, ::1, ::ffff:1.2.3.4
|
||||
// Three alternative shapes:
|
||||
// (a) full 8-group form (no `::`),
|
||||
// (b) compressed form with `::` somewhere,
|
||||
// (c) `::1` / `::` alone anchored by a non-hex lookbehind so it
|
||||
// doesn't greedily extend `fe80::` into `fe80::foo`.
|
||||
[
|
||||
"IPV6",
|
||||
new RegExp(
|
||||
[
|
||||
// (a) Full 8-group: 1:2:3:4:5:6:7:8
|
||||
"\\b(?:[A-Fa-f0-9]{1,4}:){7}[A-Fa-f0-9]{1,4}\\b",
|
||||
// (b) Compressed with `::` somewhere in the middle (left side 1+ groups)
|
||||
"\\b(?:[A-Fa-f0-9]{1,4}:){1,6}[A-Fa-f0-9]{1,4}::[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6}\\b",
|
||||
"\\b(?:[A-Fa-f0-9]{1,4}:){1,7}:[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6}\\b",
|
||||
// (c) Leading `::` (no left side): ::1, ::1:2, ::ffff:1.2.3.4
|
||||
"(?<![A-Fa-f0-9:])::(?:[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6})?\\b",
|
||||
].join("|"),
|
||||
"g"
|
||||
),
|
||||
"[REDACTED_IPV6]",
|
||||
],
|
||||
]);
|
||||
|
||||
// ── Library API ──────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Redact all PII tokens in a string. Returns the redacted string and the
|
||||
* match counts so the caller can decide whether to fail in strict mode.
|
||||
*
|
||||
* @param {string} input
|
||||
* @returns {{ output: string, counts: Record<string, number> }}
|
||||
*/
|
||||
export function redactString(input) {
|
||||
if (typeof input !== "string" || input.length === 0) {
|
||||
return { output: input ?? "", counts: {} };
|
||||
}
|
||||
const counts = {};
|
||||
let output = input;
|
||||
for (const [name, regex, marker] of REDACT_PATTERNS) {
|
||||
output = output.replace(regex, () => {
|
||||
counts[name] = (counts[name] ?? 0) + 1;
|
||||
return marker;
|
||||
});
|
||||
}
|
||||
return { output, counts };
|
||||
}
|
||||
|
||||
/**
|
||||
* Redact a single line. Convenience wrapper around redactString that does
|
||||
* not allocate an intermediate object for the counts.
|
||||
*
|
||||
* @param {string} line
|
||||
* @returns {string}
|
||||
*/
|
||||
export function redact(line) {
|
||||
return redactString(line).output;
|
||||
}
|
||||
|
||||
/**
|
||||
* Redact TransformStream.
|
||||
*
|
||||
* Implements the WHATWG TransformStream API so callers can do:
|
||||
* await src.pipeThrough(new TextDecoderStream()).pipeThrough(new RedactTransform()).pipeTo(sink)
|
||||
*
|
||||
* Counts are accumulated on the stream instance (`.counts`).
|
||||
*/
|
||||
export class RedactTransform extends TransformStream {
|
||||
constructor() {
|
||||
const counts = {};
|
||||
const decoder = new TextDecoder("utf-8");
|
||||
const encoder = new TextEncoder();
|
||||
let pendingLine = "";
|
||||
|
||||
const emitRedacted = (text, controller) => {
|
||||
if (text.length === 0) return;
|
||||
|
||||
const { output, counts: localCounts } = redactString(text);
|
||||
for (const [name, n] of Object.entries(localCounts)) {
|
||||
counts[name] = (counts[name] ?? 0) + n;
|
||||
}
|
||||
controller.enqueue(encoder.encode(output));
|
||||
};
|
||||
|
||||
super({
|
||||
transform(chunk, controller) {
|
||||
const text =
|
||||
typeof chunk === "string"
|
||||
? decoder.decode() + chunk
|
||||
: decoder.decode(chunk, { stream: true });
|
||||
pendingLine += text;
|
||||
|
||||
const lastNewline = pendingLine.lastIndexOf("\n");
|
||||
if (lastNewline >= 0) {
|
||||
emitRedacted(pendingLine.slice(0, lastNewline + 1), controller);
|
||||
pendingLine = pendingLine.slice(lastNewline + 1);
|
||||
}
|
||||
},
|
||||
flush(controller) {
|
||||
pendingLine += decoder.decode();
|
||||
emitRedacted(pendingLine, controller);
|
||||
},
|
||||
});
|
||||
// Attach counts as an enumerable own property so tests can read it.
|
||||
Object.defineProperty(this, "counts", {
|
||||
value: counts,
|
||||
writable: false,
|
||||
enumerable: true,
|
||||
configurable: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ── CLI ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { files: [], output: null, strict: false, help: false };
|
||||
for (let i = 2; i < argv.length; i += 1) {
|
||||
const a = argv[i];
|
||||
if (a === "--file") {
|
||||
args.files.push(argv[++i]);
|
||||
} else if (a === "--output" || a === "-o") {
|
||||
args.output = argv[++i];
|
||||
} else if (a === "--strict") {
|
||||
args.strict = true;
|
||||
} else if (a === "--help" || a === "-h") {
|
||||
args.help = true;
|
||||
} else {
|
||||
process.stderr.write(`unknown argument: ${a}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
process.stdout.write(`Usage: redact-logs.mjs [options]
|
||||
|
||||
Options:
|
||||
--file <path> Read from file (repeatable). Defaults to stdin.
|
||||
--output, -o <p> Write to file. Defaults to stdout.
|
||||
--strict Exit non-zero if any PII is detected.
|
||||
--help, -h Show this help.
|
||||
|
||||
Library:
|
||||
import { redact, redactString, RedactTransform } from "./scripts/sre/redact-logs.mjs";
|
||||
`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv);
|
||||
if (args.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const transform = new RedactTransform();
|
||||
|
||||
// Build a WHATWG ReadableStream from each input source. We open files /
|
||||
// stdin as a Node Readable and convert it via Readable.toWeb().
|
||||
const { Readable } = await import("node:stream");
|
||||
const { Writable: WritableStreamWeb } = await import("node:stream/web");
|
||||
const sources = args.files.length > 0 ? args.files : ["-"];
|
||||
|
||||
for (const source of sources) {
|
||||
const nodeSrc = source === "-" ? process.stdin : createReadStream(source, "utf8");
|
||||
const webSrc = Readable.toWeb(nodeSrc);
|
||||
const webDecoded = webSrc.pipeThrough(new TextDecoderStream("utf-8"));
|
||||
const webEncoded = webDecoded.pipeThrough(transform);
|
||||
|
||||
const sink = args.output ? createWriteStream(args.output, "utf8") : process.stdout;
|
||||
const webSink = WritableStreamWeb.toWeb(sink);
|
||||
|
||||
try {
|
||||
await webEncoded.pipeTo(webSink);
|
||||
} catch (err) {
|
||||
process.stderr.write(`redact-logs: ${err.message}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
const totals = transform.counts;
|
||||
if (Object.keys(totals).length > 0) {
|
||||
const summary = Object.entries(totals)
|
||||
.map(([k, v]) => `${k}=${v}`)
|
||||
.join(" ");
|
||||
process.stderr.write(`redact-logs: redacted ${summary}\n`);
|
||||
if (args.strict) {
|
||||
process.exit(3);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only run as CLI when this module is the entrypoint (not when imported as a
|
||||
// library). `import.meta.url === pathToFileURL(process.argv[1]).href` is the
|
||||
// canonical ESM check.
|
||||
import { pathToFileURL } from "node:url";
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user