#!/usr/bin/env node /** * OmniRoute — UI i18n key sync (next-intl message catalogs). * * Source of truth: `src/i18n/messages/en.json`. Every other locale JSON in * `src/i18n/messages/` should mirror the same key tree. This script replicates * any keys that are missing in a target locale, marking them with a * `__MISSING__:` sentinel so reviewers (and the optional LLM * pass below) can spot them. It never overwrites an existing translated value. * * Usage (driven by npm scripts in package.json): * npm run i18n:sync-ui * npm run i18n:sync-ui -- --locale=pt-BR,zh-CN * npm run i18n:sync-ui -- --dry-run * npm run i18n:sync-ui -- --translate-markers * npm run i18n:sync-ui -- --translate-markers --locale=pt-BR --concurrency=4 * npm run i18n:sync-ui -- --translate-markers --batch-size=40 * * --translate-markers calls the OmniRoute translation backend (same env vars * as `run-translation.mjs`; the client lives in `lib/translate-backend.mjs`) * and replaces every `__MISSING__:` placeholder with a translated string. * Missing env vars cause the script to fail fast — the markers stay in place * for a later run. * * --batch-size=N (default 1) translates up to N placeholders per request as * one JSON object instead of one request per string. A batch whose response * cannot be parsed (or whose upstream call fails) is retried string by * string, so the worst case degrades to the default per-string behaviour. * * Output examples: * [i18n-ui-sync] pt-BR: +589 missing keys (589 __MISSING__, 0 translated) * [i18n-ui-sync] pt-BR: +0 missing keys (already in sync) */ import { promises as fs, existsSync, readFileSync } from "node:fs"; import path from "node:path"; import process from "node:process"; import { fileURLToPath, pathToFileURL } from "node:url"; import { backendConfig, translateBatch, translateString } from "./lib/translate-backend.mjs"; // ----- .env loader -------------------------------------------------------- // Loads variables from a local `.env` (gitignored) into process.env without // pulling dotenv as a dependency. Already-set env vars take precedence so the // shell / CI environment can still override. (function loadDotEnv() { const envPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".env"); if (!existsSync(envPath)) return; try { const raw = readFileSync(envPath, "utf8"); for (const rawLine of raw.split(/\r?\n/)) { const line = rawLine.trim(); if (!line || line.startsWith("#")) continue; const eq = line.indexOf("="); if (eq <= 0) continue; const key = line.slice(0, eq).trim(); if (!key || process.env[key] !== undefined) continue; let value = line.slice(eq + 1); if ( (value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'")) ) { value = value.slice(1, -1); } process.env[key] = value; } } catch { /* ignore — script will fall back to the requireEnv error path */ } })(); const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(SCRIPT_DIR, "..", ".."); const CONFIG_PATH = path.join(ROOT, "config", "i18n.json"); const MESSAGES_DIR = path.join(ROOT, "src", "i18n", "messages"); const SOURCE_LOCALE = "en"; const PLACEHOLDER_PREFIX = "__MISSING__:"; // ----- Helpers ------------------------------------------------------------- function logInfo(...parts) { console.log("[i18n-ui-sync]", ...parts); } function logWarn(...parts) { console.warn("[i18n-ui-sync] WARN", ...parts); } function logError(...parts) { console.error("[i18n-ui-sync] ERROR", ...parts); } function parseArgs(argv) { const opts = { locales: null, dryRun: false, translateMarkers: false, concurrency: null, batchSize: 1, }; for (const arg of argv.slice(2)) { if (arg === "--dry-run" || arg === "--dryrun") opts.dryRun = true; else if (arg === "--translate-markers") opts.translateMarkers = true; else if (arg.startsWith("--locale=")) { opts.locales = arg .slice(9) .split(",") .map((s) => s.trim()) .filter(Boolean); } else if (arg.startsWith("--locales=")) { opts.locales = arg .slice(10) .split(",") .map((s) => s.trim()) .filter(Boolean); } else if (arg.startsWith("--concurrency=")) { opts.concurrency = Number(arg.slice(14)); } else if (arg.startsWith("--batch-size=")) { // Whole numbers only: a fractional size would make the slice windows // overlap; NaN / 0 / negatives mean "per-string" (1). opts.batchSize = Math.max(1, Math.floor(Number(arg.slice(13))) || 1); } else if (arg === "--help" || arg === "-h") { console.log( [ "Usage: node scripts/i18n/sync-ui-keys.mjs [options]", "", " --locale= Target locales (default: all except `en`)", " --dry-run Report what would change, write nothing", " --translate-markers Call the translation backend to translate every", " __MISSING__: placeholder", " --concurrency= Parallel translation requests (default: env or 4)", " --batch-size= Placeholders per translation request (default: 1).", " n>1 sends up to n strings as one JSON object; a batch", " that fails or cannot be parsed falls back to one-by-one", ].join("\n") ); process.exit(0); } } return opts; } async function loadConfig() { const raw = await fs.readFile(CONFIG_PATH, "utf8"); const cfg = JSON.parse(raw); if (!cfg.default || !Array.isArray(cfg.locales)) { throw new Error("config/i18n.json: invalid shape (need `default` and `locales[]`)"); } return cfg; } async function loadJson(filePath) { const raw = await fs.readFile(filePath, "utf8"); return JSON.parse(raw); } function isPlainObject(value) { return value !== null && typeof value === "object" && !Array.isArray(value); } // Defensive: reject any key that could traverse into the object prototype // chain when we copy/merge values across JSON trees. Our inputs are // authored JSON we already control, but excluding these keys is a cheap // safety net. const FORBIDDEN_KEYS = new Set(["__proto__", "prototype", "constructor"]); /** * Walks the source tree key-by-key. For each leaf in `source` that is not * present in `target` (or whose corresponding target path is an object when * source is a leaf, etc.), copies the source value into a new merged object, * prefixing scalar values with PLACEHOLDER_PREFIX. Existing translated keys * are preserved verbatim. * * Returns a tuple: { merged, addedPaths } so the caller can report the * additions and (optionally) translate them. */ function mergeMissing(source, target) { const addedPaths = []; function walk(srcNode, tgtNode, prefix) { if (!isPlainObject(srcNode)) { // Source is a leaf. If target is missing or shape-mismatched, insert. if (tgtNode === undefined) { addedPaths.push(prefix); return typeof srcNode === "string" ? `${PLACEHOLDER_PREFIX}${srcNode}` : srcNode; } // Existing value (even if string starts with placeholder) is kept. return tgtNode; } // Source is an object — produce a prototype-less object preserving source // key order. Using Object.create(null) guarantees no inherited keys can // leak through later lookups, and we skip any key that resolves to a // built-in prototype property name as a defense in depth. const out = Object.create(null); for (const [key, value] of Object.entries(srcNode)) { if (FORBIDDEN_KEYS.has(key)) continue; const nextPrefix = prefix ? `${prefix}.${key}` : key; let tgtChild; if (isPlainObject(tgtNode) && Object.prototype.hasOwnProperty.call(tgtNode, key)) { // Read the property via Object.entries instead of dynamic bracket // access to keep static analyzers happy. const entry = Object.entries(tgtNode).find(([k]) => k === key); tgtChild = entry ? entry[1] : undefined; } out[key] = walk(value, tgtChild, nextPrefix); } return out; } const merged = walk(source, target, ""); return { merged, addedPaths }; } function countPlaceholders(node) { if (typeof node === "string") return node.startsWith(PLACEHOLDER_PREFIX) ? 1 : 0; if (!isPlainObject(node)) return 0; let total = 0; for (const value of Object.values(node)) total += countPlaceholders(value); return total; } // ----- Translator backend -------------------------------------------------- // The chat-completions client (`backendConfig`, `translateString`, // `translateBatch`) lives in `./lib/translate-backend.mjs` so the other i18n // tooling can share it. Only the concurrency limiter stays here. // Simple promise-based semaphore (avoid runtime deps). function createLimiter(max) { let active = 0; const queue = []; const next = () => { if (!queue.length || active >= max) return; active++; const { fn, resolve, reject } = queue.shift(); fn() .then((v) => { active--; resolve(v); next(); }) .catch((err) => { active--; reject(err); next(); }); }; return (fn) => new Promise((resolve, reject) => { queue.push({ fn, resolve, reject }); next(); }); } /** * Walks a merged tree, finding every leaf that starts with PLACEHOLDER_PREFIX * and replacing it with the translation produced by the backend. * * Translations happen with bounded concurrency. On failure, the placeholder * is preserved so a later run can retry. * * With `batchSize > 1` the placeholders are grouped into requests of up to * `batchSize` strings (one JSON object per request). A batch whose response * cannot be parsed — or whose upstream call fails — is retried one string at * a time, so a bad batch never loses more than the per-string path would. */ async function translatePlaceholders(merged, localeEntry, backend, concurrency, batchSize = 1) { const tasks = []; function collect(node, parent, key) { if (typeof node === "string") { if (node.startsWith(PLACEHOLDER_PREFIX)) { const englishValue = node.slice(PLACEHOLDER_PREFIX.length); tasks.push({ parent, key, englishValue }); } return; } if (!isPlainObject(node)) return; for (const [k, v] of Object.entries(node)) { collect(v, node, k); } } collect(merged, null, null); if (tasks.length === 0) return { translated: 0, failed: 0 }; const limit = createLimiter(concurrency); let translatedCount = 0; let failed = 0; if (batchSize > 1) { const groups = []; for (let i = 0; i < tasks.length; i += batchSize) groups.push(tasks.slice(i, i + batchSize)); await Promise.all( groups.map((group) => limit(async () => { const entries = group.map((task, i) => ({ id: `s${i}`, text: task.englishValue })); try { const translated = await translateBatch(entries, localeEntry, backend); group.forEach((task, i) => { task.parent[task.key] = translated.get(`s${i}`); translatedCount++; }); } catch (err) { logWarn( `batch of ${group.length} failed for ${localeEntry.code} (${err.message}) — retrying one by one` ); for (const task of group) { try { task.parent[task.key] = await translateString( task.englishValue, localeEntry, backend ); translatedCount++; } catch (inner) { failed++; logWarn(`translation failed for ${localeEntry.code}: ${inner.message}`); } } } }) ) ); return { translated: translatedCount, failed }; } await Promise.all( tasks.map((task) => limit(async () => { try { const value = await translateString(task.englishValue, localeEntry, backend); task.parent[task.key] = value; translatedCount++; } catch (err) { // Keep the __MISSING__ marker so subsequent runs can retry. failed++; logWarn(`translation failed for ${localeEntry.code}: ${err.message}`); } }) ) ); return { translated: translatedCount, failed }; } // ----- Main ---------------------------------------------------------------- async function processLocale(locale, source, config, opts, backend) { const localePath = path.join(MESSAGES_DIR, `${locale}.json`); let target = {}; if (existsSync(localePath)) { try { target = await loadJson(localePath); } catch (err) { logWarn(`${locale}: failed to parse existing JSON — starting fresh (${err.message})`); target = {}; } } else { logWarn(`${locale}: messages file did not exist — creating it`); } const { merged, addedPaths } = mergeMissing(source, target); const placeholderCountBefore = countPlaceholders(merged); let translateStats = { translated: 0, failed: 0 }; if (opts.translateMarkers && placeholderCountBefore > 0 && backend) { const localeEntry = config.locales.find((l) => l.code === locale); if (!localeEntry) { logWarn(`${locale}: not present in config/i18n.json — skipping translation`); } else { const concurrency = opts.concurrency ?? Number(process.env.OMNIROUTE_TRANSLATION_CONCURRENCY || 4); translateStats = await translatePlaceholders( merged, localeEntry, backend, concurrency, opts.batchSize ); } } const placeholderCountAfter = countPlaceholders(merged); const totalMissing = addedPaths.length; const stillPlaceholder = placeholderCountAfter; const summary = `${locale}: +${totalMissing} missing keys (${stillPlaceholder} __MISSING__, ${translateStats.translated} translated${translateStats.failed ? `, ${translateStats.failed} failed` : ""})`; if (opts.dryRun) { logInfo(`[DRY] ${summary}`); return { addedPaths, translated: translateStats.translated }; } // Only write when something changed. (json-stable serialization) const before = existsSync(localePath) ? await fs.readFile(localePath, "utf8") : ""; const after = JSON.stringify(merged, null, 2) + "\n"; if (before === after) { logInfo(`${locale}: already in sync (no changes)`); return { addedPaths, translated: translateStats.translated }; } await fs.writeFile(localePath, after, "utf8"); logInfo(summary); return { addedPaths, translated: translateStats.translated }; } async function main() { const opts = parseArgs(process.argv); const config = await loadConfig(); const sourcePath = path.join(MESSAGES_DIR, `${SOURCE_LOCALE}.json`); if (!existsSync(sourcePath)) { throw new Error(`Source messages file not found: ${sourcePath}`); } const source = await loadJson(sourcePath); // Locales = every code in config except `en`, intersected with locales that // already exist on disk (so we never silently create unknown locale files). const onDisk = new Set( (await fs.readdir(MESSAGES_DIR)).filter((f) => f.endsWith(".json")).map((f) => f.slice(0, -5)) ); let targetLocales = config.locales .map((l) => l.code) .filter((code) => code !== SOURCE_LOCALE && onDisk.has(code)); if (opts.locales) { const missingFromConfig = opts.locales.filter((c) => !config.locales.some((l) => l.code === c)); if (missingFromConfig.length) { logWarn(`--locale contains codes not in config/i18n.json: ${missingFromConfig.join(", ")}`); } targetLocales = targetLocales.filter((code) => opts.locales.includes(code)); } logInfo(`source: ${path.relative(ROOT, sourcePath)}`); logInfo(`locales: ${targetLocales.length} (${targetLocales.join(", ")})`); logInfo( `dry-run: ${opts.dryRun ? "yes" : "no"}, translate-markers: ${opts.translateMarkers ? "yes" : "no"}` ); let backend = null; if (opts.translateMarkers && !opts.dryRun) { backend = backendConfig(); backend.concurrency = opts.concurrency ?? Number(process.env.OMNIROUTE_TRANSLATION_CONCURRENCY || 4); const batchInfo = opts.batchSize > 1 ? `, batch=${opts.batchSize}` : ""; logInfo( `backend: ${backend.apiUrl} (model=${backend.model}, concurrency=${backend.concurrency}${batchInfo}, timeout=${backend.timeoutMs}ms)` ); } const startMs = Date.now(); let totalAdded = 0; let totalTranslated = 0; for (const locale of targetLocales) { const result = await processLocale(locale, source, config, opts, backend); totalAdded += result.addedPaths.length; totalTranslated += result.translated; } const elapsedSec = ((Date.now() - startMs) / 1000).toFixed(1); logInfo( `summary: locales=${targetLocales.length}, added=${totalAdded}, translated=${totalTranslated}, elapsed=${elapsedSec}s` ); } const isDirectRun = import.meta.url === pathToFileURL(process.argv[1]).href; if (isDirectRun) { main().catch((err) => { logError(err?.stack || err?.message || String(err)); process.exit(1); }); }