From 9e36dde2f76d6e4d6a1dbbd2d4c5ff83286c9ed6 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Wed, 16 Sep 2026 12:19:50 -0300 Subject: [PATCH] feat(i18n): retranslate-site rewrites the verbatim-English leaves of the site catalogs (#13886) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/i18n/retranslate-site.mjs + untranslatable-site-keys.json (22 keys) + tests; the run landed on the site as OmniRouteSite#8 (2,059 leaves, mean English residue 10.3 % → 6.3 %). ⚠️ base-red inherited: #12732 --- .../features/13886-site-retranslate.md | 1 + scripts/i18n/retranslate-site.mjs | 213 ++++++++++++++++++ scripts/i18n/untranslatable-site-keys.json | 27 +++ tests/unit/i18n-retranslate-site.test.ts | 59 +++++ 4 files changed, 300 insertions(+) create mode 100644 changelog.d/features/13886-site-retranslate.md create mode 100644 scripts/i18n/retranslate-site.mjs create mode 100644 scripts/i18n/untranslatable-site-keys.json create mode 100644 tests/unit/i18n-retranslate-site.test.ts diff --git a/changelog.d/features/13886-site-retranslate.md b/changelog.d/features/13886-site-retranslate.md new file mode 100644 index 0000000000..671bb3ed69 --- /dev/null +++ b/changelog.d/features/13886-site-retranslate.md @@ -0,0 +1 @@ +- **feat(i18n):** `retranslate-site` rewrites the site catalogs' verbatim-English leaves (2,059 across 63 catalogs; mean English residue 10.3 % → 6.3 %, the rest being brand names kept on purpose). (#13886) diff --git a/scripts/i18n/retranslate-site.mjs b/scripts/i18n/retranslate-site.mjs new file mode 100644 index 0000000000..f427248228 --- /dev/null +++ b/scripts/i18n/retranslate-site.mjs @@ -0,0 +1,213 @@ +#!/usr/bin/env node +/** + * OmniRoute — site catalog retranslator (omnirouteSite/lang/.json). + * + * The site catalogs are flat { "dotted.key": "text" } files translated once by + * add-locale; 10 % of their leaves were still verbatim English on 2026-09-16 + * (th 25 %). For every locale, the leaves equal to lang/_source.en.json — + * outside untranslatable-site-keys.json — are sent to the translation backend + * in batches and written back in place (key order preserved). Same backend + * env as sync-ui-keys (`OMNIROUTE_TRANSLATION_*`, loaded from the repo-root + * `.env` when present). + * + * A batch whose answer cannot be trusted (see `parseBatchResponse`) is retried + * one string at a time; a leaf that still fails keeps its English value so the + * next run picks it up again. Each catalog is written as soon as its locale is + * done, so an aborted run keeps the locales already finished. + * + * Usage: + * node scripts/i18n/retranslate-site.mjs --site-dir=../omnirouteSite \ + * [--locale=th,phi] [--dry-run] [--batch-size=40] + */ +import { existsSync, readFileSync } from "node:fs"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { backendConfig, translateBatch, translateString } from "./lib/translate-backend.mjs"; + +const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.resolve(SCRIPT_DIR, "..", ".."); +const ALLOWLIST = path.join(SCRIPT_DIR, "untranslatable-site-keys.json"); +const LOG_PREFIX = "[site-retranslate]"; + +// ----- .env loader -------------------------------------------------------- +// Same loader as sync-ui-keys.mjs: variables from the repo-root `.env` +// (gitignored) land in process.env unless the shell already set them. +function loadDotEnv() { + const envPath = path.join(ROOT, ".env"); + if (!existsSync(envPath)) return; + try { + const raw = readFileSync(envPath, "utf8"); + for (const rawLine of raw.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith("#")) continue; + const eq = line.indexOf("="); + if (eq <= 0) continue; + const key = line.slice(0, eq).trim(); + if (!key || process.env[key] !== undefined) continue; + let value = line.slice(eq + 1); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + process.env[key] = value; + } + } catch { + /* ignore — backendConfig() reports the missing variables */ + } +} + +/** + * Keys of `source` whose `target` value is still the verbatim English string, + * outside `allow`. Missing target keys are not "identical copies" (that is a + * sync problem, not a translation one); empty and non-string source leaves are + * skipped. Sorted so the batches are deterministic. + * + * @param {Record} source lang/_source.en.json + * @param {Record} target lang/.json + * @param {Set} allow keys that must stay English + * @returns {string[]} + */ +export function findSiteIdenticalKeys(source, target, allow) { + return Object.keys(source) + .filter( + (k) => + k in target && + typeof source[k] === "string" && + source[k] !== "" && + target[k] === source[k] && + !allow.has(k) + ) + .sort(); +} + +function parseArgs(argv) { + const o = { siteDir: null, locales: null, dryRun: false, batchSize: 40 }; + for (const a of argv.slice(2)) { + if (a.startsWith("--site-dir=")) o.siteDir = path.resolve(ROOT, a.slice("--site-dir=".length)); + else if (a.startsWith("--locale=")) + o.locales = a + .slice("--locale=".length) + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + else if (a === "--dry-run") o.dryRun = true; + else if (a.startsWith("--batch-size=")) + o.batchSize = Math.max(1, Number(a.slice("--batch-size=".length)) || 40); + else throw new Error(`unknown argument: ${a}`); + } + if (!o.siteDir) throw new Error("--site-dir= is required"); + return o; +} + +async function readJson(file) { + return JSON.parse(await fs.readFile(file, "utf8")); +} + +/** + * Translates `keys` of `source` for one locale, writing into `target` in place. + * Returns { translated, failed } counts. + */ +async function translateLocale(keys, source, target, localeEntry, backend, batchSize) { + let translated = 0; + let failed = 0; + for (let i = 0; i < keys.length; i += batchSize) { + const slice = keys.slice(i, i + batchSize); + try { + const out = await translateBatch( + slice.map((id) => ({ id, text: source[id] })), + localeEntry, + backend + ); + for (const id of slice) { + const value = out.get(id); + if (typeof value === "string" && value.trim()) { + target[id] = value.trim(); + translated++; + } else { + failed++; + } + } + } catch (err) { + console.warn( + `${LOG_PREFIX} ${localeEntry.code}: batch of ${slice.length} failed (${err.message}) — retrying one by one` + ); + for (const id of slice) { + try { + const value = await translateString(source[id], localeEntry, backend); + if (typeof value === "string" && value.trim()) { + target[id] = value.trim(); + translated++; + } else { + failed++; + } + } catch (inner) { + failed++; + console.warn(`${LOG_PREFIX} ${localeEntry.code}: ${id} failed (${inner.message})`); + } + } + } + } + return { translated, failed }; +} + +async function main() { + const o = parseArgs(process.argv); + if (!o.dryRun) loadDotEnv(); + const langDir = path.join(o.siteDir, "lang"); + const source = await readJson(path.join(langDir, "_source.en.json")); + const allow = new Set((await readJson(ALLOWLIST)).keys ?? []); + const config = await readJson(path.join(ROOT, "config", "i18n.json")); + const codes = (o.locales ?? config.locales.map((l) => l.code)).filter((c) => c !== "en"); + const backend = o.dryRun ? null : backendConfig(); + let total = 0; + let failedTotal = 0; + for (const code of codes) { + const file = path.join(langDir, `${code}.json`); + const localeEntry = config.locales.find((l) => l.code === code); + if (!localeEntry) { + console.warn(`${LOG_PREFIX} ${code}: not in config/i18n.json, skipped`); + continue; + } + let target; + try { + target = await readJson(file); + } catch { + console.warn(`${LOG_PREFIX} ${code}: no catalog, skipped`); + continue; + } + const keys = findSiteIdenticalKeys(source, target, allow); + console.log( + `${LOG_PREFIX} ${code}: ${keys.length} English leaves${o.dryRun ? " (dry-run)" : ""}` + ); + if (o.dryRun || keys.length === 0) continue; + const { translated, failed } = await translateLocale( + keys, + source, + target, + localeEntry, + backend, + o.batchSize + ); + total += translated; + failedTotal += failed; + if (translated > 0) await fs.writeFile(file, JSON.stringify(target, null, 2) + "\n", "utf8"); + if (failed > 0) console.warn(`${LOG_PREFIX} ${code}: ${failed} leaves still English`); + } + console.log( + `${LOG_PREFIX} done — ${total} leaves rewritten across ${codes.length} locales` + + (failedTotal > 0 ? ` (${failedTotal} failed)` : "") + ); + if (failedTotal > 0) process.exitCode = 1; +} + +const isDirectRun = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; +if (isDirectRun) { + main().catch((e) => { + console.error(`${LOG_PREFIX} ${e.message}`); + process.exitCode = 1; + }); +} diff --git a/scripts/i18n/untranslatable-site-keys.json b/scripts/i18n/untranslatable-site-keys.json new file mode 100644 index 0000000000..95fa2046ef --- /dev/null +++ b/scripts/i18n/untranslatable-site-keys.json @@ -0,0 +1,27 @@ +{ + "description": "Site catalog keys (omnirouteSite/lang/.json) that must stay identical to lang/_source.en.json — brand and product names, package names, URLs, protocol acronyms and literal config values. retranslate-site.mjs never sends these to the translation backend.", + "keys": [ + "combos.mode.auto", + "compare.ops.oauth.lite", + "compare.res.tls.cli", + "compare.res.tls.own", + "cta.community.whatsapp.brazil", + "deploy.arm", + "deploy.docker", + "deploy.npm", + "deploy.opencode.cmd", + "deploy.pwa", + "deploy.termux", + "deploy.vscode", + "footer.github", + "footer.protocols", + "hero.cta.github", + "providers.cat.local.ex", + "providers.cat.oauth", + "viral.footer.github", + "viral.footer.home", + "why.flow.ide", + "why.where.reddit.title", + "why.where.x.title" + ] +} diff --git a/tests/unit/i18n-retranslate-site.test.ts b/tests/unit/i18n-retranslate-site.test.ts new file mode 100644 index 0000000000..9978b29015 --- /dev/null +++ b/tests/unit/i18n-retranslate-site.test.ts @@ -0,0 +1,59 @@ +/** + * `scripts/i18n/retranslate-site.mjs` — the site catalog retranslator + * (omnirouteSite/lang/.json vs lang/_source.en.json). + * + * `findSiteIdenticalKeys` is the pure selector behind it: the keys of a locale + * catalog whose value is still a verbatim copy of the English source, outside + * the allowlist (`scripts/i18n/untranslatable-site-keys.json`). + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { findSiteIdenticalKeys } from "../../scripts/i18n/retranslate-site.mjs"; + +const source = { + "hero.title": "Route every model", + brand: "OmniRoute", + cta: "Get started", + n: "66 languages", +}; + +test("keys still equal to the English source are listed, allowlisted ones are not", () => { + const target = { + "hero.title": "Route every model", + brand: "OmniRoute", + cta: "Começar", + n: "66 idiomas", + }; + assert.deepEqual(findSiteIdenticalKeys(source, target, new Set(["brand"])), ["hero.title"]); +}); + +test("a key missing from the target is not an identical copy (that is a sync problem, not ours)", () => { + assert.deepEqual(findSiteIdenticalKeys(source, { brand: "OmniRoute" }, new Set(["brand"])), []); +}); + +test("empty source strings are ignored", () => { + assert.deepEqual(findSiteIdenticalKeys({ a: "" }, { a: "" }, new Set()), []); +}); + +test("the result is sorted and only string leaves count", () => { + const src = { z: "Zulu", a: "Alpha", m: "Mike", obj: { nested: "x" } as unknown as string }; + const tgt = { z: "Zulu", a: "Alpha", m: "Mike", obj: { nested: "x" } }; + assert.deepEqual(findSiteIdenticalKeys(src, tgt, new Set()), ["a", "m", "z"]); +}); + +test("untranslatable-site-keys.json has the { keys: string[] } shape with unique keys", () => { + const file = path.resolve( + import.meta.dirname, + "../../scripts/i18n/untranslatable-site-keys.json" + ); + const allow = JSON.parse(readFileSync(file, "utf8")) as { description?: string; keys: string[] }; + assert.ok(Array.isArray(allow.keys), "keys must be an array"); + assert.ok(allow.keys.length > 0, "the allowlist must not be empty"); + assert.ok( + allow.keys.every((k) => typeof k === "string" && k.trim() === k && k.length > 0), + "every key is a trimmed non-empty string" + ); + assert.equal(new Set(allow.keys).size, allow.keys.length, "no duplicated keys"); +});