OmniRoute/scripts/i18n/sync-ui-keys.mjs

#!/usr/bin/env node
/**
 * OmniRoute — UI i18n key sync (next-intl message catalogs).
 *
 * Source of truth: `src/i18n/messages/en.json`. Every other locale JSON in
 * `src/i18n/messages/` should mirror the same key tree. This script replicates
 * any keys that are missing in a target locale, marking them with a
 * `__MISSING__:<english_value>` sentinel so reviewers (and the optional LLM
 * pass below) can spot them. It never overwrites an existing translated value.
 *
 * Usage (driven by npm scripts in package.json):
 *   npm run i18n:sync-ui
 *   npm run i18n:sync-ui -- --locale=pt-BR,zh-CN
 *   npm run i18n:sync-ui -- --dry-run
 *   npm run i18n:sync-ui -- --translate-markers
 *   npm run i18n:sync-ui -- --translate-markers --locale=pt-BR --concurrency=4
 *   npm run i18n:sync-ui -- --translate-markers --batch-size=40
 *
 * --translate-markers calls the OmniRoute translation backend (same env vars
 * as `run-translation.mjs`; the client lives in `lib/translate-backend.mjs`)
 * and replaces every `__MISSING__:<en>` placeholder with a translated string.
 * Missing env vars cause the script to fail fast — the markers stay in place
 * for a later run.
 *
 * --batch-size=N (default 1) translates up to N placeholders per request as
 * one JSON object instead of one request per string. A batch whose response
 * cannot be parsed (or whose upstream call fails) is retried string by
 * string, so the worst case degrades to the default per-string behaviour.
 *
 * Output examples:
 *   [i18n-ui-sync] pt-BR: +589 missing keys (589 __MISSING__, 0 translated)
 *   [i18n-ui-sync] pt-BR: +0 missing keys (already in sync)
 */

import { promises as fs, existsSync, readFileSync } from "node:fs";
import path from "node:path";
import process from "node:process";
import { fileURLToPath, pathToFileURL } from "node:url";

import { backendConfig, translateBatch, translateString } from "./lib/translate-backend.mjs";

// ----- .env loader --------------------------------------------------------
// Loads variables from a local `.env` (gitignored) into process.env without
// pulling dotenv as a dependency. Already-set env vars take precedence so the
// shell / CI environment can still override.
(function loadDotEnv() {
  const envPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".env");
  if (!existsSync(envPath)) return;
  try {
    const raw = readFileSync(envPath, "utf8");
    for (const rawLine of raw.split(/\r?\n/)) {
      const line = rawLine.trim();
      if (!line || line.startsWith("#")) continue;
      const eq = line.indexOf("=");
      if (eq <= 0) continue;
      const key = line.slice(0, eq).trim();
      if (!key || process.env[key] !== undefined) continue;
      let value = line.slice(eq + 1);
      if (
        (value.startsWith('"') && value.endsWith('"')) ||
        (value.startsWith("'") && value.endsWith("'"))
      ) {
        value = value.slice(1, -1);
      }
      process.env[key] = value;
    }
  } catch {
    /* ignore — script will fall back to the requireEnv error path */
  }
})();

const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.resolve(SCRIPT_DIR, "..", "..");
const CONFIG_PATH = path.join(ROOT, "config", "i18n.json");
const MESSAGES_DIR = path.join(ROOT, "src", "i18n", "messages");
const SOURCE_LOCALE = "en";
const PLACEHOLDER_PREFIX = "__MISSING__:";

// ----- Helpers -------------------------------------------------------------

function logInfo(...parts) {
  console.log("[i18n-ui-sync]", ...parts);
}
function logWarn(...parts) {
  console.warn("[i18n-ui-sync] WARN", ...parts);
}
function logError(...parts) {
  console.error("[i18n-ui-sync] ERROR", ...parts);
}

function parseArgs(argv) {
  const opts = {
    locales: null,
    dryRun: false,
    translateMarkers: false,
    concurrency: null,
    batchSize: 1,
  };
  for (const arg of argv.slice(2)) {
    if (arg === "--dry-run" || arg === "--dryrun") opts.dryRun = true;
    else if (arg === "--translate-markers") opts.translateMarkers = true;
    else if (arg.startsWith("--locale=")) {
      opts.locales = arg
        .slice(9)
        .split(",")
        .map((s) => s.trim())
        .filter(Boolean);
    } else if (arg.startsWith("--locales=")) {
      opts.locales = arg
        .slice(10)
        .split(",")
        .map((s) => s.trim())
        .filter(Boolean);
    } else if (arg.startsWith("--concurrency=")) {
      opts.concurrency = Number(arg.slice(14));
    } else if (arg.startsWith("--batch-size=")) {
      // Whole numbers only: a fractional size would make the slice windows
      // overlap; NaN / 0 / negatives mean "per-string" (1).
      opts.batchSize = Math.max(1, Math.floor(Number(arg.slice(13))) || 1);
    } else if (arg === "--help" || arg === "-h") {
      console.log(
        [
          "Usage: node scripts/i18n/sync-ui-keys.mjs [options]",
          "",
          "  --locale=<csv>          Target locales (default: all except `en`)",
          "  --dry-run               Report what would change, write nothing",
          "  --translate-markers     Call the translation backend to translate every",
          "                          __MISSING__:<en> placeholder",
          "  --concurrency=<n>       Parallel translation requests (default: env or 4)",
          "  --batch-size=<n>        Placeholders per translation request (default: 1).",
          "                          n>1 sends up to n strings as one JSON object; a batch",
          "                          that fails or cannot be parsed falls back to one-by-one",
        ].join("\n")
      );
      process.exit(0);
    }
  }
  return opts;
}

async function loadConfig() {
  const raw = await fs.readFile(CONFIG_PATH, "utf8");
  const cfg = JSON.parse(raw);
  if (!cfg.default || !Array.isArray(cfg.locales)) {
    throw new Error("config/i18n.json: invalid shape (need `default` and `locales[]`)");
  }
  return cfg;
}

async function loadJson(filePath) {
  const raw = await fs.readFile(filePath, "utf8");
  return JSON.parse(raw);
}

function isPlainObject(value) {
  return value !== null && typeof value === "object" && !Array.isArray(value);
}

// Defensive: reject any key that could traverse into the object prototype
// chain when we copy/merge values across JSON trees. Our inputs are
// authored JSON we already control, but excluding these keys is a cheap
// safety net.
const FORBIDDEN_KEYS = new Set(["__proto__", "prototype", "constructor"]);

/**
 * Walks the source tree key-by-key. For each leaf in `source` that is not
 * present in `target` (or whose corresponding target path is an object when
 * source is a leaf, etc.), copies the source value into a new merged object,
 * prefixing scalar values with PLACEHOLDER_PREFIX. Existing translated keys
 * are preserved verbatim.
 *
 * Returns a tuple: { merged, addedPaths } so the caller can report the
 * additions and (optionally) translate them.
 */
function mergeMissing(source, target) {
  const addedPaths = [];

  function walk(srcNode, tgtNode, prefix) {
    if (!isPlainObject(srcNode)) {
      // Source is a leaf. If target is missing or shape-mismatched, insert.
      if (tgtNode === undefined) {
        addedPaths.push(prefix);
        return typeof srcNode === "string" ? `${PLACEHOLDER_PREFIX}${srcNode}` : srcNode;
      }
      // Existing value (even if string starts with placeholder) is kept.
      return tgtNode;
    }

    // Source is an object — produce a prototype-less object preserving source
    // key order. Using Object.create(null) guarantees no inherited keys can
    // leak through later lookups, and we skip any key that resolves to a
    // built-in prototype property name as a defense in depth.
    const out = Object.create(null);
    for (const [key, value] of Object.entries(srcNode)) {
      if (FORBIDDEN_KEYS.has(key)) continue;
      const nextPrefix = prefix ? `${prefix}.${key}` : key;
      let tgtChild;
      if (isPlainObject(tgtNode) && Object.prototype.hasOwnProperty.call(tgtNode, key)) {
        // Read the property via Object.entries instead of dynamic bracket
        // access to keep static analyzers happy.
        const entry = Object.entries(tgtNode).find(([k]) => k === key);
        tgtChild = entry ? entry[1] : undefined;
      }
      out[key] = walk(value, tgtChild, nextPrefix);
    }
    return out;
  }

  const merged = walk(source, target, "");
  return { merged, addedPaths };
}

function countPlaceholders(node) {
  if (typeof node === "string") return node.startsWith(PLACEHOLDER_PREFIX) ? 1 : 0;
  if (!isPlainObject(node)) return 0;
  let total = 0;
  for (const value of Object.values(node)) total += countPlaceholders(value);
  return total;
}

// ----- Translator backend --------------------------------------------------
// The chat-completions client (`backendConfig`, `translateString`,
// `translateBatch`) lives in `./lib/translate-backend.mjs` so the other i18n
// tooling can share it. Only the concurrency limiter stays here.

// Simple promise-based semaphore (avoid runtime deps).
function createLimiter(max) {
  let active = 0;
  const queue = [];
  const next = () => {
    if (!queue.length || active >= max) return;
    active++;
    const { fn, resolve, reject } = queue.shift();
    fn()
      .then((v) => {
        active--;
        resolve(v);
        next();
      })
      .catch((err) => {
        active--;
        reject(err);
        next();
      });
  };
  return (fn) =>
    new Promise((resolve, reject) => {
      queue.push({ fn, resolve, reject });
      next();
    });
}

/**
 * Walks a merged tree, finding every leaf that starts with PLACEHOLDER_PREFIX
 * and replacing it with the translation produced by the backend.
 *
 * Translations happen with bounded concurrency. On failure, the placeholder
 * is preserved so a later run can retry.
 *
 * With `batchSize > 1` the placeholders are grouped into requests of up to
 * `batchSize` strings (one JSON object per request). A batch whose response
 * cannot be parsed — or whose upstream call fails — is retried one string at
 * a time, so a bad batch never loses more than the per-string path would.
 */
async function translatePlaceholders(merged, localeEntry, backend, concurrency, batchSize = 1) {
  const tasks = [];
  function collect(node, parent, key) {
    if (typeof node === "string") {
      if (node.startsWith(PLACEHOLDER_PREFIX)) {
        const englishValue = node.slice(PLACEHOLDER_PREFIX.length);
        tasks.push({ parent, key, englishValue });
      }
      return;
    }
    if (!isPlainObject(node)) return;
    for (const [k, v] of Object.entries(node)) {
      collect(v, node, k);
    }
  }
  collect(merged, null, null);

  if (tasks.length === 0) return { translated: 0, failed: 0 };

  const limit = createLimiter(concurrency);
  let translatedCount = 0;
  let failed = 0;

  if (batchSize > 1) {
    const groups = [];
    for (let i = 0; i < tasks.length; i += batchSize) groups.push(tasks.slice(i, i + batchSize));
    await Promise.all(
      groups.map((group) =>
        limit(async () => {
          const entries = group.map((task, i) => ({ id: `s${i}`, text: task.englishValue }));
          try {
            const translated = await translateBatch(entries, localeEntry, backend);
            group.forEach((task, i) => {
              task.parent[task.key] = translated.get(`s${i}`);
              translatedCount++;
            });
          } catch (err) {
            logWarn(
              `batch of ${group.length} failed for ${localeEntry.code} (${err.message}) — retrying one by one`
            );
            for (const task of group) {
              try {
                task.parent[task.key] = await translateString(
                  task.englishValue,
                  localeEntry,
                  backend
                );
                translatedCount++;
              } catch (inner) {
                failed++;
                logWarn(`translation failed for ${localeEntry.code}: ${inner.message}`);
              }
            }
          }
        })
      )
    );
    return { translated: translatedCount, failed };
  }

  await Promise.all(
    tasks.map((task) =>
      limit(async () => {
        try {
          const value = await translateString(task.englishValue, localeEntry, backend);
          task.parent[task.key] = value;
          translatedCount++;
        } catch (err) {
          // Keep the __MISSING__ marker so subsequent runs can retry.
          failed++;
          logWarn(`translation failed for ${localeEntry.code}: ${err.message}`);
        }
      })
    )
  );
  return { translated: translatedCount, failed };
}

// ----- Main ----------------------------------------------------------------

async function processLocale(locale, source, config, opts, backend) {
  const localePath = path.join(MESSAGES_DIR, `${locale}.json`);
  let target = {};
  if (existsSync(localePath)) {
    try {
      target = await loadJson(localePath);
    } catch (err) {
      logWarn(`${locale}: failed to parse existing JSON — starting fresh (${err.message})`);
      target = {};
    }
  } else {
    logWarn(`${locale}: messages file did not exist — creating it`);
  }

  const { merged, addedPaths } = mergeMissing(source, target);
  const placeholderCountBefore = countPlaceholders(merged);

  let translateStats = { translated: 0, failed: 0 };
  if (opts.translateMarkers && placeholderCountBefore > 0 && backend) {
    const localeEntry = config.locales.find((l) => l.code === locale);
    if (!localeEntry) {
      logWarn(`${locale}: not present in config/i18n.json — skipping translation`);
    } else {
      const concurrency =
        opts.concurrency ?? Number(process.env.OMNIROUTE_TRANSLATION_CONCURRENCY || 4);
      translateStats = await translatePlaceholders(
        merged,
        localeEntry,
        backend,
        concurrency,
        opts.batchSize
      );
    }
  }

  const placeholderCountAfter = countPlaceholders(merged);
  const totalMissing = addedPaths.length;
  const stillPlaceholder = placeholderCountAfter;

  const summary = `${locale}: +${totalMissing} missing keys (${stillPlaceholder} __MISSING__, ${translateStats.translated} translated${translateStats.failed ? `, ${translateStats.failed} failed` : ""})`;

  if (opts.dryRun) {
    logInfo(`[DRY] ${summary}`);
    return { addedPaths, translated: translateStats.translated };
  }

  // Only write when something changed. (json-stable serialization)
  const before = existsSync(localePath) ? await fs.readFile(localePath, "utf8") : "";
  const after = JSON.stringify(merged, null, 2) + "\n";
  if (before === after) {
    logInfo(`${locale}: already in sync (no changes)`);
    return { addedPaths, translated: translateStats.translated };
  }
  await fs.writeFile(localePath, after, "utf8");
  logInfo(summary);
  return { addedPaths, translated: translateStats.translated };
}

async function main() {
  const opts = parseArgs(process.argv);
  const config = await loadConfig();

  const sourcePath = path.join(MESSAGES_DIR, `${SOURCE_LOCALE}.json`);
  if (!existsSync(sourcePath)) {
    throw new Error(`Source messages file not found: ${sourcePath}`);
  }
  const source = await loadJson(sourcePath);

  // Locales = every code in config except `en`, intersected with locales that
  // already exist on disk (so we never silently create unknown locale files).
  const onDisk = new Set(
    (await fs.readdir(MESSAGES_DIR)).filter((f) => f.endsWith(".json")).map((f) => f.slice(0, -5))
  );

  let targetLocales = config.locales
    .map((l) => l.code)
    .filter((code) => code !== SOURCE_LOCALE && onDisk.has(code));

  if (opts.locales) {
    const missingFromConfig = opts.locales.filter((c) => !config.locales.some((l) => l.code === c));
    if (missingFromConfig.length) {
      logWarn(`--locale contains codes not in config/i18n.json: ${missingFromConfig.join(", ")}`);
    }
    targetLocales = targetLocales.filter((code) => opts.locales.includes(code));
  }

  logInfo(`source: ${path.relative(ROOT, sourcePath)}`);
  logInfo(`locales: ${targetLocales.length} (${targetLocales.join(", ")})`);
  logInfo(
    `dry-run: ${opts.dryRun ? "yes" : "no"}, translate-markers: ${opts.translateMarkers ? "yes" : "no"}`
  );

  let backend = null;
  if (opts.translateMarkers && !opts.dryRun) {
    backend = backendConfig();
    backend.concurrency =
      opts.concurrency ?? Number(process.env.OMNIROUTE_TRANSLATION_CONCURRENCY || 4);
    const batchInfo = opts.batchSize > 1 ? `, batch=${opts.batchSize}` : "";
    logInfo(
      `backend: ${backend.apiUrl} (model=${backend.model}, concurrency=${backend.concurrency}${batchInfo}, timeout=${backend.timeoutMs}ms)`
    );
  }

  const startMs = Date.now();
  let totalAdded = 0;
  let totalTranslated = 0;
  for (const locale of targetLocales) {
    const result = await processLocale(locale, source, config, opts, backend);
    totalAdded += result.addedPaths.length;
    totalTranslated += result.translated;
  }
  const elapsedSec = ((Date.now() - startMs) / 1000).toFixed(1);
  logInfo(
    `summary: locales=${targetLocales.length}, added=${totalAdded}, translated=${totalTranslated}, elapsed=${elapsedSec}s`
  );
}

const isDirectRun = import.meta.url === pathToFileURL(process.argv[1]).href;
if (isDirectRun) {
  main().catch((err) => {
    logError(err?.stack || err?.message || String(err));
    process.exit(1);
  });
}