feat(i18n): 8 new locales — Kannada, Malayalam, Odia, Punjabi, Nepali, Sinhala, Burmese, Khmer (59 locales) (#13660)

Batch 2 of the locale expansion across the dashboard catalog, docs mirrors, CLI catalog, README, locale index and the site. 51 → 59 locales.

Also: the translator restores the ICU literal escape around angle placeholders, and the docs chunker splits oversized sections before translating.

⚠️ base-red inherited: #12732 — the eight red checks fail identically on unrelated PRs cut from the same base (e.g. #13197); every gate is green locally after merging the base.

scripts/i18n/lib/translate-backend.mjs

@@ -119,6 +119,32 @@ export const TRANSLATION_SYSTEM = (englishName, native) =>
     `Keep punctuation and trailing whitespace identical to the source.`,
   ].join(" ");
 
+/**
+ * Restores the ICU literal escape the backends drop around angle placeholders.
+ *
+ * English writes `'<name>'`: those single quotes are ICU's escape, so the span
+ * renders as the literal text `<name>`. Translations come back as a bare
+ * `<nome>`, which ICU then parses as an (unclosed) tag and the message fails to
+ * compile — every locale in the first batch shipped two of these.
+ *
+ * Only messages whose English side quotes EVERY angle span are touched: when the
+ * source mixes real markup (`<b>`) with a literal span there is no safe way to
+ * tell which is which, so the translation is left exactly as it came back. An
+ * apostrophe inside the span is doubled, otherwise it closes the literal early.
+ */
+export function preserveIcuLiteralQuotes(englishValue, translated) {
+  if (typeof englishValue !== "string" || typeof translated !== "string") return translated;
+  if (!englishValue.includes("'<")) return translated;
+  // Every "<" in the source must be the start of an escaped span.
+  for (let i = 0; i < englishValue.length; i++) {
+    if (englishValue[i] === "<" && englishValue[i - 1] !== "'") return translated;
+  }
+  return translated.replace(
+    /(?<!')<([^<>]*)>(?!')/g,
+    (_m, inner) => `'<${inner.replace(/'/g, "''")}>'`
+  );
+}
+
 export async function translateString(englishValue, localeEntry, backend) {
   const englishName = localeEntry.english ?? localeEntry.name;
   const native = localeEntry.native ?? localeEntry.name;
@@ -127,7 +153,7 @@ export async function translateString(englishValue, localeEntry, backend) {
     { role: "user", content: englishValue },
   ];
   const out = await callChat(messages, backend);
-  return out.trim();
+  return preserveIcuLiteralQuotes(englishValue, out.trim());
 }
 
 // ----- Batch mode ----------------------------------------------------------
@@ -198,8 +224,14 @@ export async function translateBatch(entries, localeEntry, backend) {
     { role: "user", content: JSON.stringify(payload) },
   ];
   const out = await callChat(messages, backend);
-  return parseBatchResponse(
+  const parsed = parseBatchResponse(
     out,
     entries.map((e) => e.id)
   );
+  for (const entry of entries) {
+    if (typeof parsed[entry.id] === "string") {
+      parsed[entry.id] = preserveIcuLiteralQuotes(entry.text, parsed[entry.id]);
+    }
+  }
+  return parsed;
 }

scripts/i18n/run-translation.mjs

@@ -380,16 +380,22 @@ const SYSTEM_PROMPT = (englishName, native) =>
     `Return ONLY the translated markdown — no preamble, no explanation, no surrounding fences.`,
   ].join(" ");
 
-// Splits a markdown body into chunks of <= maxChars, breaking on top-level `## ` headings only.
-function chunkMarkdown(markdown, maxChars = 6000) {
+// Splits a markdown body into chunks of <= maxChars. Top-level `## ` headings
+// are the preferred cut; a section that is still longer than maxChars is then
+// split again on `### ` headings and paragraph boundaries, never inside a
+// fenced code block. Before the second pass a single long section (README.md
+// has a 16 KB one, USER_GUIDE.md a 20 KB one) became one oversized request
+// that the slow fallback model could not answer inside the backend's
+// 10-minute fetch timeout, and the biggest docs failed on every retry.
+export function chunkMarkdown(markdown, maxChars = 6000) {
   if (markdown.length <= maxChars) return [markdown];
   const lines = markdown.split("\n");
-  const chunks = [];
+  const sections = [];
   let buf = [];
   let size = 0;
   for (const line of lines) {
     if (line.startsWith("## ") && size > maxChars * 0.5) {
-      chunks.push(buf.join("\n"));
+      sections.push(buf.join("\n"));
       buf = [line];
       size = line.length;
     } else {
@@ -397,7 +403,65 @@ function chunkMarkdown(markdown, maxChars = 6000) {
       size += line.length + 1;
     }
   }
-  if (buf.length) chunks.push(buf.join("\n"));
+  if (buf.length) sections.push(buf.join("\n"));
+  return sections.flatMap((section) =>
+    section.length <= maxChars ? [section] : splitOversizedSection(section, maxChars)
+  );
+}
+
+const FENCE_LINE = /^\s*(```|~~~)/;
+
+// Groups a section into blocks — a whole fenced code block, a heading-led run,
+// or a paragraph ending at a blank line — and packs them greedily. A block that
+// is itself larger than maxChars stays whole: cutting mid-paragraph or inside a
+// fence would hand the model a fragment it cannot translate faithfully.
+function splitOversizedSection(section, maxChars) {
+  const blocks = [];
+  let block = [];
+  let inFence = false;
+  for (const line of section.split("\n")) {
+    const isFence = FENCE_LINE.test(line);
+    if (inFence) {
+      block.push(line);
+      if (isFence) {
+        inFence = false;
+        blocks.push(block);
+        block = [];
+      }
+      continue;
+    }
+    if (isFence) {
+      if (block.length) blocks.push(block);
+      block = [line];
+      inFence = true;
+      continue;
+    }
+    if (/^##+ /.test(line) && block.length) {
+      blocks.push(block);
+      block = [];
+    }
+    block.push(line);
+    if (line.trim() === "") {
+      blocks.push(block);
+      block = [];
+    }
+  }
+  if (block.length) blocks.push(block);
+
+  const chunks = [];
+  let current = [];
+  let size = 0;
+  for (const lines of blocks) {
+    const length = lines.join("\n").length + 1;
+    if (size > 0 && size + length > maxChars) {
+      chunks.push(current.join("\n"));
+      current = [];
+      size = 0;
+    }
+    current.push(...lines);
+    size += length;
+  }
+  if (current.length) chunks.push(current.join("\n"));
   return chunks;
 }