diff --git a/changelog.d/fixes/12359-preserve-zwnj-zwj.md b/changelog.d/fixes/12359-preserve-zwnj-zwj.md new file mode 100644 index 0000000000..31d61a0c9d --- /dev/null +++ b/changelog.d/fixes/12359-preserve-zwnj-zwj.md @@ -0,0 +1 @@ +- **fix(sse):** Keep ZWNJ (U+200C) and ZWJ (U+200D) in assistant text, reasoning and tool-call arguments — Persian/Kurdish half-space (`ارائه‌دهنده`), Arabic/Indic shaping and emoji sequences no longer lose them; the response de-obfuscation now removes joiners only between ASCII word characters, where the request side inserts them ([#12186](https://github.com/diegosouzapw/OmniRoute/issues/12186)) — thanks @rezjalibd diff --git a/open-sse/executors/antigravity/sseCollect.ts b/open-sse/executors/antigravity/sseCollect.ts index d84b1fb077..630b091aab 100644 --- a/open-sse/executors/antigravity/sseCollect.ts +++ b/open-sse/executors/antigravity/sseCollect.ts @@ -1,6 +1,7 @@ // Pure SSE-payload -> collected-stream parsing for the Antigravity executor. // Extracted verbatim from antigravity.ts (no host state, no fetch/auth). import { normalizeOpenAICompatibleFinishReasonString } from "../../utils/finishReason.ts"; +import { stripObfuscationZeroWidth } from "../../utils/zeroWidth.ts"; export type AntigravityCollectedStream = { textContent: string; @@ -17,7 +18,7 @@ export type AntigravityCollectedStream = { export function stripZeroWidth(value: unknown): unknown { if (typeof value === "string") { - return value.replace(/[\u200B-\u200D\uFEFF]/g, ""); + return stripObfuscationZeroWidth(value); } if (Array.isArray(value)) { return value.map((item) => stripZeroWidth(item)); @@ -37,7 +38,7 @@ export function parseAntigravityTextualToolCall( text: unknown ): { name: string; args: unknown } | null { if (typeof text !== "string") return null; - const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, ""); + const normalized = stripObfuscationZeroWidth(text); const match = normalized.match( /^[\s\S]*?\[Tool call:\s*([^\]\n]+)\]\s*\nArguments:\s*([\s\S]+?)\s*$/ ); diff --git a/open-sse/handlers/responseSanitizer.ts b/open-sse/handlers/responseSanitizer.ts index a2210681d7..ce2d2af227 100644 --- a/open-sse/handlers/responseSanitizer.ts +++ b/open-sse/handlers/responseSanitizer.ts @@ -12,6 +12,7 @@ import { applyCacheHitTokensToUsage, applyCacheHitTokensToResponsesUsage, } from "./responseSanitizer/cacheHitTokens.ts"; +import { stripObfuscationZeroWidth } from "../utils/zeroWidth.ts"; export { extractThinkingFromContent, shouldParseTextualReasoningTags, @@ -85,7 +86,7 @@ function deleteOpenAICompatibleReasoningFields(record: JsonRecord): void { } function stripZeroWidthText(value: string): string { - return value.replace(/[\u200B-\u200D\uFEFF]/g, ""); + return stripObfuscationZeroWidth(value); } function stripZeroWidthToolArgumentJson(value: unknown): string { diff --git a/open-sse/handlers/responseTranslator.ts b/open-sse/handlers/responseTranslator.ts index 0038f7625b..05195b8011 100644 --- a/open-sse/handlers/responseTranslator.ts +++ b/open-sse/handlers/responseTranslator.ts @@ -5,6 +5,7 @@ import { } from "../services/geminiThoughtSignatureStore.ts"; import { normalizeOpenAICompatibleFinishReasonString } from "../utils/finishReason.ts"; import { containsTextualToolCallMarker } from "../utils/textualToolCall.ts"; +import { stripObfuscationZeroWidth } from "../utils/zeroWidth.ts"; import { getAnyReasoningValue } from "../utils/reasoningFields.ts"; import { caseInsensitiveToolNameLookup, @@ -63,7 +64,7 @@ function parseTextualToolCall(text: unknown): { name: string; args: unknown } | // variations, e.g. a leading "(empty)" marker or zero-width chars inserted // into argument strings. Normalize those variants before parsing so the // response is still surfaced as a structured OpenAI tool call. - const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, ""); + const normalized = stripObfuscationZeroWidth(text); const match = normalized.match( /^[\s\S]*?\[Tool call:\s*([^\]\n]+)\]\s*\nArguments:\s*([\s\S]+?)\s*$/ ); diff --git a/open-sse/handlers/sseParser/geminiResponse.ts b/open-sse/handlers/sseParser/geminiResponse.ts index 1657cf2030..df18008c04 100644 --- a/open-sse/handlers/sseParser/geminiResponse.ts +++ b/open-sse/handlers/sseParser/geminiResponse.ts @@ -2,6 +2,7 @@ // Extracted verbatim from sseParser.ts (file-size cap): pure parsing, no host // state, following the handlers submodule pattern (chatCore/, responseSanitizer/). import { normalizeOpenAICompatibleFinishReasonString } from "../../utils/finishReason.ts"; +import { stripObfuscationZeroWidth } from "../../utils/zeroWidth.ts"; type AccumulatedToolCall = { id: string; @@ -20,7 +21,7 @@ type GeminiSSEAccumulator = { }; function stripZeroWidth(value: unknown): unknown { - if (typeof value === "string") return value.replace(/[\u200B-\u200D\uFEFF]/g, ""); + if (typeof value === "string") return stripObfuscationZeroWidth(value); return value; } @@ -29,7 +30,7 @@ function stripZeroWidth(value: unknown): unknown { * Gemini/Antigravity models emit instead of a native functionCall part. */ function tryParseTextualToolCall(text: string): { name: string; args: unknown } | null { - const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, ""); + const normalized = stripObfuscationZeroWidth(text); const match = normalized.match( /^[\s\S]*?\[Tool call:\s*([^\]\n]+)\]\s*\nArguments:\s*([\s\S]+?)\s*$/ ); diff --git a/open-sse/translator/response/gemini-to-openai.ts b/open-sse/translator/response/gemini-to-openai.ts index 2a8240e290..98808808e1 100644 --- a/open-sse/translator/response/gemini-to-openai.ts +++ b/open-sse/translator/response/gemini-to-openai.ts @@ -14,6 +14,7 @@ import { isMalformedToolCallFinishReason, } from "../../utils/finishReason.ts"; import { stripAnsiCodes } from "../../utils/streamHelpers.ts"; +import { stripObfuscationZeroWidth } from "../../utils/zeroWidth.ts"; type GeminiToOpenAIState = { functionIndex: number; @@ -483,7 +484,7 @@ export function geminiToOpenAIResponse(chunk, state) { let candidate = parseTextualToolCallCandidate(accumulated); if (candidate) { - accumulated = accumulated.replace(/[\u200B-\u200D\uFEFF]/g, ""); + accumulated = stripObfuscationZeroWidth(accumulated); let toolCallIndex = accumulated.lastIndexOf("(empty)[Tool call:"); if (toolCallIndex < 0) { toolCallIndex = accumulated.lastIndexOf("[Tool call:"); diff --git a/open-sse/utils/stream.ts b/open-sse/utils/stream.ts index 16572b18b3..dd37eda217 100644 --- a/open-sse/utils/stream.ts +++ b/open-sse/utils/stream.ts @@ -47,6 +47,7 @@ import { } from "./responsesCommentaryDrop.ts"; import { buildErrorBody } from "./error.ts"; import { parseTextualToolCallCandidate, isValidToolCallHeaderPrefix } from "./textualToolCall.ts"; +import { stripObfuscationZeroWidth } from "./zeroWidth.ts"; import { formatTranslatedStreamError, normalizeStreamFailurePayload, @@ -272,7 +273,7 @@ function containsMalformedTextualToolCall( allowedToolNames?: Set | null ): boolean { if (typeof text !== "string") return false; - const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, ""); + const normalized = stripObfuscationZeroWidth(text); let searchIdx = 0; while (true) { @@ -1637,10 +1638,7 @@ export function createSSEStream(options: StreamOptions = {}) { isResponsesCommentaryMessageItem ).items : passthroughResponsesOutputItems; - const backfilled = backfillResponsesCompletedOutput( - parsed, - backfillCandidates - ); + const backfilled = backfillResponsesCompletedOutput(parsed, backfillCandidates); const usageNormalized = normalizeUsage(parsed); if ( stripped || @@ -1760,7 +1758,11 @@ export function createSSEStream(options: StreamOptions = {}) { ) { const pt = emptyChoicesUsage.prompt_tokens ?? 0; if (pt === 0) { - const estimated = estimateUsage(body, totalContentLength, sourceFormat || FORMATS.OPENAI); + const estimated = estimateUsage( + body, + totalContentLength, + sourceFormat || FORMATS.OPENAI + ); if (estimated?.prompt_tokens > 0) { emptyChoicesUsage.prompt_tokens = estimated.prompt_tokens; emptyChoicesUsage.total_tokens = @@ -2519,11 +2521,7 @@ export function createSSEStream(options: StreamOptions = {}) { // [DONE], so metered clients still see token counts. When the // upstream DID send usage (trailing or in-band), it was forwarded // already and passthroughForwardedUsage guards this off. - if ( - shouldEmitDoneTerminator && - !passthroughForwardedUsage && - hasValidUsage(usage) - ) { + if (shouldEmitDoneTerminator && !passthroughForwardedUsage && hasValidUsage(usage)) { const usageOnlyChunk = { id: passthroughLastChatId ?? passthroughResponsesId ?? `chatcmpl-${Date.now()}`, object: "chat.completion.chunk", diff --git a/open-sse/utils/textualToolCall.ts b/open-sse/utils/textualToolCall.ts index 67c7fde1a2..36b5668fe3 100644 --- a/open-sse/utils/textualToolCall.ts +++ b/open-sse/utils/textualToolCall.ts @@ -1,6 +1,8 @@ +import { stripObfuscationZeroWidth } from "./zeroWidth.ts"; + export function stripZeroWidth(value: unknown): unknown { if (typeof value === "string") { - return value.replace(/[\u200B-\u200D\uFEFF]/g, ""); + return stripObfuscationZeroWidth(value); } if (Array.isArray(value)) { return value.map((item) => stripZeroWidth(item)); @@ -58,7 +60,7 @@ export function parseTextualToolCallCandidate( text: unknown ): { kind: "complete"; name: string; args: unknown } | { kind: "partial" } | null { if (typeof text !== "string") return null; - const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, ""); + const normalized = stripObfuscationZeroWidth(text); const toolCallIndex = normalized.lastIndexOf("[Tool call:"); if (toolCallIndex < 0) { const lastParen = normalized.lastIndexOf("("); @@ -102,7 +104,7 @@ export function parseTextualToolCallCandidate( export function containsTextualToolCallMarker(text: unknown): boolean { if (typeof text !== "string") return false; - const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, ""); + const normalized = stripObfuscationZeroWidth(text); if (!normalized.includes("[Tool call:")) return false; if (normalized.includes("Arguments:")) return true; diff --git a/open-sse/utils/zeroWidth.ts b/open-sse/utils/zeroWidth.ts new file mode 100644 index 0000000000..ecce702bc3 --- /dev/null +++ b/open-sse/utils/zeroWidth.ts @@ -0,0 +1,38 @@ +/** + * Zero-width character cleanup for model output. + * + * The request side obfuscates configurable agent words by inserting a + * U+200D ZERO WIDTH JOINER after their first letter (`o\u200Dpencode`, see + * `services/claudeCodeObfuscation.ts` and `services/systemTransforms.ts`), and + * the response side removes zero-width code points again so an echoed word is + * not corrupted. Removing every U+200B..U+200D also deletes U+200C ZERO WIDTH + * NON-JOINER and U+200D where they belong to the text itself: the Persian and + * Kurdish half-space (ارائه\u200Cدهنده, می\u200Cروم, کتاب\u200Cها), Arabic and Indic shaping, + * and emoji ZWJ sequences (👨\u200D👩\u200D👧). See #12186. + * + * The obfuscator only ever places a joiner between two ASCII word characters, + * so a joiner is removed only there. A joiner touching the edge of the string + * is removed as well when its other neighbour is an ASCII word character, so + * an obfuscated word split across streaming deltas (`o\u200D` + `pencode`) + * is still cleaned. A joiner next to non-ASCII text, and a delta that consists + * of nothing but a joiner (an emoji sequence split by the tokenizer), pass + * through untouched. + * + * U+200B ZERO WIDTH SPACE and U+FEFF have no shaping role and keep the + * unconditional removal they always had. + */ + +const ANY_ZERO_WIDTH = /[\u200B-\u200D\uFEFF]/; +const ZERO_WIDTH_SPACE_OR_BOM = /[\u200B\uFEFF]/g; +const JOINER_BETWEEN_ASCII_WORD_CHARS = + /(?<=[A-Za-z0-9_])[\u200C\u200D]+(?=[A-Za-z0-9_]|$)|^[\u200C\u200D]+(?=[A-Za-z0-9_])/g; + +/** + * Strip the zero-width markers used for agent-word obfuscation while keeping + * ZWNJ/ZWJ that are part of the text (Persian half-space, Arabic/Indic + * shaping, emoji sequences). + */ +export function stripObfuscationZeroWidth(text: string): string { + if (!text || !ANY_ZERO_WIDTH.test(text)) return text; + return text.replace(ZERO_WIDTH_SPACE_OR_BOM, "").replace(JOINER_BETWEEN_ASCII_WORD_CHARS, ""); +} diff --git a/tests/unit/12186-preserve-zwnj-zwj.test.ts b/tests/unit/12186-preserve-zwnj-zwj.test.ts new file mode 100644 index 0000000000..d60c595a9b --- /dev/null +++ b/tests/unit/12186-preserve-zwnj-zwj.test.ts @@ -0,0 +1,337 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +// #12186 — the response pipeline strips every zero-width code point in +// U+200B..U+200D to undo the request-side agent-word obfuscation (which inserts +// one U+200D after the first letter of an ASCII word). That blanket strip also +// deletes U+200C ZERO WIDTH NON-JOINER and U+200D ZERO WIDTH JOINER where they +// are part of the text itself: Persian half-space, Arabic/Indic shaping and +// emoji ZWJ sequences. These tests pin that linguistic joiners survive while the +// ASCII obfuscation marker is still removed. + +const { sanitizeOpenAIResponse, sanitizeStreamingChunk } = + await import("../../open-sse/handlers/responseSanitizer.ts"); +const { parseTextualToolCallCandidate } = await import("../../open-sse/utils/textualToolCall.ts"); +const { parseAntigravityTextualToolCall } = + await import("../../open-sse/executors/antigravity/sseCollect.ts"); +const { parseSSEToGeminiResponse } = + await import("../../open-sse/handlers/sseParser/geminiResponse.ts"); +const { translateNonStreamingResponse } = + await import("../../open-sse/handlers/responseTranslator.ts"); +const { geminiToOpenAIResponse } = + await import("../../open-sse/translator/response/gemini-to-openai.ts"); +const { FORMATS } = await import("../../open-sse/translator/formats.ts"); +const { stripObfuscationZeroWidth } = await import("../../open-sse/utils/zeroWidth.ts"); +const { obfuscateSensitiveWords, getSensitiveWords } = + await import("../../open-sse/services/claudeCodeObfuscation.ts"); + +// Exact word from the issue report: ارائه + U+200C + دهنده ("provider"). +const PERSIAN_WORD = "ارائه\u200Cدهنده"; +// Family emoji: MAN + ZWJ + WOMAN + ZWJ + GIRL. +const FAMILY_EMOJI = "\u{1F468}\u200D\u{1F469}\u200D\u{1F467}"; + +function openAIChunk(delta: Record) { + return { + id: "chatcmpl_12186", + object: "chat.completion.chunk", + created: 1, + model: "auto", + choices: [{ index: 0, delta, finish_reason: null }], + }; +} + +test("#12186 sanitizeOpenAIResponse keeps Persian ZWNJ in non-stream message content", () => { + const sanitized = sanitizeOpenAIResponse({ + id: "chatcmpl_12186", + model: "auto", + choices: [ + { + index: 0, + finish_reason: "stop", + message: { role: "assistant", content: PERSIAN_WORD }, + }, + ], + }) as unknown as { choices: { message: { content: string } }[] }; + + assert.equal(sanitized.choices[0].message.content, PERSIAN_WORD); +}); + +test("#12186 sanitizeOpenAIResponse keeps joiners in text but still de-obfuscates ASCII agent words", () => { + const sanitized = sanitizeOpenAIResponse({ + id: "chatcmpl_12186_mixed", + model: "auto", + choices: [ + { + index: 0, + finish_reason: "stop", + message: { + role: "assistant", + content: `o\u200Dpencode ${PERSIAN_WORD} می\u200Cروم کتاب\u200Cها ${FAMILY_EMOJI} c\u200Dursor`, + }, + }, + ], + }) as unknown as { choices: { message: { content: string } }[] }; + + assert.equal( + sanitized.choices[0].message.content, + `opencode ${PERSIAN_WORD} می\u200Cروم کتاب\u200Cها ${FAMILY_EMOJI} cursor` + ); +}); + +test("#12186 sanitizeOpenAIResponse still strips U+200B and U+FEFF from message content", () => { + const sanitized = sanitizeOpenAIResponse({ + id: "chatcmpl_12186_zwsp", + model: "auto", + choices: [ + { + index: 0, + finish_reason: "stop", + message: { role: "assistant", content: "\uFEFFhello\u200B world o\u200Bpencode" }, + }, + ], + }) as unknown as { choices: { message: { content: string } }[] }; + + assert.equal(sanitized.choices[0].message.content, "hello world opencode"); +}); + +test("#12186 sanitizeOpenAIResponse keeps Persian ZWNJ inside tool-call arguments", () => { + const args = JSON.stringify({ command: `echo ${PERSIAN_WORD}`, note: "o\u200Dpencode" }); + const sanitized = sanitizeOpenAIResponse({ + id: "chatcmpl_12186_tool", + model: "auto", + choices: [ + { + index: 0, + finish_reason: "tool_calls", + message: { + role: "assistant", + content: "", + tool_calls: [ + { id: "call_1", type: "function", function: { name: "run", arguments: args } }, + ], + }, + }, + ], + }) as unknown as { + choices: { message: { tool_calls: { function: { arguments: string } }[] } }[]; + }; + + assert.equal( + sanitized.choices[0].message.tool_calls[0].function.arguments, + JSON.stringify({ command: `echo ${PERSIAN_WORD}`, note: "opencode" }) + ); +}); + +test("#12186 sanitizeStreamingChunk keeps Persian ZWNJ in OpenAI delta content", () => { + const sanitized = sanitizeStreamingChunk(openAIChunk({ content: PERSIAN_WORD })) as unknown as { + choices: { delta: { content: string } }[]; + }; + + assert.equal(sanitized.choices[0].delta.content, PERSIAN_WORD); +}); + +test("#12186 sanitizeStreamingChunk keeps an emoji ZWJ sequence in OpenAI delta content", () => { + const sanitized = sanitizeStreamingChunk(openAIChunk({ content: FAMILY_EMOJI })) as unknown as { + choices: { delta: { content: string } }[]; + }; + + assert.equal(sanitized.choices[0].delta.content, FAMILY_EMOJI); +}); + +test("#12186 sanitizeStreamingChunk keeps a delta that is only a ZWJ (emoji sequence split by the tokenizer)", () => { + const sanitized = sanitizeStreamingChunk(openAIChunk({ content: "\u200D" })) as unknown as { + choices: { delta: { content: string } }[]; + }; + + assert.equal(sanitized.choices[0].delta.content, "\u200D"); +}); + +test("#12186 sanitizeStreamingChunk still de-obfuscates an ASCII word split across deltas", () => { + const first = sanitizeStreamingChunk(openAIChunk({ content: "o\u200D" })) as unknown as { + choices: { delta: { content: string } }[]; + }; + const second = sanitizeStreamingChunk(openAIChunk({ content: "\u200Dpencode" })) as unknown as { + choices: { delta: { content: string } }[]; + }; + + assert.equal(first.choices[0].delta.content, "o"); + assert.equal(second.choices[0].delta.content, "pencode"); +}); + +test("#12186 sanitizeStreamingChunk keeps Persian ZWNJ in reasoning_content deltas", () => { + const sanitized = sanitizeStreamingChunk( + openAIChunk({ reasoning_content: `${PERSIAN_WORD} c\u200Dursor` }) + ) as unknown as { choices: { delta: { reasoning_content: string } }[] }; + + assert.equal(sanitized.choices[0].delta.reasoning_content, `${PERSIAN_WORD} cursor`); +}); + +test("#12186 sanitizeStreamingChunk keeps Persian ZWNJ in Anthropic text_delta events", () => { + const sanitized = sanitizeStreamingChunk({ + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: `${PERSIAN_WORD} ${FAMILY_EMOJI} a\u200Dider` }, + }) as unknown as { delta: { text: string } }; + + assert.equal(sanitized.delta.text, `${PERSIAN_WORD} ${FAMILY_EMOJI} aider`); +}); + +test("#12186 sanitizeStreamingChunk keeps Persian ZWNJ in native response.output_text.delta", () => { + const sanitized = sanitizeStreamingChunk({ + type: "response.output_text.delta", + delta: PERSIAN_WORD, + }) as unknown as { delta: string }; + + assert.equal(sanitized.delta, PERSIAN_WORD); +}); + +test("#12186 sanitizeStreamingChunk keeps Persian ZWNJ in native response.output_text.done", () => { + const sanitized = sanitizeStreamingChunk({ + type: "response.output_text.done", + text: `${PERSIAN_WORD} o\u200Dpencode`, + }) as unknown as { text: string }; + + assert.equal(sanitized.text, `${PERSIAN_WORD} opencode`); +}); + +test("#12186 parseTextualToolCallCandidate keeps Persian ZWNJ in textual tool-call arguments", () => { + const parsed = parseTextualToolCallCandidate( + `[Tool call: terminal]\nArguments: {"command":"echo ${PERSIAN_WORD} o\u200Dpencode"}` + ); + + assert.ok(parsed && parsed.kind === "complete"); + assert.equal(parsed.name, "terminal"); + assert.deepEqual(parsed.args, { command: `echo ${PERSIAN_WORD} opencode` }); +}); + +test("#12186 parseAntigravityTextualToolCall keeps Persian ZWNJ in textual tool-call arguments", () => { + const parsed = parseAntigravityTextualToolCall( + `[Tool call: terminal]\nArguments: {"command":"echo ${PERSIAN_WORD} o\u200Dpencode"}` + ); + + assert.ok(parsed); + assert.equal(parsed.name, "terminal"); + assert.deepEqual(parsed.args, { command: `echo ${PERSIAN_WORD} opencode` }); +}); + +test("#12186 parseSSEToGeminiResponse keeps Persian ZWNJ in textual tool-call arguments", () => { + const text = `[Tool call: terminal]\nArguments: {"command":"echo ${PERSIAN_WORD}"}`; + const rawSSE = `data: ${JSON.stringify({ + response: { + candidates: [{ content: { parts: [{ text }] }, finishReason: "STOP" }], + }, + })}`; + + const parsed = parseSSEToGeminiResponse(rawSSE, "gemini-2.5-flash") as { + choices: { message: { tool_calls: { function: { arguments: string } }[] } }[]; + }; + + assert.ok(parsed); + assert.equal( + parsed.choices[0].message.tool_calls[0].function.arguments, + JSON.stringify({ command: `echo ${PERSIAN_WORD}` }) + ); +}); + +test("#12186 Gemini non-stream translation keeps Persian ZWNJ in textual tool-call arguments", () => { + const result = translateNonStreamingResponse( + { + responseId: "resp-12186", + modelVersion: "gemini-2.5-flash", + candidates: [ + { + content: { + parts: [ + { + text: `[Tool call: terminal]\nArguments: {"command":"echo ${PERSIAN_WORD} o\u200Dpencode"}`, + }, + ], + }, + finishReason: "STOP", + }, + ], + }, + FORMATS.GEMINI, + FORMATS.OPENAI + ) as { choices: { message: { tool_calls: { function: { arguments: string } }[] } }[] }; + + assert.equal( + result.choices[0].message.tool_calls[0].function.arguments, + JSON.stringify({ command: `echo ${PERSIAN_WORD} opencode` }) + ); +}); + +test("#12186 Gemini stream translation keeps Persian ZWNJ in text emitted before a textual tool call", () => { + const result = geminiToOpenAIResponse( + { + responseId: "resp-12186-stream", + modelVersion: "gemini-2.5-flash", + candidates: [ + { + content: { + parts: [ + { text: `${PERSIAN_WORD}: [Tool call: terminal]\nArguments: {"command":"whoami"}` }, + ], + }, + finishReason: "STOP", + }, + ], + }, + { toolCalls: new Map() } + ) as Array<{ choices?: { delta?: { content?: string; tool_calls?: unknown[] } }[] }>; + + const leakedContent = result.map((event) => event.choices?.[0]?.delta?.content || "").join(""); + assert.equal(leakedContent, `${PERSIAN_WORD}: `); + + const toolCalls = result.flatMap((event) => event.choices?.[0]?.delta?.tool_calls || []); + assert.equal(toolCalls.length, 1); +}); + +test("#12186 stripObfuscationZeroWidth keeps ZWNJ/ZWJ that belong to the text", () => { + for (const text of [ + PERSIAN_WORD, + "می\u200Cروم نمی\u200Cدانم کتاب\u200Cها", + FAMILY_EMOJI, + "\u{1F3F3}\u{FE0F}\u200D\u{1F308}", + "\u200D", + "\u{1F468}\u200D", + "\u200D\u{1F469}", + "\u200C", + ]) { + assert.equal(stripObfuscationZeroWidth(text), text); + } +}); + +test("#12186 stripObfuscationZeroWidth reverses the request-side obfuscation for every default agent word", () => { + const original = `Use ${getSensitiveWords().join(", ")} in ${PERSIAN_WORD} ${FAMILY_EMOJI}`; + const obfuscated = obfuscateSensitiveWords(original); + + assert.notEqual(obfuscated, original); + assert.equal(stripObfuscationZeroWidth(obfuscated), original); +}); + +test("#12186 stripObfuscationZeroWidth removes joiners only between ASCII word characters", () => { + assert.equal(stripObfuscationZeroWidth("o\u200Dpencode"), "opencode"); + assert.equal(stripObfuscationZeroWidth("roo_\u200Dcline 4\u200D2"), "roo_cline 42"); + assert.equal(stripObfuscationZeroWidth("a\u200Cb"), "ab"); + assert.equal(stripObfuscationZeroWidth("o\u200D\u200D\u200Cpencode"), "opencode"); + assert.equal(stripObfuscationZeroWidth("o\u200D"), "o"); + assert.equal(stripObfuscationZeroWidth("\u200Dpencode"), "pencode"); + assert.equal(stripObfuscationZeroWidth("x \u200D y"), "x \u200D y"); + // Neither side ASCII-adjacent on both ends: a joiner next to whitespace is not + // an obfuscation marker and is left alone. + assert.equal(stripObfuscationZeroWidth("x\u200D \u200Dy"), "x\u200D \u200Dy"); +}); + +test("#12186 stripObfuscationZeroWidth still removes U+200B and U+FEFF anywhere", () => { + assert.equal(stripObfuscationZeroWidth(`\u200B${PERSIAN_WORD}\uFEFF`), PERSIAN_WORD); + assert.equal(stripObfuscationZeroWidth("\uFEFF"), ""); + assert.equal(stripObfuscationZeroWidth("o\u200B\u200Dp"), "op"); + assert.equal(stripObfuscationZeroWidth("\u200BКак исправить"), "Как исправить"); +}); + +test("#12186 stripObfuscationZeroWidth returns the same reference when nothing needs stripping", () => { + const text = `plain ${PERSIAN_WORD}`; + assert.equal(stripObfuscationZeroWidth(text), text); + assert.equal(stripObfuscationZeroWidth(""), ""); +});