fix(sse): preserve ZWNJ and ZWJ in sanitized responses (#12359)

The response de-obfuscation stripped the whole U+200B..U+200D range, so Persian/Kurdish half-spaces (U+200C), Arabic/Indic shaping and emoji ZWJ sequences (U+200D) were deleted from every assistant response — text, reasoning and tool-call arguments, streaming and non-streaming, every provider: ارائه‌دهنده came back as ارائهدهنده.

The request side only ever inserts a U+200D between two ASCII word characters, so the new stripObfuscationZeroWidth() removes a joiner only there, or at a string edge next to one so a word split across streaming deltas is still cleaned; U+200B and U+FEFF keep their unconditional removal. All seven copies of the old regex now go through the helper.

Validated in a combined worktree with all 25 PRs of this batch boarded together: typecheck:core clean, 443/443 node-runner tests plus 14/14 vitest across every test file the batch touches, and check-changelog-integrity, check:cycles (418 files), check:provider-consistency (272 REGISTRY entries, 355 canonical providers), check:docs-counts, check:docs-sync (42 locales) and check-file-size all green.

Thanks @pacocartones.
This commit is contained in:
Paco Cartones
2026-09-02 08:15:19 +02:00
committed by GitHub
parent 5a34111125
commit 0ec7504024
10 changed files with 402 additions and 21 deletions

View File

@@ -0,0 +1 @@
- **fix(sse):** Keep ZWNJ (U+200C) and ZWJ (U+200D) in assistant text, reasoning and tool-call arguments — Persian/Kurdish half-space (`ارائه‌دهنده`), Arabic/Indic shaping and emoji sequences no longer lose them; the response de-obfuscation now removes joiners only between ASCII word characters, where the request side inserts them ([#12186](https://github.com/diegosouzapw/OmniRoute/issues/12186)) — thanks @rezjalibd

View File

@@ -1,6 +1,7 @@
// Pure SSE-payload -> collected-stream parsing for the Antigravity executor.
// Extracted verbatim from antigravity.ts (no host state, no fetch/auth).
import { normalizeOpenAICompatibleFinishReasonString } from "../../utils/finishReason.ts";
import { stripObfuscationZeroWidth } from "../../utils/zeroWidth.ts";
export type AntigravityCollectedStream = {
textContent: string;
@@ -17,7 +18,7 @@ export type AntigravityCollectedStream = {
export function stripZeroWidth(value: unknown): unknown {
if (typeof value === "string") {
return value.replace(/[\u200B-\u200D\uFEFF]/g, "");
return stripObfuscationZeroWidth(value);
}
if (Array.isArray(value)) {
return value.map((item) => stripZeroWidth(item));
@@ -37,7 +38,7 @@ export function parseAntigravityTextualToolCall(
text: unknown
): { name: string; args: unknown } | null {
if (typeof text !== "string") return null;
const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, "");
const normalized = stripObfuscationZeroWidth(text);
const match = normalized.match(
/^[\s\S]*?\[Tool call:\s*([^\]\n]+)\]\s*\nArguments:\s*([\s\S]+?)\s*$/
);

View File

@@ -12,6 +12,7 @@ import {
applyCacheHitTokensToUsage,
applyCacheHitTokensToResponsesUsage,
} from "./responseSanitizer/cacheHitTokens.ts";
import { stripObfuscationZeroWidth } from "../utils/zeroWidth.ts";
export {
extractThinkingFromContent,
shouldParseTextualReasoningTags,
@@ -85,7 +86,7 @@ function deleteOpenAICompatibleReasoningFields(record: JsonRecord): void {
}
function stripZeroWidthText(value: string): string {
return value.replace(/[\u200B-\u200D\uFEFF]/g, "");
return stripObfuscationZeroWidth(value);
}
function stripZeroWidthToolArgumentJson(value: unknown): string {

View File

@@ -5,6 +5,7 @@ import {
} from "../services/geminiThoughtSignatureStore.ts";
import { normalizeOpenAICompatibleFinishReasonString } from "../utils/finishReason.ts";
import { containsTextualToolCallMarker } from "../utils/textualToolCall.ts";
import { stripObfuscationZeroWidth } from "../utils/zeroWidth.ts";
import { getAnyReasoningValue } from "../utils/reasoningFields.ts";
import {
caseInsensitiveToolNameLookup,
@@ -63,7 +64,7 @@ function parseTextualToolCall(text: unknown): { name: string; args: unknown } |
// variations, e.g. a leading "(empty)" marker or zero-width chars inserted
// into argument strings. Normalize those variants before parsing so the
// response is still surfaced as a structured OpenAI tool call.
const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, "");
const normalized = stripObfuscationZeroWidth(text);
const match = normalized.match(
/^[\s\S]*?\[Tool call:\s*([^\]\n]+)\]\s*\nArguments:\s*([\s\S]+?)\s*$/
);

View File

@@ -2,6 +2,7 @@
// Extracted verbatim from sseParser.ts (file-size cap): pure parsing, no host
// state, following the handlers submodule pattern (chatCore/, responseSanitizer/).
import { normalizeOpenAICompatibleFinishReasonString } from "../../utils/finishReason.ts";
import { stripObfuscationZeroWidth } from "../../utils/zeroWidth.ts";
type AccumulatedToolCall = {
id: string;
@@ -20,7 +21,7 @@ type GeminiSSEAccumulator = {
};
function stripZeroWidth(value: unknown): unknown {
if (typeof value === "string") return value.replace(/[\u200B-\u200D\uFEFF]/g, "");
if (typeof value === "string") return stripObfuscationZeroWidth(value);
return value;
}
@@ -29,7 +30,7 @@ function stripZeroWidth(value: unknown): unknown {
* Gemini/Antigravity models emit instead of a native functionCall part.
*/
function tryParseTextualToolCall(text: string): { name: string; args: unknown } | null {
const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, "");
const normalized = stripObfuscationZeroWidth(text);
const match = normalized.match(
/^[\s\S]*?\[Tool call:\s*([^\]\n]+)\]\s*\nArguments:\s*([\s\S]+?)\s*$/
);

View File

@@ -14,6 +14,7 @@ import {
isMalformedToolCallFinishReason,
} from "../../utils/finishReason.ts";
import { stripAnsiCodes } from "../../utils/streamHelpers.ts";
import { stripObfuscationZeroWidth } from "../../utils/zeroWidth.ts";
type GeminiToOpenAIState = {
functionIndex: number;
@@ -483,7 +484,7 @@ export function geminiToOpenAIResponse(chunk, state) {
let candidate = parseTextualToolCallCandidate(accumulated);
if (candidate) {
accumulated = accumulated.replace(/[\u200B-\u200D\uFEFF]/g, "");
accumulated = stripObfuscationZeroWidth(accumulated);
let toolCallIndex = accumulated.lastIndexOf("(empty)[Tool call:");
if (toolCallIndex < 0) {
toolCallIndex = accumulated.lastIndexOf("[Tool call:");

View File

@@ -47,6 +47,7 @@ import {
} from "./responsesCommentaryDrop.ts";
import { buildErrorBody } from "./error.ts";
import { parseTextualToolCallCandidate, isValidToolCallHeaderPrefix } from "./textualToolCall.ts";
import { stripObfuscationZeroWidth } from "./zeroWidth.ts";
import {
formatTranslatedStreamError,
normalizeStreamFailurePayload,
@@ -272,7 +273,7 @@ function containsMalformedTextualToolCall(
allowedToolNames?: Set<string> | null
): boolean {
if (typeof text !== "string") return false;
const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, "");
const normalized = stripObfuscationZeroWidth(text);
let searchIdx = 0;
while (true) {
@@ -1637,10 +1638,7 @@ export function createSSEStream(options: StreamOptions = {}) {
isResponsesCommentaryMessageItem
).items
: passthroughResponsesOutputItems;
const backfilled = backfillResponsesCompletedOutput(
parsed,
backfillCandidates
);
const backfilled = backfillResponsesCompletedOutput(parsed, backfillCandidates);
const usageNormalized = normalizeUsage(parsed);
if (
stripped ||
@@ -1760,7 +1758,11 @@ export function createSSEStream(options: StreamOptions = {}) {
) {
const pt = emptyChoicesUsage.prompt_tokens ?? 0;
if (pt === 0) {
const estimated = estimateUsage(body, totalContentLength, sourceFormat || FORMATS.OPENAI);
const estimated = estimateUsage(
body,
totalContentLength,
sourceFormat || FORMATS.OPENAI
);
if (estimated?.prompt_tokens > 0) {
emptyChoicesUsage.prompt_tokens = estimated.prompt_tokens;
emptyChoicesUsage.total_tokens =
@@ -2519,11 +2521,7 @@ export function createSSEStream(options: StreamOptions = {}) {
// [DONE], so metered clients still see token counts. When the
// upstream DID send usage (trailing or in-band), it was forwarded
// already and passthroughForwardedUsage guards this off.
if (
shouldEmitDoneTerminator &&
!passthroughForwardedUsage &&
hasValidUsage(usage)
) {
if (shouldEmitDoneTerminator && !passthroughForwardedUsage && hasValidUsage(usage)) {
const usageOnlyChunk = {
id: passthroughLastChatId ?? passthroughResponsesId ?? `chatcmpl-${Date.now()}`,
object: "chat.completion.chunk",

View File

@@ -1,6 +1,8 @@
import { stripObfuscationZeroWidth } from "./zeroWidth.ts";
export function stripZeroWidth(value: unknown): unknown {
if (typeof value === "string") {
return value.replace(/[\u200B-\u200D\uFEFF]/g, "");
return stripObfuscationZeroWidth(value);
}
if (Array.isArray(value)) {
return value.map((item) => stripZeroWidth(item));
@@ -58,7 +60,7 @@ export function parseTextualToolCallCandidate(
text: unknown
): { kind: "complete"; name: string; args: unknown } | { kind: "partial" } | null {
if (typeof text !== "string") return null;
const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, "");
const normalized = stripObfuscationZeroWidth(text);
const toolCallIndex = normalized.lastIndexOf("[Tool call:");
if (toolCallIndex < 0) {
const lastParen = normalized.lastIndexOf("(");
@@ -102,7 +104,7 @@ export function parseTextualToolCallCandidate(
export function containsTextualToolCallMarker(text: unknown): boolean {
if (typeof text !== "string") return false;
const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, "");
const normalized = stripObfuscationZeroWidth(text);
if (!normalized.includes("[Tool call:")) return false;
if (normalized.includes("Arguments:")) return true;

View File

@@ -0,0 +1,38 @@
/**
* Zero-width character cleanup for model output.
*
* The request side obfuscates configurable agent words by inserting a
* U+200D ZERO WIDTH JOINER after their first letter (`o\u200Dpencode`, see
* `services/claudeCodeObfuscation.ts` and `services/systemTransforms.ts`), and
* the response side removes zero-width code points again so an echoed word is
* not corrupted. Removing every U+200B..U+200D also deletes U+200C ZERO WIDTH
* NON-JOINER and U+200D where they belong to the text itself: the Persian and
* Kurdish half-space (ارائه\u200Cدهنده, می\u200Cروم, کتاب\u200Cها), Arabic and Indic shaping,
* and emoji ZWJ sequences (👨\u200D👩\u200D👧). See #12186.
*
* The obfuscator only ever places a joiner between two ASCII word characters,
* so a joiner is removed only there. A joiner touching the edge of the string
* is removed as well when its other neighbour is an ASCII word character, so
* an obfuscated word split across streaming deltas (`o\u200D` + `pencode`)
* is still cleaned. A joiner next to non-ASCII text, and a delta that consists
* of nothing but a joiner (an emoji sequence split by the tokenizer), pass
* through untouched.
*
* U+200B ZERO WIDTH SPACE and U+FEFF have no shaping role and keep the
* unconditional removal they always had.
*/
const ANY_ZERO_WIDTH = /[\u200B-\u200D\uFEFF]/;
const ZERO_WIDTH_SPACE_OR_BOM = /[\u200B\uFEFF]/g;
const JOINER_BETWEEN_ASCII_WORD_CHARS =
/(?<=[A-Za-z0-9_])[\u200C\u200D]+(?=[A-Za-z0-9_]|$)|^[\u200C\u200D]+(?=[A-Za-z0-9_])/g;
/**
* Strip the zero-width markers used for agent-word obfuscation while keeping
* ZWNJ/ZWJ that are part of the text (Persian half-space, Arabic/Indic
* shaping, emoji sequences).
*/
export function stripObfuscationZeroWidth(text: string): string {
if (!text || !ANY_ZERO_WIDTH.test(text)) return text;
return text.replace(ZERO_WIDTH_SPACE_OR_BOM, "").replace(JOINER_BETWEEN_ASCII_WORD_CHARS, "");
}

View File

@@ -0,0 +1,337 @@
import test from "node:test";
import assert from "node:assert/strict";
// #12186 — the response pipeline strips every zero-width code point in
// U+200B..U+200D to undo the request-side agent-word obfuscation (which inserts
// one U+200D after the first letter of an ASCII word). That blanket strip also
// deletes U+200C ZERO WIDTH NON-JOINER and U+200D ZERO WIDTH JOINER where they
// are part of the text itself: Persian half-space, Arabic/Indic shaping and
// emoji ZWJ sequences. These tests pin that linguistic joiners survive while the
// ASCII obfuscation marker is still removed.
const { sanitizeOpenAIResponse, sanitizeStreamingChunk } =
await import("../../open-sse/handlers/responseSanitizer.ts");
const { parseTextualToolCallCandidate } = await import("../../open-sse/utils/textualToolCall.ts");
const { parseAntigravityTextualToolCall } =
await import("../../open-sse/executors/antigravity/sseCollect.ts");
const { parseSSEToGeminiResponse } =
await import("../../open-sse/handlers/sseParser/geminiResponse.ts");
const { translateNonStreamingResponse } =
await import("../../open-sse/handlers/responseTranslator.ts");
const { geminiToOpenAIResponse } =
await import("../../open-sse/translator/response/gemini-to-openai.ts");
const { FORMATS } = await import("../../open-sse/translator/formats.ts");
const { stripObfuscationZeroWidth } = await import("../../open-sse/utils/zeroWidth.ts");
const { obfuscateSensitiveWords, getSensitiveWords } =
await import("../../open-sse/services/claudeCodeObfuscation.ts");
// Exact word from the issue report: ارائه + U+200C + دهنده ("provider").
const PERSIAN_WORD = "ارائه\u200Cدهنده";
// Family emoji: MAN + ZWJ + WOMAN + ZWJ + GIRL.
const FAMILY_EMOJI = "\u{1F468}\u200D\u{1F469}\u200D\u{1F467}";
function openAIChunk(delta: Record<string, unknown>) {
return {
id: "chatcmpl_12186",
object: "chat.completion.chunk",
created: 1,
model: "auto",
choices: [{ index: 0, delta, finish_reason: null }],
};
}
test("#12186 sanitizeOpenAIResponse keeps Persian ZWNJ in non-stream message content", () => {
const sanitized = sanitizeOpenAIResponse({
id: "chatcmpl_12186",
model: "auto",
choices: [
{
index: 0,
finish_reason: "stop",
message: { role: "assistant", content: PERSIAN_WORD },
},
],
}) as unknown as { choices: { message: { content: string } }[] };
assert.equal(sanitized.choices[0].message.content, PERSIAN_WORD);
});
test("#12186 sanitizeOpenAIResponse keeps joiners in text but still de-obfuscates ASCII agent words", () => {
const sanitized = sanitizeOpenAIResponse({
id: "chatcmpl_12186_mixed",
model: "auto",
choices: [
{
index: 0,
finish_reason: "stop",
message: {
role: "assistant",
content: `o\u200Dpencode ${PERSIAN_WORD} می\u200Cروم کتاب\u200Cها ${FAMILY_EMOJI} c\u200Dursor`,
},
},
],
}) as unknown as { choices: { message: { content: string } }[] };
assert.equal(
sanitized.choices[0].message.content,
`opencode ${PERSIAN_WORD} می\u200Cروم کتاب\u200Cها ${FAMILY_EMOJI} cursor`
);
});
test("#12186 sanitizeOpenAIResponse still strips U+200B and U+FEFF from message content", () => {
const sanitized = sanitizeOpenAIResponse({
id: "chatcmpl_12186_zwsp",
model: "auto",
choices: [
{
index: 0,
finish_reason: "stop",
message: { role: "assistant", content: "\uFEFFhello\u200B world o\u200Bpencode" },
},
],
}) as unknown as { choices: { message: { content: string } }[] };
assert.equal(sanitized.choices[0].message.content, "hello world opencode");
});
test("#12186 sanitizeOpenAIResponse keeps Persian ZWNJ inside tool-call arguments", () => {
const args = JSON.stringify({ command: `echo ${PERSIAN_WORD}`, note: "o\u200Dpencode" });
const sanitized = sanitizeOpenAIResponse({
id: "chatcmpl_12186_tool",
model: "auto",
choices: [
{
index: 0,
finish_reason: "tool_calls",
message: {
role: "assistant",
content: "",
tool_calls: [
{ id: "call_1", type: "function", function: { name: "run", arguments: args } },
],
},
},
],
}) as unknown as {
choices: { message: { tool_calls: { function: { arguments: string } }[] } }[];
};
assert.equal(
sanitized.choices[0].message.tool_calls[0].function.arguments,
JSON.stringify({ command: `echo ${PERSIAN_WORD}`, note: "opencode" })
);
});
test("#12186 sanitizeStreamingChunk keeps Persian ZWNJ in OpenAI delta content", () => {
const sanitized = sanitizeStreamingChunk(openAIChunk({ content: PERSIAN_WORD })) as unknown as {
choices: { delta: { content: string } }[];
};
assert.equal(sanitized.choices[0].delta.content, PERSIAN_WORD);
});
test("#12186 sanitizeStreamingChunk keeps an emoji ZWJ sequence in OpenAI delta content", () => {
const sanitized = sanitizeStreamingChunk(openAIChunk({ content: FAMILY_EMOJI })) as unknown as {
choices: { delta: { content: string } }[];
};
assert.equal(sanitized.choices[0].delta.content, FAMILY_EMOJI);
});
test("#12186 sanitizeStreamingChunk keeps a delta that is only a ZWJ (emoji sequence split by the tokenizer)", () => {
const sanitized = sanitizeStreamingChunk(openAIChunk({ content: "\u200D" })) as unknown as {
choices: { delta: { content: string } }[];
};
assert.equal(sanitized.choices[0].delta.content, "\u200D");
});
test("#12186 sanitizeStreamingChunk still de-obfuscates an ASCII word split across deltas", () => {
const first = sanitizeStreamingChunk(openAIChunk({ content: "o\u200D" })) as unknown as {
choices: { delta: { content: string } }[];
};
const second = sanitizeStreamingChunk(openAIChunk({ content: "\u200Dpencode" })) as unknown as {
choices: { delta: { content: string } }[];
};
assert.equal(first.choices[0].delta.content, "o");
assert.equal(second.choices[0].delta.content, "pencode");
});
test("#12186 sanitizeStreamingChunk keeps Persian ZWNJ in reasoning_content deltas", () => {
const sanitized = sanitizeStreamingChunk(
openAIChunk({ reasoning_content: `${PERSIAN_WORD} c\u200Dursor` })
) as unknown as { choices: { delta: { reasoning_content: string } }[] };
assert.equal(sanitized.choices[0].delta.reasoning_content, `${PERSIAN_WORD} cursor`);
});
test("#12186 sanitizeStreamingChunk keeps Persian ZWNJ in Anthropic text_delta events", () => {
const sanitized = sanitizeStreamingChunk({
type: "content_block_delta",
index: 0,
delta: { type: "text_delta", text: `${PERSIAN_WORD} ${FAMILY_EMOJI} a\u200Dider` },
}) as unknown as { delta: { text: string } };
assert.equal(sanitized.delta.text, `${PERSIAN_WORD} ${FAMILY_EMOJI} aider`);
});
test("#12186 sanitizeStreamingChunk keeps Persian ZWNJ in native response.output_text.delta", () => {
const sanitized = sanitizeStreamingChunk({
type: "response.output_text.delta",
delta: PERSIAN_WORD,
}) as unknown as { delta: string };
assert.equal(sanitized.delta, PERSIAN_WORD);
});
test("#12186 sanitizeStreamingChunk keeps Persian ZWNJ in native response.output_text.done", () => {
const sanitized = sanitizeStreamingChunk({
type: "response.output_text.done",
text: `${PERSIAN_WORD} o\u200Dpencode`,
}) as unknown as { text: string };
assert.equal(sanitized.text, `${PERSIAN_WORD} opencode`);
});
test("#12186 parseTextualToolCallCandidate keeps Persian ZWNJ in textual tool-call arguments", () => {
const parsed = parseTextualToolCallCandidate(
`[Tool call: terminal]\nArguments: {"command":"echo ${PERSIAN_WORD} o\u200Dpencode"}`
);
assert.ok(parsed && parsed.kind === "complete");
assert.equal(parsed.name, "terminal");
assert.deepEqual(parsed.args, { command: `echo ${PERSIAN_WORD} opencode` });
});
test("#12186 parseAntigravityTextualToolCall keeps Persian ZWNJ in textual tool-call arguments", () => {
const parsed = parseAntigravityTextualToolCall(
`[Tool call: terminal]\nArguments: {"command":"echo ${PERSIAN_WORD} o\u200Dpencode"}`
);
assert.ok(parsed);
assert.equal(parsed.name, "terminal");
assert.deepEqual(parsed.args, { command: `echo ${PERSIAN_WORD} opencode` });
});
test("#12186 parseSSEToGeminiResponse keeps Persian ZWNJ in textual tool-call arguments", () => {
const text = `[Tool call: terminal]\nArguments: {"command":"echo ${PERSIAN_WORD}"}`;
const rawSSE = `data: ${JSON.stringify({
response: {
candidates: [{ content: { parts: [{ text }] }, finishReason: "STOP" }],
},
})}`;
const parsed = parseSSEToGeminiResponse(rawSSE, "gemini-2.5-flash") as {
choices: { message: { tool_calls: { function: { arguments: string } }[] } }[];
};
assert.ok(parsed);
assert.equal(
parsed.choices[0].message.tool_calls[0].function.arguments,
JSON.stringify({ command: `echo ${PERSIAN_WORD}` })
);
});
test("#12186 Gemini non-stream translation keeps Persian ZWNJ in textual tool-call arguments", () => {
const result = translateNonStreamingResponse(
{
responseId: "resp-12186",
modelVersion: "gemini-2.5-flash",
candidates: [
{
content: {
parts: [
{
text: `[Tool call: terminal]\nArguments: {"command":"echo ${PERSIAN_WORD} o\u200Dpencode"}`,
},
],
},
finishReason: "STOP",
},
],
},
FORMATS.GEMINI,
FORMATS.OPENAI
) as { choices: { message: { tool_calls: { function: { arguments: string } }[] } }[] };
assert.equal(
result.choices[0].message.tool_calls[0].function.arguments,
JSON.stringify({ command: `echo ${PERSIAN_WORD} opencode` })
);
});
test("#12186 Gemini stream translation keeps Persian ZWNJ in text emitted before a textual tool call", () => {
const result = geminiToOpenAIResponse(
{
responseId: "resp-12186-stream",
modelVersion: "gemini-2.5-flash",
candidates: [
{
content: {
parts: [
{ text: `${PERSIAN_WORD}: [Tool call: terminal]\nArguments: {"command":"whoami"}` },
],
},
finishReason: "STOP",
},
],
},
{ toolCalls: new Map() }
) as Array<{ choices?: { delta?: { content?: string; tool_calls?: unknown[] } }[] }>;
const leakedContent = result.map((event) => event.choices?.[0]?.delta?.content || "").join("");
assert.equal(leakedContent, `${PERSIAN_WORD}: `);
const toolCalls = result.flatMap((event) => event.choices?.[0]?.delta?.tool_calls || []);
assert.equal(toolCalls.length, 1);
});
test("#12186 stripObfuscationZeroWidth keeps ZWNJ/ZWJ that belong to the text", () => {
for (const text of [
PERSIAN_WORD,
"می\u200Cروم نمی\u200Cدانم کتاب\u200Cها",
FAMILY_EMOJI,
"\u{1F3F3}\u{FE0F}\u200D\u{1F308}",
"\u200D",
"\u{1F468}\u200D",
"\u200D\u{1F469}",
"\u200C",
]) {
assert.equal(stripObfuscationZeroWidth(text), text);
}
});
test("#12186 stripObfuscationZeroWidth reverses the request-side obfuscation for every default agent word", () => {
const original = `Use ${getSensitiveWords().join(", ")} in ${PERSIAN_WORD} ${FAMILY_EMOJI}`;
const obfuscated = obfuscateSensitiveWords(original);
assert.notEqual(obfuscated, original);
assert.equal(stripObfuscationZeroWidth(obfuscated), original);
});
test("#12186 stripObfuscationZeroWidth removes joiners only between ASCII word characters", () => {
assert.equal(stripObfuscationZeroWidth("o\u200Dpencode"), "opencode");
assert.equal(stripObfuscationZeroWidth("roo_\u200Dcline 4\u200D2"), "roo_cline 42");
assert.equal(stripObfuscationZeroWidth("a\u200Cb"), "ab");
assert.equal(stripObfuscationZeroWidth("o\u200D\u200D\u200Cpencode"), "opencode");
assert.equal(stripObfuscationZeroWidth("o\u200D"), "o");
assert.equal(stripObfuscationZeroWidth("\u200Dpencode"), "pencode");
assert.equal(stripObfuscationZeroWidth("x \u200D y"), "x \u200D y");
// Neither side ASCII-adjacent on both ends: a joiner next to whitespace is not
// an obfuscation marker and is left alone.
assert.equal(stripObfuscationZeroWidth("x\u200D \u200Dy"), "x\u200D \u200Dy");
});
test("#12186 stripObfuscationZeroWidth still removes U+200B and U+FEFF anywhere", () => {
assert.equal(stripObfuscationZeroWidth(`\u200B${PERSIAN_WORD}\uFEFF`), PERSIAN_WORD);
assert.equal(stripObfuscationZeroWidth("\uFEFF"), "");
assert.equal(stripObfuscationZeroWidth("o\u200B\u200Dp"), "op");
assert.equal(stripObfuscationZeroWidth("\u200BКак исправить"), "Как исправить");
});
test("#12186 stripObfuscationZeroWidth returns the same reference when nothing needs stripping", () => {
const text = `plain ${PERSIAN_WORD}`;
assert.equal(stripObfuscationZeroWidth(text), text);
assert.equal(stripObfuscationZeroWidth(""), "");
});