mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-14 02:42:24 +03:00
fix(sse): preserve ZWNJ and ZWJ in sanitized responses (#12359)
The response de-obfuscation stripped the whole U+200B..U+200D range, so Persian/Kurdish half-spaces (U+200C), Arabic/Indic shaping and emoji ZWJ sequences (U+200D) were deleted from every assistant response — text, reasoning and tool-call arguments, streaming and non-streaming, every provider: ارائهدهنده came back as ارائهدهنده. The request side only ever inserts a U+200D between two ASCII word characters, so the new stripObfuscationZeroWidth() removes a joiner only there, or at a string edge next to one so a word split across streaming deltas is still cleaned; U+200B and U+FEFF keep their unconditional removal. All seven copies of the old regex now go through the helper. Validated in a combined worktree with all 25 PRs of this batch boarded together: typecheck:core clean, 443/443 node-runner tests plus 14/14 vitest across every test file the batch touches, and check-changelog-integrity, check:cycles (418 files), check:provider-consistency (272 REGISTRY entries, 355 canonical providers), check:docs-counts, check:docs-sync (42 locales) and check-file-size all green. Thanks @pacocartones.
This commit is contained in:
1
changelog.d/fixes/12359-preserve-zwnj-zwj.md
Normal file
1
changelog.d/fixes/12359-preserve-zwnj-zwj.md
Normal file
@@ -0,0 +1 @@
|
||||
- **fix(sse):** Keep ZWNJ (U+200C) and ZWJ (U+200D) in assistant text, reasoning and tool-call arguments — Persian/Kurdish half-space (`ارائهدهنده`), Arabic/Indic shaping and emoji sequences no longer lose them; the response de-obfuscation now removes joiners only between ASCII word characters, where the request side inserts them ([#12186](https://github.com/diegosouzapw/OmniRoute/issues/12186)) — thanks @rezjalibd
|
||||
@@ -1,6 +1,7 @@
|
||||
// Pure SSE-payload -> collected-stream parsing for the Antigravity executor.
|
||||
// Extracted verbatim from antigravity.ts (no host state, no fetch/auth).
|
||||
import { normalizeOpenAICompatibleFinishReasonString } from "../../utils/finishReason.ts";
|
||||
import { stripObfuscationZeroWidth } from "../../utils/zeroWidth.ts";
|
||||
|
||||
export type AntigravityCollectedStream = {
|
||||
textContent: string;
|
||||
@@ -17,7 +18,7 @@ export type AntigravityCollectedStream = {
|
||||
|
||||
export function stripZeroWidth(value: unknown): unknown {
|
||||
if (typeof value === "string") {
|
||||
return value.replace(/[\u200B-\u200D\uFEFF]/g, "");
|
||||
return stripObfuscationZeroWidth(value);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((item) => stripZeroWidth(item));
|
||||
@@ -37,7 +38,7 @@ export function parseAntigravityTextualToolCall(
|
||||
text: unknown
|
||||
): { name: string; args: unknown } | null {
|
||||
if (typeof text !== "string") return null;
|
||||
const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, "");
|
||||
const normalized = stripObfuscationZeroWidth(text);
|
||||
const match = normalized.match(
|
||||
/^[\s\S]*?\[Tool call:\s*([^\]\n]+)\]\s*\nArguments:\s*([\s\S]+?)\s*$/
|
||||
);
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
applyCacheHitTokensToUsage,
|
||||
applyCacheHitTokensToResponsesUsage,
|
||||
} from "./responseSanitizer/cacheHitTokens.ts";
|
||||
import { stripObfuscationZeroWidth } from "../utils/zeroWidth.ts";
|
||||
export {
|
||||
extractThinkingFromContent,
|
||||
shouldParseTextualReasoningTags,
|
||||
@@ -85,7 +86,7 @@ function deleteOpenAICompatibleReasoningFields(record: JsonRecord): void {
|
||||
}
|
||||
|
||||
function stripZeroWidthText(value: string): string {
|
||||
return value.replace(/[\u200B-\u200D\uFEFF]/g, "");
|
||||
return stripObfuscationZeroWidth(value);
|
||||
}
|
||||
|
||||
function stripZeroWidthToolArgumentJson(value: unknown): string {
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
} from "../services/geminiThoughtSignatureStore.ts";
|
||||
import { normalizeOpenAICompatibleFinishReasonString } from "../utils/finishReason.ts";
|
||||
import { containsTextualToolCallMarker } from "../utils/textualToolCall.ts";
|
||||
import { stripObfuscationZeroWidth } from "../utils/zeroWidth.ts";
|
||||
import { getAnyReasoningValue } from "../utils/reasoningFields.ts";
|
||||
import {
|
||||
caseInsensitiveToolNameLookup,
|
||||
@@ -63,7 +64,7 @@ function parseTextualToolCall(text: unknown): { name: string; args: unknown } |
|
||||
// variations, e.g. a leading "(empty)" marker or zero-width chars inserted
|
||||
// into argument strings. Normalize those variants before parsing so the
|
||||
// response is still surfaced as a structured OpenAI tool call.
|
||||
const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, "");
|
||||
const normalized = stripObfuscationZeroWidth(text);
|
||||
const match = normalized.match(
|
||||
/^[\s\S]*?\[Tool call:\s*([^\]\n]+)\]\s*\nArguments:\s*([\s\S]+?)\s*$/
|
||||
);
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// Extracted verbatim from sseParser.ts (file-size cap): pure parsing, no host
|
||||
// state, following the handlers submodule pattern (chatCore/, responseSanitizer/).
|
||||
import { normalizeOpenAICompatibleFinishReasonString } from "../../utils/finishReason.ts";
|
||||
import { stripObfuscationZeroWidth } from "../../utils/zeroWidth.ts";
|
||||
|
||||
type AccumulatedToolCall = {
|
||||
id: string;
|
||||
@@ -20,7 +21,7 @@ type GeminiSSEAccumulator = {
|
||||
};
|
||||
|
||||
function stripZeroWidth(value: unknown): unknown {
|
||||
if (typeof value === "string") return value.replace(/[\u200B-\u200D\uFEFF]/g, "");
|
||||
if (typeof value === "string") return stripObfuscationZeroWidth(value);
|
||||
return value;
|
||||
}
|
||||
|
||||
@@ -29,7 +30,7 @@ function stripZeroWidth(value: unknown): unknown {
|
||||
* Gemini/Antigravity models emit instead of a native functionCall part.
|
||||
*/
|
||||
function tryParseTextualToolCall(text: string): { name: string; args: unknown } | null {
|
||||
const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, "");
|
||||
const normalized = stripObfuscationZeroWidth(text);
|
||||
const match = normalized.match(
|
||||
/^[\s\S]*?\[Tool call:\s*([^\]\n]+)\]\s*\nArguments:\s*([\s\S]+?)\s*$/
|
||||
);
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
isMalformedToolCallFinishReason,
|
||||
} from "../../utils/finishReason.ts";
|
||||
import { stripAnsiCodes } from "../../utils/streamHelpers.ts";
|
||||
import { stripObfuscationZeroWidth } from "../../utils/zeroWidth.ts";
|
||||
|
||||
type GeminiToOpenAIState = {
|
||||
functionIndex: number;
|
||||
@@ -483,7 +484,7 @@ export function geminiToOpenAIResponse(chunk, state) {
|
||||
let candidate = parseTextualToolCallCandidate(accumulated);
|
||||
|
||||
if (candidate) {
|
||||
accumulated = accumulated.replace(/[\u200B-\u200D\uFEFF]/g, "");
|
||||
accumulated = stripObfuscationZeroWidth(accumulated);
|
||||
let toolCallIndex = accumulated.lastIndexOf("(empty)[Tool call:");
|
||||
if (toolCallIndex < 0) {
|
||||
toolCallIndex = accumulated.lastIndexOf("[Tool call:");
|
||||
|
||||
@@ -47,6 +47,7 @@ import {
|
||||
} from "./responsesCommentaryDrop.ts";
|
||||
import { buildErrorBody } from "./error.ts";
|
||||
import { parseTextualToolCallCandidate, isValidToolCallHeaderPrefix } from "./textualToolCall.ts";
|
||||
import { stripObfuscationZeroWidth } from "./zeroWidth.ts";
|
||||
import {
|
||||
formatTranslatedStreamError,
|
||||
normalizeStreamFailurePayload,
|
||||
@@ -272,7 +273,7 @@ function containsMalformedTextualToolCall(
|
||||
allowedToolNames?: Set<string> | null
|
||||
): boolean {
|
||||
if (typeof text !== "string") return false;
|
||||
const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, "");
|
||||
const normalized = stripObfuscationZeroWidth(text);
|
||||
|
||||
let searchIdx = 0;
|
||||
while (true) {
|
||||
@@ -1637,10 +1638,7 @@ export function createSSEStream(options: StreamOptions = {}) {
|
||||
isResponsesCommentaryMessageItem
|
||||
).items
|
||||
: passthroughResponsesOutputItems;
|
||||
const backfilled = backfillResponsesCompletedOutput(
|
||||
parsed,
|
||||
backfillCandidates
|
||||
);
|
||||
const backfilled = backfillResponsesCompletedOutput(parsed, backfillCandidates);
|
||||
const usageNormalized = normalizeUsage(parsed);
|
||||
if (
|
||||
stripped ||
|
||||
@@ -1760,7 +1758,11 @@ export function createSSEStream(options: StreamOptions = {}) {
|
||||
) {
|
||||
const pt = emptyChoicesUsage.prompt_tokens ?? 0;
|
||||
if (pt === 0) {
|
||||
const estimated = estimateUsage(body, totalContentLength, sourceFormat || FORMATS.OPENAI);
|
||||
const estimated = estimateUsage(
|
||||
body,
|
||||
totalContentLength,
|
||||
sourceFormat || FORMATS.OPENAI
|
||||
);
|
||||
if (estimated?.prompt_tokens > 0) {
|
||||
emptyChoicesUsage.prompt_tokens = estimated.prompt_tokens;
|
||||
emptyChoicesUsage.total_tokens =
|
||||
@@ -2519,11 +2521,7 @@ export function createSSEStream(options: StreamOptions = {}) {
|
||||
// [DONE], so metered clients still see token counts. When the
|
||||
// upstream DID send usage (trailing or in-band), it was forwarded
|
||||
// already and passthroughForwardedUsage guards this off.
|
||||
if (
|
||||
shouldEmitDoneTerminator &&
|
||||
!passthroughForwardedUsage &&
|
||||
hasValidUsage(usage)
|
||||
) {
|
||||
if (shouldEmitDoneTerminator && !passthroughForwardedUsage && hasValidUsage(usage)) {
|
||||
const usageOnlyChunk = {
|
||||
id: passthroughLastChatId ?? passthroughResponsesId ?? `chatcmpl-${Date.now()}`,
|
||||
object: "chat.completion.chunk",
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { stripObfuscationZeroWidth } from "./zeroWidth.ts";
|
||||
|
||||
export function stripZeroWidth(value: unknown): unknown {
|
||||
if (typeof value === "string") {
|
||||
return value.replace(/[\u200B-\u200D\uFEFF]/g, "");
|
||||
return stripObfuscationZeroWidth(value);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((item) => stripZeroWidth(item));
|
||||
@@ -58,7 +60,7 @@ export function parseTextualToolCallCandidate(
|
||||
text: unknown
|
||||
): { kind: "complete"; name: string; args: unknown } | { kind: "partial" } | null {
|
||||
if (typeof text !== "string") return null;
|
||||
const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, "");
|
||||
const normalized = stripObfuscationZeroWidth(text);
|
||||
const toolCallIndex = normalized.lastIndexOf("[Tool call:");
|
||||
if (toolCallIndex < 0) {
|
||||
const lastParen = normalized.lastIndexOf("(");
|
||||
@@ -102,7 +104,7 @@ export function parseTextualToolCallCandidate(
|
||||
|
||||
export function containsTextualToolCallMarker(text: unknown): boolean {
|
||||
if (typeof text !== "string") return false;
|
||||
const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, "");
|
||||
const normalized = stripObfuscationZeroWidth(text);
|
||||
|
||||
if (!normalized.includes("[Tool call:")) return false;
|
||||
if (normalized.includes("Arguments:")) return true;
|
||||
|
||||
38
open-sse/utils/zeroWidth.ts
Normal file
38
open-sse/utils/zeroWidth.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* Zero-width character cleanup for model output.
|
||||
*
|
||||
* The request side obfuscates configurable agent words by inserting a
|
||||
* U+200D ZERO WIDTH JOINER after their first letter (`o\u200Dpencode`, see
|
||||
* `services/claudeCodeObfuscation.ts` and `services/systemTransforms.ts`), and
|
||||
* the response side removes zero-width code points again so an echoed word is
|
||||
* not corrupted. Removing every U+200B..U+200D also deletes U+200C ZERO WIDTH
|
||||
* NON-JOINER and U+200D where they belong to the text itself: the Persian and
|
||||
* Kurdish half-space (ارائه\u200Cدهنده, می\u200Cروم, کتاب\u200Cها), Arabic and Indic shaping,
|
||||
* and emoji ZWJ sequences (👨\u200D👩\u200D👧). See #12186.
|
||||
*
|
||||
* The obfuscator only ever places a joiner between two ASCII word characters,
|
||||
* so a joiner is removed only there. A joiner touching the edge of the string
|
||||
* is removed as well when its other neighbour is an ASCII word character, so
|
||||
* an obfuscated word split across streaming deltas (`o\u200D` + `pencode`)
|
||||
* is still cleaned. A joiner next to non-ASCII text, and a delta that consists
|
||||
* of nothing but a joiner (an emoji sequence split by the tokenizer), pass
|
||||
* through untouched.
|
||||
*
|
||||
* U+200B ZERO WIDTH SPACE and U+FEFF have no shaping role and keep the
|
||||
* unconditional removal they always had.
|
||||
*/
|
||||
|
||||
const ANY_ZERO_WIDTH = /[\u200B-\u200D\uFEFF]/;
|
||||
const ZERO_WIDTH_SPACE_OR_BOM = /[\u200B\uFEFF]/g;
|
||||
const JOINER_BETWEEN_ASCII_WORD_CHARS =
|
||||
/(?<=[A-Za-z0-9_])[\u200C\u200D]+(?=[A-Za-z0-9_]|$)|^[\u200C\u200D]+(?=[A-Za-z0-9_])/g;
|
||||
|
||||
/**
|
||||
* Strip the zero-width markers used for agent-word obfuscation while keeping
|
||||
* ZWNJ/ZWJ that are part of the text (Persian half-space, Arabic/Indic
|
||||
* shaping, emoji sequences).
|
||||
*/
|
||||
export function stripObfuscationZeroWidth(text: string): string {
|
||||
if (!text || !ANY_ZERO_WIDTH.test(text)) return text;
|
||||
return text.replace(ZERO_WIDTH_SPACE_OR_BOM, "").replace(JOINER_BETWEEN_ASCII_WORD_CHARS, "");
|
||||
}
|
||||
337
tests/unit/12186-preserve-zwnj-zwj.test.ts
Normal file
337
tests/unit/12186-preserve-zwnj-zwj.test.ts
Normal file
@@ -0,0 +1,337 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
// #12186 — the response pipeline strips every zero-width code point in
|
||||
// U+200B..U+200D to undo the request-side agent-word obfuscation (which inserts
|
||||
// one U+200D after the first letter of an ASCII word). That blanket strip also
|
||||
// deletes U+200C ZERO WIDTH NON-JOINER and U+200D ZERO WIDTH JOINER where they
|
||||
// are part of the text itself: Persian half-space, Arabic/Indic shaping and
|
||||
// emoji ZWJ sequences. These tests pin that linguistic joiners survive while the
|
||||
// ASCII obfuscation marker is still removed.
|
||||
|
||||
const { sanitizeOpenAIResponse, sanitizeStreamingChunk } =
|
||||
await import("../../open-sse/handlers/responseSanitizer.ts");
|
||||
const { parseTextualToolCallCandidate } = await import("../../open-sse/utils/textualToolCall.ts");
|
||||
const { parseAntigravityTextualToolCall } =
|
||||
await import("../../open-sse/executors/antigravity/sseCollect.ts");
|
||||
const { parseSSEToGeminiResponse } =
|
||||
await import("../../open-sse/handlers/sseParser/geminiResponse.ts");
|
||||
const { translateNonStreamingResponse } =
|
||||
await import("../../open-sse/handlers/responseTranslator.ts");
|
||||
const { geminiToOpenAIResponse } =
|
||||
await import("../../open-sse/translator/response/gemini-to-openai.ts");
|
||||
const { FORMATS } = await import("../../open-sse/translator/formats.ts");
|
||||
const { stripObfuscationZeroWidth } = await import("../../open-sse/utils/zeroWidth.ts");
|
||||
const { obfuscateSensitiveWords, getSensitiveWords } =
|
||||
await import("../../open-sse/services/claudeCodeObfuscation.ts");
|
||||
|
||||
// Exact word from the issue report: ارائه + U+200C + دهنده ("provider").
|
||||
const PERSIAN_WORD = "ارائه\u200Cدهنده";
|
||||
// Family emoji: MAN + ZWJ + WOMAN + ZWJ + GIRL.
|
||||
const FAMILY_EMOJI = "\u{1F468}\u200D\u{1F469}\u200D\u{1F467}";
|
||||
|
||||
function openAIChunk(delta: Record<string, unknown>) {
|
||||
return {
|
||||
id: "chatcmpl_12186",
|
||||
object: "chat.completion.chunk",
|
||||
created: 1,
|
||||
model: "auto",
|
||||
choices: [{ index: 0, delta, finish_reason: null }],
|
||||
};
|
||||
}
|
||||
|
||||
test("#12186 sanitizeOpenAIResponse keeps Persian ZWNJ in non-stream message content", () => {
|
||||
const sanitized = sanitizeOpenAIResponse({
|
||||
id: "chatcmpl_12186",
|
||||
model: "auto",
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
finish_reason: "stop",
|
||||
message: { role: "assistant", content: PERSIAN_WORD },
|
||||
},
|
||||
],
|
||||
}) as unknown as { choices: { message: { content: string } }[] };
|
||||
|
||||
assert.equal(sanitized.choices[0].message.content, PERSIAN_WORD);
|
||||
});
|
||||
|
||||
test("#12186 sanitizeOpenAIResponse keeps joiners in text but still de-obfuscates ASCII agent words", () => {
|
||||
const sanitized = sanitizeOpenAIResponse({
|
||||
id: "chatcmpl_12186_mixed",
|
||||
model: "auto",
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
finish_reason: "stop",
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: `o\u200Dpencode ${PERSIAN_WORD} می\u200Cروم کتاب\u200Cها ${FAMILY_EMOJI} c\u200Dursor`,
|
||||
},
|
||||
},
|
||||
],
|
||||
}) as unknown as { choices: { message: { content: string } }[] };
|
||||
|
||||
assert.equal(
|
||||
sanitized.choices[0].message.content,
|
||||
`opencode ${PERSIAN_WORD} می\u200Cروم کتاب\u200Cها ${FAMILY_EMOJI} cursor`
|
||||
);
|
||||
});
|
||||
|
||||
test("#12186 sanitizeOpenAIResponse still strips U+200B and U+FEFF from message content", () => {
|
||||
const sanitized = sanitizeOpenAIResponse({
|
||||
id: "chatcmpl_12186_zwsp",
|
||||
model: "auto",
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
finish_reason: "stop",
|
||||
message: { role: "assistant", content: "\uFEFFhello\u200B world o\u200Bpencode" },
|
||||
},
|
||||
],
|
||||
}) as unknown as { choices: { message: { content: string } }[] };
|
||||
|
||||
assert.equal(sanitized.choices[0].message.content, "hello world opencode");
|
||||
});
|
||||
|
||||
test("#12186 sanitizeOpenAIResponse keeps Persian ZWNJ inside tool-call arguments", () => {
|
||||
const args = JSON.stringify({ command: `echo ${PERSIAN_WORD}`, note: "o\u200Dpencode" });
|
||||
const sanitized = sanitizeOpenAIResponse({
|
||||
id: "chatcmpl_12186_tool",
|
||||
model: "auto",
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
finish_reason: "tool_calls",
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "",
|
||||
tool_calls: [
|
||||
{ id: "call_1", type: "function", function: { name: "run", arguments: args } },
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}) as unknown as {
|
||||
choices: { message: { tool_calls: { function: { arguments: string } }[] } }[];
|
||||
};
|
||||
|
||||
assert.equal(
|
||||
sanitized.choices[0].message.tool_calls[0].function.arguments,
|
||||
JSON.stringify({ command: `echo ${PERSIAN_WORD}`, note: "opencode" })
|
||||
);
|
||||
});
|
||||
|
||||
test("#12186 sanitizeStreamingChunk keeps Persian ZWNJ in OpenAI delta content", () => {
|
||||
const sanitized = sanitizeStreamingChunk(openAIChunk({ content: PERSIAN_WORD })) as unknown as {
|
||||
choices: { delta: { content: string } }[];
|
||||
};
|
||||
|
||||
assert.equal(sanitized.choices[0].delta.content, PERSIAN_WORD);
|
||||
});
|
||||
|
||||
test("#12186 sanitizeStreamingChunk keeps an emoji ZWJ sequence in OpenAI delta content", () => {
|
||||
const sanitized = sanitizeStreamingChunk(openAIChunk({ content: FAMILY_EMOJI })) as unknown as {
|
||||
choices: { delta: { content: string } }[];
|
||||
};
|
||||
|
||||
assert.equal(sanitized.choices[0].delta.content, FAMILY_EMOJI);
|
||||
});
|
||||
|
||||
test("#12186 sanitizeStreamingChunk keeps a delta that is only a ZWJ (emoji sequence split by the tokenizer)", () => {
|
||||
const sanitized = sanitizeStreamingChunk(openAIChunk({ content: "\u200D" })) as unknown as {
|
||||
choices: { delta: { content: string } }[];
|
||||
};
|
||||
|
||||
assert.equal(sanitized.choices[0].delta.content, "\u200D");
|
||||
});
|
||||
|
||||
test("#12186 sanitizeStreamingChunk still de-obfuscates an ASCII word split across deltas", () => {
|
||||
const first = sanitizeStreamingChunk(openAIChunk({ content: "o\u200D" })) as unknown as {
|
||||
choices: { delta: { content: string } }[];
|
||||
};
|
||||
const second = sanitizeStreamingChunk(openAIChunk({ content: "\u200Dpencode" })) as unknown as {
|
||||
choices: { delta: { content: string } }[];
|
||||
};
|
||||
|
||||
assert.equal(first.choices[0].delta.content, "o");
|
||||
assert.equal(second.choices[0].delta.content, "pencode");
|
||||
});
|
||||
|
||||
test("#12186 sanitizeStreamingChunk keeps Persian ZWNJ in reasoning_content deltas", () => {
|
||||
const sanitized = sanitizeStreamingChunk(
|
||||
openAIChunk({ reasoning_content: `${PERSIAN_WORD} c\u200Dursor` })
|
||||
) as unknown as { choices: { delta: { reasoning_content: string } }[] };
|
||||
|
||||
assert.equal(sanitized.choices[0].delta.reasoning_content, `${PERSIAN_WORD} cursor`);
|
||||
});
|
||||
|
||||
test("#12186 sanitizeStreamingChunk keeps Persian ZWNJ in Anthropic text_delta events", () => {
|
||||
const sanitized = sanitizeStreamingChunk({
|
||||
type: "content_block_delta",
|
||||
index: 0,
|
||||
delta: { type: "text_delta", text: `${PERSIAN_WORD} ${FAMILY_EMOJI} a\u200Dider` },
|
||||
}) as unknown as { delta: { text: string } };
|
||||
|
||||
assert.equal(sanitized.delta.text, `${PERSIAN_WORD} ${FAMILY_EMOJI} aider`);
|
||||
});
|
||||
|
||||
test("#12186 sanitizeStreamingChunk keeps Persian ZWNJ in native response.output_text.delta", () => {
|
||||
const sanitized = sanitizeStreamingChunk({
|
||||
type: "response.output_text.delta",
|
||||
delta: PERSIAN_WORD,
|
||||
}) as unknown as { delta: string };
|
||||
|
||||
assert.equal(sanitized.delta, PERSIAN_WORD);
|
||||
});
|
||||
|
||||
test("#12186 sanitizeStreamingChunk keeps Persian ZWNJ in native response.output_text.done", () => {
|
||||
const sanitized = sanitizeStreamingChunk({
|
||||
type: "response.output_text.done",
|
||||
text: `${PERSIAN_WORD} o\u200Dpencode`,
|
||||
}) as unknown as { text: string };
|
||||
|
||||
assert.equal(sanitized.text, `${PERSIAN_WORD} opencode`);
|
||||
});
|
||||
|
||||
test("#12186 parseTextualToolCallCandidate keeps Persian ZWNJ in textual tool-call arguments", () => {
|
||||
const parsed = parseTextualToolCallCandidate(
|
||||
`[Tool call: terminal]\nArguments: {"command":"echo ${PERSIAN_WORD} o\u200Dpencode"}`
|
||||
);
|
||||
|
||||
assert.ok(parsed && parsed.kind === "complete");
|
||||
assert.equal(parsed.name, "terminal");
|
||||
assert.deepEqual(parsed.args, { command: `echo ${PERSIAN_WORD} opencode` });
|
||||
});
|
||||
|
||||
test("#12186 parseAntigravityTextualToolCall keeps Persian ZWNJ in textual tool-call arguments", () => {
|
||||
const parsed = parseAntigravityTextualToolCall(
|
||||
`[Tool call: terminal]\nArguments: {"command":"echo ${PERSIAN_WORD} o\u200Dpencode"}`
|
||||
);
|
||||
|
||||
assert.ok(parsed);
|
||||
assert.equal(parsed.name, "terminal");
|
||||
assert.deepEqual(parsed.args, { command: `echo ${PERSIAN_WORD} opencode` });
|
||||
});
|
||||
|
||||
test("#12186 parseSSEToGeminiResponse keeps Persian ZWNJ in textual tool-call arguments", () => {
|
||||
const text = `[Tool call: terminal]\nArguments: {"command":"echo ${PERSIAN_WORD}"}`;
|
||||
const rawSSE = `data: ${JSON.stringify({
|
||||
response: {
|
||||
candidates: [{ content: { parts: [{ text }] }, finishReason: "STOP" }],
|
||||
},
|
||||
})}`;
|
||||
|
||||
const parsed = parseSSEToGeminiResponse(rawSSE, "gemini-2.5-flash") as {
|
||||
choices: { message: { tool_calls: { function: { arguments: string } }[] } }[];
|
||||
};
|
||||
|
||||
assert.ok(parsed);
|
||||
assert.equal(
|
||||
parsed.choices[0].message.tool_calls[0].function.arguments,
|
||||
JSON.stringify({ command: `echo ${PERSIAN_WORD}` })
|
||||
);
|
||||
});
|
||||
|
||||
test("#12186 Gemini non-stream translation keeps Persian ZWNJ in textual tool-call arguments", () => {
|
||||
const result = translateNonStreamingResponse(
|
||||
{
|
||||
responseId: "resp-12186",
|
||||
modelVersion: "gemini-2.5-flash",
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
parts: [
|
||||
{
|
||||
text: `[Tool call: terminal]\nArguments: {"command":"echo ${PERSIAN_WORD} o\u200Dpencode"}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
finishReason: "STOP",
|
||||
},
|
||||
],
|
||||
},
|
||||
FORMATS.GEMINI,
|
||||
FORMATS.OPENAI
|
||||
) as { choices: { message: { tool_calls: { function: { arguments: string } }[] } }[] };
|
||||
|
||||
assert.equal(
|
||||
result.choices[0].message.tool_calls[0].function.arguments,
|
||||
JSON.stringify({ command: `echo ${PERSIAN_WORD} opencode` })
|
||||
);
|
||||
});
|
||||
|
||||
test("#12186 Gemini stream translation keeps Persian ZWNJ in text emitted before a textual tool call", () => {
|
||||
const result = geminiToOpenAIResponse(
|
||||
{
|
||||
responseId: "resp-12186-stream",
|
||||
modelVersion: "gemini-2.5-flash",
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
parts: [
|
||||
{ text: `${PERSIAN_WORD}: [Tool call: terminal]\nArguments: {"command":"whoami"}` },
|
||||
],
|
||||
},
|
||||
finishReason: "STOP",
|
||||
},
|
||||
],
|
||||
},
|
||||
{ toolCalls: new Map() }
|
||||
) as Array<{ choices?: { delta?: { content?: string; tool_calls?: unknown[] } }[] }>;
|
||||
|
||||
const leakedContent = result.map((event) => event.choices?.[0]?.delta?.content || "").join("");
|
||||
assert.equal(leakedContent, `${PERSIAN_WORD}: `);
|
||||
|
||||
const toolCalls = result.flatMap((event) => event.choices?.[0]?.delta?.tool_calls || []);
|
||||
assert.equal(toolCalls.length, 1);
|
||||
});
|
||||
|
||||
test("#12186 stripObfuscationZeroWidth keeps ZWNJ/ZWJ that belong to the text", () => {
|
||||
for (const text of [
|
||||
PERSIAN_WORD,
|
||||
"می\u200Cروم نمی\u200Cدانم کتاب\u200Cها",
|
||||
FAMILY_EMOJI,
|
||||
"\u{1F3F3}\u{FE0F}\u200D\u{1F308}",
|
||||
"\u200D",
|
||||
"\u{1F468}\u200D",
|
||||
"\u200D\u{1F469}",
|
||||
"\u200C",
|
||||
]) {
|
||||
assert.equal(stripObfuscationZeroWidth(text), text);
|
||||
}
|
||||
});
|
||||
|
||||
test("#12186 stripObfuscationZeroWidth reverses the request-side obfuscation for every default agent word", () => {
|
||||
const original = `Use ${getSensitiveWords().join(", ")} in ${PERSIAN_WORD} ${FAMILY_EMOJI}`;
|
||||
const obfuscated = obfuscateSensitiveWords(original);
|
||||
|
||||
assert.notEqual(obfuscated, original);
|
||||
assert.equal(stripObfuscationZeroWidth(obfuscated), original);
|
||||
});
|
||||
|
||||
test("#12186 stripObfuscationZeroWidth removes joiners only between ASCII word characters", () => {
|
||||
assert.equal(stripObfuscationZeroWidth("o\u200Dpencode"), "opencode");
|
||||
assert.equal(stripObfuscationZeroWidth("roo_\u200Dcline 4\u200D2"), "roo_cline 42");
|
||||
assert.equal(stripObfuscationZeroWidth("a\u200Cb"), "ab");
|
||||
assert.equal(stripObfuscationZeroWidth("o\u200D\u200D\u200Cpencode"), "opencode");
|
||||
assert.equal(stripObfuscationZeroWidth("o\u200D"), "o");
|
||||
assert.equal(stripObfuscationZeroWidth("\u200Dpencode"), "pencode");
|
||||
assert.equal(stripObfuscationZeroWidth("x \u200D y"), "x \u200D y");
|
||||
// Neither side ASCII-adjacent on both ends: a joiner next to whitespace is not
|
||||
// an obfuscation marker and is left alone.
|
||||
assert.equal(stripObfuscationZeroWidth("x\u200D \u200Dy"), "x\u200D \u200Dy");
|
||||
});
|
||||
|
||||
test("#12186 stripObfuscationZeroWidth still removes U+200B and U+FEFF anywhere", () => {
|
||||
assert.equal(stripObfuscationZeroWidth(`\u200B${PERSIAN_WORD}\uFEFF`), PERSIAN_WORD);
|
||||
assert.equal(stripObfuscationZeroWidth("\uFEFF"), "");
|
||||
assert.equal(stripObfuscationZeroWidth("o\u200B\u200Dp"), "op");
|
||||
assert.equal(stripObfuscationZeroWidth("\u200BКак исправить"), "Как исправить");
|
||||
});
|
||||
|
||||
test("#12186 stripObfuscationZeroWidth returns the same reference when nothing needs stripping", () => {
|
||||
const text = `plain ${PERSIAN_WORD}`;
|
||||
assert.equal(stripObfuscationZeroWidth(text), text);
|
||||
assert.equal(stripObfuscationZeroWidth(""), "");
|
||||
});
|
||||
Reference in New Issue
Block a user