From 6d02e0572eb6814c0eb2432fdeca152981562cef Mon Sep 17 00:00:00 2001 From: Denis Kotsyuba Date: Thu, 2 Jul 2026 03:05:01 +0200 Subject: [PATCH] fix(sse): strip zero-width markers from streamed responses (parity with non-streaming) (#5857) Integrated into release/v3.8.43 --- open-sse/handlers/responseSanitizer.ts | 43 ++++++++- tests/unit/response-sanitizer.test.ts | 126 +++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 3 deletions(-) diff --git a/open-sse/handlers/responseSanitizer.ts b/open-sse/handlers/responseSanitizer.ts index 1ae64a6670..04d25871a6 100644 --- a/open-sse/handlers/responseSanitizer.ts +++ b/open-sse/handlers/responseSanitizer.ts @@ -594,7 +594,7 @@ function sanitizeResponsesStreamingOutputItem(item: unknown): JsonRecord | null return { ...partRecord, type: toString(partRecord.type) || "summary_text", - text: collapseExcessiveNewlines(toString(partRecord.text) || ""), + text: collapseExcessiveNewlines(stripZeroWidthText(toString(partRecord.text) || "")), }; }) .filter((part) => part !== null) @@ -624,7 +624,7 @@ function sanitizeResponsesStreamingOutputItem(item: unknown): JsonRecord | null type: "function_call_output", output: typeof itemRecord.output === "string" - ? collapseExcessiveNewlines(itemRecord.output) + ? collapseExcessiveNewlines(stripZeroWidthText(itemRecord.output)) : JSON.stringify(itemRecord.output ?? ""), }; } @@ -640,10 +640,39 @@ function sanitizeResponsesStreamingOutput(output: unknown): JsonRecord[] { .filter((item): item is JsonRecord => item !== null); } +// Native Responses streaming events that carry raw model text directly on the +// root `delta` field. These must get the same zero-width-joiner stripping as the +// non-streaming path so agent words (opencode/cursor/aider) are not corrupted. +// Scoped as an allow-list on purpose: `response.function_call_arguments.delta` +// carries tool-call argument JSON that must pass through byte-exact. +const RESPONSES_STREAMING_TEXT_DELTA_EVENTS = new Set([ + "response.output_text.delta", + "response.reasoning_summary_text.delta", + "response.reasoning_text.delta", +]); + +// Matching `*.done` events that carry the finalized text on the root `text` field. +const RESPONSES_STREAMING_TEXT_DONE_EVENTS = new Set([ + "response.output_text.done", + "response.reasoning_summary_text.done", + "response.reasoning_text.done", +]); + function sanitizeResponsesStreamingEvent(parsedRecord: JsonRecord): JsonRecord { const sanitized: JsonRecord = { ...parsedRecord }; const eventType = toString(parsedRecord.type) || ""; + // Root-level text events (output_text / reasoning_summary_text / reasoning_text) + // carry the model text directly on the event, not under item/output. Strip ZWJ + // there too. Only touch string values and only the allow-listed event types — + // never function-call argument events. + if (RESPONSES_STREAMING_TEXT_DELTA_EVENTS.has(eventType) && typeof sanitized.delta === "string") { + sanitized.delta = stripZeroWidthText(sanitized.delta); + } + if (RESPONSES_STREAMING_TEXT_DONE_EVENTS.has(eventType) && typeof sanitized.text === "string") { + sanitized.text = stripZeroWidthText(sanitized.text); + } + if (parsedRecord.item !== undefined) { const sanitizedItem = sanitizeResponsesStreamingOutputItem(parsedRecord.item); if (sanitizedItem) { @@ -957,10 +986,18 @@ export function sanitizeStreamingChunk(parsed: unknown): unknown { if (deltaRecord.content !== undefined) { delta.content = typeof deltaRecord.content === "string" - ? collapseExcessiveNewlines(deltaRecord.content) + ? collapseExcessiveNewlines(stripZeroWidthText(deltaRecord.content)) : deltaRecord.content; } copyOpenAICompatibleReasoningFields(deltaRecord, delta); + // Parity with the non-streaming path: strip the zero-width joiners that the + // request side injects into agent words. copyOpenAICompatibleReasoningFields + // is shared, so strip locally on the fields it writes (string values only). + for (const reasoningKey of ["reasoning_content", "reasoning", "reasoning_text"]) { + if (typeof delta[reasoningKey] === "string") { + delta[reasoningKey] = stripZeroWidthText(delta[reasoningKey] as string); + } + } if (deltaRecord.tool_calls !== undefined) { delta.tool_calls = Array.isArray(deltaRecord.tool_calls) ? deltaRecord.tool_calls.map((tc) => { diff --git a/tests/unit/response-sanitizer.test.ts b/tests/unit/response-sanitizer.test.ts index d46bc14f1d..18ad4d6283 100644 --- a/tests/unit/response-sanitizer.test.ts +++ b/tests/unit/response-sanitizer.test.ts @@ -777,3 +777,129 @@ test("sanitizeResponsesApiResponse strips leaked multi_tool_use envelopes from R assert.equal(JSON.stringify(sanitized).includes("to=multi_tool_use.parallel"), false); assert.equal(JSON.stringify(sanitized).includes("recipient_name"), false); }); + +test("sanitizeStreamingChunk strips zero-width joiners from delta content", () => { + const sanitized = sanitizeStreamingChunk({ + choices: [ + { + index: 0, + delta: { + content: "o\u200dpencode", + }, + }, + ], + }) as any; + const output = JSON.stringify(sanitized); + + assert.equal(sanitized.choices[0].delta.content, "opencode"); + assert.equal(output.includes("\u200d"), false); +}); + +test("sanitizeStreamingChunk leaves delta content without zero-width joiners unchanged", () => { + const sanitized = sanitizeStreamingChunk({ + choices: [ + { + index: 0, + delta: { + content: "opncode", + }, + }, + ], + }) as any; + const output = JSON.stringify(sanitized); + + assert.equal(sanitized.choices[0].delta.content, "opncode"); + assert.equal(output.includes("\u200d"), false); +}); + +test("sanitizeStreamingChunk strips inline zero-width joiners from sentence content", () => { + const sanitized = sanitizeStreamingChunk({ + choices: [ + { + index: 0, + delta: { + content: "hello o\u200dpencode world", + }, + }, + ], + }) as any; + const output = JSON.stringify(sanitized); + + assert.equal(sanitized.choices[0].delta.content, "hello opencode world"); + assert.equal(output.includes("\u200d"), false); +}); + +test("sanitizeStreamingChunk strips zero-width joiners from reasoning_content deltas", () => { + const sanitized = sanitizeStreamingChunk({ + choices: [ + { + index: 0, + delta: { + reasoning_content: "c\u200dursor plan", + }, + }, + ], + }) as any; + const output = JSON.stringify(sanitized); + + assert.equal(sanitized.choices[0].delta.reasoning_content, "cursor plan"); + assert.equal(output.includes("\u200d"), false); +}); + +test("sanitizeStreamingChunk strips zero-width joiners from Responses reasoning summaries", () => { + const sanitized = sanitizeStreamingChunk({ + type: "response.output_item.done", + item: { + id: "rs_1", + type: "reasoning", + summary: [{ type: "summary_text", text: "a\u200dider note" }], + }, + }) as any; + const output = JSON.stringify(sanitized); + + assert.equal(sanitized.item.summary[0].text, "aider note"); + assert.equal(output.includes("\u200d"), false); +}); + +test("sanitizeStreamingChunk strips zero-width joiners from native response.output_text.delta", () => { + const sanitized = sanitizeStreamingChunk({ + type: "response.output_text.delta", + delta: "o\u200dpencode", + }) as any; + const output = JSON.stringify(sanitized); + + assert.equal(sanitized.delta, "opencode"); + assert.equal(output.includes("\u200d"), false); +}); + +test("sanitizeStreamingChunk strips zero-width joiners from native response.output_text.done", () => { + const sanitized = sanitizeStreamingChunk({ + type: "response.output_text.done", + text: "c\u200dursor done", + }) as any; + const output = JSON.stringify(sanitized); + + assert.equal(sanitized.text, "cursor done"); + assert.equal(output.includes("\u200d"), false); +}); + +test("sanitizeStreamingChunk strips zero-width joiners from response.reasoning_summary_text.delta", () => { + const sanitized = sanitizeStreamingChunk({ + type: "response.reasoning_summary_text.delta", + delta: "a\u200dider", + }) as any; + const output = JSON.stringify(sanitized); + + assert.equal(sanitized.delta, "aider"); + assert.equal(output.includes("\u200d"), false); +}); + +test("sanitizeStreamingChunk leaves function_call_arguments.delta byte-exact (tool args must not be corrupted)", () => { + const sanitized = sanitizeStreamingChunk({ + type: "response.function_call_arguments.delta", + delta: '{"path":"o\u200dpencode"}', + }) as any; + + assert.equal(sanitized.delta, '{"path":"o\u200dpencode"}'); + assert.equal((sanitized.delta as string).includes("\u200d"), true); +});