diff --git a/changelog.d/fixes/responses-think-close-marker.md b/changelog.d/fixes/responses-think-close-marker.md new file mode 100644 index 0000000000..0d5177ec33 --- /dev/null +++ b/changelog.d/fixes/responses-think-close-marker.md @@ -0,0 +1 @@ +- **fix(stream):** Responses API clients (`/v1/responses`) no longer receive a stray `` text delta at the start of the assistant message on Claude-format upstreams (observed with kimi-coding). The `` close marker (#4633) exists for Chat Completions clients that scan content for it; Responses API clients receive reasoning as structured reasoning items, so the marker is now always suppressed on that path — including the GLM and zed-hosted executors, which do their own Claude→OpenAI translation. diff --git a/open-sse/executors/base.ts b/open-sse/executors/base.ts index 04d40336ad..607fde251e 100644 --- a/open-sse/executors/base.ts +++ b/open-sse/executors/base.ts @@ -169,6 +169,11 @@ export type ExecuteInput = { upstreamExtraHeaders?: Record | null; /** Original client request headers (read-only). Executors may forward select headers upstream. */ clientHeaders?: Record | null; + /** Response format the end client expects (e.g. "openai-responses"). Executors + * that do their own Claude→OpenAI stream translation (GLM, zed-hosted) use + * this to apply client-format-aware policies such as `` close-marker + * suppression. */ + clientResponseFormat?: string | null; /** Callback to persist tokens that are proactively refreshed during execution. * Accepts a partial credentials patch (e.g. `{ accessToken, refreshToken }` or * `{ testStatus: "expired", isActive: false }`); the caller merges into the diff --git a/open-sse/executors/glm.ts b/open-sse/executors/glm.ts index 1fb438aa82..945dbe82cb 100644 --- a/open-sse/executors/glm.ts +++ b/open-sse/executors/glm.ts @@ -424,33 +424,7 @@ export class GlmExecutor extends DefaultExecutor { const result = { response, url, headers, transformedBody }; if (transport === "anthropic") { - // Resolve whether the `` close marker should be suppressed for - // this client. GLM's Anthropic transport does its own Claude→OpenAI - // translation (bypassing chatCore's stream), so we must resolve the flag - // here from the original client headers (#5245 / #5312). - const clientHeaders = input.clientHeaders ?? {}; - const suppressThinkClose = resolveSuppressThinkClose({ - userAgent: clientHeaders["user-agent"] ?? clientHeaders["User-Agent"] ?? null, - thinkingMarkerHeader: - clientHeaders[THINKING_MARKER_HEADER] ?? - clientHeaders["x-omniroute-thinking-marker"] ?? - null, - }); - - const translatedResponse = - input.stream && result.response.ok - ? translateSseResponse(result.response, this.provider, input.model, suppressThinkClose) - : isJsonResponse(result.response) - ? await translateAnthropicJsonResponse(result.response) - : result.response; - return { - ...result, - response: translatedResponse, - url, - headers, - transformedBody, - targetFormat: FORMATS.OPENAI, - }; + return this.finalizeAnthropicTransportResult(input, result); } return { @@ -462,6 +436,44 @@ export class GlmExecutor extends DefaultExecutor { }; } + /** + * GLM's Anthropic transport does its own Claude→OpenAI translation + * (bypassing chatCore's stream), so the `` close-marker + * suppression flag and the response translation both have to be resolved + * here from the original client headers (#5245 / #5312). Extracted from + * `executeTransport` to keep that method's cyclomatic complexity under the + * project cap. + */ + private async finalizeAnthropicTransportResult( + input: ExecuteInput, + result: { response: Response; url: string; headers: Record; transformedBody: unknown } + ): Promise { + const { response: rawResponse, url, headers, transformedBody } = result; + const clientHeaders = input.clientHeaders ?? {}; + const suppressThinkClose = resolveSuppressThinkClose({ + userAgent: clientHeaders["user-agent"] ?? clientHeaders["User-Agent"] ?? null, + thinkingMarkerHeader: + clientHeaders[THINKING_MARKER_HEADER] ?? + clientHeaders["x-omniroute-thinking-marker"] ?? + null, + clientResponseFormat: input.clientResponseFormat ?? null, + }); + + const translatedResponse = + input.stream && rawResponse.ok + ? translateSseResponse(rawResponse, this.provider, input.model, suppressThinkClose) + : isJsonResponse(rawResponse) + ? await translateAnthropicJsonResponse(rawResponse) + : rawResponse; + return { + response: translatedResponse, + url, + headers, + transformedBody, + targetFormat: FORMATS.OPENAI, + }; + } + async execute(input: ExecuteInput): Promise { const effortTier = parseGlm52Effort(input.model); diff --git a/open-sse/executors/zed-hosted.ts b/open-sse/executors/zed-hosted.ts index 842ce834e6..3ad0cc8740 100644 --- a/open-sse/executors/zed-hosted.ts +++ b/open-sse/executors/zed-hosted.ts @@ -39,6 +39,7 @@ import { claudeToOpenAIResponse } from "../translator/response/claude-to-openai. import { geminiToOpenAIResponse } from "../translator/response/gemini-to-openai.ts"; import { openaiResponsesToOpenAIResponse } from "../translator/response/openai-responses.ts"; import { ZED_HEADERS, resolveZedModels, zedLlmFetch, type ZedCredentials } from "../shared/zedAuth.ts"; +import { resolveSuppressThinkClose, THINKING_MARKER_HEADER } from "../utils/thinkCloseMarker.ts"; const ZED_PROVIDER = { anthropic: "Anthropic", @@ -162,16 +163,42 @@ function normalizeStatus(status: unknown): Record | null { return null; } +/** + * Resolves `` close-marker suppression from the incoming client + * headers / response format, extracted from `ZedHostedExecutor.execute` to + * keep that method's cyclomatic complexity under the project cap. + */ +function resolveZedSuppressThinkClose( + clientHeaders: ExecuteInput["clientHeaders"], + clientResponseFormat: ExecuteInput["clientResponseFormat"] +): boolean { + return resolveSuppressThinkClose({ + userAgent: clientHeaders?.["user-agent"] ?? clientHeaders?.["User-Agent"] ?? null, + thinkingMarkerHeader: + clientHeaders?.[THINKING_MARKER_HEADER] ?? + clientHeaders?.["x-omniroute-thinking-marker"] ?? + null, + clientResponseFormat: clientResponseFormat ?? null, + }); +} + function wrapZedCompletionStream( response: Response, provider: ZedProviderName, - model: string + model: string, + options?: { suppressThinkClose?: boolean } ): Response { if (!response.ok || !response.body) return response; const decoder = new TextDecoder(); const encoder = new TextEncoder(); const state = initProviderState(provider, model); + if (options?.suppressThinkClose) { + // Responses API clients (and UA/header-opted-out clients) must not see the + // textual `` close marker — same policy chatCore applies (#4633 / + // #5245 / kimi-coding stray marker on /v1/responses). + state.suppressThinkClose = true; + } let buffer = ""; let done = false; @@ -272,7 +299,16 @@ export class ZedHostedExecutor extends BaseExecutor { } } - async execute({ model, body, stream, credentials, signal, log }: ExecuteInput): Promise<{ + async execute({ + model, + body, + stream, + credentials, + signal, + log, + clientHeaders, + clientResponseFormat, + }: ExecuteInput): Promise<{ response: Response; url: string; headers: Record; @@ -307,7 +343,15 @@ export class ZedHostedExecutor extends BaseExecutor { }, }); - const wrapped = response.ok ? wrapZedCompletionStream(response, provider, model) : response; + // The Anthropic backend converts Claude events to OpenAI chunks inside + // wrapZedCompletionStream, bypassing chatCore's marker policy — resolve + // `` close-marker suppression here from the client format / + // headers (same policy as chatCore / GLM, #5245 / kimi-coding leak). + const suppressThinkClose = resolveZedSuppressThinkClose(clientHeaders, clientResponseFormat); + + const wrapped = response.ok + ? wrapZedCompletionStream(response, provider, model, { suppressThinkClose }) + : response; return { response: wrapped, url: `${(this.config as Record)?.llmBaseUrl || "https://cloud.zed.dev"}/completions`, diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 6070d61efd..abe197ebf9 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -2566,6 +2566,7 @@ export async function handleChatCore({ clientRawRequest?.headers, userAgent ), + clientResponseFormat, onCredentialsRefreshed, skipUpstreamRetry, contextEditing: { enabled: contextEditingEnabled }, @@ -2812,6 +2813,7 @@ export async function handleChatCore({ clientRawRequest?.headers, userAgent ), + clientResponseFormat, onCredentialsRefreshed, skipUpstreamRetry, contextEditing: { enabled: contextEditingEnabled }, @@ -3307,6 +3309,7 @@ export async function handleChatCore({ extendedContext, upstreamExtraHeaders: buildUpstreamHeadersForExecute(retryModelId), clientHeaders: buildExecutorClientHeaders(clientRawRequest?.headers, userAgent), + clientResponseFormat, onCredentialsRefreshed, skipUpstreamRetry: isCombo, contextEditing: { enabled: contextEditingEnabled }, @@ -4649,10 +4652,13 @@ export async function handleChatCore({ // Suppress the `` close marker for clients that render it verbatim // (e.g. OpenCode by UA; any client via `x-omniroute-thinking-marker: off`); // preserved for Claude Code / Cursor and unknown clients by default (#5245 / - // #5312). The header wins over the UA allowlist. + // #5312). Responses API clients always suppress it (structured reasoning + // items make the marker meaningless); otherwise the header wins over the + // UA allowlist. resolveSuppressThinkClose({ userAgent: streamUserAgent, thinkingMarkerHeader, + clientResponseFormat, }) ); } else { diff --git a/open-sse/utils/thinkCloseMarker.ts b/open-sse/utils/thinkCloseMarker.ts index 44ac02c4e2..06a6a46774 100644 --- a/open-sse/utils/thinkCloseMarker.ts +++ b/open-sse/utils/thinkCloseMarker.ts @@ -21,8 +21,16 @@ * which suppresses the marker regardless of User-Agent. `on` forces it kept * (overriding the UA allowlist). The default (header absent) is byte-identical * to the UA-only policy, so #4633 / #5123 are never regressed. + * + * Responses API clients (`openai-responses`) are always suppressed: the + * Responses transformer maps `reasoning_content` to structured reasoning items + * natively, so no consumer on that path scans content for the marker — it can + * only leak verbatim into `response.output_text.delta` (observed with + * kimi-coding: a stray `` at the start of the assistant text). */ +import { FORMATS } from "../translator/formats.ts"; + /** Header clients send to explicitly opt in/out of the `` close marker. */ export const THINKING_MARKER_HEADER = "x-omniroute-thinking-marker"; @@ -70,7 +78,13 @@ export function thinkingMarkerHeaderSignal( export function resolveSuppressThinkClose(opts: { userAgent?: string | null; thinkingMarkerHeader?: string | null; + clientResponseFormat?: string | null; }): boolean { + // The marker only exists for Chat Completions clients that scan content for + // it; Responses API clients receive reasoning as structured items instead. + // This wins over the UA allowlist AND the explicit header: there is no + // legitimate marker consumer in the Responses format. + if (opts.clientResponseFormat === FORMATS.OPENAI_RESPONSES) return true; const headerSignal = thinkingMarkerHeaderSignal(opts.thinkingMarkerHeader); if (headerSignal !== null) return headerSignal; return shouldSuppressThinkCloseMarker(opts.userAgent); diff --git a/tests/unit/think-close-marker-responses-format.test.ts b/tests/unit/think-close-marker-responses-format.test.ts new file mode 100644 index 0000000000..002045ae86 --- /dev/null +++ b/tests/unit/think-close-marker-responses-format.test.ts @@ -0,0 +1,58 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { FORMATS } = await import("../../open-sse/translator/formats.ts"); +const { resolveSuppressThinkClose } = await import("../../open-sse/utils/thinkCloseMarker.ts"); + +// kimi-coding via /v1/responses: the Claude→OpenAI `` close marker +// (#4633) exists for Chat Completions clients that scan content for the marker +// (Claude Code / Cursor). Responses API clients receive reasoning as +// structured reasoning items (responsesTransformer maps reasoning_content +// natively), so the textual marker has no consumer on this path and always +// leaks verbatim into `response.output_text.delta`. + +test("openai-responses client format always suppresses the close marker", () => { + assert.equal( + resolveSuppressThinkClose({ + userAgent: "OpenAI/JS 6.26.0", + thinkingMarkerHeader: null, + clientResponseFormat: FORMATS.OPENAI_RESPONSES, + }), + true + ); +}); + +test("openai-responses suppression wins over an explicit keep header", () => { + // There is no legitimate marker consumer in the Responses API format; an + // explicit `x-omniroute-thinking-marker: on` would only re-create the leak. + assert.equal( + resolveSuppressThinkClose({ + userAgent: null, + thinkingMarkerHeader: "on", + clientResponseFormat: FORMATS.OPENAI_RESPONSES, + }), + true + ); +}); + +test("openai chat format keeps the conservative default (marker on)", () => { + assert.equal( + resolveSuppressThinkClose({ + userAgent: "OpenAI/JS 6.26.0", + thinkingMarkerHeader: null, + clientResponseFormat: FORMATS.OPENAI, + }), + false + ); +}); + +test("absent client format preserves the UA/header policy", () => { + assert.equal( + resolveSuppressThinkClose({ userAgent: "opencode/1.0", thinkingMarkerHeader: null }), + true + ); + assert.equal( + resolveSuppressThinkClose({ userAgent: "unknown-client", thinkingMarkerHeader: null }), + false + ); +}); diff --git a/tests/unit/zed-hosted-think-close-marker.test.ts b/tests/unit/zed-hosted-think-close-marker.test.ts new file mode 100644 index 0000000000..09bb442136 --- /dev/null +++ b/tests/unit/zed-hosted-think-close-marker.test.ts @@ -0,0 +1,70 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { __test__ } = await import("../../open-sse/executors/zed-hosted.ts"); +const { wrapZedCompletionStream } = __test__; + +// zed-hosted's Anthropic backend converts Claude events to OpenAI chunks +// inside the executor (wrapZedCompletionStream → claudeToOpenAIResponse), +// bypassing chatCore's suppressThinkClose wiring. Responses API clients +// receive reasoning as structured items, so the textual `` close +// marker must be suppressed on that path (same policy as chatCore / GLM). + +function buildZedAnthropicNdjson(): string { + const lines = [ + { event: { type: "message_start", message: { id: "msg_zed", model: "claude-test" } } }, + { event: { type: "content_block_start", index: 0, content_block: { type: "thinking" } } }, + { + event: { + type: "content_block_delta", + index: 0, + delta: { type: "thinking_delta", thinking: "plan" }, + }, + }, + { event: { type: "content_block_stop", index: 0 } }, + { event: { type: "content_block_start", index: 1, content_block: { type: "text", text: "" } } }, + { + event: { type: "content_block_delta", index: 1, delta: { type: "text_delta", text: "Hi" } }, + }, + { event: { type: "content_block_stop", index: 1 } }, + { + event: { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { output_tokens: 3 }, + }, + }, + { event: { type: "message_stop" } }, + ]; + return lines.map((l) => JSON.stringify(l)).join("\n") + "\n"; +} + +async function readAll(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let out = ""; + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + out += decoder.decode(value, { stream: true }); + } + return out; +} + +function wrapAnthropic(options?: Record): Promise { + const response = new Response(buildZedAnthropicNdjson(), { status: 200 }); + const wrapped = wrapZedCompletionStream(response, "Anthropic", "claude-test", options); + return readAll(wrapped.body as ReadableStream); +} + +test("zed anthropic stream keeps the close marker by default (#4633)", async () => { + const out = await wrapAnthropic(); + assert.ok(out.includes('"content":""'), "expected default marker emission"); +}); + +test("zed anthropic stream suppresses the close marker when asked", async () => { + const out = await wrapAnthropic({ suppressThinkClose: true }); + assert.ok(!out.includes(""), "marker must not leak into output"); + assert.ok(out.includes('"content":"Hi"'), "text content still flows"); + assert.ok(out.includes('"reasoning_content":"plan"'), "reasoning still flows"); +});