From cc544db38b962dedff4299e20006cc7492db529e Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Wed, 19 Aug 2026 11:08:22 -0300 Subject: [PATCH] fix: fail over streaming combo responses terminated with empty completions (#10404) (#10744) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validateResponseQuality's streaming-SSE peek only flagged an OpenAI-shape stream as invalid when it closed WITHOUT ever reaching finish_reason/[DONE] (#7285 truncation guard). A stream that DOES reach finish_reason: "stop" but never carries any real content, reasoning, or tool_calls in any chunk fell through as valid, exactly reproducing the reported content:null / completion_tokens:0 HTTP 200 for cmd/meta/muse-spark-1.2-contributor. Add a sibling failover branch for the terminated-but-empty case, mirroring the existing truncation branch. Tool-calls-only streams are unaffected — they already short-circuit through the earlier content-detection branch. Co-authored-by: Markus Hartung --- ...ng-terminated-empty-completion-failover.md | 1 + open-sse/services/combo/validateQuality.ts | 18 +++ ...ompletion-with-finish-reason-10404.test.ts | 103 ++++++++++++++++++ tests/unit/validate-response-quality.test.ts | 17 +-- 4 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 changelog.d/fixes/10404-streaming-terminated-empty-completion-failover.md create mode 100644 tests/unit/combo-streaming-empty-completion-with-finish-reason-10404.test.ts diff --git a/changelog.d/fixes/10404-streaming-terminated-empty-completion-failover.md b/changelog.d/fixes/10404-streaming-terminated-empty-completion-failover.md new file mode 100644 index 0000000000..d8441a21c1 --- /dev/null +++ b/changelog.d/fixes/10404-streaming-terminated-empty-completion-failover.md @@ -0,0 +1 @@ +- fix(sse): fail over combo streaming responses that reach `finish_reason` with zero content, reasoning, or tool_calls instead of forwarding a terminated-but-empty completion (#10404) diff --git a/open-sse/services/combo/validateQuality.ts b/open-sse/services/combo/validateQuality.ts index 79f742b2c2..7c7f0ce8a9 100644 --- a/open-sse/services/combo/validateQuality.ts +++ b/open-sse/services/combo/validateQuality.ts @@ -516,6 +516,24 @@ export async function validateResponseQuality( return { valid: false, reason: "streaming openai truncated without finish_reason" }; } + // Issue #10404: an OpenAI-shape stream that DOES reach a terminal + // marker (finish_reason / [DONE]) but never carried any real + // content, reasoning, or tool_calls in any chunk — an upstream + // that burns the whole generation budget and returns + // completion_tokens:0 with an HTTP 200. `anyContentFound` only + // flips true via `isKnownNonClaudeStreamPayload` detecting + // content/reasoning/tool_calls (`hasOpenAICompatibleStreamValue`), + // so a tool_calls-only stream already exits early via the + // `outcome === "content"` branch above and never reaches here — + // this branch only fires on genuinely empty completions. + if (openAi.hasChoicePayload && openAi.hasTerminalMarker && !anyContentFound) { + log.warn?.( + "COMBO", + "Streaming OpenAI-shape response reached finish_reason/[DONE] with no content, reasoning, or tool_calls — marking as invalid for combo failover" + ); + return { valid: false, reason: "streaming openai terminated with empty completion" }; + } + // Incomplete lifecycle or non-Claude stream — replay all buffered // bytes. The reader is exhausted so the forwarding reader will // immediately signal done. diff --git a/tests/unit/combo-streaming-empty-completion-with-finish-reason-10404.test.ts b/tests/unit/combo-streaming-empty-completion-with-finish-reason-10404.test.ts new file mode 100644 index 0000000000..bbbe456f41 --- /dev/null +++ b/tests/unit/combo-streaming-empty-completion-with-finish-reason-10404.test.ts @@ -0,0 +1,103 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { validateResponseQuality } = await import("../../open-sse/services/combo.ts"); + +const encoder = new TextEncoder(); +const silentLog = { warn: () => {} }; + +function sseStream(body: string): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(body)); + controller.close(); + }, + }); +} + +function makeEmptyButTerminatedOpenAiStream(): Response { + const chunks = [ + JSON.stringify({ + id: "chatcmpl-10404", + object: "chat.completion.chunk", + choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], + }), + JSON.stringify({ + id: "chatcmpl-10404", + object: "chat.completion.chunk", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 56447, completion_tokens: 0, total_tokens: 56447 }, + }), + ]; + const body = chunks.map((c) => `data: ${c}\n\n`).join("") + "data: [DONE]\n\n"; + return new Response(sseStream(body), { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); +} + +function makeToolCallsOnlyOpenAiStream(): Response { + const chunks = [ + JSON.stringify({ + id: "chatcmpl-10404-tool", + object: "chat.completion.chunk", + choices: [ + { + index: 0, + delta: { + role: "assistant", + tool_calls: [ + { + index: 0, + id: "call_1", + type: "function", + function: { name: "get_weather", arguments: "" }, + }, + ], + }, + finish_reason: null, + }, + ], + }), + JSON.stringify({ + id: "chatcmpl-10404-tool", + object: "chat.completion.chunk", + choices: [ + { + index: 0, + delta: { + tool_calls: [{ index: 0, function: { arguments: '{"city":"SF"}' } }], + }, + finish_reason: null, + }, + ], + }), + JSON.stringify({ + id: "chatcmpl-10404-tool", + object: "chat.completion.chunk", + choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], + usage: { prompt_tokens: 100, completion_tokens: 12, total_tokens: 112 }, + }), + ]; + const body = chunks.map((c) => `data: ${c}\n\n`).join("") + "data: [DONE]\n\n"; + return new Response(sseStream(body), { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); +} + +test("#10404: OpenAI-shape stream that reaches finish_reason:stop with zero content/tool_calls/reasoning should fail over, not pass as valid", async () => { + const res = makeEmptyButTerminatedOpenAiStream(); + const out = await validateResponseQuality(res, true, silentLog); + assert.equal( + out.valid, + false, + "expected failover (valid:false) for a terminated-but-empty streaming completion" + ); +}); + +test("#10404 no-regression: OpenAI-shape stream carrying only tool_calls deltas + finish_reason:tool_calls must still pass as valid", async () => { + const res = makeToolCallsOnlyOpenAiStream(); + const out = await validateResponseQuality(res, true, silentLog); + assert.equal(out.valid, true, "tool_calls-only streams must not be treated as empty"); +}); diff --git a/tests/unit/validate-response-quality.test.ts b/tests/unit/validate-response-quality.test.ts index 7ea04a15ba..de4b5f47dc 100644 --- a/tests/unit/validate-response-quality.test.ts +++ b/tests/unit/validate-response-quality.test.ts @@ -137,15 +137,18 @@ test("streaming event: ping only (no content, no terminator) → still valid (re assert.strictEqual(verdict.valid, true); }); -test("streaming OpenAI finish_reason-only chunk (no content delta) → valid (recognised terminator)", async () => { - // Some reasoning models emit a final `finish_reason: "stop"` chunk with - // no content and no follow-up `data: [DONE]`. That's a legitimate empty - // completion, not a truncation. Sending the `finish_reason` chunk - // WITHOUT a trailing `[DONE]` isolates the new finish_reason check — - // removing it would flip this test to invalid. +test("streaming OpenAI finish_reason-only chunk (no content delta) → invalid (#10404 terminated-but-empty failover)", async () => { + // Some upstreams reach a terminal `finish_reason: "stop"` chunk while + // never emitting any content, reasoning, or tool_calls in any chunk — + // an upstream that burns the whole generation budget and returns + // completion_tokens:0 with an HTTP 200 (#10404). The stream is + // well-formed and properly terminated (not a #7285 truncation), but it + // carries zero usable output, so combo must fail over to a sibling + // target rather than forward the empty completion as a success. const res = makeSseResponse( 'data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n' ); const verdict = await validateResponseQuality(res, true, {}); - assert.strictEqual(verdict.valid, true); + assert.strictEqual(verdict.valid, false); + assert.match(verdict.reason ?? "", /streaming openai terminated with empty completion/); });