diff --git a/changelog.d/fixes/11915-openai-to-claude-usage-emission-ordering.md b/changelog.d/fixes/11915-openai-to-claude-usage-emission-ordering.md new file mode 100644 index 0000000000..3ff9cee01e --- /dev/null +++ b/changelog.d/fixes/11915-openai-to-claude-usage-emission-ordering.md @@ -0,0 +1 @@ +- **fix(sse):** the OpenAI→Claude stream translator now defers the terminal `message_delta`/`message_stop` emission until the real usage block has arrived (or a genuine end-of-stream flush forces it) instead of emitting it immediately on `finish_reason` — previously, when the trailing usage-only chunk (`{"choices":[],"usage":{...}}`) arrived *after* the `finish_reason` chunk (the normal order for Fireworks/vLLM/Together and other `stream_options.include_usage` upstreams), the client-visible `message_delta` still carried stale/zero usage even though `state.usage` was internally corrected too late to matter (ported from [#11915](https://github.com/diegosouzapw/OmniRoute/pull/11915) — thanks @HouMinXi). diff --git a/open-sse/translator/response/openai-to-claude.ts b/open-sse/translator/response/openai-to-claude.ts index 5a841f1cb4..5ecdd2348a 100644 --- a/open-sse/translator/response/openai-to-claude.ts +++ b/open-sse/translator/response/openai-to-claude.ts @@ -234,7 +234,11 @@ function trackUsageFromChunk(chunk, state) { // Convert OpenAI stream chunk to Claude format export function openaiToClaudeResponse(chunk, state) { - if (!chunk) return null; + if (!chunk && !state.pendingClaudeFinishChoice) return null; + + const results = []; + const chunkUsage = chunk?.usage; + const hasChunkUsage = chunkUsage && typeof chunkUsage === "object"; // Usage must be harvested BEFORE the choices guard: many OpenAI-compatible // upstreams (Fireworks, vLLM, Together, …) deliver the authoritative usage @@ -242,14 +246,23 @@ export function openaiToClaudeResponse(chunk, state) { // usage-only chunk shaped `{"choices":[],"usage":{...}}`. Returning early on // that chunk discarded the real numbers and left downstream accounting on // OmniRoute's own tokenizer estimate (#11817). - trackUsageFromChunk(chunk, state); + // + // Harvesting alone is not enough: if the finish_reason chunk arrives BEFORE + // this trailing usage chunk (the normal order for these upstreams), the + // finish block below fires immediately and emits message_delta with + // whatever state.usage held at that moment — zero/stale, since the real + // trailing chunk hasn't been seen yet. The finish deferral below + // (pendingClaudeFinishChoice) holds the terminal emission open until either + // real usage has arrived or a genuine flush forces it, so the message_delta + // actually sent to the client carries the correct numbers (#11817 follow-up). + if (chunk) trackUsageFromChunk(chunk, state); - if (!chunk.choices?.[0]) return null; - - const results = []; - const choice = chunk.choices[0]; + const chunkChoice = chunk?.choices?.[0]; + const flushingPendingFinish = !chunkChoice && Boolean(state.pendingClaudeFinishChoice); + const choice = chunkChoice || state.pendingClaudeFinishChoice; + if (!choice) return null; + if (flushingPendingFinish) state.pendingClaudeFinishChoice = null; const delta = choice.delta; - // First chunk - ALWAYS send message_start first if (!state.messageStartSent) { state.messageStartSent = true; @@ -501,6 +514,11 @@ export function openaiToClaudeResponse(chunk, state) { // guard therefore misfired and silently dropped the terminal message_delta/message_stop // for Responses→Claude streams (#5828 regression). if (choice.finish_reason && !state.claudeFinishEmitted) { + if (!hasChunkUsage && !flushingPendingFinish) { + state.pendingClaudeFinishChoice = choice; + return results.length > 0 ? results : null; + } + state.claudeFinishEmitted = true; stopThinkingBlock(state, results); stopTextBlock(state, results); diff --git a/tests/unit/translator/openai-to-claude-trailing-usage.test.ts b/tests/unit/translator/openai-to-claude-trailing-usage.test.ts new file mode 100644 index 0000000000..75a207b698 --- /dev/null +++ b/tests/unit/translator/openai-to-claude-trailing-usage.test.ts @@ -0,0 +1,229 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { openaiToClaudeResponse } from "../../../open-sse/translator/response/openai-to-claude.ts"; + +type ClaudeUsage = { + input_tokens: number; + output_tokens: number; + cache_read_input_tokens?: number; + cache_creation_input_tokens?: number; +}; + +type TranslatorState = Record & { + toolCalls: Map; + usage?: ClaudeUsage; +}; + +const TRAILING_USAGE = { + prompt_tokens: 6103, + completion_tokens: 16, + total_tokens: 6119, + prompt_tokens_details: { + cached_tokens: 6000, + cache_creation_tokens: 100, + }, +}; + +function createState(): TranslatorState { + return { toolCalls: new Map() }; +} + +function collectEvents( + chunks: Array | null>, + state: TranslatorState +): Array> { + return chunks.flatMap((chunk) => openaiToClaudeResponse(chunk, state) ?? []); +} + +test("usage-only choices-empty chunk updates Claude usage without emitting a content delta", () => { + const state = createState(); + + const events = openaiToClaudeResponse( + { + id: "chatcmpl-11817", + model: "accounts/fireworks/models/kimi-k3", + choices: [], + usage: TRAILING_USAGE, + }, + state + ); + + assert.equal(events, null); + assert.deepEqual(state.usage, { + input_tokens: 3, + output_tokens: 16, + cache_read_input_tokens: 6000, + cache_creation_input_tokens: 100, + }); +}); + +test("trailing choices-empty usage completes the stream with real cache accounting", () => { + const state = createState(); + const events = collectEvents( + [ + { + id: "chatcmpl-11817", + model: "accounts/fireworks/models/kimi-k3", + choices: [{ index: 0, delta: { content: "OK" }, finish_reason: null }], + }, + { + id: "chatcmpl-11817", + model: "accounts/fireworks/models/kimi-k3", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + }, + { + id: "chatcmpl-11817", + model: "accounts/fireworks/models/kimi-k3", + choices: [], + usage: TRAILING_USAGE, + }, + ], + state + ); + + assert.equal(events[0].type, "message_start"); + assert.equal(events[1].type, "content_block_start"); + assert.equal(events[2].type, "content_block_delta"); + assert.equal(events[2].delta?.text, "OK"); + assert.equal(events[3].type, "content_block_stop"); + assert.equal(events[4].type, "message_delta"); + assert.equal(events[4].delta?.stop_reason, "end_turn"); + assert.deepEqual(events[4].usage, { + input_tokens: 3, + output_tokens: 16, + cache_read_input_tokens: 6000, + cache_creation_input_tokens: 100, + }); + assert.equal(events[5].type, "message_stop"); + assert.equal(events.length, 6); +}); + +test("stream-end flush still emits terminal events when upstream omits usage", () => { + const state = createState(); + const events = collectEvents( + [ + { + id: "chatcmpl-11817-no-usage", + model: "accounts/fireworks/models/kimi-k3", + choices: [{ index: 0, delta: { content: "OK" }, finish_reason: null }], + }, + { + id: "chatcmpl-11817-no-usage", + model: "accounts/fireworks/models/kimi-k3", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + }, + null, + ], + state + ); + + assert.deepEqual( + events.filter((event) => event.type === "message_delta" || event.type === "message_stop"), + [ + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { input_tokens: 0, output_tokens: 0 }, + }, + { type: "message_stop" }, + ] + ); +}); + +test("trailing chunk without choices property updates usage and flushes finish", () => { + const state = createState(); + const events = collectEvents( + [ + { + id: "chatcmpl-11817-no-choices-key", + model: "accounts/fireworks/models/kimi-k3", + choices: [{ index: 0, delta: { content: "Done" }, finish_reason: null }], + }, + { + id: "chatcmpl-11817-no-choices-key", + model: "accounts/fireworks/models/kimi-k3", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + }, + { + id: "chatcmpl-11817-no-choices-key", + model: "accounts/fireworks/models/kimi-k3", + usage: TRAILING_USAGE, + }, + ], + state + ); + + const terminalEvents = events.filter( + (event) => event.type === "message_delta" || event.type === "message_stop" + ); + assert.deepEqual(terminalEvents, [ + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { + input_tokens: 3, + output_tokens: 16, + cache_read_input_tokens: 6000, + cache_creation_input_tokens: 100, + }, + }, + { type: "message_stop" }, + ]); +}); + +test("trailing choices-empty chunk with tool_calls finish_reason preserves tool_use stop_reason and usage", () => { + const state = createState(); + const events = collectEvents( + [ + { + id: "chatcmpl-11817-tool", + model: "accounts/fireworks/models/kimi-k3", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + id: "call_123", + function: { name: "get_weather", arguments: "{\"city\":\"Beijing\"}" }, + }, + ], + }, + finish_reason: null, + }, + ], + }, + { + id: "chatcmpl-11817-tool", + model: "accounts/fireworks/models/kimi-k3", + choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], + }, + { + id: "chatcmpl-11817-tool", + model: "accounts/fireworks/models/kimi-k3", + choices: [], + usage: TRAILING_USAGE, + }, + ], + state + ); + + const terminalEvents = events.filter( + (event) => event.type === "message_delta" || event.type === "message_stop" + ); + assert.deepEqual(terminalEvents, [ + { + type: "message_delta", + delta: { stop_reason: "tool_use" }, + usage: { + input_tokens: 3, + output_tokens: 16, + cache_read_input_tokens: 6000, + cache_creation_input_tokens: 100, + }, + }, + { type: "message_stop" }, + ]); +});