diff --git a/changelog.d/fixes/11883-openai-to-claude-trailing-usage.md b/changelog.d/fixes/11883-openai-to-claude-trailing-usage.md new file mode 100644 index 0000000000..2ab09c4d96 --- /dev/null +++ b/changelog.d/fixes/11883-openai-to-claude-trailing-usage.md @@ -0,0 +1 @@ +- **fix(translator):** the streaming OpenAI→Claude translator keeps upstream usage, including prompt-cache tokens, when it arrives on a trailing `choices: []` chunk (Fireworks and any upstream using `stream_options.include_usage`) ([#11883](https://github.com/diegosouzapw/OmniRoute/pull/11883)) — thanks @NoxzRCW diff --git a/open-sse/translator/response/openai-to-claude.ts b/open-sse/translator/response/openai-to-claude.ts index f1683d55ab..5a841f1cb4 100644 --- a/open-sse/translator/response/openai-to-claude.ts +++ b/open-sse/translator/response/openai-to-claude.ts @@ -194,50 +194,62 @@ function stopTextBlock(state, results) { state.textBlockStarted = false; } +// Harvest the upstream usage block from any chunk, including trailing +// usage-only chunks that carry `choices: []` (#11817). +function trackUsageFromChunk(chunk, state) { + if (!chunk.usage || typeof chunk.usage !== "object") return; + const promptTokens = + typeof chunk.usage.prompt_tokens === "number" ? chunk.usage.prompt_tokens : 0; + const outputTokens = + typeof chunk.usage.completion_tokens === "number" ? chunk.usage.completion_tokens : 0; + + // Extract cache tokens from prompt_tokens_details + const cachedTokens = chunk.usage.prompt_tokens_details?.cached_tokens; + const cacheCreationTokens = chunk.usage.prompt_tokens_details?.cache_creation_tokens; + const cacheReadTokens = typeof cachedTokens === "number" ? cachedTokens : 0; + const cacheCreateTokens = typeof cacheCreationTokens === "number" ? cacheCreationTokens : 0; + + // input_tokens = prompt_tokens - cached_tokens - cache_creation_tokens + // Because OpenAI's prompt_tokens includes all prompt-side tokens + const inputTokens = promptTokens - cacheReadTokens - cacheCreateTokens; + + state.usage = { + input_tokens: inputTokens, + output_tokens: outputTokens, + }; + + // Add cache_read_input_tokens if present + if (cacheReadTokens > 0) { + state.usage.cache_read_input_tokens = cacheReadTokens; + } + + // Add cache_creation_input_tokens if present + if (cacheCreateTokens > 0) { + state.usage.cache_creation_input_tokens = cacheCreateTokens; + } + + // Note: completion_tokens_details.reasoning_tokens is already included in output_tokens + // No need to add separately as Claude expects total output_tokens +} + // Convert OpenAI stream chunk to Claude format export function openaiToClaudeResponse(chunk, state) { - if (!chunk || !chunk.choices?.[0]) return null; + if (!chunk) return null; + + // Usage must be harvested BEFORE the choices guard: many OpenAI-compatible + // upstreams (Fireworks, vLLM, Together, …) deliver the authoritative usage + // block — including prompt_tokens_details.cached_tokens — on a trailing + // usage-only chunk shaped `{"choices":[],"usage":{...}}`. Returning early on + // that chunk discarded the real numbers and left downstream accounting on + // OmniRoute's own tokenizer estimate (#11817). + trackUsageFromChunk(chunk, state); + + if (!chunk.choices?.[0]) return null; const results = []; const choice = chunk.choices[0]; const delta = choice.delta; - // Track usage from OpenAI chunk if available - if (chunk.usage && typeof chunk.usage === "object") { - const promptTokens = - typeof chunk.usage.prompt_tokens === "number" ? chunk.usage.prompt_tokens : 0; - const outputTokens = - typeof chunk.usage.completion_tokens === "number" ? chunk.usage.completion_tokens : 0; - - // Extract cache tokens from prompt_tokens_details - const cachedTokens = chunk.usage.prompt_tokens_details?.cached_tokens; - const cacheCreationTokens = chunk.usage.prompt_tokens_details?.cache_creation_tokens; - const cacheReadTokens = typeof cachedTokens === "number" ? cachedTokens : 0; - const cacheCreateTokens = typeof cacheCreationTokens === "number" ? cacheCreationTokens : 0; - - // input_tokens = prompt_tokens - cached_tokens - cache_creation_tokens - // Because OpenAI's prompt_tokens includes all prompt-side tokens - const inputTokens = promptTokens - cacheReadTokens - cacheCreateTokens; - - state.usage = { - input_tokens: inputTokens, - output_tokens: outputTokens, - }; - - // Add cache_read_input_tokens if present - if (cacheReadTokens > 0) { - state.usage.cache_read_input_tokens = cacheReadTokens; - } - - // Add cache_creation_input_tokens if present - if (cacheCreateTokens > 0) { - state.usage.cache_creation_input_tokens = cacheCreateTokens; - } - - // Note: completion_tokens_details.reasoning_tokens is already included in output_tokens - // No need to add separately as Claude expects total output_tokens - } - // First chunk - ALWAYS send message_start first if (!state.messageStartSent) { state.messageStartSent = true; diff --git a/tests/unit/openai-to-claude-trailing-usage-11817.test.ts b/tests/unit/openai-to-claude-trailing-usage-11817.test.ts new file mode 100644 index 0000000000..624a692ade --- /dev/null +++ b/tests/unit/openai-to-claude-trailing-usage-11817.test.ts @@ -0,0 +1,104 @@ +/** + * Regression for #11817 — the streaming OpenAI→Claude translator dropped the + * upstream usage block (including prompt-cache accounting) whenever it arrived + * on a trailing usage-only chunk shaped `{"choices":[],"usage":{...}}`. + * + * Many OpenAI-compatible upstreams (confirmed: Fireworks / kimi-k3, also vLLM + * and Together with `stream_options.include_usage`) emit usage exactly that + * way. `openaiToClaudeResponse()` returned early on `!chunk.choices?.[0]` + * BEFORE reading `chunk.usage`, so `state.usage` stayed undefined and every + * downstream consumer fell back to OmniRoute's own tokenizer estimate — no + * cache_read_input_tokens, no cache_creation_input_tokens, and an input_tokens + * figure that disagreed with the provider's own count. + * + * Impact was silent over-billing: a session served ~75% from prompt cache was + * metered at the full uncached rate. + * + * Runner: node --import tsx/esm --test tests/unit/openai-to-claude-trailing-usage-11817.test.ts + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { openaiToClaudeResponse } = + await import("../../open-sse/translator/response/openai-to-claude.ts"); + +function newState() { + return { toolCalls: new Map(), messageId: "msg_11817", model: "kimi-k3" } as Record< + string, + unknown + >; +} + +test("#11817 — usage on a trailing choices:[] chunk is harvested, with cache split", () => { + const state = newState(); + + openaiToClaudeResponse({ choices: [{ index: 0, delta: { content: "ok" } }] }, state); + openaiToClaudeResponse({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }, state); + openaiToClaudeResponse( + { + choices: [], + usage: { + prompt_tokens: 6103, + completion_tokens: 24, + prompt_tokens_details: { cached_tokens: 6102 }, + }, + }, + state + ); + + assert.deepEqual(state.usage, { + input_tokens: 1, // 6103 - 6102 cached + output_tokens: 24, + cache_read_input_tokens: 6102, + }); +}); + +test("#11817 — cache_creation_tokens on a trailing chunk is mapped too", () => { + const state = newState(); + openaiToClaudeResponse( + { + choices: [], + usage: { + prompt_tokens: 1000, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: 400, cache_creation_tokens: 100 }, + }, + }, + state + ); + + assert.deepEqual(state.usage, { + input_tokens: 500, + output_tokens: 5, + cache_read_input_tokens: 400, + cache_creation_input_tokens: 100, + }); +}); + +test("#11817 — a usage-only chunk still emits no Claude events", () => { + const state = newState(); + const out = openaiToClaudeResponse( + { choices: [], usage: { prompt_tokens: 10, completion_tokens: 1 } }, + state + ); + assert.equal(out, null); +}); + +test("#11817 — no regression: usage carried inline on the finish chunk", () => { + const state = newState(); + openaiToClaudeResponse({ choices: [{ index: 0, delta: { content: "hi" } }] }, state); + openaiToClaudeResponse( + { + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 100, completion_tokens: 10 }, + }, + state + ); + assert.deepEqual(state.usage, { input_tokens: 100, output_tokens: 10 }); +}); + +test("#11817 — no regression: empty and nullish chunks are still ignored", () => { + assert.equal(openaiToClaudeResponse(null, newState()), null); + assert.equal(openaiToClaudeResponse({ choices: [] }, newState()), null); + assert.equal(openaiToClaudeResponse({}, newState()), null); +});