fix(translator): keep upstream usage from trailing empty-choices chunks (#11883)

openaiToClaudeResponse() returned early on !chunk.choices?.[0], dropping the trailing usage-only chunk many OpenAI-compatible upstreams send when stream_options.include_usage is set (confirmed on Fireworks kimi-k3) — state.usage stayed undefined and billing fell back to an uncached token estimate. 154/154 focused assertions across the fix + regression suite. Thanks for tracking down the billing impact!

changelog.d/fixes/11883-openai-to-claude-trailing-usage.md

@@ -0,0 +1 @@
+- **fix(translator):** the streaming OpenAI→Claude translator keeps upstream usage, including prompt-cache tokens, when it arrives on a trailing `choices: []` chunk (Fireworks and any upstream using `stream_options.include_usage`) ([#11883](https://github.com/diegosouzapw/OmniRoute/pull/11883)) — thanks @NoxzRCW

open-sse/translator/response/openai-to-claude.ts

@@ -194,50 +194,62 @@ function stopTextBlock(state, results) {
   state.textBlockStarted = false;
 }
 
+// Harvest the upstream usage block from any chunk, including trailing
+// usage-only chunks that carry `choices: []` (#11817).
+function trackUsageFromChunk(chunk, state) {
+  if (!chunk.usage || typeof chunk.usage !== "object") return;
+  const promptTokens =
+    typeof chunk.usage.prompt_tokens === "number" ? chunk.usage.prompt_tokens : 0;
+  const outputTokens =
+    typeof chunk.usage.completion_tokens === "number" ? chunk.usage.completion_tokens : 0;
+
+  // Extract cache tokens from prompt_tokens_details
+  const cachedTokens = chunk.usage.prompt_tokens_details?.cached_tokens;
+  const cacheCreationTokens = chunk.usage.prompt_tokens_details?.cache_creation_tokens;
+  const cacheReadTokens = typeof cachedTokens === "number" ? cachedTokens : 0;
+  const cacheCreateTokens = typeof cacheCreationTokens === "number" ? cacheCreationTokens : 0;
+
+  // input_tokens = prompt_tokens - cached_tokens - cache_creation_tokens
+  // Because OpenAI's prompt_tokens includes all prompt-side tokens
+  const inputTokens = promptTokens - cacheReadTokens - cacheCreateTokens;
+
+  state.usage = {
+    input_tokens: inputTokens,
+    output_tokens: outputTokens,
+  };
+
+  // Add cache_read_input_tokens if present
+  if (cacheReadTokens > 0) {
+    state.usage.cache_read_input_tokens = cacheReadTokens;
+  }
+
+  // Add cache_creation_input_tokens if present
+  if (cacheCreateTokens > 0) {
+    state.usage.cache_creation_input_tokens = cacheCreateTokens;
+  }
+
+  // Note: completion_tokens_details.reasoning_tokens is already included in output_tokens
+  // No need to add separately as Claude expects total output_tokens
+}
+
 // Convert OpenAI stream chunk to Claude format
 export function openaiToClaudeResponse(chunk, state) {
-  if (!chunk || !chunk.choices?.[0]) return null;
+  if (!chunk) return null;
+
+  // Usage must be harvested BEFORE the choices guard: many OpenAI-compatible
+  // upstreams (Fireworks, vLLM, Together, …) deliver the authoritative usage
+  // block — including prompt_tokens_details.cached_tokens — on a trailing
+  // usage-only chunk shaped `{"choices":[],"usage":{...}}`. Returning early on
+  // that chunk discarded the real numbers and left downstream accounting on
+  // OmniRoute's own tokenizer estimate (#11817).
+  trackUsageFromChunk(chunk, state);
+
+  if (!chunk.choices?.[0]) return null;
 
   const results = [];
   const choice = chunk.choices[0];
   const delta = choice.delta;
 
-  // Track usage from OpenAI chunk if available
-  if (chunk.usage && typeof chunk.usage === "object") {
-    const promptTokens =
-      typeof chunk.usage.prompt_tokens === "number" ? chunk.usage.prompt_tokens : 0;
-    const outputTokens =
-      typeof chunk.usage.completion_tokens === "number" ? chunk.usage.completion_tokens : 0;
-
-    // Extract cache tokens from prompt_tokens_details
-    const cachedTokens = chunk.usage.prompt_tokens_details?.cached_tokens;
-    const cacheCreationTokens = chunk.usage.prompt_tokens_details?.cache_creation_tokens;
-    const cacheReadTokens = typeof cachedTokens === "number" ? cachedTokens : 0;
-    const cacheCreateTokens = typeof cacheCreationTokens === "number" ? cacheCreationTokens : 0;
-
-    // input_tokens = prompt_tokens - cached_tokens - cache_creation_tokens
-    // Because OpenAI's prompt_tokens includes all prompt-side tokens
-    const inputTokens = promptTokens - cacheReadTokens - cacheCreateTokens;
-
-    state.usage = {
-      input_tokens: inputTokens,
-      output_tokens: outputTokens,
-    };
-
-    // Add cache_read_input_tokens if present
-    if (cacheReadTokens > 0) {
-      state.usage.cache_read_input_tokens = cacheReadTokens;
-    }
-
-    // Add cache_creation_input_tokens if present
-    if (cacheCreateTokens > 0) {
-      state.usage.cache_creation_input_tokens = cacheCreateTokens;
-    }
-
-    // Note: completion_tokens_details.reasoning_tokens is already included in output_tokens
-    // No need to add separately as Claude expects total output_tokens
-  }
-
   // First chunk - ALWAYS send message_start first
   if (!state.messageStartSent) {
     state.messageStartSent = true;

tests/unit/openai-to-claude-trailing-usage-11817.test.ts

@@ -0,0 +1,104 @@
+/**
+ * Regression for #11817 — the streaming OpenAI→Claude translator dropped the
+ * upstream usage block (including prompt-cache accounting) whenever it arrived
+ * on a trailing usage-only chunk shaped `{"choices":[],"usage":{...}}`.
+ *
+ * Many OpenAI-compatible upstreams (confirmed: Fireworks / kimi-k3, also vLLM
+ * and Together with `stream_options.include_usage`) emit usage exactly that
+ * way. `openaiToClaudeResponse()` returned early on `!chunk.choices?.[0]`
+ * BEFORE reading `chunk.usage`, so `state.usage` stayed undefined and every
+ * downstream consumer fell back to OmniRoute's own tokenizer estimate — no
+ * cache_read_input_tokens, no cache_creation_input_tokens, and an input_tokens
+ * figure that disagreed with the provider's own count.
+ *
+ * Impact was silent over-billing: a session served ~75% from prompt cache was
+ * metered at the full uncached rate.
+ *
+ * Runner: node --import tsx/esm --test tests/unit/openai-to-claude-trailing-usage-11817.test.ts
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const { openaiToClaudeResponse } =
+  await import("../../open-sse/translator/response/openai-to-claude.ts");
+
+function newState() {
+  return { toolCalls: new Map(), messageId: "msg_11817", model: "kimi-k3" } as Record<
+    string,
+    unknown
+  >;
+}
+
+test("#11817 — usage on a trailing choices:[] chunk is harvested, with cache split", () => {
+  const state = newState();
+
+  openaiToClaudeResponse({ choices: [{ index: 0, delta: { content: "ok" } }] }, state);
+  openaiToClaudeResponse({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }, state);
+  openaiToClaudeResponse(
+    {
+      choices: [],
+      usage: {
+        prompt_tokens: 6103,
+        completion_tokens: 24,
+        prompt_tokens_details: { cached_tokens: 6102 },
+      },
+    },
+    state
+  );
+
+  assert.deepEqual(state.usage, {
+    input_tokens: 1, // 6103 - 6102 cached
+    output_tokens: 24,
+    cache_read_input_tokens: 6102,
+  });
+});
+
+test("#11817 — cache_creation_tokens on a trailing chunk is mapped too", () => {
+  const state = newState();
+  openaiToClaudeResponse(
+    {
+      choices: [],
+      usage: {
+        prompt_tokens: 1000,
+        completion_tokens: 5,
+        prompt_tokens_details: { cached_tokens: 400, cache_creation_tokens: 100 },
+      },
+    },
+    state
+  );
+
+  assert.deepEqual(state.usage, {
+    input_tokens: 500,
+    output_tokens: 5,
+    cache_read_input_tokens: 400,
+    cache_creation_input_tokens: 100,
+  });
+});
+
+test("#11817 — a usage-only chunk still emits no Claude events", () => {
+  const state = newState();
+  const out = openaiToClaudeResponse(
+    { choices: [], usage: { prompt_tokens: 10, completion_tokens: 1 } },
+    state
+  );
+  assert.equal(out, null);
+});
+
+test("#11817 — no regression: usage carried inline on the finish chunk", () => {
+  const state = newState();
+  openaiToClaudeResponse({ choices: [{ index: 0, delta: { content: "hi" } }] }, state);
+  openaiToClaudeResponse(
+    {
+      choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
+      usage: { prompt_tokens: 100, completion_tokens: 10 },
+    },
+    state
+  );
+  assert.deepEqual(state.usage, { input_tokens: 100, output_tokens: 10 });
+});
+
+test("#11817 — no regression: empty and nullish chunks are still ignored", () => {
+  assert.equal(openaiToClaudeResponse(null, newState()), null);
+  assert.equal(openaiToClaudeResponse({ choices: [] }, newState()), null);
+  assert.equal(openaiToClaudeResponse({}, newState()), null);
+});