diff --git a/open-sse/executors/commandCode.ts b/open-sse/executors/commandCode.ts index fe056eab8f..cc2642de5c 100644 --- a/open-sse/executors/commandCode.ts +++ b/open-sse/executors/commandCode.ts @@ -461,14 +461,31 @@ function applyEventToAggregateOrThrow(event: JsonRecord, state: AggregateState): function usageFromCommandCode(usage: JsonRecord | null) { if (!usage) return undefined; const details = isRecord(usage.inputTokenDetails) ? usage.inputTokenDetails : {}; - const prompt = - (numberValue(usage.inputTokens) || 0) + (numberValue(details.cacheReadTokens) || 0); + const cacheRead = numberValue(details.cacheReadTokens) || 0; + const noCache = numberValue(details.noCacheTokens) || 0; + // Command Code's totalUsage.inputTokens is the FULL prompt total and already + // includes the cached portion (noCacheTokens + cacheReadTokens = inputTokens), + // so we must NOT add cacheRead back — that would double-count. There is no + // cache-write field in the upstream payload, so cache creation stays unset. + const inputTokens = numberValue(usage.inputTokens) || 0; + const prompt = inputTokens; const completion = numberValue(usage.outputTokens) || 0; - return { + const result: JsonRecord = { prompt_tokens: prompt, completion_tokens: completion, total_tokens: prompt + completion, }; + // Surface the cache breakdown as informational fields so logUsage prints + // `| cache_read=X | no_cache=Y` and appendRequestLog persists them. These are + // NOT added to prompt_tokens (already included) — metering stays accurate. + if (cacheRead > 0) result.cache_read_input_tokens = cacheRead; + if (noCache > 0) result.no_cache_tokens = noCache; + // Keep reasoning_token_details (reasoningTokens) when present so stream.ts's + // extractUsage can surface it as reasoning_tokens. + const reasoningDetails = isRecord(usage.reasoningTokenDetails) ? usage.reasoningTokenDetails : {}; + const reasoning = numberValue(reasoningDetails.reasoningTokens); + if (reasoning !== undefined && reasoning > 0) result.reasoning_tokens = reasoning; + return result; } function createStreamResponse( @@ -549,6 +566,22 @@ function createStreamResponse( state.finishReason = mapFinishReason(event.finishReason); state.usage = isRecord(event.totalUsage) ? event.totalUsage : null; controller.enqueue(sse(chatCompletionChunk(id, model, {}, state.finishReason))); + // Emit a standards-compliant usage-only chunk (choices: []) before + // [DONE] when upstream reported usage. stream.ts's extractUsage + // recognizes this shape (see stream.ts:1661) and logs the ACTUAL + // token counts (in/out/cache_read/no_cache) instead of estimates. + const usagePayload = usageFromCommandCode(state.usage); + if (usagePayload) { + controller.enqueue( + sse({ + id, + object: "chat.completion.chunk", + model, + usage: usagePayload, + choices: [], + }) + ); + } controller.enqueue(encoder.encode("data: [DONE]\n\n")); closed = true; controller.close(); diff --git a/open-sse/utils/usageTracking.ts b/open-sse/utils/usageTracking.ts index 93d41c83cc..398d1d5906 100644 --- a/open-sse/utils/usageTracking.ts +++ b/open-sse/utils/usageTracking.ts @@ -6,6 +6,7 @@ import { appendRequestLog } from "@/lib/usageDb"; import { getLoggedInputTokens, getLoggedOutputTokens, + getNoCacheTokens, getPromptCacheCreationTokens, getPromptCacheReadTokens, } from "@/lib/usage/tokenAccounting"; @@ -290,6 +291,7 @@ export function normalizeUsage(usage) { assignNumber("cache_read_input_tokens", usage?.cache_read_input_tokens); assignNumber("cache_creation_input_tokens", usage?.cache_creation_input_tokens); assignNumber("cached_tokens", usage?.cached_tokens); + assignNumber("no_cache_tokens", usage?.no_cache_tokens); assignNumber("reasoning_tokens", usage?.reasoning_tokens); // xAI's exact provider-reported cost (port of decolua/9router#2453, capability A — // @ryanngit). Ticks → USD conversion happens in costCalculator.ts, not here. @@ -416,6 +418,9 @@ export function extractUsage(chunk) { chunk.usage.input_tokens_details?.cached_tokens ?? chunk.usage.prompt_cache_hit_tokens ?? chunk.usage.cached_tokens, + cache_read_input_tokens: chunk.usage.cache_read_input_tokens, + cache_creation_input_tokens: chunk.usage.cache_creation_input_tokens, + no_cache_tokens: chunk.usage.no_cache_tokens, reasoning_tokens: chunk.usage.completion_tokens_details?.reasoning_tokens ?? chunk.usage.output_tokens_details?.reasoning_tokens ?? @@ -609,6 +614,11 @@ export function logUsage( const cacheCreation = getPromptCacheCreationTokens(usage); if (cacheCreation) msg += ` | cache_create=${cacheCreation}`; + // Non-cached (fresh) input tokens — informational only, already included in + // prompt_tokens (Command Code reports inputTokenDetails.noCacheTokens). + const noCache = getNoCacheTokens(usage); + if (noCache) msg += ` | no_cache=${noCache}`; + const reasoning = usage.reasoning_tokens; if (reasoning) msg += ` | reasoning=${reasoning}`; diff --git a/src/lib/usage/tokenAccounting.ts b/src/lib/usage/tokenAccounting.ts index 4131107902..932d71223a 100644 --- a/src/lib/usage/tokenAccounting.ts +++ b/src/lib/usage/tokenAccounting.ts @@ -191,6 +191,23 @@ export function getReasoningTokensOrNull(tokens: unknown): number | null { return null; } +/** + * Return non-cached (fresh) input tokens, or `null` if the provider didn't + * report any. Command Code reports this as `inputTokenDetails.noCacheTokens`. + * Informational only — the value is already included in prompt_tokens, so it + * must never be added to metering totals (see commandCode.ts usageFromCommandCode). + */ +export function getNoCacheTokens(tokens: unknown): number | null { + const tokenRecord = asRecord(tokens); + const promptDetails = getPromptTokenDetails(tokenRecord); + if (hasAnyKey(tokenRecord, ["no_cache_tokens"]) || hasAnyKey(promptDetails, ["noCacheTokens"])) { + return toFiniteNumber( + tokenRecord.no_cache_tokens ?? promptDetails.noCacheTokens ?? tokenRecord.noCacheTokens + ); + } + return null; +} + export function formatUsageLog(tokens: unknown): string { const input = getLoggedInputTokens(tokens); const output = getLoggedOutputTokens(tokens); diff --git a/tests/unit/command-code-executor.test.ts b/tests/unit/command-code-executor.test.ts index 35ad263257..3fd1dacac2 100644 --- a/tests/unit/command-code-executor.test.ts +++ b/tests/unit/command-code-executor.test.ts @@ -268,7 +268,12 @@ test("Command Code data: SSE lines aggregate into non-stream ChatCompletion JSON assert.equal(json.choices[0].message.reasoning_content, "because"); assert.equal(json.choices[0].message.tool_calls[0].function.arguments, JSON.stringify({ id: 7 })); assert.equal(json.choices[0].finish_reason, "length"); - assert.deepEqual(json.usage, { prompt_tokens: 5, completion_tokens: 5, total_tokens: 10 }); + assert.deepEqual(json.usage, { + prompt_tokens: 3, + completion_tokens: 5, + total_tokens: 8, + cache_read_input_tokens: 2, + }); }); test("Command Code executor surfaces upstream and streamed errors", async () => { @@ -385,3 +390,125 @@ test("Command Code non-stream aggregation throws when the final error event lack }); }, /boom/); }); + +test("Command Code usage chunk surfaces cache_read and no_cache for the stream pipeline", async () => { + globalThis.fetch = async () => + commandCodeStream([ + { type: "text-delta", text: "Hi" }, + { + type: "finish", + finishReason: "stop", + totalUsage: { + inputTokens: 10, + inputTokenDetails: { noCacheTokens: 6, cacheReadTokens: 4 }, + outputTokens: 6, + }, + }, + ]); + + const { response } = await getExecutor("command-code").execute({ + model: "gpt-5.4-mini", + stream: true, + credentials: { apiKey: "cc_test_key" }, + body: { messages: [{ role: "user", content: "Hi" }] }, + }); + + const sse = await response.text(); + const chunks = parseSsePayloads(sse); + const usageChunk = chunks.find( + (chunk) => Array.isArray(chunk.choices) && chunk.choices.length === 0 + ); + assert.ok(usageChunk, "expected a usage-only chunk (choices: []) in the stream"); + + // The usage-only chunk feeds stream.ts's extractUsage, which surfaces + // cache_read_input_tokens / no_cache_tokens into the [USAGE] line. + const { extractUsage } = await import("../../open-sse/utils/usageTracking.ts"); + const extracted = extractUsage(usageChunk); + assert.ok(extracted, "extractUsage should recognize the usage-only chunk"); + assert.equal(extracted.prompt_tokens, 10); + assert.equal(extracted.completion_tokens, 6); + assert.equal(extracted.cache_read_input_tokens, 4); + assert.equal(extracted.no_cache_tokens, 6); +}); + +test("Command Code stream emits a usage-only chunk with actual tokens before [DONE]", async () => { + globalThis.fetch = async () => + commandCodeStream([ + { type: "text-delta", text: "Hi" }, + { + type: "finish", + finishReason: "stop", + totalUsage: { + inputTokens: 10, + inputTokenDetails: { cacheReadTokens: 4, cacheCreationTokens: 2 }, + outputTokens: 6, + reasoningTokenDetails: { reasoningTokens: 1 }, + }, + }, + ]); + + const { response } = await getExecutor("command-code").execute({ + model: "gpt-5.4-mini", + stream: true, + credentials: { apiKey: "cc_test_key" }, + body: { messages: [{ role: "user", content: "Hi" }] }, + }); + + const sse = await response.text(); + const chunks = parseSsePayloads(sse); + + // Find the usage-only chunk: choices must be [] and usage must carry the + // actual upstream numbers. prompt_tokens = inputTokens (10) — cacheRead 4 is + // already included in that 10, so it is reported separately, NOT re-added. + const usageChunk = chunks.find( + (chunk) => Array.isArray(chunk.choices) && chunk.choices.length === 0 + ); + assert.ok(usageChunk, "expected a usage-only chunk (choices: []) in the stream"); + assert.deepEqual(usageChunk.usage, { + prompt_tokens: 10, + completion_tokens: 6, + total_tokens: 16, + cache_read_input_tokens: 4, + reasoning_tokens: 1, + }); + // The usage chunk must come before the [DONE] marker. + assert.match(sse, /"usage":/); + const doneIndex = sse.indexOf("data: [DONE]"); + const usageIndex = sse.indexOf(`"choices":[]`); + assert.ok(usageIndex > -1 && usageIndex < doneIndex, "usage chunk must precede [DONE]"); +}); + +test("Command Code non-stream usage keeps inputTokens as prompt_tokens and reports cache separately", async () => { + globalThis.fetch = async () => + commandCodeStream( + [ + { type: "text-delta", text: "ok" }, + { + type: "finish", + finishReason: "stop", + totalUsage: { + inputTokens: 5, + inputTokenDetails: { noCacheTokens: 2, cacheReadTokens: 3 }, + outputTokens: 2, + }, + }, + ], + { sse: true } + ); + + const { response } = await getExecutor("command-code").execute({ + model: "gpt-5.4-mini", + stream: false, + credentials: { apiKey: "cc_test_key" }, + body: { messages: [{ role: "user", content: "Hi" }] }, + }); + + const json = await response.json(); + assert.deepEqual(json.usage, { + prompt_tokens: 5, + completion_tokens: 2, + total_tokens: 7, + cache_read_input_tokens: 3, + no_cache_tokens: 2, + }); +});