From eb704d64208f4604bd890058660eda9a6c9af719 Mon Sep 17 00:00:00 2001 From: Randi <55005611+rdself@users.noreply.github.com> Date: Tue, 7 Apr 2026 16:26:46 -0400 Subject: [PATCH] feat: detailed token tracking in call logs + fix Anthropic input undercount (#1017) Fixes critical Anthropic streaming input token undercount + adds detailed token tracking with DB migration 018. Includes 18 new unit tests. Integrated into release/v3.5.4. --- open-sse/utils/usageTracking.ts | 17 +- .../018_call_logs_detailed_tokens.sql | 6 + src/lib/usage/callLogs.ts | 59 ++++- src/lib/usage/tokenAccounting.ts | 96 ++++++- src/lib/usage/usageHistory.ts | 3 +- src/shared/components/RequestLoggerDetail.tsx | 28 +- src/shared/components/RequestLoggerV2.tsx | 8 +- src/shared/constants/colors.ts | 3 +- tests/unit/call-log-detailed-tokens.test.mjs | 247 ++++++++++++++++++ .../unit/token-accounting-input-fix.test.mjs | 130 +++++++++ 10 files changed, 570 insertions(+), 27 deletions(-) create mode 100644 src/lib/db/migrations/018_call_logs_detailed_tokens.sql create mode 100644 tests/unit/call-log-detailed-tokens.test.mjs create mode 100644 tests/unit/token-accounting-input-fix.test.mjs diff --git a/open-sse/utils/usageTracking.ts b/open-sse/utils/usageTracking.ts index 3eb5b8cc76..2eca04bb27 100644 --- a/open-sse/utils/usageTracking.ts +++ b/open-sse/utils/usageTracking.ts @@ -299,12 +299,18 @@ export function extractUsage(chunk) { // Claude/Antigravity streaming: message_start event carries INPUT tokens // FIX #74: This event was not handled — input_tokens were being dropped // Structure: { type: "message_start", message: { usage: { input_tokens: N, output_tokens: 0 } } } + // + // Note: Claude's input_tokens is only the non-cached portion. + // Sum cache tokens into prompt_tokens for a correct total (consistent with + // extractUsageFromResponse in usageExtractor.ts for non-streaming). if (chunk.type === "message_start" && chunk.message?.usage) { const u = chunk.message.usage; const inputTokens = u.input_tokens || u.prompt_tokens || 0; - if (inputTokens > 0) { + const cacheRead = u.cache_read_input_tokens || 0; + const cacheCreation = u.cache_creation_input_tokens || 0; + if (inputTokens > 0 || cacheRead > 0 || cacheCreation > 0) { return normalizeUsage({ - prompt_tokens: inputTokens, + prompt_tokens: inputTokens + cacheRead + cacheCreation, completion_tokens: u.output_tokens || u.completion_tokens || 0, cache_read_input_tokens: u.cache_read_input_tokens, cache_creation_input_tokens: u.cache_creation_input_tokens, @@ -312,10 +318,13 @@ export function extractUsage(chunk) { } } - // Claude format (message_delta event) — carries OUTPUT tokens + // Claude format (message_delta event) — typically carries OUTPUT tokens if (chunk.type === "message_delta" && chunk.usage && typeof chunk.usage === "object") { + const deltaInput = chunk.usage.input_tokens || 0; + const deltaCacheRead = chunk.usage.cache_read_input_tokens || 0; + const deltaCacheCreation = chunk.usage.cache_creation_input_tokens || 0; return normalizeUsage({ - prompt_tokens: chunk.usage.input_tokens || 0, + prompt_tokens: deltaInput + deltaCacheRead + deltaCacheCreation, completion_tokens: chunk.usage.output_tokens || 0, cache_read_input_tokens: chunk.usage.cache_read_input_tokens, cache_creation_input_tokens: chunk.usage.cache_creation_input_tokens, diff --git a/src/lib/db/migrations/018_call_logs_detailed_tokens.sql b/src/lib/db/migrations/018_call_logs_detailed_tokens.sql new file mode 100644 index 0000000000..46cd0d2dce --- /dev/null +++ b/src/lib/db/migrations/018_call_logs_detailed_tokens.sql @@ -0,0 +1,6 @@ +-- Add detailed token breakdown columns to call_logs. +-- These are NULL when the provider does not report the field, +-- or an integer (including 0) when the provider explicitly returned a value. +ALTER TABLE call_logs ADD COLUMN tokens_cache_read INTEGER DEFAULT NULL; +ALTER TABLE call_logs ADD COLUMN tokens_cache_creation INTEGER DEFAULT NULL; +ALTER TABLE call_logs ADD COLUMN tokens_reasoning INTEGER DEFAULT NULL; diff --git a/src/lib/usage/callLogs.ts b/src/lib/usage/callLogs.ts index abb7344d57..334e2ea779 100644 --- a/src/lib/usage/callLogs.ts +++ b/src/lib/usage/callLogs.ts @@ -13,7 +13,13 @@ import type { RequestPipelinePayloads } from "@omniroute/open-sse/utils/requestL import { getDbInstance } from "../db/core"; import { getRequestDetailLogByCallLogId } from "../db/detailedLogs"; import { shouldPersistToDisk, CALL_LOGS_DIR } from "./migrations"; -import { getLoggedInputTokens, getLoggedOutputTokens } from "./tokenAccounting"; +import { + getLoggedInputTokens, + getLoggedOutputTokens, + getPromptCacheReadTokensOrNull, + getPromptCacheCreationTokensOrNull, + getReasoningTokensOrNull, +} from "./tokenAccounting"; import { isNoLog } from "../compliance"; import { sanitizePII } from "../piiSanitizer"; import { @@ -26,7 +32,7 @@ import { getCallLogMaxEntries, getCallLogRetentionDays } from "../logEnv"; type JsonRecord = Record; type CallLogArtifact = { - schemaVersion: 2; + schemaVersion: 3; summary: { id: string; timestamp: string; @@ -39,7 +45,13 @@ type CallLogArtifact = { account: string; connectionId: string | null; duration: number; - tokens: { in: number; out: number }; + tokens: { + in: number; + out: number; + cacheRead: number | null; + cacheWrite: number | null; + reasoning: number | null; + }; requestType: string | null; sourceFormat: string | null; targetFormat: string | null; @@ -180,6 +192,9 @@ function buildArtifact( duration: number; tokensIn: number; tokensOut: number; + tokensCacheRead: number | null; + tokensCacheCreation: number | null; + tokensReasoning: number | null; requestType: string | null; sourceFormat: string | null; targetFormat: string | null; @@ -193,7 +208,7 @@ function buildArtifact( pipelinePayloads: RequestPipelinePayloads | null ): CallLogArtifact { return { - schemaVersion: 2, + schemaVersion: 3, summary: { id: logEntry.id, timestamp: logEntry.timestamp, @@ -206,7 +221,13 @@ function buildArtifact( account: logEntry.account, connectionId: logEntry.connectionId, duration: logEntry.duration, - tokens: { in: logEntry.tokensIn, out: logEntry.tokensOut }, + tokens: { + in: logEntry.tokensIn, + out: logEntry.tokensOut, + cacheRead: logEntry.tokensCacheRead, + cacheWrite: logEntry.tokensCacheCreation, + reasoning: logEntry.tokensReasoning, + }, requestType: logEntry.requestType, sourceFormat: logEntry.sourceFormat, targetFormat: logEntry.targetFormat, @@ -387,6 +408,9 @@ export async function saveCallLog(entry: any) { duration: entry.duration || 0, tokensIn: toNumber(getLoggedInputTokens(entry.tokens)), tokensOut: toNumber(getLoggedOutputTokens(entry.tokens)), + tokensCacheRead: getPromptCacheReadTokensOrNull(entry.tokens), + tokensCacheCreation: getPromptCacheCreationTokensOrNull(entry.tokens), + tokensReasoning: getReasoningTokensOrNull(entry.tokens), requestType: entry.requestType || null, sourceFormat: entry.sourceFormat || null, targetFormat: entry.targetFormat || null, @@ -403,13 +427,17 @@ export async function saveCallLog(entry: any) { ` INSERT INTO call_logs ( id, timestamp, method, path, status, model, requested_model, provider, - account, connection_id, duration, tokens_in, tokens_out, request_type, source_format, + account, connection_id, duration, tokens_in, tokens_out, + tokens_cache_read, tokens_cache_creation, tokens_reasoning, + request_type, source_format, target_format, api_key_id, api_key_name, combo_name, request_body, response_body, error, artifact_relpath, has_pipeline_details ) VALUES ( @id, @timestamp, @method, @path, @status, @model, @requestedModel, @provider, - @account, @connectionId, @duration, @tokensIn, @tokensOut, @requestType, @sourceFormat, + @account, @connectionId, @duration, @tokensIn, @tokensOut, + @tokensCacheRead, @tokensCacheCreation, @tokensReasoning, + @requestType, @sourceFormat, @targetFormat, @apiKeyId, @apiKeyName, @comboName, @requestBody, @responseBody, @error, NULL, 0 ) @@ -541,7 +569,13 @@ export async function getCallLogs(filter: any = {}) { provider: toStringOrNull(l.provider), account: toStringOrNull(l.account), duration: toNumber(l.duration), - tokens: { in: toNumber(l.tokens_in), out: toNumber(l.tokens_out) }, + tokens: { + in: toNumber(l.tokens_in), + out: toNumber(l.tokens_out), + cacheRead: l.tokens_cache_read != null ? toNumber(l.tokens_cache_read) : null, + cacheWrite: l.tokens_cache_creation != null ? toNumber(l.tokens_cache_creation) : null, + reasoning: l.tokens_reasoning != null ? toNumber(l.tokens_reasoning) : null, + }, sourceFormat: toStringOrNull(l.source_format), targetFormat: toStringOrNull(l.target_format), error: toStringOrNull(l.error), @@ -586,7 +620,14 @@ export async function getCallLogById(id: string) { account: toStringOrNull(entryRow.account), connectionId: toStringOrNull(entryRow.connection_id), duration: toNumber(entryRow.duration), - tokens: { in: toNumber(entryRow.tokens_in), out: toNumber(entryRow.tokens_out) }, + tokens: { + in: toNumber(entryRow.tokens_in), + out: toNumber(entryRow.tokens_out), + cacheRead: entryRow.tokens_cache_read != null ? toNumber(entryRow.tokens_cache_read) : null, + cacheWrite: + entryRow.tokens_cache_creation != null ? toNumber(entryRow.tokens_cache_creation) : null, + reasoning: entryRow.tokens_reasoning != null ? toNumber(entryRow.tokens_reasoning) : null, + }, sourceFormat: toStringOrNull(entryRow.source_format), targetFormat: toStringOrNull(entryRow.target_format), apiKeyId: toStringOrNull(entryRow.api_key_id), diff --git a/src/lib/usage/tokenAccounting.ts b/src/lib/usage/tokenAccounting.ts index bb4da25f3c..179795a432 100644 --- a/src/lib/usage/tokenAccounting.ts +++ b/src/lib/usage/tokenAccounting.ts @@ -49,11 +49,22 @@ export function getLoggedInputTokens(tokens: unknown): number { } if (tokenRecord.input_tokens !== undefined && tokenRecord.input_tokens !== null) { - return toFiniteNumber(tokenRecord.input_tokens); + // Anthropic / anthropic-compatible-cc streaming: input_tokens is only the + // non-cached portion. The cache counters sit as separate top-level fields + // (cache_read_input_tokens, cache_creation_input_tokens). We need to add + // them to get the true total input. + return ( + toFiniteNumber(tokenRecord.input_tokens) + + toFiniteNumber(tokenRecord.cache_read_input_tokens) + + toFiniteNumber(tokenRecord.cache_creation_input_tokens) + ); } - // prompt_tokens from translator already includes input + cache_read + cache_creation - // Do NOT subtract cached tokens - we want the total billable prompt tokens + // prompt_tokens from translator/extractor already includes cache tokens: + // - OpenAI format: prompt_tokens inherently includes cached + // - Claude non-streaming: extractUsageFromResponse sums input + cache_read + cache_creation + // - Claude streaming: extractUsage (after fix) sums input + cache_read + cache_creation + // Do NOT add cache fields here — would double-count. const promptTokens = toFiniteNumber(tokenRecord.prompt_tokens); return promptTokens; } @@ -66,14 +77,93 @@ export function getLoggedOutputTokens(tokens: unknown): number { return toFiniteNumber(tokenRecord.completion_tokens ?? tokenRecord.output_tokens); } +/** + * Return the reasoning/thinking output token count. + * Checks multiple field locations used by different providers: + * - completion_tokens_details.reasoning_tokens (OpenAI, OpenRouter) + * - reasoning_tokens (GitHub — top-level) + * - reasoning (usage_history DB format) + */ +export function getReasoningTokens(tokens: unknown): number { + const tokenRecord = asRecord(tokens); + const completionDetails = asRecord(tokenRecord.completion_tokens_details); + return toFiniteNumber( + tokenRecord.reasoning ?? tokenRecord.reasoning_tokens ?? completionDetails.reasoning_tokens + ); +} + +// ─── Nullable variants ────────────────────────────────────────────────── +// Return `null` when the provider simply doesn't report the field, +// vs `0` when the provider explicitly reported zero. + +function hasAnyKey(record: JsonRecord, keys: string[]): boolean { + return keys.some((k) => record[k] !== undefined && record[k] !== null); +} + +/** + * Return prompt cache-read tokens, or `null` if the provider didn't + * report any cache-read field at all. + */ +export function getPromptCacheReadTokensOrNull(tokens: unknown): number | null { + const tokenRecord = asRecord(tokens); + const promptDetails = getPromptTokenDetails(tokenRecord); + if ( + hasAnyKey(tokenRecord, ["cacheRead", "cache_read_input_tokens", "cached_tokens"]) || + hasAnyKey(promptDetails, ["cached_tokens"]) + ) { + return getPromptCacheReadTokens(tokens); + } + return null; +} + +/** + * Return prompt cache-creation (write) tokens, or `null` if the + * provider didn't report any cache-creation field at all. + */ +export function getPromptCacheCreationTokensOrNull(tokens: unknown): number | null { + const tokenRecord = asRecord(tokens); + const promptDetails = getPromptTokenDetails(tokenRecord); + if ( + hasAnyKey(tokenRecord, ["cacheCreation", "cache_creation_input_tokens"]) || + hasAnyKey(promptDetails, ["cache_creation_tokens"]) + ) { + return getPromptCacheCreationTokens(tokens); + } + return null; +} + +/** + * Return reasoning tokens, or `null` if the provider didn't report + * any reasoning field at all. + */ +export function getReasoningTokensOrNull(tokens: unknown): number | null { + const tokenRecord = asRecord(tokens); + const completionDetails = asRecord(tokenRecord.completion_tokens_details); + if ( + hasAnyKey(tokenRecord, ["reasoning", "reasoning_tokens"]) || + hasAnyKey(completionDetails, ["reasoning_tokens"]) + ) { + return getReasoningTokens(tokens); + } + return null; +} + export function formatUsageLog(tokens: unknown): string { const input = getLoggedInputTokens(tokens); const output = getLoggedOutputTokens(tokens); const cacheRead = getPromptCacheReadTokens(tokens); + const cacheWrite = getPromptCacheCreationTokens(tokens); + const reasoning = getReasoningTokens(tokens); let msg = `in=${input} | out=${output}`; if (cacheRead > 0) { msg += ` | CR=${cacheRead}`; } + if (cacheWrite > 0) { + msg += ` | CW=${cacheWrite}`; + } + if (reasoning > 0) { + msg += ` | R=${reasoning}`; + } return msg; } diff --git a/src/lib/usage/usageHistory.ts b/src/lib/usage/usageHistory.ts index 8a8ee52f3f..56bc075f98 100644 --- a/src/lib/usage/usageHistory.ts +++ b/src/lib/usage/usageHistory.ts @@ -14,6 +14,7 @@ import { getLoggedOutputTokens, getPromptCacheCreationTokens, getPromptCacheReadTokens, + getReasoningTokens, } from "./tokenAccounting"; type JsonRecord = Record; @@ -167,7 +168,7 @@ export async function saveRequestUsage(entry: any) { getLoggedOutputTokens(entry.tokens), getPromptCacheReadTokens(entry.tokens), getPromptCacheCreationTokens(entry.tokens), - entry.tokens?.reasoning ?? entry.tokens?.reasoning_tokens ?? 0, + getReasoningTokens(entry.tokens), entry.status || null, entry.success === false ? 0 : 1, Number.isFinite(Number(entry.latencyMs)) ? Number(entry.latencyMs) : 0, diff --git a/src/shared/components/RequestLoggerDetail.tsx b/src/shared/components/RequestLoggerDetail.tsx index 8b231465a8..c1b2b0aa84 100644 --- a/src/shared/components/RequestLoggerDetail.tsx +++ b/src/shared/components/RequestLoggerDetail.tsx @@ -159,14 +159,32 @@ export default function RequestLoggerDetail({ log, detail, loading, onClose, onC
- Tokens (I/O) + Tokens
-
+
- In: {(detail?.tokens?.in || log.tokens?.in || 0).toLocaleString()} + Total In: {(detail?.tokens?.in ?? log.tokens?.in ?? 0).toLocaleString()} - Out: {(detail?.tokens?.out || log.tokens?.out || 0).toLocaleString()} + Total Out: {(detail?.tokens?.out ?? log.tokens?.out ?? 0).toLocaleString()} + + + Cache Read:{" "} + {(detail?.tokens?.cacheRead ?? log.tokens?.cacheRead) != null + ? (detail?.tokens?.cacheRead ?? log.tokens?.cacheRead).toLocaleString() + : "N/A"} + + + Cache Write:{" "} + {(detail?.tokens?.cacheWrite ?? log.tokens?.cacheWrite) != null + ? (detail?.tokens?.cacheWrite ?? log.tokens?.cacheWrite).toLocaleString() + : "N/A"} + + + Reasoning:{" "} + {(detail?.tokens?.reasoning ?? log.tokens?.reasoning) != null + ? (detail?.tokens?.reasoning ?? log.tokens?.reasoning).toLocaleString() + : "N/A"}
@@ -202,7 +220,7 @@ export default function RequestLoggerDetail({ log, detail, loading, onClose, onC
- Protocol + Req Protocol
- Protocol + Req Protocol )} {visibleColumns.account && ( @@ -728,12 +728,12 @@ export default function RequestLoggerV2() { )} {visibleColumns.tokens && ( - I:{" "} + TI:{" "} {log.tokens?.in?.toLocaleString() || 0} | - O:{" "} + TO:{" "} {log.tokens?.out?.toLocaleString() || 0} diff --git a/src/shared/constants/colors.ts b/src/shared/constants/colors.ts index 93b690ab2d..356508b25b 100644 --- a/src/shared/constants/colors.ts +++ b/src/shared/constants/colors.ts @@ -30,7 +30,8 @@ export const PROVIDER_COLORS = { // ═══════════════════════════════════════════ export const PROTOCOL_COLORS = { - openai: { bg: "#10A37F", text: "#fff", label: "OpenAI" }, + openai: { bg: "#1A1A2E", text: "#fff", label: "OpenAI-Chat" }, + "openai-responses": { bg: "#1A1A2E", text: "#fff", label: "OpenAI-Responses" }, claude: { bg: "#D97757", text: "#fff", label: "Claude" }, gemini: { bg: "#4285F4", text: "#fff", label: "Gemini" }, warmup: { bg: "#F59E0B", text: "#000", label: "Warmup" }, diff --git a/tests/unit/call-log-detailed-tokens.test.mjs b/tests/unit/call-log-detailed-tokens.test.mjs new file mode 100644 index 0000000000..160f213edd --- /dev/null +++ b/tests/unit/call-log-detailed-tokens.test.mjs @@ -0,0 +1,247 @@ +/** + * Unit tests for detailed token tracking in call logs. + * + * Verifies that getPromptCacheReadTokensOrNull, getPromptCacheCreationTokensOrNull, + * and getReasoningTokensOrNull correctly distinguish between: + * - Provider didn't report the field → null + * - Provider reported zero → 0 + * + * Also tests getLoggedInputTokens for each provider format. + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; + +// ── Inline the logic from tokenAccounting.ts ──────────────────────────── + +function asRecord(value) { + return value && typeof value === "object" && !Array.isArray(value) ? value : {}; +} + +function toFiniteNumber(value) { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string" && value.trim().length > 0) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : 0; + } + return 0; +} + +function getPromptTokenDetails(tokens) { + const tokenRecord = asRecord(tokens); + const promptDetails = asRecord(tokenRecord.prompt_tokens_details); + if (Object.keys(promptDetails).length > 0) return promptDetails; + return asRecord(tokenRecord.input_tokens_details); +} + +function getPromptCacheReadTokens(tokens) { + const tokenRecord = asRecord(tokens); + const promptDetails = getPromptTokenDetails(tokenRecord); + return toFiniteNumber( + tokenRecord.cacheRead ?? + tokenRecord.cache_read_input_tokens ?? + tokenRecord.cached_tokens ?? + promptDetails.cached_tokens + ); +} + +function getPromptCacheCreationTokens(tokens) { + const tokenRecord = asRecord(tokens); + const promptDetails = getPromptTokenDetails(tokenRecord); + return toFiniteNumber( + tokenRecord.cacheCreation ?? + tokenRecord.cache_creation_input_tokens ?? + promptDetails.cache_creation_tokens + ); +} + +function getReasoningTokens(tokens) { + const tokenRecord = asRecord(tokens); + const completionDetails = asRecord(tokenRecord.completion_tokens_details); + return toFiniteNumber( + tokenRecord.reasoning ?? tokenRecord.reasoning_tokens ?? completionDetails.reasoning_tokens + ); +} + +function hasAnyKey(record, keys) { + return keys.some((k) => record[k] !== undefined && record[k] !== null); +} + +function getPromptCacheReadTokensOrNull(tokens) { + const tokenRecord = asRecord(tokens); + const promptDetails = getPromptTokenDetails(tokenRecord); + if ( + hasAnyKey(tokenRecord, ["cacheRead", "cache_read_input_tokens", "cached_tokens"]) || + hasAnyKey(promptDetails, ["cached_tokens"]) + ) { + return getPromptCacheReadTokens(tokens); + } + return null; +} + +function getPromptCacheCreationTokensOrNull(tokens) { + const tokenRecord = asRecord(tokens); + const promptDetails = getPromptTokenDetails(tokenRecord); + if ( + hasAnyKey(tokenRecord, ["cacheCreation", "cache_creation_input_tokens"]) || + hasAnyKey(promptDetails, ["cache_creation_tokens"]) + ) { + return getPromptCacheCreationTokens(tokens); + } + return null; +} + +function getReasoningTokensOrNull(tokens) { + const tokenRecord = asRecord(tokens); + const completionDetails = asRecord(tokenRecord.completion_tokens_details); + if ( + hasAnyKey(tokenRecord, ["reasoning", "reasoning_tokens"]) || + hasAnyKey(completionDetails, ["reasoning_tokens"]) + ) { + return getReasoningTokens(tokens); + } + return null; +} + +function getLoggedInputTokens(tokens) { + const tokenRecord = asRecord(tokens); + if (tokenRecord.input !== undefined && tokenRecord.input !== null) { + return toFiniteNumber(tokenRecord.input); + } + if (tokenRecord.input_tokens !== undefined && tokenRecord.input_tokens !== null) { + return ( + toFiniteNumber(tokenRecord.input_tokens) + + toFiniteNumber(tokenRecord.cache_read_input_tokens) + + toFiniteNumber(tokenRecord.cache_creation_input_tokens) + ); + } + const promptTokens = toFiniteNumber(tokenRecord.prompt_tokens); + return promptTokens; +} + +// ── Provider format tests ─────────────────────────────────────────────── + +describe("detailed token extraction — per provider format", () => { + it("Anthropic (streaming extracted): input_tokens=3, cache_creation=113613, cache_read=0", () => { + // Raw Anthropic streaming usage (from message_start event) + const tokens = { + input_tokens: 3, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 113613, + output_tokens: 6921, + }; + assert.equal(getLoggedInputTokens(tokens), 113616, "Total input = 3 + 0 + 113613"); + assert.equal(getPromptCacheReadTokensOrNull(tokens), 0, "Cache read reported as 0"); + assert.equal(getPromptCacheCreationTokensOrNull(tokens), 113613, "Cache write = 113613"); + assert.equal(getReasoningTokensOrNull(tokens), null, "No reasoning field"); + }); + + it("anthropic-compatible-cc: same format as Anthropic", () => { + const tokens = { + input_tokens: 3, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 113613, + output_tokens: 6921, + }; + assert.equal(getLoggedInputTokens(tokens), 113616); + assert.equal(getPromptCacheReadTokensOrNull(tokens), 0); + assert.equal(getPromptCacheCreationTokensOrNull(tokens), 113613); + assert.equal(getReasoningTokensOrNull(tokens), null); + }); + + it("openai-compatible-aio: prompt_tokens=54042, cached=53221, reasoning=6433", () => { + const tokens = { + prompt_tokens: 54042, + completion_tokens: 8000, + prompt_tokens_details: { cached_tokens: 53221 }, + completion_tokens_details: { reasoning_tokens: 6433 }, + }; + assert.equal(getLoggedInputTokens(tokens), 54042, "prompt_tokens already includes cached"); + assert.equal(getPromptCacheReadTokensOrNull(tokens), 53221, "Cache read from details"); + assert.equal(getPromptCacheCreationTokensOrNull(tokens), null, "No cache creation field"); + assert.equal(getReasoningTokensOrNull(tokens), 6433, "Reasoning from completion details"); + }); + + it("OpenRouter: prompt_tokens=5, cached=0, cache_write=0, reasoning=60", () => { + const tokens = { + prompt_tokens: 5, + completion_tokens: 100, + prompt_tokens_details: { cached_tokens: 0, cache_write_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 60 }, + }; + assert.equal(getLoggedInputTokens(tokens), 5); + assert.equal(getPromptCacheReadTokensOrNull(tokens), 0, "Cache read = 0 (reported)"); + // cache_write_tokens is in prompt_tokens_details but our function checks + // cache_creation_input_tokens / cache_creation_tokens + // OpenRouter uses cache_write_tokens which is NOT recognized → null + assert.equal( + getPromptCacheCreationTokensOrNull(tokens), + null, + "OpenRouter cache_write_tokens not mapped to creation" + ); + assert.equal(getReasoningTokensOrNull(tokens), 60, "Reasoning = 60"); + }); + + it("GitHub: prompt_tokens=5, cached=0, reasoning_tokens=57", () => { + const tokens = { + prompt_tokens: 5, + completion_tokens: 100, + prompt_tokens_details: { cached_tokens: 0 }, + reasoning_tokens: 57, + }; + assert.equal(getLoggedInputTokens(tokens), 5); + assert.equal(getPromptCacheReadTokensOrNull(tokens), 0, "Cache read = 0 (reported)"); + assert.equal(getPromptCacheCreationTokensOrNull(tokens), null, "No cache creation"); + assert.equal(getReasoningTokensOrNull(tokens), 57, "Reasoning from top-level"); + }); + + it("Codex: only prompt_tokens/completion_tokens, no breakdowns", () => { + const tokens = { + prompt_tokens: 500, + completion_tokens: 200, + total_tokens: 700, + }; + assert.equal(getLoggedInputTokens(tokens), 500); + assert.equal(getPromptCacheReadTokensOrNull(tokens), null, "No cache read field"); + assert.equal(getPromptCacheCreationTokensOrNull(tokens), null, "No cache creation field"); + assert.equal(getReasoningTokensOrNull(tokens), null, "No reasoning field"); + }); + + it("Antigravity / openai-compatible-sp: same as Codex (no breakdowns)", () => { + const tokens = { + prompt_tokens: 300, + completion_tokens: 150, + }; + assert.equal(getPromptCacheReadTokensOrNull(tokens), null); + assert.equal(getPromptCacheCreationTokensOrNull(tokens), null); + assert.equal(getReasoningTokensOrNull(tokens), null); + }); +}); + +describe("null vs 0 distinction", () => { + it("explicit 0 is preserved (not collapsed to null)", () => { + const tokens = { + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + reasoning_tokens: 0, + }; + assert.equal(getPromptCacheReadTokensOrNull(tokens), 0); + assert.equal(getPromptCacheCreationTokensOrNull(tokens), 0); + assert.equal(getReasoningTokensOrNull(tokens), 0); + }); + + it("missing fields return null", () => { + const tokens = { prompt_tokens: 100, completion_tokens: 50 }; + assert.equal(getPromptCacheReadTokensOrNull(tokens), null); + assert.equal(getPromptCacheCreationTokensOrNull(tokens), null); + assert.equal(getReasoningTokensOrNull(tokens), null); + }); + + it("undefined fields return null (not 0)", () => { + const tokens = { + prompt_tokens: 100, + cache_read_input_tokens: undefined, + }; + assert.equal(getPromptCacheReadTokensOrNull(tokens), null); + }); +}); diff --git a/tests/unit/token-accounting-input-fix.test.mjs b/tests/unit/token-accounting-input-fix.test.mjs new file mode 100644 index 0000000000..fb154286a2 --- /dev/null +++ b/tests/unit/token-accounting-input-fix.test.mjs @@ -0,0 +1,130 @@ +/** + * Unit tests for getLoggedInputTokens fix — Anthropic / anthropic-compatible-cc + * + * The bug: Claude streaming sets prompt_tokens = input_tokens (non-cached only). + * Fix: extractUsage in usageTracking.ts now sums input + cache_read + cache_creation + * into prompt_tokens, consistent with the non-streaming extractor. + * + * getLoggedInputTokens itself also has a safety-net: when raw `input_tokens` + * is present (e.g. from a raw API response), it adds cache tokens too. + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; + +// ── Inline the fixed logic from tokenAccounting.ts ────────────────────── + +function asRecord(value) { + return value && typeof value === "object" && !Array.isArray(value) ? value : {}; +} + +function toFiniteNumber(value) { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string" && value.trim().length > 0) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : 0; + } + return 0; +} + +function getLoggedInputTokens(tokens) { + const tokenRecord = asRecord(tokens); + + if (tokenRecord.input !== undefined && tokenRecord.input !== null) { + return toFiniteNumber(tokenRecord.input); + } + + if (tokenRecord.input_tokens !== undefined && tokenRecord.input_tokens !== null) { + return ( + toFiniteNumber(tokenRecord.input_tokens) + + toFiniteNumber(tokenRecord.cache_read_input_tokens) + + toFiniteNumber(tokenRecord.cache_creation_input_tokens) + ); + } + + // prompt_tokens from translator/extractor already includes cache tokens + const promptTokens = toFiniteNumber(tokenRecord.prompt_tokens); + return promptTokens; +} + +// ── Tests ──────────────────────────────────────────────────────────────── + +describe("getLoggedInputTokens — input fix for Anthropic streaming", () => { + it("raw Anthropic usage with input_tokens: adds cache for correct total", () => { + // Raw API response shape (before extractUsage processes it) + const tokens = { + input_tokens: 3, + cache_read_input_tokens: 500, + cache_creation_input_tokens: 100, + output_tokens: 200, + }; + assert.equal(getLoggedInputTokens(tokens), 603); + }); + + it("raw Anthropic usage: input_tokens=3, cache_creation=113613 → 113616", () => { + const tokens = { + input_tokens: 3, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 113613, + output_tokens: 6921, + }; + assert.equal(getLoggedInputTokens(tokens), 113616); + }); + + it("extracted streaming usage (after fix): prompt_tokens is total, no double-count", () => { + // After the streaming extractor fix, message_start produces: + // prompt_tokens = input_tokens + cache_read + cache_creation + const tokens = { + prompt_tokens: 113616, // already total (3 + 0 + 113613) + completion_tokens: 6921, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 113613, + }; + // No input_tokens field → falls to prompt_tokens → returns 113616 (no double-count) + assert.equal(getLoggedInputTokens(tokens), 113616); + }); + + it("extracted non-streaming usage: prompt_tokens is total, no double-count", () => { + // extractUsageFromResponse sets prompt_tokens = input + cacheRead + cacheCreation + const tokens = { + prompt_tokens: 113616, + completion_tokens: 6921, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 113613, + }; + assert.equal(getLoggedInputTokens(tokens), 113616); + }); + + it("OpenAI format: prompt_tokens=1000, no cache top-level fields → 1000", () => { + const tokens = { + prompt_tokens: 1000, + completion_tokens: 500, + total_tokens: 1500, + }; + assert.equal(getLoggedInputTokens(tokens), 1000); + }); + + it("OpenAI format with cached_tokens in details (no top-level cache fields) → prompt_tokens", () => { + const tokens = { + prompt_tokens: 54042, + completion_tokens: 8000, + prompt_tokens_details: { cached_tokens: 53221 }, + }; + assert.equal(getLoggedInputTokens(tokens), 54042); + }); + + it("pre-computed 'input' field takes precedence over everything", () => { + const tokens = { + input: 999, + prompt_tokens: 100, + input_tokens: 50, + }; + assert.equal(getLoggedInputTokens(tokens), 999); + }); + + it("handles null/undefined gracefully", () => { + assert.equal(getLoggedInputTokens(null), 0); + assert.equal(getLoggedInputTokens(undefined), 0); + assert.equal(getLoggedInputTokens({}), 0); + }); +});