From 41d16c9bb4dea65beeaa783c521c6f6f640df8d9 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Fri, 7 Aug 2026 18:08:22 -0300 Subject: [PATCH] fix(api/analytics): stop charging :free models at arbitrary fallback price (#9054) resolveModelPricing() in analytics route fell back to Object.keys(providerPricing)[0] when a model had no pricing entry. For OpenRouter, the defaults layer always contributes an 'auto' record as the first key, so every :free model was charged at that arbitrary rate in the analytics dashboard. Fix: short-circuit :free models to return null before the last-resort fallback, and remove the Object.keys(...)[0] arbitrary-substitution fallback. Closes #9054 Co-authored-by: diegosouzapw --- changelog.d/fixes/9054-fix.plan.md | 1 + src/app/api/usage/analytics/route.ts | 11 +- .../analytics-free-model-cost-9054.test.ts | 210 ++++++++++++++++++ 3 files changed, 217 insertions(+), 5 deletions(-) create mode 100644 changelog.d/fixes/9054-fix.plan.md create mode 100644 tests/unit/analytics-free-model-cost-9054.test.ts diff --git a/changelog.d/fixes/9054-fix.plan.md b/changelog.d/fixes/9054-fix.plan.md new file mode 100644 index 0000000000..2efd3cf8f4 --- /dev/null +++ b/changelog.d/fixes/9054-fix.plan.md @@ -0,0 +1 @@ +- fix(api/analytics): stop charging :free models at arbitrary fallback price (#9054) diff --git a/src/app/api/usage/analytics/route.ts b/src/app/api/usage/analytics/route.ts index 04a05bab30..d90480964e 100644 --- a/src/app/api/usage/analytics/route.ts +++ b/src/app/api/usage/analytics/route.ts @@ -216,7 +216,12 @@ function resolveModelPricing( } } - // Last resort fallback for historical usage (e.g. "gpt-4" missing, matches "gpt-4.1" or first available) + // Short-circuit :free models to $0 (they have no pricing entry → should not fall back to arbitrary rates) + if (!pricing && model.endsWith(":free")) { + return null; + } + + // Last resort fallback for historical usage (e.g. "gpt-4" missing, matches "gpt-4.1") if (!pricing && providerPricing && typeof providerPricing === "object") { for (const [key, val] of Object.entries(providerPricing as Record)) { const lm = model.toLowerCase(); @@ -225,10 +230,6 @@ function resolveModelPricing( break; } } - if (!pricing) { - const keys = Object.keys(providerPricing as Record); - if (keys.length > 0) pricing = (providerPricing as Record)[keys[0]]; - } } return pricing as Record | null; diff --git a/tests/unit/analytics-free-model-cost-9054.test.ts b/tests/unit/analytics-free-model-cost-9054.test.ts new file mode 100644 index 0000000000..5db852929c --- /dev/null +++ b/tests/unit/analytics-free-model-cost-9054.test.ts @@ -0,0 +1,210 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +/** + * Tests the fix for #9054: resolveModelPricing() in route.ts must not fall back + * to Object.keys(providerPricing)[0] for :free models (or any unpriced model). + * + * This test validates the fix logic inline without importing the full analytics + * route (which hangs outside Next.js context due to next/headers imports). + * The actual fix is in src/app/api/usage/analytics/route.ts: + * 1. Short-circuit :free models to return null before the last-resort fallback + * 2. Remove the Object.keys(providerPricing)[0] arbitrary-substitution fallback + */ + +type Pricing = Record | null; + +function findKeyInsensitive(obj: Record | undefined | null, key: string): unknown { + if (!obj || !key) return undefined; + return obj[key.toLowerCase()]; +} + +/** + * Replicates the FIXED resolveModelPricing logic from route.ts. + * The key changes (compared to the buggy version): + * - :free models short-circuit to null before the last-resort fallback + * - No Object.keys(providerPricing)[0] fallback + */ +function resolveModelPricingFixed( + pricingByProvider: Record>>, + providerRaw: string, + model: string +): Pricing { + const pLower = (providerRaw || "").toLowerCase(); + const providerPricing = findKeyInsensitive(pricingByProvider, pLower); + + // Exact match in provider's pricing + if (providerPricing) { + const pricing = findKeyInsensitive(providerPricing as Record, model.toLowerCase()); + if (pricing) return pricing as Record; + } + + // Global fallback: search all providers for exact match + for (const prov of Object.values(pricingByProvider)) { + if (prov && typeof prov === "object") { + const found = findKeyInsensitive(prov as Record, model.toLowerCase()); + if (found) return found as Record; + } + } + + // FIX: :free models have no pricing entry — return null instead of arbitrary fallback + if (model.endsWith(":free")) { + return null; + } + + // Last resort: substring matching (historical usage patterns like "gpt-4" -> "gpt-4.1") + // Note: removed Object.keys(providerPricing)[0] fallback (the root cause of the bug) + if (providerPricing && typeof providerPricing === "object") { + for (const [key, val] of Object.entries(providerPricing as Record)) { + const lm = model.toLowerCase(); + if (key.includes(lm) || lm.includes(key)) { + return val as Record; + } + } + } + + return null; +} + +/** + * Replicates the BUGGY resolveModelPricing logic from route.ts (before fix). + * This is the version that had the Object.keys(providerPricing)[0] fallback. + */ +function resolveModelPricingBuggy( + pricingByProvider: Record>>, + providerRaw: string, + model: string +): Pricing { + const pLower = (providerRaw || "").toLowerCase(); + const providerPricing = findKeyInsensitive(pricingByProvider, pLower); + + // Exact match in provider's pricing + if (providerPricing) { + const pricing = findKeyInsensitive(providerPricing as Record, model.toLowerCase()); + if (pricing) return pricing as Record; + } + + // Global fallback: search all providers for exact match + for (const prov of Object.values(pricingByProvider)) { + if (prov && typeof prov === "object") { + const found = findKeyInsensitive(prov as Record, model.toLowerCase()); + if (found) return found as Record; + } + } + + // Last resort fallback (BUGGY): substring matching + first-key fallback + if (providerPricing && typeof providerPricing === "object") { + for (const [key, val] of Object.entries(providerPricing as Record)) { + const lm = model.toLowerCase(); + if (key.includes(lm) || lm.includes(key)) { + return val as Record; + } + } + // BUG: falls back to the first key of the provider's pricing map + const keys = Object.keys(providerPricing as Record); + if (keys.length > 0) { + return (providerPricing as Record)[keys[0]] as Record; + } + } + + return null; +} + +// Simulates the pricing data structure from getPricing() merge. +// openrouter has the defaults-layer "auto" record + user-paid models. +const OPENROUTER_PRICING_WITH_AUTO = { + openrouter: { + auto: { input: 2.0, output: 8.0, cached: 1.0, reasoning: 12.0, cache_creation: 2.0 }, + "anthropic/claude-3-haiku": { input: 0.25, output: 1.25 }, + "anthropic/claude-3.5-sonnet": { input: 3.0, output: 15.0 }, + "openai/gpt-4o": { input: 2.5, output: 10.0 }, + }, +}; + +test("fixed: :free model returns null pricing (not arbitrary fallback)", () => { + const pricing = resolveModelPricingFixed( + OPENROUTER_PRICING_WITH_AUTO, + "openrouter", + "nvidia/nemotron-3-ultra-550b-a55b:free" + ); + assert.equal(pricing, null, ":free model must get null pricing, not the arbitrary 'auto' rate"); +}); + +test("fixed: known paid model still resolves correctly (non-regression)", () => { + const pricing = resolveModelPricingFixed( + OPENROUTER_PRICING_WITH_AUTO, + "openrouter", + "anthropic/claude-3-haiku" + ); + assert.notEqual(pricing, null, "known paid model should resolve pricing"); + assert.equal(pricing?.input, 0.25); + assert.equal(pricing?.output, 1.25); +}); + +test("fixed: unknown model with no pricing entry returns null (not arbitrary fallback)", () => { + const pricing = resolveModelPricingFixed( + OPENROUTER_PRICING_WITH_AUTO, + "openrouter", + "some-unknown-model-no-pricing" + ); + assert.equal( + pricing, + null, + "unknown model with no pricing entry should get null pricing" + ); +}); + +test("fixed: :free model with no provider pricing returns null", () => { + const pricing = resolveModelPricingFixed( + { openrouter: {} }, + "openrouter", + "some-model:free" + ); + assert.equal(pricing, null, ":free model with empty provider pricing should return null"); +}); + +test("buggy: :free model gets arbitrary first-key pricing (the bug)", () => { + const pricing = resolveModelPricingBuggy( + OPENROUTER_PRICING_WITH_AUTO, + "openrouter", + "nvidia/nemotron-3-ultra-550b-a55b:free" + ); + // The bug: keys[0] is "auto" with {input: 2, output: 8} + assert.notEqual(pricing, null, "buggy version resolves pricing for :free model"); + assert.equal( + pricing?.input, + 2.0, + "buggy version charges :free model at the arbitrary 'auto' rate (first key)" + ); +}); + +test("buggy: unknown model gets arbitrary first-key pricing (the bug)", () => { + const pricing = resolveModelPricingBuggy( + OPENROUTER_PRICING_WITH_AUTO, + "openrouter", + "some-unknown-model" + ); + assert.notEqual(pricing, null, "buggy version resolves pricing for unknown model"); + assert.equal( + pricing?.input, + 2.0, + "buggy version charges unknown model at the arbitrary 'auto' rate (first key)" + ); +}); + +test("fixed: other providers without 'auto' default also work correctly", () => { + const pricingByProvider = { + someprovider: { + "gpt-4o": { input: 2.5, output: 10.0 }, + "claude-3.5-sonnet": { input: 3.0, output: 15.0 }, + }, + }; + + // :free model should return null even for providers without a default 'auto' entry + const freePricing = resolveModelPricingFixed( + pricingByProvider as Record>>, + "someprovider", + "test-model:free" + ); + assert.equal(freePricing, null, ":free model should return null for any provider"); +}); \ No newline at end of file