diff --git a/changelog.d/fixes/13446-arena-elo-sync-fetch-first.md b/changelog.d/fixes/13446-arena-elo-sync-fetch-first.md new file mode 100644 index 0000000000..cbdad0a3fc --- /dev/null +++ b/changelog.d/fixes/13446-arena-elo-sync-fetch-first.md @@ -0,0 +1 @@ +- **fix(db):** Arena ELO sync now fetches and validates the leaderboards before touching `model_intelligence`, and applies the upsert + prune of expired rows inside a single atomic transaction. Previously, an unavailable/rate-limited Arena API left the table pruned with nothing written back, and since the sync runs on every boot, repeated restarts against a rate-limited upstream permanently drained the table to zero ([#13446](https://github.com/diegosouzapw/OmniRoute/pull/13446)) — thanks @CrashCartCapital diff --git a/changelog.d/fixes/13446-compression-writer-flat-rate-as-zero.md b/changelog.d/fixes/13446-compression-writer-flat-rate-as-zero.md new file mode 100644 index 0000000000..919360586d --- /dev/null +++ b/changelog.d/fixes/13446-compression-writer-flat-rate-as-zero.md @@ -0,0 +1 @@ +- **fix(analytics):** the compression analytics writer now passes `flatRateAsZero: true` to `calculateCost`, matching `/api/usage/analytics`. Flat-rate subscription lanes (minimax, glm, kimi, bailian, xiaomi, web-cookie) no longer report a dollar "savings" figure that was never actually payable ([#13446](https://github.com/diegosouzapw/OmniRoute/pull/13446)) — thanks @CrashCartCapital diff --git a/open-sse/handlers/chatCore/compressionAnalyticsWrite.ts b/open-sse/handlers/chatCore/compressionAnalyticsWrite.ts index ecb9f11f6a..c3b0882e03 100644 --- a/open-sse/handlers/chatCore/compressionAnalyticsWrite.ts +++ b/open-sse/handlers/chatCore/compressionAnalyticsWrite.ts @@ -149,7 +149,10 @@ export function writeCompressionAnalytics( opts.provider ?? "", opts.effectiveModel ?? "", { input: tokensSaved }, - { serviceTier: opts.effectiveServiceTier } + // Flat-rate (subscription / cookie-web) lanes don't bill per token, so + // their per-token pricing rows must not book dollar "savings" here + // (same opt-in as /api/usage/analytics, #5552). + { serviceTier: opts.effectiveServiceTier, flatRateAsZero: true } ); } catch (err) { opts.log?.debug?.( diff --git a/src/lib/arenaEloSync.ts b/src/lib/arenaEloSync.ts index 15ed583df6..032de78b4c 100644 --- a/src/lib/arenaEloSync.ts +++ b/src/lib/arenaEloSync.ts @@ -14,11 +14,12 @@ import { resolveScoresAs } from "@omniroute/open-sse/services/autoCombo/scoresAs import { isArenaEloSyncEnabled } from "@/shared/utils/featureFlags"; +import { getDbInstance } from "./db/core"; import { backupDbFile } from "./db/backup"; import { - bulkUpsertModelIntelligence, - deleteExpiredIntelligence, + applyArenaEloRefresh, deleteModelIntelligenceBySource, + getLatestSyncedAt, type ModelIntelligenceEntry, } from "./db/modelIntelligence"; @@ -167,6 +168,49 @@ function getErrorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } +// ─── Failure backoff (persisted in key_value) ──────────── +// After a failed sync we stamp arena_elo/lastFailedAt so restarts and the +// periodic timer skip retrying a rate-limited/dead upstream within the +// window (remediation 2026-09-12: prevents boot-loop retry storms). +const BACKOFF_MS = SYNC_INTERVAL_MS / 4; // 6h for the default 24h interval + +const KV_READ_SQL = `SELECT value FROM key_value WHERE namespace = ? AND key = ? LIMIT 1`; +const KV_WRITE_SQL = `INSERT OR REPLACE INTO key_value (namespace, key, value) VALUES (?, ?, ?)`; + +function readLastFailedAt(): number | null { + try { + const db = getDbInstance(); + const row = db.prepare(KV_READ_SQL).get("arena_elo", "lastFailedAt") as + { value: string } | undefined; + if (!row?.value) return null; + const ts = Date.parse(row.value); + return Number.isFinite(ts) ? ts : null; + } catch { + return null; // kv problems must never break the sync itself + } +} + +function writeLastFailedAt(ts: number): void { + try { + const db = getDbInstance(); + db.prepare(KV_WRITE_SQL).run("arena_elo", "lastFailedAt", new Date(ts).toISOString()); + } catch (err) { + console.warn(`[ARENA_ELO_SYNC] Failed to persist lastFailedAt: ${getErrorMessage(err)}`); + } +} + +function clearLastFailedAt(): void { + try { + const db = getDbInstance(); + db.prepare(`DELETE FROM key_value WHERE namespace = ? AND key = ?`).run( + "arena_elo", + "lastFailedAt" + ); + } catch { + // Swallow: a stuck kv row only costs one redundant fetch later. + } +} + function getEffectiveArenaEloSyncEnabled(): boolean { try { return isArenaEloSyncEnabled(); @@ -449,25 +493,48 @@ export async function syncArenaElo(dryRun = false): Promise { firstSyncDone = true; } - // Clean up stale entries before writing new ones if (!dryRun) { - try { - deleteExpiredIntelligence(); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - console.warn(`[ARENA_ELO_SYNC] Failed to delete expired intelligence: ${message}`); + // Freshness guard: a non-empty dataset synced within the interval is + // still good — skip the fetch entirely (cheap fast path on boot). + const latest = getLatestSyncedAt("arena_elo"); + if (latest) { + const age = Date.now() - Date.parse(latest); + if (Number.isFinite(age) && age >= 0 && age < SYNC_INTERVAL_MS) { + return { + success: true, + modelCount: lastSyncModelCount, + source: "arena_elo", + }; + } + } + + // Failure backoff: a recent failed sync means the upstream is probably + // still rate-limited or down — skip instead of hammering it. + const failedAt = readLastFailedAt(); + if (failedAt !== null && Date.now() - failedAt < BACKOFF_MS) { + return { + success: false, + modelCount: 0, + source: "arena_elo", + error: "Skipping sync: recent failure within backoff window", + }; } } + // Fetch FIRST — a failed fetch must never mutate stored intelligence + // (the old delete-before-fetch order drained the table on every failed + // sync while the upstream was rate-limiting; remediation 2026-09-12). const leaderboards = await fetchArenaLeaderboards(); const entries = transformToModelIntelligence(leaderboards); if (!dryRun && entries.length > 0) { try { - bulkUpsertModelIntelligence(entries); + // Atomic replace: upsert all + prune not-in-refreshed-set in one tx. + applyArenaEloRefresh(entries); + clearLastFailedAt(); } catch (err) { - const message = err instanceof Error ? err.message : String(err); - console.warn(`[ARENA_ELO_SYNC] Failed to bulk upsert intelligence: ${message}`); + const message = getErrorMessage(err); + console.warn(`[ARENA_ELO_SYNC] Failed to apply intelligence refresh: ${message}`); return { success: false, modelCount: 0, @@ -493,8 +560,13 @@ export async function syncArenaElo(dryRun = false): Promise { source: "arena_elo", }; } catch (err) { - const message = err instanceof Error ? err.message : String(err); + const message = getErrorMessage(err); console.warn("[ARENA_ELO_SYNC] Sync failed:", message); + if (!dryRun) { + // Persist the failure so restarts/periodic ticks back off instead of + // retrying a dead upstream in a loop. + writeLastFailedAt(Date.now()); + } return { success: false, modelCount: 0, diff --git a/src/lib/db/modelIntelligence.ts b/src/lib/db/modelIntelligence.ts index f5ed208e69..e14504d45f 100644 --- a/src/lib/db/modelIntelligence.ts +++ b/src/lib/db/modelIntelligence.ts @@ -42,7 +42,10 @@ function rowToEntry(row: Record): ModelIntelligenceEntry { // ──────────────── CRUD ──────────────── -export function getModelIntelligence(model: string, category: string): ModelIntelligenceEntry | null { +export function getModelIntelligence( + model: string, + category: string +): ModelIntelligenceEntry | null { const db = getDbInstance(); const row = db .prepare( @@ -119,17 +122,13 @@ export function deleteExpiredIntelligence(source?: string): number { } const where = conditions.join(" AND "); - const result = db - .prepare(`DELETE FROM model_intelligence WHERE ${where}`) - .run(...params); + const result = db.prepare(`DELETE FROM model_intelligence WHERE ${where}`).run(...params); return result.changes ?? 0; } export function deleteModelIntelligenceBySource(source: string): number { const db = getDbInstance(); - const result = db - .prepare(`DELETE FROM model_intelligence WHERE source = ?`) - .run(source); + const result = db.prepare(`DELETE FROM model_intelligence WHERE source = ?`).run(source); return result.changes ?? 0; } @@ -158,7 +157,9 @@ export function listModelIntelligence(filters?: { return rows.map(rowToEntry); } -export function bulkUpsertModelIntelligence(entries: Array>): number { +export function bulkUpsertModelIntelligence( + entries: Array> +): number { if (entries.length === 0) return 0; const db = getDbInstance(); @@ -193,6 +194,69 @@ export function getResolvedTaskFitness(model: string, category: string): number return entry ? entry.score : null; } +/** + * Latest synced_at for a source, as an ISO string (or null when the source + * has no rows). Backs the arenaEloSync freshness guard: a non-empty dataset + * synced within the sync interval does not need re-fetching. + */ +export function getLatestSyncedAt(source: string): string | null { + const db = getDbInstance(); + const row = db + .prepare(`SELECT MAX(synced_at) as latest FROM model_intelligence WHERE source = ?`) + .get(source) as { latest: string | null } | undefined; + return row?.latest ?? null; +} + +/** + * Atomically replace a source's dataset: upsert every refreshed entry and + * prune rows for that source that are not in the refreshed set — one + * transaction, so a crash mid-refresh can never leave a half-replaced table. + * + * This is the safe replacement for the old delete-expired-then-upsert flow, + * which lost data when a fetch failed after the delete (remediation 2026-09-12). + */ +export function applyArenaEloRefresh( + entries: Array>, + source = "arena_elo" +): { upserted: number; pruned: number } { + const db = getDbInstance(); + const upsertStmt = db.prepare( + `INSERT OR REPLACE INTO model_intelligence + (model, source, category, score, elo_raw, confidence, synced_at, expires_at) + VALUES (?, ?, ?, ?, ?, ?, datetime('now'), ?)` + ); + + const refresh = db.transaction(() => { + let upserted = 0; + for (const entry of entries) { + upsertStmt.run( + entry.model, + entry.source, + entry.category, + entry.score, + entry.eloRaw ?? null, + entry.confidence ?? null, + entry.expiresAt ?? null + ); + upserted++; + } + // Membership prune via a JSON array of "model\0category" pair keys: + // correct for composite keys (a plain NOT IN over model names is not). + const refreshedPairs = JSON.stringify(entries.map((e) => `${e.model}\u0000${e.category}`)); + const pruneByPair = db.prepare( + `DELETE FROM model_intelligence + WHERE source = ? + AND model || char(0) || category NOT IN ( + SELECT value FROM json_each(?) + )` + ); + const pruneResult = pruneByPair.run(source, refreshedPairs); + return { upserted, pruned: pruneResult.changes ?? 0 }; + }); + + return refresh(); +} + /** * Write a user_override entry for a model × category combination. * Used by taskFitness.ts resolution chain as Layer 1 (highest priority). @@ -201,11 +265,7 @@ export function getResolvedTaskFitness(model: string, category: string): number * @param category - Task category * @param score - Fitness score [0..1] */ -export function setUserFitnessOverrideEntry( - model: string, - category: string, - score: number, -): void { +export function setUserFitnessOverrideEntry(model: string, category: string, score: number): void { upsertModelIntelligence({ model: model.toLowerCase(), source: "user_override", @@ -224,9 +284,6 @@ export function setUserFitnessOverrideEntry( * @param category - Task category * @returns true if an entry was deleted */ -export function deleteUserFitnessOverrideEntry( - model: string, - category: string, -): boolean { +export function deleteUserFitnessOverrideEntry(model: string, category: string): boolean { return deleteModelIntelligence(model.toLowerCase(), "user_override", category.toLowerCase()); } diff --git a/tests/unit/arena-elo-sync-redesign.test.ts b/tests/unit/arena-elo-sync-redesign.test.ts new file mode 100644 index 0000000000..9a97cba518 --- /dev/null +++ b/tests/unit/arena-elo-sync-redesign.test.ts @@ -0,0 +1,265 @@ +/** + * Redesign tests for src/lib/arenaEloSync.ts (remediation plan 2026-09-12). + * + * New invariants: + * 1. A failing fetch must NEVER delete existing entries (the old + * delete-before-fetch bug drained the table on every failed sync). + * 2. A successful sync replaces the dataset atomically: upsert all + + * prune entries not in the refreshed set, in one transaction. + * 3. Freshness guard: skip syncing when the table is non-empty and the + * latest synced_at is within the sync interval. + * 4. Failure backoff: a failed sync persists arena_elo/lastFailedAt in + * key_value; subsequent syncs within the backoff window are skipped + * (prevents boot-loop retry storms against a rate-limited upstream). + * 5. A successful sync clears any lastFailedAt stamp. + * + * Mirrors tests/unit/arena-elo-sync.test.ts patterns: node:test, real + * in-memory SQLite via tryOpenSync + globalThis.__omnirouteDb, globalThis.fetch + * mocks, patched migration 097 (no DEFAULT now on synced_at). + */ +import { describe, it, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-arena-redesign-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const MIGRATION_SQL = fs.readFileSync( + path.resolve( + import.meta.dirname ?? __dirname, + "../../src/lib/db/migrations/097_model_intelligence.sql" + ), + "utf8" +); + +import { tryOpenSync } from "../../src/lib/db/adapters/driverFactory"; +import type { SqliteAdapter } from "../../src/lib/db/adapters/types"; +const core = await import("../../src/lib/db/core.ts"); +const { syncArenaElo, getArenaEloSyncStatus, stopArenaEloSync } = + await import("../../src/lib/arenaEloSync.ts"); +import type { + ArenaLeaderboardData, + ArenaLeaderboardMap, + ArenaModelEntry, +} from "../../src/lib/arenaEloSync.ts"; + +const originalFetch = globalThis.fetch; +function mockFetch(impl: (url: string, opts?: RequestInit) => Promise): void { + globalThis.fetch = impl as typeof fetch; +} +function restoreFetch(): void { + globalThis.fetch = originalFetch; +} +function jsonResponse(data: unknown, status = 200): Response { + return new Response(JSON.stringify(data), { + status, + headers: { "Content-Type": "application/json" }, + }); +} +function makeModelEntry(overrides: Partial = {}): ArenaModelEntry { + return { + rank: 1, + model: "anthropic/claude-sonnet", + vendor: "Anthropic", + score: 1350, + ci: 10, + votes: 5000, + license: "proprietary", + ...overrides, + }; +} +function makeLeaderboardData( + models: ArenaModelEntry[] = [], + category = "text" +): ArenaLeaderboardData { + return { meta: { leaderboard: category, model_count: models.length }, models }; +} +function makeLeaderboardMap( + categories: Partial> +): ArenaLeaderboardMap { + const map: ArenaLeaderboardMap = {}; + for (const [cat, models] of Object.entries(categories)) { + map[cat] = makeLeaderboardData(models ?? [], cat); + } + return map; +} + +let testAdapter: SqliteAdapter; +function createTestAdapter(): SqliteAdapter { + const patchedSql = MIGRATION_SQL.replace( + /\n\s*synced_at TEXT NOT NULL DEFAULT \(datetime\('now'\)\)/, + "\n synced_at TEXT NOT NULL" + ); + const adapter = tryOpenSync(":memory:")!; + adapter.exec(` + CREATE TABLE IF NOT EXISTS key_value ( + namespace TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (namespace, key) + ); + `); + adapter.exec(patchedSql); + return adapter; +} + +function seedEntry( + model: string, + opts: { syncedAt?: string; category?: string; expiresAt?: string | null } = {} +): void { + const now = Date.now(); + testAdapter + .prepare( + `INSERT OR REPLACE INTO model_intelligence + (model, source, category, score, elo_raw, confidence, synced_at, expires_at) + VALUES (?, 'arena_elo', ?, ?, ?, ?, ?, ?)` + ) + .run( + model, + opts.category ?? "default", + 0.5, + 1200, + "high", + // Default: synced 2 days ago (stale) so the freshness guard does NOT + // skip and tests actually reach the fetch path. Tests that WANT the + // guard to fire pass an explicit fresh syncedAt. + opts.syncedAt ?? new Date(now - 2 * 86400_000).toISOString(), + opts.expiresAt ?? new Date(now + 86400_000).toISOString() + ); +} + +function countArenaEloEntries(): number { + const row = testAdapter + .prepare("SELECT COUNT(*) as cnt FROM model_intelligence WHERE source = 'arena_elo'") + .get() as Record | undefined; + return Number(row?.cnt ?? 0); +} + +function getLastFailedAt(): string | null { + const row = testAdapter + .prepare("SELECT value FROM key_value WHERE namespace = 'arena_elo' AND key = 'lastFailedAt'") + .get() as { value: string } | undefined; + return row?.value ?? null; +} + +/** Fetch mock that answers all categories with the given model lists. */ +function fetchServing(modelsByCategory: Record) { + return async (url: string): Promise => { + for (const [cat, models] of Object.entries(modelsByCategory)) { + if (url.includes(`name=${cat}`)) return jsonResponse(makeLeaderboardData(models, cat)); + } + return new Response("", { status: 404 }); + }; +} + +beforeEach(() => { + core.resetDbInstance(); + testAdapter = createTestAdapter(); + globalThis.__omnirouteDb = testAdapter as never; + stopArenaEloSync(); + delete process.env.ARENA_ELO_SYNC_INTERVAL; +}); + +afterEach(() => { + restoreFetch(); + stopArenaEloSync(); + delete globalThis.__omnirouteDb; +}); + +describe("arenaEloSync redesign", () => { + it("a failing fetch never deletes existing entries", async () => { + seedEntry("existing-model"); + assert.equal(countArenaEloEntries(), 1); + + mockFetch(async () => { + throw new Error("network down"); + }); + const result = await syncArenaElo(); + + assert.equal(result.success, false); + assert.equal(countArenaEloEntries(), 1, "entries must survive a failed fetch"); + assert.ok(getLastFailedAt(), "failure must persist arena_elo/lastFailedAt"); + }); + + it("a successful sync replaces the dataset atomically (upsert + prune)", async () => { + seedEntry("stale-model"); // in DB, absent from the refreshed leaderboard + + mockFetch( + fetchServing({ + text: [makeModelEntry({ model: "fresh-model" })], + code: [], + }) + ); + const result = await syncArenaElo(); + + assert.equal(result.success, true); + const models = getAllEntries().map((r) => r.model as string); + assert.ok(!models.includes("stale-model"), "stale entry must be pruned"); + assert.ok(models.includes("fresh-model"), "fresh entry must be present"); + assert.equal(getLastFailedAt(), null, "success must clear lastFailedAt"); + }); + + it("freshness guard: skips sync when table non-empty and synced_at within interval", async () => { + seedEntry("recent-model", { syncedAt: new Date(Date.now() - 60_000).toISOString() }); // 1 min ago + + let fetchCalled = false; + mockFetch(async () => { + fetchCalled = true; + return jsonResponse({}); + }); + const result = await syncArenaElo(); + + assert.equal(fetchCalled, false, "fresh table must not trigger a fetch"); + assert.equal(result.success, true, "skip is a success (fast path)"); + }); + + it("backoff wins even on an empty table: fresh lastFailedAt skips the fetch", async () => { + // Empty table + fresh failure stamp = the exact boot-loop condition that + // drained the live table (restart → sync → 429 → repeat). Backoff must + // skip; the retry happens when the window (6h) elapses. + testAdapter + .prepare( + "INSERT OR REPLACE INTO key_value (namespace, key, value) VALUES ('arena_elo','lastFailedAt',?)" + ) + .run(new Date().toISOString()); + + let fetchCalled = false; + mockFetch(async () => { + fetchCalled = true; + return jsonResponse({}); + }); + const result = await syncArenaElo(); + + assert.equal(fetchCalled, false, "backoff must skip the fetch even when the table is empty"); + assert.equal(result.success, false, "backoff skip reports not-success (no data change)"); + assert.equal(countArenaEloEntries(), 0, "table untouched during backoff"); + }); + + it("failure backoff: skips sync within backoff window after a failure", async () => { + seedEntry("existing-model"); + mockFetch(async () => { + throw new Error("429 rate limited"); + }); + await syncArenaElo(); + assert.ok(getLastFailedAt(), "lastFailedAt persisted after failure"); + + // Second attempt within the backoff window: fetch must NOT be called. + let fetchCalled = false; + mockFetch(async () => { + fetchCalled = true; + return jsonResponse({}); + }); + const result2 = await syncArenaElo(); + assert.equal(fetchCalled, false, "backoff must skip the fetch"); + assert.equal(result2.success, false, "backoff skip reports not-success (no data change)"); + assert.equal(countArenaEloEntries(), 1, "entries untouched during backoff"); + }); +}); + +function getAllEntries(): Array> { + return testAdapter + .prepare("SELECT * FROM model_intelligence WHERE source = 'arena_elo' ORDER BY model, category") + .all() as Array>; +} diff --git a/tests/unit/compression/compressionAnalyticsWriterFlatRate.test.ts b/tests/unit/compression/compressionAnalyticsWriterFlatRate.test.ts new file mode 100644 index 0000000000..af82bf497d --- /dev/null +++ b/tests/unit/compression/compressionAnalyticsWriterFlatRate.test.ts @@ -0,0 +1,134 @@ +/** + * Regression test (audit 2026-09-12): the compression analytics writer must + * opt into flatRateAsZero so flat-rate subscription lanes (minimax, glm, kimi, + * bailian, xiaomi, web-cookie) never book a non-zero dollar "savings" estimate. + * Cost rows on those lanes are for pre-flight estimates only — booking them as + * compression savings invents money the operator never pays (audit §5.1). + * + * Mirrors the flat-rate convention established by tests/unit/flat-rate-cost-5552.test.ts + * (upstream #5552) and the opt-in used by src/app/api/usage/analytics/route.ts. + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const tmpDir = mkdtempSync(join(tmpdir(), "omniroute-caw-")); +process.env.DATA_DIR = tmpDir; + +const core = await import("../../../src/lib/db/core.ts"); +core.resetDbInstance(); +const { getDbInstance } = core; +const { writeCompressionAnalytics } = + await import("../../../open-sse/handlers/chatCore/compressionAnalyticsWrite.ts"); + +function ensureTables() { + const db = getDbInstance(); + db.exec(` + CREATE TABLE IF NOT EXISTS compression_analytics ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + combo_id TEXT, + provider TEXT, + mode TEXT NOT NULL, + original_tokens INTEGER NOT NULL, + compressed_tokens INTEGER NOT NULL, + tokens_saved INTEGER NOT NULL, + duration_ms INTEGER, + request_id TEXT, + estimated_usd_saved REAL + ) + `); + // costCalculator reads provider pricing through the layered pricing settings; + // an empty pricing namespace forces the defaults layer, which has non-zero + // rates for minimax — exactly the condition under which the bug books money. + db.exec(` + CREATE TABLE IF NOT EXISTS key_value ( + namespace TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT, + PRIMARY KEY (namespace, key) + ) + `); +} + +function lastRow() { + const db = getDbInstance(); + return db + .prepare( + "SELECT provider, tokens_saved, estimated_usd_saved FROM compression_analytics ORDER BY id DESC LIMIT 1" + ) + .get() as { provider: string; tokens_saved: number; estimated_usd_saved: number | null }; +} + +function makeStats(originalTokens: number, compressedTokens: number) { + // Minimal CompressionStats shape used by the writer: originalTokens, + // compressedTokens, durationMs, rtkRawOutputPointers. + return { + originalTokens, + compressedTokens, + durationMs: 123, + rtkRawOutputPointers: [] as Array<{ id?: string | null; bytes?: number | null }>, + engine: "rtk", + } as never; +} + +test("writeCompressionAnalytics books $0 savings for flat-rate providers (minimax)", async () => { + ensureTables(); + getDbInstance().exec("DELETE FROM compression_analytics"); + + await writeCompressionAnalytics({ + stats: makeStats(10_000, 2_000), + provider: "minimax", + effectiveModel: "MiniMax-M3", + effectiveServiceTier: undefined, + comboName: null, + mode: "chat", + compressionComboId: null, + skillRequestId: "req-flat-rate-test", + cavemanOutputModeApplied: false, + cavemanOutputModeIntensity: null, + log: null, + }); + + const row = lastRow(); + assert.ok(row, "a compression_analytics row should have been written"); + assert.equal(row.provider, "minimax"); + assert.equal(row.tokens_saved, 8_000); + // THE assertion: flat-rate lanes must not invent dollar savings. + assert.ok( + row.estimated_usd_saved === null || row.estimated_usd_saved === 0, + `expected null/0 estimated_usd_saved for flat-rate provider, got ${row.estimated_usd_saved}` + ); +}); + +test("writeCompressionAnalytics still books real savings for metered providers (openai)", async () => { + ensureTables(); + getDbInstance().exec("DELETE FROM compression_analytics"); + + await writeCompressionAnalytics({ + stats: makeStats(10_000, 2_000), + provider: "openai", + effectiveModel: "gpt-5.5", + effectiveServiceTier: undefined, + comboName: null, + mode: "chat", + compressionComboId: null, + skillRequestId: "req-metered-test", + cavemanOutputModeApplied: false, + cavemanOutputModeIntensity: null, + log: null, + }); + + const row = lastRow(); + assert.ok(row, "a compression_analytics row should have been written"); + assert.equal(row.provider, "openai"); + assert.equal(row.tokens_saved, 8_000); + // Metered providers keep a non-zero estimate (real per-token money avoided). + assert.ok( + row.estimated_usd_saved !== null && row.estimated_usd_saved > 0, + `expected positive estimated_usd_saved for metered provider, got ${row.estimated_usd_saved}` + ); +});