mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-19 05:12:16 +03:00
fix(arena+analytics): atomic ELO sync (fetch-first) + flatRateAsZero in compression writer (#13446)
* fix(analytics+arena): flatRateAsZero in compression writer; atomic arena sync redesign * docs(changelog): add fragments for arena ELO sync and compression flat-rate fixes Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --------- Co-authored-by: CrashCartCapital <crashcartcapital@gmail.com> Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
This commit is contained in:
1
changelog.d/fixes/13446-arena-elo-sync-fetch-first.md
Normal file
1
changelog.d/fixes/13446-arena-elo-sync-fetch-first.md
Normal file
@@ -0,0 +1 @@
|
||||
- **fix(db):** Arena ELO sync now fetches and validates the leaderboards before touching `model_intelligence`, and applies the upsert + prune of expired rows inside a single atomic transaction. Previously, an unavailable/rate-limited Arena API left the table pruned with nothing written back, and since the sync runs on every boot, repeated restarts against a rate-limited upstream permanently drained the table to zero ([#13446](https://github.com/diegosouzapw/OmniRoute/pull/13446)) — thanks @CrashCartCapital
|
||||
@@ -0,0 +1 @@
|
||||
- **fix(analytics):** the compression analytics writer now passes `flatRateAsZero: true` to `calculateCost`, matching `/api/usage/analytics`. Flat-rate subscription lanes (minimax, glm, kimi, bailian, xiaomi, web-cookie) no longer report a dollar "savings" figure that was never actually payable ([#13446](https://github.com/diegosouzapw/OmniRoute/pull/13446)) — thanks @CrashCartCapital
|
||||
@@ -149,7 +149,10 @@ export function writeCompressionAnalytics(
|
||||
opts.provider ?? "",
|
||||
opts.effectiveModel ?? "",
|
||||
{ input: tokensSaved },
|
||||
{ serviceTier: opts.effectiveServiceTier }
|
||||
// Flat-rate (subscription / cookie-web) lanes don't bill per token, so
|
||||
// their per-token pricing rows must not book dollar "savings" here
|
||||
// (same opt-in as /api/usage/analytics, #5552).
|
||||
{ serviceTier: opts.effectiveServiceTier, flatRateAsZero: true }
|
||||
);
|
||||
} catch (err) {
|
||||
opts.log?.debug?.(
|
||||
|
||||
@@ -14,11 +14,12 @@ import { resolveScoresAs } from "@omniroute/open-sse/services/autoCombo/scoresAs
|
||||
|
||||
import { isArenaEloSyncEnabled } from "@/shared/utils/featureFlags";
|
||||
|
||||
import { getDbInstance } from "./db/core";
|
||||
import { backupDbFile } from "./db/backup";
|
||||
import {
|
||||
bulkUpsertModelIntelligence,
|
||||
deleteExpiredIntelligence,
|
||||
applyArenaEloRefresh,
|
||||
deleteModelIntelligenceBySource,
|
||||
getLatestSyncedAt,
|
||||
type ModelIntelligenceEntry,
|
||||
} from "./db/modelIntelligence";
|
||||
|
||||
@@ -167,6 +168,49 @@ function getErrorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
// ─── Failure backoff (persisted in key_value) ────────────
|
||||
// After a failed sync we stamp arena_elo/lastFailedAt so restarts and the
|
||||
// periodic timer skip retrying a rate-limited/dead upstream within the
|
||||
// window (remediation 2026-09-12: prevents boot-loop retry storms).
|
||||
const BACKOFF_MS = SYNC_INTERVAL_MS / 4; // 6h for the default 24h interval
|
||||
|
||||
const KV_READ_SQL = `SELECT value FROM key_value WHERE namespace = ? AND key = ? LIMIT 1`;
|
||||
const KV_WRITE_SQL = `INSERT OR REPLACE INTO key_value (namespace, key, value) VALUES (?, ?, ?)`;
|
||||
|
||||
function readLastFailedAt(): number | null {
|
||||
try {
|
||||
const db = getDbInstance();
|
||||
const row = db.prepare(KV_READ_SQL).get("arena_elo", "lastFailedAt") as
|
||||
{ value: string } | undefined;
|
||||
if (!row?.value) return null;
|
||||
const ts = Date.parse(row.value);
|
||||
return Number.isFinite(ts) ? ts : null;
|
||||
} catch {
|
||||
return null; // kv problems must never break the sync itself
|
||||
}
|
||||
}
|
||||
|
||||
function writeLastFailedAt(ts: number): void {
|
||||
try {
|
||||
const db = getDbInstance();
|
||||
db.prepare(KV_WRITE_SQL).run("arena_elo", "lastFailedAt", new Date(ts).toISOString());
|
||||
} catch (err) {
|
||||
console.warn(`[ARENA_ELO_SYNC] Failed to persist lastFailedAt: ${getErrorMessage(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function clearLastFailedAt(): void {
|
||||
try {
|
||||
const db = getDbInstance();
|
||||
db.prepare(`DELETE FROM key_value WHERE namespace = ? AND key = ?`).run(
|
||||
"arena_elo",
|
||||
"lastFailedAt"
|
||||
);
|
||||
} catch {
|
||||
// Swallow: a stuck kv row only costs one redundant fetch later.
|
||||
}
|
||||
}
|
||||
|
||||
function getEffectiveArenaEloSyncEnabled(): boolean {
|
||||
try {
|
||||
return isArenaEloSyncEnabled();
|
||||
@@ -449,25 +493,48 @@ export async function syncArenaElo(dryRun = false): Promise<SyncResult> {
|
||||
firstSyncDone = true;
|
||||
}
|
||||
|
||||
// Clean up stale entries before writing new ones
|
||||
if (!dryRun) {
|
||||
try {
|
||||
deleteExpiredIntelligence();
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.warn(`[ARENA_ELO_SYNC] Failed to delete expired intelligence: ${message}`);
|
||||
// Freshness guard: a non-empty dataset synced within the interval is
|
||||
// still good — skip the fetch entirely (cheap fast path on boot).
|
||||
const latest = getLatestSyncedAt("arena_elo");
|
||||
if (latest) {
|
||||
const age = Date.now() - Date.parse(latest);
|
||||
if (Number.isFinite(age) && age >= 0 && age < SYNC_INTERVAL_MS) {
|
||||
return {
|
||||
success: true,
|
||||
modelCount: lastSyncModelCount,
|
||||
source: "arena_elo",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Failure backoff: a recent failed sync means the upstream is probably
|
||||
// still rate-limited or down — skip instead of hammering it.
|
||||
const failedAt = readLastFailedAt();
|
||||
if (failedAt !== null && Date.now() - failedAt < BACKOFF_MS) {
|
||||
return {
|
||||
success: false,
|
||||
modelCount: 0,
|
||||
source: "arena_elo",
|
||||
error: "Skipping sync: recent failure within backoff window",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch FIRST — a failed fetch must never mutate stored intelligence
|
||||
// (the old delete-before-fetch order drained the table on every failed
|
||||
// sync while the upstream was rate-limiting; remediation 2026-09-12).
|
||||
const leaderboards = await fetchArenaLeaderboards();
|
||||
const entries = transformToModelIntelligence(leaderboards);
|
||||
|
||||
if (!dryRun && entries.length > 0) {
|
||||
try {
|
||||
bulkUpsertModelIntelligence(entries);
|
||||
// Atomic replace: upsert all + prune not-in-refreshed-set in one tx.
|
||||
applyArenaEloRefresh(entries);
|
||||
clearLastFailedAt();
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.warn(`[ARENA_ELO_SYNC] Failed to bulk upsert intelligence: ${message}`);
|
||||
const message = getErrorMessage(err);
|
||||
console.warn(`[ARENA_ELO_SYNC] Failed to apply intelligence refresh: ${message}`);
|
||||
return {
|
||||
success: false,
|
||||
modelCount: 0,
|
||||
@@ -493,8 +560,13 @@ export async function syncArenaElo(dryRun = false): Promise<SyncResult> {
|
||||
source: "arena_elo",
|
||||
};
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
const message = getErrorMessage(err);
|
||||
console.warn("[ARENA_ELO_SYNC] Sync failed:", message);
|
||||
if (!dryRun) {
|
||||
// Persist the failure so restarts/periodic ticks back off instead of
|
||||
// retrying a dead upstream in a loop.
|
||||
writeLastFailedAt(Date.now());
|
||||
}
|
||||
return {
|
||||
success: false,
|
||||
modelCount: 0,
|
||||
|
||||
@@ -42,7 +42,10 @@ function rowToEntry(row: Record<string, unknown>): ModelIntelligenceEntry {
|
||||
|
||||
// ──────────────── CRUD ────────────────
|
||||
|
||||
export function getModelIntelligence(model: string, category: string): ModelIntelligenceEntry | null {
|
||||
export function getModelIntelligence(
|
||||
model: string,
|
||||
category: string
|
||||
): ModelIntelligenceEntry | null {
|
||||
const db = getDbInstance();
|
||||
const row = db
|
||||
.prepare(
|
||||
@@ -119,17 +122,13 @@ export function deleteExpiredIntelligence(source?: string): number {
|
||||
}
|
||||
|
||||
const where = conditions.join(" AND ");
|
||||
const result = db
|
||||
.prepare(`DELETE FROM model_intelligence WHERE ${where}`)
|
||||
.run(...params);
|
||||
const result = db.prepare(`DELETE FROM model_intelligence WHERE ${where}`).run(...params);
|
||||
return result.changes ?? 0;
|
||||
}
|
||||
|
||||
export function deleteModelIntelligenceBySource(source: string): number {
|
||||
const db = getDbInstance();
|
||||
const result = db
|
||||
.prepare(`DELETE FROM model_intelligence WHERE source = ?`)
|
||||
.run(source);
|
||||
const result = db.prepare(`DELETE FROM model_intelligence WHERE source = ?`).run(source);
|
||||
return result.changes ?? 0;
|
||||
}
|
||||
|
||||
@@ -158,7 +157,9 @@ export function listModelIntelligence(filters?: {
|
||||
return rows.map(rowToEntry);
|
||||
}
|
||||
|
||||
export function bulkUpsertModelIntelligence(entries: Array<Omit<ModelIntelligenceEntry, "syncedAt">>): number {
|
||||
export function bulkUpsertModelIntelligence(
|
||||
entries: Array<Omit<ModelIntelligenceEntry, "syncedAt">>
|
||||
): number {
|
||||
if (entries.length === 0) return 0;
|
||||
|
||||
const db = getDbInstance();
|
||||
@@ -193,6 +194,69 @@ export function getResolvedTaskFitness(model: string, category: string): number
|
||||
return entry ? entry.score : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Latest synced_at for a source, as an ISO string (or null when the source
|
||||
* has no rows). Backs the arenaEloSync freshness guard: a non-empty dataset
|
||||
* synced within the sync interval does not need re-fetching.
|
||||
*/
|
||||
export function getLatestSyncedAt(source: string): string | null {
|
||||
const db = getDbInstance();
|
||||
const row = db
|
||||
.prepare(`SELECT MAX(synced_at) as latest FROM model_intelligence WHERE source = ?`)
|
||||
.get(source) as { latest: string | null } | undefined;
|
||||
return row?.latest ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomically replace a source's dataset: upsert every refreshed entry and
|
||||
* prune rows for that source that are not in the refreshed set — one
|
||||
* transaction, so a crash mid-refresh can never leave a half-replaced table.
|
||||
*
|
||||
* This is the safe replacement for the old delete-expired-then-upsert flow,
|
||||
* which lost data when a fetch failed after the delete (remediation 2026-09-12).
|
||||
*/
|
||||
export function applyArenaEloRefresh(
|
||||
entries: Array<Omit<ModelIntelligenceEntry, "syncedAt">>,
|
||||
source = "arena_elo"
|
||||
): { upserted: number; pruned: number } {
|
||||
const db = getDbInstance();
|
||||
const upsertStmt = db.prepare(
|
||||
`INSERT OR REPLACE INTO model_intelligence
|
||||
(model, source, category, score, elo_raw, confidence, synced_at, expires_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, datetime('now'), ?)`
|
||||
);
|
||||
|
||||
const refresh = db.transaction(() => {
|
||||
let upserted = 0;
|
||||
for (const entry of entries) {
|
||||
upsertStmt.run(
|
||||
entry.model,
|
||||
entry.source,
|
||||
entry.category,
|
||||
entry.score,
|
||||
entry.eloRaw ?? null,
|
||||
entry.confidence ?? null,
|
||||
entry.expiresAt ?? null
|
||||
);
|
||||
upserted++;
|
||||
}
|
||||
// Membership prune via a JSON array of "model\0category" pair keys:
|
||||
// correct for composite keys (a plain NOT IN over model names is not).
|
||||
const refreshedPairs = JSON.stringify(entries.map((e) => `${e.model}\u0000${e.category}`));
|
||||
const pruneByPair = db.prepare(
|
||||
`DELETE FROM model_intelligence
|
||||
WHERE source = ?
|
||||
AND model || char(0) || category NOT IN (
|
||||
SELECT value FROM json_each(?)
|
||||
)`
|
||||
);
|
||||
const pruneResult = pruneByPair.run(source, refreshedPairs);
|
||||
return { upserted, pruned: pruneResult.changes ?? 0 };
|
||||
});
|
||||
|
||||
return refresh();
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a user_override entry for a model × category combination.
|
||||
* Used by taskFitness.ts resolution chain as Layer 1 (highest priority).
|
||||
@@ -201,11 +265,7 @@ export function getResolvedTaskFitness(model: string, category: string): number
|
||||
* @param category - Task category
|
||||
* @param score - Fitness score [0..1]
|
||||
*/
|
||||
export function setUserFitnessOverrideEntry(
|
||||
model: string,
|
||||
category: string,
|
||||
score: number,
|
||||
): void {
|
||||
export function setUserFitnessOverrideEntry(model: string, category: string, score: number): void {
|
||||
upsertModelIntelligence({
|
||||
model: model.toLowerCase(),
|
||||
source: "user_override",
|
||||
@@ -224,9 +284,6 @@ export function setUserFitnessOverrideEntry(
|
||||
* @param category - Task category
|
||||
* @returns true if an entry was deleted
|
||||
*/
|
||||
export function deleteUserFitnessOverrideEntry(
|
||||
model: string,
|
||||
category: string,
|
||||
): boolean {
|
||||
export function deleteUserFitnessOverrideEntry(model: string, category: string): boolean {
|
||||
return deleteModelIntelligence(model.toLowerCase(), "user_override", category.toLowerCase());
|
||||
}
|
||||
|
||||
265
tests/unit/arena-elo-sync-redesign.test.ts
Normal file
265
tests/unit/arena-elo-sync-redesign.test.ts
Normal file
@@ -0,0 +1,265 @@
|
||||
/**
|
||||
* Redesign tests for src/lib/arenaEloSync.ts (remediation plan 2026-09-12).
|
||||
*
|
||||
* New invariants:
|
||||
* 1. A failing fetch must NEVER delete existing entries (the old
|
||||
* delete-before-fetch bug drained the table on every failed sync).
|
||||
* 2. A successful sync replaces the dataset atomically: upsert all +
|
||||
* prune entries not in the refreshed set, in one transaction.
|
||||
* 3. Freshness guard: skip syncing when the table is non-empty and the
|
||||
* latest synced_at is within the sync interval.
|
||||
* 4. Failure backoff: a failed sync persists arena_elo/lastFailedAt in
|
||||
* key_value; subsequent syncs within the backoff window are skipped
|
||||
* (prevents boot-loop retry storms against a rate-limited upstream).
|
||||
* 5. A successful sync clears any lastFailedAt stamp.
|
||||
*
|
||||
* Mirrors tests/unit/arena-elo-sync.test.ts patterns: node:test, real
|
||||
* in-memory SQLite via tryOpenSync + globalThis.__omnirouteDb, globalThis.fetch
|
||||
* mocks, patched migration 097 (no DEFAULT now on synced_at).
|
||||
*/
|
||||
import { describe, it, beforeEach, afterEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-arena-redesign-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
|
||||
const MIGRATION_SQL = fs.readFileSync(
|
||||
path.resolve(
|
||||
import.meta.dirname ?? __dirname,
|
||||
"../../src/lib/db/migrations/097_model_intelligence.sql"
|
||||
),
|
||||
"utf8"
|
||||
);
|
||||
|
||||
import { tryOpenSync } from "../../src/lib/db/adapters/driverFactory";
|
||||
import type { SqliteAdapter } from "../../src/lib/db/adapters/types";
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const { syncArenaElo, getArenaEloSyncStatus, stopArenaEloSync } =
|
||||
await import("../../src/lib/arenaEloSync.ts");
|
||||
import type {
|
||||
ArenaLeaderboardData,
|
||||
ArenaLeaderboardMap,
|
||||
ArenaModelEntry,
|
||||
} from "../../src/lib/arenaEloSync.ts";
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
function mockFetch(impl: (url: string, opts?: RequestInit) => Promise<Response>): void {
|
||||
globalThis.fetch = impl as typeof fetch;
|
||||
}
|
||||
function restoreFetch(): void {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
function jsonResponse(data: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(data), {
|
||||
status,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
function makeModelEntry(overrides: Partial<ArenaModelEntry> = {}): ArenaModelEntry {
|
||||
return {
|
||||
rank: 1,
|
||||
model: "anthropic/claude-sonnet",
|
||||
vendor: "Anthropic",
|
||||
score: 1350,
|
||||
ci: 10,
|
||||
votes: 5000,
|
||||
license: "proprietary",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
function makeLeaderboardData(
|
||||
models: ArenaModelEntry[] = [],
|
||||
category = "text"
|
||||
): ArenaLeaderboardData {
|
||||
return { meta: { leaderboard: category, model_count: models.length }, models };
|
||||
}
|
||||
function makeLeaderboardMap(
|
||||
categories: Partial<Record<string, ArenaModelEntry[]>>
|
||||
): ArenaLeaderboardMap {
|
||||
const map: ArenaLeaderboardMap = {};
|
||||
for (const [cat, models] of Object.entries(categories)) {
|
||||
map[cat] = makeLeaderboardData(models ?? [], cat);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
let testAdapter: SqliteAdapter;
|
||||
function createTestAdapter(): SqliteAdapter {
|
||||
const patchedSql = MIGRATION_SQL.replace(
|
||||
/\n\s*synced_at TEXT NOT NULL DEFAULT \(datetime\('now'\)\)/,
|
||||
"\n synced_at TEXT NOT NULL"
|
||||
);
|
||||
const adapter = tryOpenSync(":memory:")!;
|
||||
adapter.exec(`
|
||||
CREATE TABLE IF NOT EXISTS key_value (
|
||||
namespace TEXT NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
value TEXT NOT NULL,
|
||||
PRIMARY KEY (namespace, key)
|
||||
);
|
||||
`);
|
||||
adapter.exec(patchedSql);
|
||||
return adapter;
|
||||
}
|
||||
|
||||
function seedEntry(
|
||||
model: string,
|
||||
opts: { syncedAt?: string; category?: string; expiresAt?: string | null } = {}
|
||||
): void {
|
||||
const now = Date.now();
|
||||
testAdapter
|
||||
.prepare(
|
||||
`INSERT OR REPLACE INTO model_intelligence
|
||||
(model, source, category, score, elo_raw, confidence, synced_at, expires_at)
|
||||
VALUES (?, 'arena_elo', ?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.run(
|
||||
model,
|
||||
opts.category ?? "default",
|
||||
0.5,
|
||||
1200,
|
||||
"high",
|
||||
// Default: synced 2 days ago (stale) so the freshness guard does NOT
|
||||
// skip and tests actually reach the fetch path. Tests that WANT the
|
||||
// guard to fire pass an explicit fresh syncedAt.
|
||||
opts.syncedAt ?? new Date(now - 2 * 86400_000).toISOString(),
|
||||
opts.expiresAt ?? new Date(now + 86400_000).toISOString()
|
||||
);
|
||||
}
|
||||
|
||||
function countArenaEloEntries(): number {
|
||||
const row = testAdapter
|
||||
.prepare("SELECT COUNT(*) as cnt FROM model_intelligence WHERE source = 'arena_elo'")
|
||||
.get() as Record<string, unknown> | undefined;
|
||||
return Number(row?.cnt ?? 0);
|
||||
}
|
||||
|
||||
function getLastFailedAt(): string | null {
|
||||
const row = testAdapter
|
||||
.prepare("SELECT value FROM key_value WHERE namespace = 'arena_elo' AND key = 'lastFailedAt'")
|
||||
.get() as { value: string } | undefined;
|
||||
return row?.value ?? null;
|
||||
}
|
||||
|
||||
/** Fetch mock that answers all categories with the given model lists. */
|
||||
function fetchServing(modelsByCategory: Record<string, ArenaModelEntry[]>) {
|
||||
return async (url: string): Promise<Response> => {
|
||||
for (const [cat, models] of Object.entries(modelsByCategory)) {
|
||||
if (url.includes(`name=${cat}`)) return jsonResponse(makeLeaderboardData(models, cat));
|
||||
}
|
||||
return new Response("", { status: 404 });
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
core.resetDbInstance();
|
||||
testAdapter = createTestAdapter();
|
||||
globalThis.__omnirouteDb = testAdapter as never;
|
||||
stopArenaEloSync();
|
||||
delete process.env.ARENA_ELO_SYNC_INTERVAL;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
restoreFetch();
|
||||
stopArenaEloSync();
|
||||
delete globalThis.__omnirouteDb;
|
||||
});
|
||||
|
||||
describe("arenaEloSync redesign", () => {
|
||||
it("a failing fetch never deletes existing entries", async () => {
|
||||
seedEntry("existing-model");
|
||||
assert.equal(countArenaEloEntries(), 1);
|
||||
|
||||
mockFetch(async () => {
|
||||
throw new Error("network down");
|
||||
});
|
||||
const result = await syncArenaElo();
|
||||
|
||||
assert.equal(result.success, false);
|
||||
assert.equal(countArenaEloEntries(), 1, "entries must survive a failed fetch");
|
||||
assert.ok(getLastFailedAt(), "failure must persist arena_elo/lastFailedAt");
|
||||
});
|
||||
|
||||
it("a successful sync replaces the dataset atomically (upsert + prune)", async () => {
|
||||
seedEntry("stale-model"); // in DB, absent from the refreshed leaderboard
|
||||
|
||||
mockFetch(
|
||||
fetchServing({
|
||||
text: [makeModelEntry({ model: "fresh-model" })],
|
||||
code: [],
|
||||
})
|
||||
);
|
||||
const result = await syncArenaElo();
|
||||
|
||||
assert.equal(result.success, true);
|
||||
const models = getAllEntries().map((r) => r.model as string);
|
||||
assert.ok(!models.includes("stale-model"), "stale entry must be pruned");
|
||||
assert.ok(models.includes("fresh-model"), "fresh entry must be present");
|
||||
assert.equal(getLastFailedAt(), null, "success must clear lastFailedAt");
|
||||
});
|
||||
|
||||
it("freshness guard: skips sync when table non-empty and synced_at within interval", async () => {
|
||||
seedEntry("recent-model", { syncedAt: new Date(Date.now() - 60_000).toISOString() }); // 1 min ago
|
||||
|
||||
let fetchCalled = false;
|
||||
mockFetch(async () => {
|
||||
fetchCalled = true;
|
||||
return jsonResponse({});
|
||||
});
|
||||
const result = await syncArenaElo();
|
||||
|
||||
assert.equal(fetchCalled, false, "fresh table must not trigger a fetch");
|
||||
assert.equal(result.success, true, "skip is a success (fast path)");
|
||||
});
|
||||
|
||||
it("backoff wins even on an empty table: fresh lastFailedAt skips the fetch", async () => {
|
||||
// Empty table + fresh failure stamp = the exact boot-loop condition that
|
||||
// drained the live table (restart → sync → 429 → repeat). Backoff must
|
||||
// skip; the retry happens when the window (6h) elapses.
|
||||
testAdapter
|
||||
.prepare(
|
||||
"INSERT OR REPLACE INTO key_value (namespace, key, value) VALUES ('arena_elo','lastFailedAt',?)"
|
||||
)
|
||||
.run(new Date().toISOString());
|
||||
|
||||
let fetchCalled = false;
|
||||
mockFetch(async () => {
|
||||
fetchCalled = true;
|
||||
return jsonResponse({});
|
||||
});
|
||||
const result = await syncArenaElo();
|
||||
|
||||
assert.equal(fetchCalled, false, "backoff must skip the fetch even when the table is empty");
|
||||
assert.equal(result.success, false, "backoff skip reports not-success (no data change)");
|
||||
assert.equal(countArenaEloEntries(), 0, "table untouched during backoff");
|
||||
});
|
||||
|
||||
it("failure backoff: skips sync within backoff window after a failure", async () => {
|
||||
seedEntry("existing-model");
|
||||
mockFetch(async () => {
|
||||
throw new Error("429 rate limited");
|
||||
});
|
||||
await syncArenaElo();
|
||||
assert.ok(getLastFailedAt(), "lastFailedAt persisted after failure");
|
||||
|
||||
// Second attempt within the backoff window: fetch must NOT be called.
|
||||
let fetchCalled = false;
|
||||
mockFetch(async () => {
|
||||
fetchCalled = true;
|
||||
return jsonResponse({});
|
||||
});
|
||||
const result2 = await syncArenaElo();
|
||||
assert.equal(fetchCalled, false, "backoff must skip the fetch");
|
||||
assert.equal(result2.success, false, "backoff skip reports not-success (no data change)");
|
||||
assert.equal(countArenaEloEntries(), 1, "entries untouched during backoff");
|
||||
});
|
||||
});
|
||||
|
||||
function getAllEntries(): Array<Record<string, unknown>> {
|
||||
return testAdapter
|
||||
.prepare("SELECT * FROM model_intelligence WHERE source = 'arena_elo' ORDER BY model, category")
|
||||
.all() as Array<Record<string, unknown>>;
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
/**
|
||||
* Regression test (audit 2026-09-12): the compression analytics writer must
|
||||
* opt into flatRateAsZero so flat-rate subscription lanes (minimax, glm, kimi,
|
||||
* bailian, xiaomi, web-cookie) never book a non-zero dollar "savings" estimate.
|
||||
* Cost rows on those lanes are for pre-flight estimates only — booking them as
|
||||
* compression savings invents money the operator never pays (audit §5.1).
|
||||
*
|
||||
* Mirrors the flat-rate convention established by tests/unit/flat-rate-cost-5552.test.ts
|
||||
* (upstream #5552) and the opt-in used by src/app/api/usage/analytics/route.ts.
|
||||
*/
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const tmpDir = mkdtempSync(join(tmpdir(), "omniroute-caw-"));
|
||||
process.env.DATA_DIR = tmpDir;
|
||||
|
||||
const core = await import("../../../src/lib/db/core.ts");
|
||||
core.resetDbInstance();
|
||||
const { getDbInstance } = core;
|
||||
const { writeCompressionAnalytics } =
|
||||
await import("../../../open-sse/handlers/chatCore/compressionAnalyticsWrite.ts");
|
||||
|
||||
function ensureTables() {
|
||||
const db = getDbInstance();
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS compression_analytics (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL,
|
||||
combo_id TEXT,
|
||||
provider TEXT,
|
||||
mode TEXT NOT NULL,
|
||||
original_tokens INTEGER NOT NULL,
|
||||
compressed_tokens INTEGER NOT NULL,
|
||||
tokens_saved INTEGER NOT NULL,
|
||||
duration_ms INTEGER,
|
||||
request_id TEXT,
|
||||
estimated_usd_saved REAL
|
||||
)
|
||||
`);
|
||||
// costCalculator reads provider pricing through the layered pricing settings;
|
||||
// an empty pricing namespace forces the defaults layer, which has non-zero
|
||||
// rates for minimax — exactly the condition under which the bug books money.
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS key_value (
|
||||
namespace TEXT NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
value TEXT,
|
||||
PRIMARY KEY (namespace, key)
|
||||
)
|
||||
`);
|
||||
}
|
||||
|
||||
function lastRow() {
|
||||
const db = getDbInstance();
|
||||
return db
|
||||
.prepare(
|
||||
"SELECT provider, tokens_saved, estimated_usd_saved FROM compression_analytics ORDER BY id DESC LIMIT 1"
|
||||
)
|
||||
.get() as { provider: string; tokens_saved: number; estimated_usd_saved: number | null };
|
||||
}
|
||||
|
||||
function makeStats(originalTokens: number, compressedTokens: number) {
|
||||
// Minimal CompressionStats shape used by the writer: originalTokens,
|
||||
// compressedTokens, durationMs, rtkRawOutputPointers.
|
||||
return {
|
||||
originalTokens,
|
||||
compressedTokens,
|
||||
durationMs: 123,
|
||||
rtkRawOutputPointers: [] as Array<{ id?: string | null; bytes?: number | null }>,
|
||||
engine: "rtk",
|
||||
} as never;
|
||||
}
|
||||
|
||||
test("writeCompressionAnalytics books $0 savings for flat-rate providers (minimax)", async () => {
|
||||
ensureTables();
|
||||
getDbInstance().exec("DELETE FROM compression_analytics");
|
||||
|
||||
await writeCompressionAnalytics({
|
||||
stats: makeStats(10_000, 2_000),
|
||||
provider: "minimax",
|
||||
effectiveModel: "MiniMax-M3",
|
||||
effectiveServiceTier: undefined,
|
||||
comboName: null,
|
||||
mode: "chat",
|
||||
compressionComboId: null,
|
||||
skillRequestId: "req-flat-rate-test",
|
||||
cavemanOutputModeApplied: false,
|
||||
cavemanOutputModeIntensity: null,
|
||||
log: null,
|
||||
});
|
||||
|
||||
const row = lastRow();
|
||||
assert.ok(row, "a compression_analytics row should have been written");
|
||||
assert.equal(row.provider, "minimax");
|
||||
assert.equal(row.tokens_saved, 8_000);
|
||||
// THE assertion: flat-rate lanes must not invent dollar savings.
|
||||
assert.ok(
|
||||
row.estimated_usd_saved === null || row.estimated_usd_saved === 0,
|
||||
`expected null/0 estimated_usd_saved for flat-rate provider, got ${row.estimated_usd_saved}`
|
||||
);
|
||||
});
|
||||
|
||||
test("writeCompressionAnalytics still books real savings for metered providers (openai)", async () => {
|
||||
ensureTables();
|
||||
getDbInstance().exec("DELETE FROM compression_analytics");
|
||||
|
||||
await writeCompressionAnalytics({
|
||||
stats: makeStats(10_000, 2_000),
|
||||
provider: "openai",
|
||||
effectiveModel: "gpt-5.5",
|
||||
effectiveServiceTier: undefined,
|
||||
comboName: null,
|
||||
mode: "chat",
|
||||
compressionComboId: null,
|
||||
skillRequestId: "req-metered-test",
|
||||
cavemanOutputModeApplied: false,
|
||||
cavemanOutputModeIntensity: null,
|
||||
log: null,
|
||||
});
|
||||
|
||||
const row = lastRow();
|
||||
assert.ok(row, "a compression_analytics row should have been written");
|
||||
assert.equal(row.provider, "openai");
|
||||
assert.equal(row.tokens_saved, 8_000);
|
||||
// Metered providers keep a non-zero estimate (real per-token money avoided).
|
||||
assert.ok(
|
||||
row.estimated_usd_saved !== null && row.estimated_usd_saved > 0,
|
||||
`expected positive estimated_usd_saved for metered provider, got ${row.estimated_usd_saved}`
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user