diff --git a/changelog.d/fixes/11506-arena-elo-base-rows.md b/changelog.d/fixes/11506-arena-elo-base-rows.md new file mode 100644 index 0000000000..c3fd744fa4 --- /dev/null +++ b/changelog.d/fixes/11506-arena-elo-base-rows.md @@ -0,0 +1 @@ +- **fix(intelligence):** Arena ELO sync stores a synthesized base-model row for every effort/harness variant (`claude-opus-5-max` → `claude-opus-5`, `gpt-5.6-sol-xhigh (codex-harness)` → `gpt-5.6-sol`), so bare model requests reach the synced intelligence layer instead of the static fallback table; the hand-maintained `MODEL_ALIAS_MAP` — which collapsed generations (`gpt-5.5` → `gpt-5`) — is gone ([#11506](https://github.com/diegosouzapw/OmniRoute/pull/11506)) diff --git a/src/lib/arenaEloSync.ts b/src/lib/arenaEloSync.ts index fa4284b2c5..427763b4a8 100644 --- a/src/lib/arenaEloSync.ts +++ b/src/lib/arenaEloSync.ts @@ -10,6 +10,8 @@ * On by default; opt out via Dashboard Feature Flags or ARENA_ELO_SYNC_ENABLED=false. */ +import { resolveScoresAs } from "@omniroute/open-sse/services/autoCombo/scoresAs.ts"; + import { isArenaEloSyncEnabled } from "@/shared/utils/featureFlags"; import { backupDbFile } from "./db/backup"; @@ -139,22 +141,6 @@ const VENDOR_PREFIXES = [ "ai21/", ] as const; -/** - * OmniRoute model aliases: canonical name → known aliases. - * Creates additional DB entries for each alias so that models - * are findable under any name OmniRoute uses internally. - */ -const MODEL_ALIAS_MAP: Record = { - "claude-opus-4-6-thinking": ["claude-opus-4", "anthropic/claude-opus-4"], - "claude-sonnet-4-5": ["claude-sonnet-4.5", "anthropic/claude-sonnet-4.5"], - "gpt-5.5": ["openai/gpt-5.5", "gpt-5"], - "gemini-3-flash": ["google/gemini-3-flash", "gemini-flash"], - "deepseek-r1": ["deepseek/deepseek-r1", "if/deepseek-r1"], - "kimi-k2-thinking": ["moonshot/kimi-k2"], - "qwen3-coder-plus": ["alibaba/qwen3-coder"], - "llama-4": ["meta/llama-4", "llama4"], -}; - /** Votes threshold for "high" confidence. */ const HIGH_CONFIDENCE_VOTES = 5000; @@ -196,17 +182,25 @@ function getEffectiveArenaEloSyncEnabled(): boolean { // ─── Model name normalization ──────────────────────────── +/** + * Trailing harness annotation the Arena leaderboard appends to some entries, + * e.g. "gpt-5.6-sol-xhigh (codex-harness)". It describes the scaffold the model + * was measured under, not the model id, so it never belongs in a stored key. + */ +const HARNESS_ANNOTATION_RE = /\s*\([^)]*\)\s*$/; + /** * Normalize a model name from the Arena leaderboard. * - * Lowercases the name and strips known vendor prefixes - * (e.g. "anthropic/claude-opus-4" → "claude-opus-4"). + * Lowercases the name, drops a trailing harness annotation + * ("gpt-5.6-sol-xhigh (codex-harness)" → "gpt-5.6-sol-xhigh") and strips known + * vendor prefixes ("anthropic/claude-opus-4" → "claude-opus-4"). * * @param rawName - The raw model name from the API response. * @returns The cleaned, lowercase model name. */ export function normalizeModelName(rawName: string): string { - let name = rawName.toLowerCase(); + let name = rawName.toLowerCase().replace(HARNESS_ANNOTATION_RE, ""); for (const prefix of VENDOR_PREFIXES) { if (name.startsWith(prefix)) { name = name.slice(prefix.length); @@ -295,7 +289,14 @@ function computeConfidence(votes: number): "high" | "medium" | "low" { * - "text" → default, review, documentation, debugging * - "code" → coding * - * Known OmniRoute model aliases are also expanded into additional entries. + * The leaderboard scores harness × effort combinations as separate entries + * ("claude-opus-5-high", "claude-opus-5-max"), so a request for the bare id + * would miss every row. Each variant that `resolveScoresAs()` resolves to a + * routable base id therefore also contributes a synthesized base row carrying + * the BEST task fit among that base's variants (the per-effort entries measure + * the same weights at different budgets — the model's ceiling is the max, and + * cost/latency are separate factors in the 12-factor score). An explicitly + * measured base row is never lowered by synthesis. * * @param data - Map of leaderboard category → Arena leaderboard data. * @returns Array of model intelligence entries ready for DB upsert. @@ -325,7 +326,7 @@ export function transformToModelIntelligence( const taskFit = 0.4 + 0.58 * ((model.score - minElo) / eloRange); for (const taskCategory of taskCategories) { - const entry: Omit = { + entries.push({ model: normalizedModel, category: taskCategory, source: "arena_elo", @@ -333,24 +334,63 @@ export function transformToModelIntelligence( eloRaw: model.score, confidence, expiresAt, - }; - entries.push(entry); - - // Expand known aliases - const aliases = MODEL_ALIAS_MAP[normalizedModel]; - if (aliases) { - for (const alias of aliases) { - entries.push({ - ...entry, - model: alias, - }); - } - } + }); } } } - return entries; + return withSynthesizedBaseRows(entries); +} + +/** + * Add one synthesized base row per (base, category) for every leaderboard entry + * that is an effort/alias variant of a routable catalog id. + * + * Runs as a pass over the finished variant rows so that all variants of a base + * are visible at once: the winner is the highest task fit, and it contributes + * its own `eloRaw` and `confidence` (no invented confidence label — the column's + * vocabulary stays high/medium/low). A base that the leaderboard measured + * directly keeps its own row unless a variant scored strictly higher. + * + * Resolution never guesses: `resolveScoresAs` returning `via: null` means the + * stripped base is not a catalog id, so nothing is synthesized for it. + * + * @param entries - Variant rows, one per (leaderboard entry, task category). + * @returns The same rows plus the synthesized base rows. + */ +function withSynthesizedBaseRows( + entries: Array> +): Array> { + const keyOf = (model: string, category: string) => `${model}|${category}`; + const indexByKey = new Map(); + entries.forEach((entry, index) => indexByKey.set(keyOf(entry.model, entry.category), index)); + + const best = new Map>(); + for (const entry of entries) { + const { base, via } = resolveScoresAs(entry.model); + if (via === null || base === entry.model) continue; + + const key = keyOf(base, entry.category); + const current = best.get(key); + if (!current || entry.score > current.score) { + best.set(key, { ...entry, model: base }); + } + } + + const result = [...entries]; + for (const [key, candidate] of best) { + const existingIndex = indexByKey.get(key); + if (existingIndex === undefined) { + result.push(candidate); + continue; + } + // The leaderboard measured the base itself — only a strictly better variant wins. + if (candidate.score > result[existingIndex].score) { + result[existingIndex] = candidate; + } + } + + return result; } // ─── Main sync function ────────────────────────────────── diff --git a/tests/unit/arena-elo-sync.test.ts b/tests/unit/arena-elo-sync.test.ts index 717963458d..0fc7b07ae9 100644 --- a/tests/unit/arena-elo-sync.test.ts +++ b/tests/unit/arena-elo-sync.test.ts @@ -40,6 +40,7 @@ const { } = await import("../../src/lib/arenaEloSync.ts"); const { setFeatureFlagOverride, removeFeatureFlagOverride } = await import("../../src/lib/db/featureFlags.ts"); +const { resolveScoresAs } = await import("../../open-sse/services/autoCombo/scoresAs.ts"); import type { ArenaLeaderboardData, @@ -328,7 +329,7 @@ describe("transformToModelIntelligence()", () => { } }); - it("expands model aliases for known models", () => { + it("does not copy a variant's score onto a different generation (MODEL_ALIAS_MAP removed, #11504)", () => { const data = makeLeaderboardMap({ text: [ makeModelEntry({ @@ -344,8 +345,10 @@ describe("transformToModelIntelligence()", () => { const models = entries.map((e) => e.model); assert.ok(models.includes("claude-opus-4-6-thinking")); - assert.ok(models.includes("claude-opus-4")); - assert.ok(models.includes("anthropic/claude-opus-4")); + // Was asserted the other way round while MODEL_ALIAS_MAP existed: it copied this + // score onto `claude-opus-4`, so a claude-opus-4 request read a 4.6-thinking ELO. + assert.ok(!models.includes("claude-opus-4")); + assert.ok(!models.includes("anthropic/claude-opus-4")); }); it("empty leaderboard → no entries", () => { @@ -409,6 +412,131 @@ describe("transformToModelIntelligence()", () => { }); }); +// ═══════════════════════════════════════════════════════════ +// 2b. Variant → base row synthesis (#11504) +// ═══════════════════════════════════════════════════════════ + +describe("transformToModelIntelligence() — base-model synthesis", () => { + it("normalizeModelName strips a trailing harness annotation", () => { + assert.strictEqual( + normalizeModelName("gpt-5.6-sol-xhigh (codex-harness)"), + "gpt-5.6-sol-xhigh" + ); + }); + + it("normalizeModelName strips vendor prefix and harness annotation together", () => { + assert.strictEqual( + normalizeModelName("OpenAI/GPT-5.6-Sol-xhigh (Codex-Harness)"), + "gpt-5.6-sol-xhigh" + ); + }); + + it("harness-annotated variant yields both the variant row and a base row", () => { + const data = makeLeaderboardMap({ + code: [ + makeModelEntry({ + model: "gpt-5.6-sol-xhigh (codex-harness)", + score: 1700, + votes: 5000, + rank: 1, + }), + ], + }); + + const entries = transformToModelIntelligence(data); + const coding = entries.filter((e) => e.category === "coding").map((e) => e.model); + + assert.ok(coding.includes("gpt-5.6-sol-xhigh"), "variant row is kept"); + assert.ok(coding.includes("gpt-5.6-sol"), "base row is synthesized"); + // `gpt-5.6` is a vendor alias of `gpt-5.6-sol`, resolved at lookup time — never stored here. + assert.ok(!coding.includes("gpt-5.6")); + }); + + it("base row carries the best variant's score and eloRaw", () => { + const data = makeLeaderboardMap({ + code: [ + makeModelEntry({ model: "claude-opus-5-max", score: 1691, votes: 5000, rank: 1 }), + makeModelEntry({ model: "claude-opus-5-high", score: 1663, votes: 5000, rank: 2 }), + ], + }); + + const entries = transformToModelIntelligence(data); + const base = entries.find((e) => e.model === "claude-opus-5" && e.category === "coding"); + const max = entries.find((e) => e.model === "claude-opus-5-max" && e.category === "coding"); + + assert.ok(base, "synthesized base row exists"); + assert.ok(max); + assert.strictEqual(base.eloRaw, 1691); + assert.strictEqual(base.score, max.score); + }); + + it("never lowers an explicitly measured base row", () => { + const data = makeLeaderboardMap({ + code: [ + makeModelEntry({ model: "claude-opus-5", score: 1700, votes: 5000, rank: 1 }), + makeModelEntry({ model: "claude-opus-5-max", score: 1691, votes: 5000, rank: 2 }), + ], + }); + + const entries = transformToModelIntelligence(data); + const base = entries.filter((e) => e.model === "claude-opus-5" && e.category === "coding"); + + assert.strictEqual(base.length, 1); + assert.strictEqual(base[0].eloRaw, 1700); + }); + + it("does not collapse generations (regression guard for the deleted alias map)", () => { + const data = makeLeaderboardMap({ + text: [makeModelEntry({ model: "openai/gpt-5.5", score: 1600, votes: 5000, rank: 1 })], + }); + + const entries = transformToModelIntelligence(data); + const models = entries.map((e) => e.model); + + assert.ok(models.includes("gpt-5.5")); + assert.ok(!models.includes("gpt-5"), "gpt-5.5 must never emit a gpt-5 row"); + }); + + it("synthesizes nothing when the stripped base is not a routable catalog id", () => { + // Catalog-anchored by construction: assert the premise instead of a frozen catalog fact. + assert.strictEqual(resolveScoresAs("grok-4.6-fast-high").via, null); + + const data = makeLeaderboardMap({ + code: [makeModelEntry({ model: "grok-4.6-fast-high", score: 1600, votes: 5000, rank: 1 })], + }); + + const entries = transformToModelIntelligence(data); + const models = entries.map((e) => e.model); + + assert.deepStrictEqual(models, ["grok-4.6-fast-high"]); + }); + + it("is idempotent and emits no duplicate (model, category) keys", () => { + const data = makeLeaderboardMap({ + code: [ + makeModelEntry({ model: "claude-opus-5-max", score: 1691, votes: 5000, rank: 1 }), + makeModelEntry({ model: "claude-opus-5-high", score: 1663, votes: 5000, rank: 2 }), + makeModelEntry({ + model: "gpt-5.6-sol-xhigh (codex-harness)", + score: 1700, + votes: 5000, + rank: 3, + }), + ], + }); + + const first = transformToModelIntelligence(data); + const second = transformToModelIntelligence(data); + + const strip = (entries: ReturnType) => + entries.map(({ expiresAt: _expiresAt, ...rest }) => rest); + assert.deepStrictEqual(strip(second), strip(first)); + + const keys = first.map((e) => `${e.model}|${e.category}`); + assert.strictEqual(new Set(keys).size, keys.length); + }); +}); + // ═══════════════════════════════════════════════════════════ // 3. fetchArenaLeaderboards() // ═══════════════════════════════════════════════════════════ @@ -693,7 +821,7 @@ describe("syncArenaElo()", () => { assert.ok(status.lastSyncModelCount > 0); }); - it("model aliases are stored in DB alongside canonical names", async () => { + it("stores no cross-generation alias rows in the DB (MODEL_ALIAS_MAP removed, #11504)", async () => { const textData = makeLeaderboardData( [ makeModelEntry({ @@ -718,7 +846,7 @@ describe("syncArenaElo()", () => { const models = entries.map((e) => String(e.model)); assert.ok(models.includes("claude-opus-4-6-thinking")); - assert.ok(models.includes("claude-opus-4")); + assert.ok(!models.includes("claude-opus-4")); }); });