fix(intelligence): synthesize base-model arena rows from effort/harness variants; drop MODEL_ALIAS_MAP (#11504) (#11506)

Validated in a combined 4-PR batch worktree off release/v3.8.51 tip (stacked on #11492, merged first).
- TDD-first: the new base-model-synthesis describe block failed 5/5 pre-fix, passes now
- Focused test: tests/unit/arena-elo-sync.test.ts — 56/56 pass, part of batch's 246/246 node:test + 126/126 vitest runs
- typecheck:core, file-size, changelog-integrity, complexity, cognitive-complexity, check:cycles — all OK

Thanks for closing the arena-lookup gap for harness/effort-annotated leaderboard rows and retiring MODEL_ALIAS_MAP's cross-generation score copying in favor of scoresAs + registry-owned aliases.

changelog.d/fixes/11506-arena-elo-base-rows.md

@@ -0,0 +1 @@
+- **fix(intelligence):** Arena ELO sync stores a synthesized base-model row for every effort/harness variant (`claude-opus-5-max` → `claude-opus-5`, `gpt-5.6-sol-xhigh (codex-harness)` → `gpt-5.6-sol`), so bare model requests reach the synced intelligence layer instead of the static fallback table; the hand-maintained `MODEL_ALIAS_MAP` — which collapsed generations (`gpt-5.5` → `gpt-5`) — is gone ([#11506](https://github.com/diegosouzapw/OmniRoute/pull/11506))

src/lib/arenaEloSync.ts

@@ -10,6 +10,8 @@
  * On by default; opt out via Dashboard Feature Flags or ARENA_ELO_SYNC_ENABLED=false.
  */
 
+import { resolveScoresAs } from "@omniroute/open-sse/services/autoCombo/scoresAs.ts";
+
 import { isArenaEloSyncEnabled } from "@/shared/utils/featureFlags";
 
 import { backupDbFile } from "./db/backup";
@@ -139,22 +141,6 @@ const VENDOR_PREFIXES = [
   "ai21/",
 ] as const;
 
-/**
- * OmniRoute model aliases: canonical name → known aliases.
- * Creates additional DB entries for each alias so that models
- * are findable under any name OmniRoute uses internally.
- */
-const MODEL_ALIAS_MAP: Record<string, string[]> = {
-  "claude-opus-4-6-thinking": ["claude-opus-4", "anthropic/claude-opus-4"],
-  "claude-sonnet-4-5": ["claude-sonnet-4.5", "anthropic/claude-sonnet-4.5"],
-  "gpt-5.5": ["openai/gpt-5.5", "gpt-5"],
-  "gemini-3-flash": ["google/gemini-3-flash", "gemini-flash"],
-  "deepseek-r1": ["deepseek/deepseek-r1", "if/deepseek-r1"],
-  "kimi-k2-thinking": ["moonshot/kimi-k2"],
-  "qwen3-coder-plus": ["alibaba/qwen3-coder"],
-  "llama-4": ["meta/llama-4", "llama4"],
-};
-
 /** Votes threshold for "high" confidence. */
 const HIGH_CONFIDENCE_VOTES = 5000;
 
@@ -196,17 +182,25 @@ function getEffectiveArenaEloSyncEnabled(): boolean {
 
 // ─── Model name normalization ────────────────────────────
 
+/**
+ * Trailing harness annotation the Arena leaderboard appends to some entries,
+ * e.g. "gpt-5.6-sol-xhigh (codex-harness)". It describes the scaffold the model
+ * was measured under, not the model id, so it never belongs in a stored key.
+ */
+const HARNESS_ANNOTATION_RE = /\s*\([^)]*\)\s*$/;
+
 /**
  * Normalize a model name from the Arena leaderboard.
  *
- * Lowercases the name and strips known vendor prefixes
- * (e.g. "anthropic/claude-opus-4" → "claude-opus-4").
+ * Lowercases the name, drops a trailing harness annotation
+ * ("gpt-5.6-sol-xhigh (codex-harness)" → "gpt-5.6-sol-xhigh") and strips known
+ * vendor prefixes ("anthropic/claude-opus-4" → "claude-opus-4").
  *
  * @param rawName - The raw model name from the API response.
  * @returns The cleaned, lowercase model name.
  */
 export function normalizeModelName(rawName: string): string {
-  let name = rawName.toLowerCase();
+  let name = rawName.toLowerCase().replace(HARNESS_ANNOTATION_RE, "");
   for (const prefix of VENDOR_PREFIXES) {
     if (name.startsWith(prefix)) {
       name = name.slice(prefix.length);
@@ -295,7 +289,14 @@ function computeConfidence(votes: number): "high" | "medium" | "low" {
  * - "text" → default, review, documentation, debugging
  * - "code" → coding
  *
- * Known OmniRoute model aliases are also expanded into additional entries.
+ * The leaderboard scores harness × effort combinations as separate entries
+ * ("claude-opus-5-high", "claude-opus-5-max"), so a request for the bare id
+ * would miss every row. Each variant that `resolveScoresAs()` resolves to a
+ * routable base id therefore also contributes a synthesized base row carrying
+ * the BEST task fit among that base's variants (the per-effort entries measure
+ * the same weights at different budgets — the model's ceiling is the max, and
+ * cost/latency are separate factors in the 12-factor score). An explicitly
+ * measured base row is never lowered by synthesis.
  *
  * @param data - Map of leaderboard category → Arena leaderboard data.
  * @returns Array of model intelligence entries ready for DB upsert.
@@ -325,7 +326,7 @@ export function transformToModelIntelligence(
       const taskFit = 0.4 + 0.58 * ((model.score - minElo) / eloRange);
 
       for (const taskCategory of taskCategories) {
-        const entry: Omit<ModelIntelligenceEntry, "syncedAt"> = {
+        entries.push({
           model: normalizedModel,
           category: taskCategory,
           source: "arena_elo",
@@ -333,24 +334,63 @@ export function transformToModelIntelligence(
           eloRaw: model.score,
           confidence,
           expiresAt,
-        };
-        entries.push(entry);
-
-        // Expand known aliases
-        const aliases = MODEL_ALIAS_MAP[normalizedModel];
-        if (aliases) {
-          for (const alias of aliases) {
-            entries.push({
-              ...entry,
-              model: alias,
-            });
-          }
-        }
+        });
       }
     }
   }
 
-  return entries;
+  return withSynthesizedBaseRows(entries);
+}
+
+/**
+ * Add one synthesized base row per (base, category) for every leaderboard entry
+ * that is an effort/alias variant of a routable catalog id.
+ *
+ * Runs as a pass over the finished variant rows so that all variants of a base
+ * are visible at once: the winner is the highest task fit, and it contributes
+ * its own `eloRaw` and `confidence` (no invented confidence label — the column's
+ * vocabulary stays high/medium/low). A base that the leaderboard measured
+ * directly keeps its own row unless a variant scored strictly higher.
+ *
+ * Resolution never guesses: `resolveScoresAs` returning `via: null` means the
+ * stripped base is not a catalog id, so nothing is synthesized for it.
+ *
+ * @param entries - Variant rows, one per (leaderboard entry, task category).
+ * @returns The same rows plus the synthesized base rows.
+ */
+function withSynthesizedBaseRows(
+  entries: Array<Omit<ModelIntelligenceEntry, "syncedAt">>
+): Array<Omit<ModelIntelligenceEntry, "syncedAt">> {
+  const keyOf = (model: string, category: string) => `${model}|${category}`;
+  const indexByKey = new Map<string, number>();
+  entries.forEach((entry, index) => indexByKey.set(keyOf(entry.model, entry.category), index));
+
+  const best = new Map<string, Omit<ModelIntelligenceEntry, "syncedAt">>();
+  for (const entry of entries) {
+    const { base, via } = resolveScoresAs(entry.model);
+    if (via === null || base === entry.model) continue;
+
+    const key = keyOf(base, entry.category);
+    const current = best.get(key);
+    if (!current || entry.score > current.score) {
+      best.set(key, { ...entry, model: base });
+    }
+  }
+
+  const result = [...entries];
+  for (const [key, candidate] of best) {
+    const existingIndex = indexByKey.get(key);
+    if (existingIndex === undefined) {
+      result.push(candidate);
+      continue;
+    }
+    // The leaderboard measured the base itself — only a strictly better variant wins.
+    if (candidate.score > result[existingIndex].score) {
+      result[existingIndex] = candidate;
+    }
+  }
+
+  return result;
 }
 
 // ─── Main sync function ──────────────────────────────────

tests/unit/arena-elo-sync.test.ts

@@ -40,6 +40,7 @@ const {
 } = await import("../../src/lib/arenaEloSync.ts");
 const { setFeatureFlagOverride, removeFeatureFlagOverride } =
   await import("../../src/lib/db/featureFlags.ts");
+const { resolveScoresAs } = await import("../../open-sse/services/autoCombo/scoresAs.ts");
 
 import type {
   ArenaLeaderboardData,
@@ -328,7 +329,7 @@ describe("transformToModelIntelligence()", () => {
     }
   });
 
-  it("expands model aliases for known models", () => {
+  it("does not copy a variant's score onto a different generation (MODEL_ALIAS_MAP removed, #11504)", () => {
     const data = makeLeaderboardMap({
       text: [
         makeModelEntry({
@@ -344,8 +345,10 @@ describe("transformToModelIntelligence()", () => {
     const models = entries.map((e) => e.model);
 
     assert.ok(models.includes("claude-opus-4-6-thinking"));
-    assert.ok(models.includes("claude-opus-4"));
-    assert.ok(models.includes("anthropic/claude-opus-4"));
+    // Was asserted the other way round while MODEL_ALIAS_MAP existed: it copied this
+    // score onto `claude-opus-4`, so a claude-opus-4 request read a 4.6-thinking ELO.
+    assert.ok(!models.includes("claude-opus-4"));
+    assert.ok(!models.includes("anthropic/claude-opus-4"));
   });
 
   it("empty leaderboard → no entries", () => {
@@ -409,6 +412,131 @@ describe("transformToModelIntelligence()", () => {
   });
 });
 
+// ═══════════════════════════════════════════════════════════
+// 2b. Variant → base row synthesis (#11504)
+// ═══════════════════════════════════════════════════════════
+
+describe("transformToModelIntelligence() — base-model synthesis", () => {
+  it("normalizeModelName strips a trailing harness annotation", () => {
+    assert.strictEqual(
+      normalizeModelName("gpt-5.6-sol-xhigh (codex-harness)"),
+      "gpt-5.6-sol-xhigh"
+    );
+  });
+
+  it("normalizeModelName strips vendor prefix and harness annotation together", () => {
+    assert.strictEqual(
+      normalizeModelName("OpenAI/GPT-5.6-Sol-xhigh (Codex-Harness)"),
+      "gpt-5.6-sol-xhigh"
+    );
+  });
+
+  it("harness-annotated variant yields both the variant row and a base row", () => {
+    const data = makeLeaderboardMap({
+      code: [
+        makeModelEntry({
+          model: "gpt-5.6-sol-xhigh (codex-harness)",
+          score: 1700,
+          votes: 5000,
+          rank: 1,
+        }),
+      ],
+    });
+
+    const entries = transformToModelIntelligence(data);
+    const coding = entries.filter((e) => e.category === "coding").map((e) => e.model);
+
+    assert.ok(coding.includes("gpt-5.6-sol-xhigh"), "variant row is kept");
+    assert.ok(coding.includes("gpt-5.6-sol"), "base row is synthesized");
+    // `gpt-5.6` is a vendor alias of `gpt-5.6-sol`, resolved at lookup time — never stored here.
+    assert.ok(!coding.includes("gpt-5.6"));
+  });
+
+  it("base row carries the best variant's score and eloRaw", () => {
+    const data = makeLeaderboardMap({
+      code: [
+        makeModelEntry({ model: "claude-opus-5-max", score: 1691, votes: 5000, rank: 1 }),
+        makeModelEntry({ model: "claude-opus-5-high", score: 1663, votes: 5000, rank: 2 }),
+      ],
+    });
+
+    const entries = transformToModelIntelligence(data);
+    const base = entries.find((e) => e.model === "claude-opus-5" && e.category === "coding");
+    const max = entries.find((e) => e.model === "claude-opus-5-max" && e.category === "coding");
+
+    assert.ok(base, "synthesized base row exists");
+    assert.ok(max);
+    assert.strictEqual(base.eloRaw, 1691);
+    assert.strictEqual(base.score, max.score);
+  });
+
+  it("never lowers an explicitly measured base row", () => {
+    const data = makeLeaderboardMap({
+      code: [
+        makeModelEntry({ model: "claude-opus-5", score: 1700, votes: 5000, rank: 1 }),
+        makeModelEntry({ model: "claude-opus-5-max", score: 1691, votes: 5000, rank: 2 }),
+      ],
+    });
+
+    const entries = transformToModelIntelligence(data);
+    const base = entries.filter((e) => e.model === "claude-opus-5" && e.category === "coding");
+
+    assert.strictEqual(base.length, 1);
+    assert.strictEqual(base[0].eloRaw, 1700);
+  });
+
+  it("does not collapse generations (regression guard for the deleted alias map)", () => {
+    const data = makeLeaderboardMap({
+      text: [makeModelEntry({ model: "openai/gpt-5.5", score: 1600, votes: 5000, rank: 1 })],
+    });
+
+    const entries = transformToModelIntelligence(data);
+    const models = entries.map((e) => e.model);
+
+    assert.ok(models.includes("gpt-5.5"));
+    assert.ok(!models.includes("gpt-5"), "gpt-5.5 must never emit a gpt-5 row");
+  });
+
+  it("synthesizes nothing when the stripped base is not a routable catalog id", () => {
+    // Catalog-anchored by construction: assert the premise instead of a frozen catalog fact.
+    assert.strictEqual(resolveScoresAs("grok-4.6-fast-high").via, null);
+
+    const data = makeLeaderboardMap({
+      code: [makeModelEntry({ model: "grok-4.6-fast-high", score: 1600, votes: 5000, rank: 1 })],
+    });
+
+    const entries = transformToModelIntelligence(data);
+    const models = entries.map((e) => e.model);
+
+    assert.deepStrictEqual(models, ["grok-4.6-fast-high"]);
+  });
+
+  it("is idempotent and emits no duplicate (model, category) keys", () => {
+    const data = makeLeaderboardMap({
+      code: [
+        makeModelEntry({ model: "claude-opus-5-max", score: 1691, votes: 5000, rank: 1 }),
+        makeModelEntry({ model: "claude-opus-5-high", score: 1663, votes: 5000, rank: 2 }),
+        makeModelEntry({
+          model: "gpt-5.6-sol-xhigh (codex-harness)",
+          score: 1700,
+          votes: 5000,
+          rank: 3,
+        }),
+      ],
+    });
+
+    const first = transformToModelIntelligence(data);
+    const second = transformToModelIntelligence(data);
+
+    const strip = (entries: ReturnType<typeof transformToModelIntelligence>) =>
+      entries.map(({ expiresAt: _expiresAt, ...rest }) => rest);
+    assert.deepStrictEqual(strip(second), strip(first));
+
+    const keys = first.map((e) => `${e.model}|${e.category}`);
+    assert.strictEqual(new Set(keys).size, keys.length);
+  });
+});
+
 // ═══════════════════════════════════════════════════════════
 // 3. fetchArenaLeaderboards()
 // ═══════════════════════════════════════════════════════════
@@ -693,7 +821,7 @@ describe("syncArenaElo()", () => {
     assert.ok(status.lastSyncModelCount > 0);
   });
 
-  it("model aliases are stored in DB alongside canonical names", async () => {
+  it("stores no cross-generation alias rows in the DB (MODEL_ALIAS_MAP removed, #11504)", async () => {
     const textData = makeLeaderboardData(
       [
         makeModelEntry({
@@ -718,7 +846,7 @@ describe("syncArenaElo()", () => {
     const models = entries.map((e) => String(e.model));
 
     assert.ok(models.includes("claude-opus-4-6-thinking"));
-    assert.ok(models.includes("claude-opus-4"));
+    assert.ok(!models.includes("claude-opus-4"));
   });
 });