From 78d10da84c09be8169de90a35823523a21099e57 Mon Sep 17 00:00:00 2001 From: MumuTW <42820974+MumuTW@users.noreply.github.com> Date: Wed, 26 Aug 2026 00:51:27 +0800 Subject: [PATCH] fix(autoCombo): inherit task fitness from base model for effort/alias variants (#11489) (#11492) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validated in a combined 4-PR batch worktree off release/v3.8.51 tip. - Focused tests: scoresAs-11489.test.ts + task-fitness-scores-as-11489.test.ts + autoCombo.test.ts — pass as part of batch's 126/126 vitest + 246/246 node:test runs - typecheck:core, file-size, changelog-integrity, complexity, cognitive-complexity, check:cycles — all OK Thanks for closing the gap where effort/alias variants fell back to the wildcard score instead of inheriting their base model's task fitness. --- .../fixes/11492-taskfitness-scores-as.md | 1 + open-sse/config/providerModels.ts | 42 +++++++++ .../config/providers/registry/cursor/index.ts | 64 ++++++++++--- .../config/providers/registry/openai/index.ts | 5 +- open-sse/config/providers/shared.ts | 11 +++ .../autoCombo/__tests__/autoCombo.test.ts | 17 ++-- .../__tests__/scoresAs-11489.test.ts | 92 +++++++++++++++++++ open-sse/services/autoCombo/scoresAs.ts | 85 +++++++++++++++++ open-sse/services/autoCombo/taskFitness.ts | 62 ++++++++----- .../unit/task-fitness-scores-as-11489.test.ts | 53 +++++++++++ 10 files changed, 386 insertions(+), 46 deletions(-) create mode 100644 changelog.d/fixes/11492-taskfitness-scores-as.md create mode 100644 open-sse/services/autoCombo/__tests__/scoresAs-11489.test.ts create mode 100644 open-sse/services/autoCombo/scoresAs.ts create mode 100644 tests/unit/task-fitness-scores-as-11489.test.ts diff --git a/changelog.d/fixes/11492-taskfitness-scores-as.md b/changelog.d/fixes/11492-taskfitness-scores-as.md new file mode 100644 index 0000000000..e1b02d4871 --- /dev/null +++ b/changelog.d/fixes/11492-taskfitness-scores-as.md @@ -0,0 +1 @@ +- **fix(autoCombo):** effort/alias model variants (`gpt-5.6-sol-xhigh`, `gpt-5.6`, cursor's `claude-4.6-opus-high`) inherit their base model's task fitness instead of falling to the wildcard 0.5 ([#11492](https://github.com/diegosouzapw/OmniRoute/pull/11492)) diff --git a/open-sse/config/providerModels.ts b/open-sse/config/providerModels.ts index 5cc9833871..3e6dffbf99 100644 --- a/open-sse/config/providerModels.ts +++ b/open-sse/config/providerModels.ts @@ -145,6 +145,48 @@ function getGlobalModel(modelId: string): RegistryModel | undefined { return bestMatch; } +/** + * Exact-id catalog lookup: the registry entry for `modelId` across every + * provider, or for its basename once a `vendor/` prefix is stripped. + * + * This is `getGlobalModel`'s steps 1–2 only. Step 3 (a `startsWith` substring + * scan) deliberately guesses at a base model, which is exactly what a + * catalog-anchor check must not do — `resolveScoresAs` (#11489) uses this to + * verify that a suffix-stripped base is a REAL routable id before inheriting + * its quality scores. + */ +export function findRegistryModelById(modelId: string): RegistryModel | undefined { + if (typeof modelId !== "string" || modelId.length === 0) return undefined; + for (const models of Object.values(PROVIDER_MODELS)) { + const found = models.find((m) => m.id === modelId); + if (found) return found; + } + const basename = modelId.split("/").pop() || modelId; + if (basename === modelId) return undefined; + for (const models of Object.values(PROVIDER_MODELS)) { + const found = models.find((m) => m.id === basename); + if (found) return found; + } + return undefined; +} + +/** + * The `scoresAs` target declared for `modelId`, if any (#11489). Scans every + * provider that ships the id rather than stopping at the first hit, so an + * unannotated duplicate of the same id in another provider's catalog cannot + * shadow the entry that actually declares the relation. + */ +export function findRegistryScoresAs(modelId: string): string | undefined { + if (typeof modelId !== "string" || modelId.length === 0) return undefined; + const basename = modelId.split("/").pop() || modelId; + for (const models of Object.values(PROVIDER_MODELS)) { + for (const m of models) { + if ((m.id === modelId || m.id === basename) && m.scoresAs) return m.scoresAs; + } + } + return undefined; +} + export function getProviderModel(aliasOrId: string, modelId: string): RegistryModel | undefined { const models = PROVIDER_MODELS[aliasOrId]; if (!models) return getGlobalModel(modelId); diff --git a/open-sse/config/providers/registry/cursor/index.ts b/open-sse/config/providers/registry/cursor/index.ts index 7d54a2252f..cc5a78f8ce 100644 --- a/open-sse/config/providers/registry/cursor/index.ts +++ b/open-sse/config/providers/registry/cursor/index.ts @@ -26,10 +26,30 @@ export const cursorProvider: RegistryEntry = { { id: "gpt-5.3-codex-spark-preview", name: "GPT 5.3 Codex Spark Preview" }, { id: "gpt-5.3-codex-spark-preview-high", name: "GPT 5.3 Codex Spark Preview High" }, { id: "gpt-5.3-codex-spark-preview-xhigh", name: "GPT 5.3 Codex Spark Preview XHigh" }, - { id: "claude-4.6-opus-high-thinking-fast", name: "Claude 4.6 Opus High Thinking Fast" }, - { id: "claude-4.6-opus-max-thinking-fast", name: "Claude 4.6 Opus Max Thinking Fast" }, - { id: "claude-4.6-sonnet-medium", name: "Claude 4.6 Sonnet Medium" }, - { id: "claude-4.6-sonnet-medium-thinking", name: "Claude 4.6 Sonnet Medium Thinking" }, + // #11489: cursor/agy spell Claude ids - ("claude-4.6-opus-high"); + // the effort splitter strips those to "claude-4.6-opus", which is not a catalog id. + // `scoresAs` points each at the canonical - spelling so quality + // scores are inherited. Operational metadata stays on these entries. + { + id: "claude-4.6-opus-high-thinking-fast", + name: "Claude 4.6 Opus High Thinking Fast", + scoresAs: "claude-opus-4-6", + }, + { + id: "claude-4.6-opus-max-thinking-fast", + name: "Claude 4.6 Opus Max Thinking Fast", + scoresAs: "claude-opus-4-6", + }, + { + id: "claude-4.6-sonnet-medium", + name: "Claude 4.6 Sonnet Medium", + scoresAs: "claude-sonnet-4-6", + }, + { + id: "claude-4.6-sonnet-medium-thinking", + name: "Claude 4.6 Sonnet Medium Thinking", + scoresAs: "claude-sonnet-4-6", + }, { id: "gemini-3.1-pro", name: "Gemini 3.1 Pro" }, { id: "gemini-3.7-flash", name: "Gemini 3.7 Flash" }, { id: "gemini-3-flash", name: "Gemini 3 Flash" }, @@ -184,12 +204,24 @@ export const cursorProvider: RegistryEntry = { { id: "gpt-5.4-high-fast", name: "GPT-5.4 High Fast" }, { id: "gpt-5.4-xhigh", name: "GPT-5.4 1M Extra High" }, { id: "gpt-5.4-xhigh-fast", name: "GPT-5.4 Extra High Fast" }, - { id: "claude-4.6-opus-high", name: "Opus 4.6 1M" }, - { id: "claude-4.6-opus-max", name: "Opus 4.6 1M Max" }, - { id: "claude-4.6-opus-high-thinking", name: "Opus 4.6 1M Thinking" }, - { id: "claude-4.6-opus-max-thinking", name: "Opus 4.6 1M Max Thinking" }, - { id: "claude-4.5-opus-high", name: "Opus 4.5" }, - { id: "claude-4.5-opus-high-thinking", name: "Opus 4.5 Thinking" }, + // #11489: cursor/agy spell Claude ids - ("claude-4.6-opus-high"); + // the effort splitter strips those to "claude-4.6-opus", which is not a catalog id. + // `scoresAs` points each at the canonical - spelling so quality + // scores are inherited. Operational metadata stays on these entries. + { id: "claude-4.6-opus-high", name: "Opus 4.6 1M", scoresAs: "claude-opus-4-6" }, + { id: "claude-4.6-opus-max", name: "Opus 4.6 1M Max", scoresAs: "claude-opus-4-6" }, + { + id: "claude-4.6-opus-high-thinking", + name: "Opus 4.6 1M Thinking", + scoresAs: "claude-opus-4-6", + }, + { + id: "claude-4.6-opus-max-thinking", + name: "Opus 4.6 1M Max Thinking", + scoresAs: "claude-opus-4-6", + }, + { id: "claude-4.5-opus-high", name: "Opus 4.5", scoresAs: "claude-opus-4-5" }, + { id: "claude-4.5-opus-high-thinking", name: "Opus 4.5 Thinking", scoresAs: "claude-opus-4-5" }, { id: "gpt-5.2-low", name: "GPT-5.2 Low" }, { id: "gpt-5.2-low-fast", name: "GPT-5.2 Low Fast" }, { id: "gpt-5.2-fast", name: "GPT-5.2 Fast" }, @@ -223,13 +255,17 @@ export const cursorProvider: RegistryEntry = { { id: "gpt-5.4-nano-medium", name: "GPT-5.4 Nano" }, { id: "gpt-5.4-nano-high", name: "GPT-5.4 Nano High" }, { id: "gpt-5.4-nano-xhigh", name: "GPT-5.4 Nano Extra High" }, - { id: "claude-4.5-sonnet", name: "Sonnet 4.5" }, - { id: "claude-4.5-sonnet-thinking", name: "Sonnet 4.5 Thinking" }, + { id: "claude-4.5-sonnet", name: "Sonnet 4.5", scoresAs: "claude-sonnet-4-5" }, + { + id: "claude-4.5-sonnet-thinking", + name: "Sonnet 4.5 Thinking", + scoresAs: "claude-sonnet-4-5", + }, { id: "gpt-5.1-low", name: "GPT-5.1 Low" }, { id: "gpt-5.1", name: "GPT-5.1" }, { id: "gpt-5.1-high", name: "GPT-5.1 High" }, - { id: "claude-4-sonnet", name: "Sonnet 4" }, - { id: "claude-4-sonnet-thinking", name: "Sonnet 4 Thinking" }, + { id: "claude-4-sonnet", name: "Sonnet 4", scoresAs: "claude-sonnet-4" }, + { id: "claude-4-sonnet-thinking", name: "Sonnet 4 Thinking", scoresAs: "claude-sonnet-4" }, { id: "gpt-5-mini", name: "GPT-5 Mini" }, { id: "kimi-k3-low", name: "Kimi K3 Low" }, { id: "kimi-k3-max", name: "Kimi K3" }, diff --git a/open-sse/config/providers/registry/openai/index.ts b/open-sse/config/providers/registry/openai/index.ts index 60a276a948..f2cd5dc2d5 100644 --- a/open-sse/config/providers/registry/openai/index.ts +++ b/open-sse/config/providers/registry/openai/index.ts @@ -12,7 +12,10 @@ export const openaiProvider: RegistryEntry = { authHeader: "bearer", defaultContextLength: 128000, models: [ - { id: "gpt-5.6", name: "GPT-5.6", ...GPT_5_6_API_CAPABILITIES }, + // #11489: per OpenAI's model reference `gpt-5.6` is an ALIAS of `gpt-5.6-sol`, + // not a distinct model — quality scores point forward, which no suffix + // stripper can express. Siblings `-terra`/`-luna` are their own models. + { id: "gpt-5.6", name: "GPT-5.6", scoresAs: "gpt-5.6-sol", ...GPT_5_6_API_CAPABILITIES }, { id: "gpt-5.6-sol", name: "GPT-5.6 Sol", ...GPT_5_6_API_CAPABILITIES }, { id: "gpt-5.6-terra", name: "GPT-5.6 Terra", ...GPT_5_6_API_CAPABILITIES }, { id: "gpt-5.6-luna", name: "GPT-5.6 Luna", ...GPT_5_6_API_CAPABILITIES }, diff --git a/open-sse/config/providers/shared.ts b/open-sse/config/providers/shared.ts index 16c0c09b41..6c67c4eb5f 100644 --- a/open-sse/config/providers/shared.ts +++ b/open-sse/config/providers/shared.ts @@ -83,6 +83,17 @@ export interface RegistryModel { /** Per-model upstream header-response timeout override — precedes * `RegistryEntry.timeoutMs` and the global `FETCH_TIMEOUT_MS` (#6354). */ timeoutMs?: number; + /** + * Id whose QUALITY scores (task fitness / arena / user overrides) this id + * inherits (#11489). Operational fields (timeoutMs, cost, context) stay on + * this entry. One hop only; the target must itself be a catalog id. + * + * Only for relations suffix-stripping cannot express: forward vendor aliases + * (`gpt-5.6` → `gpt-5.6-sol`) and cross-provider spellings of the same model + * (cursor's `claude-4.6-opus-high` → `claude-opus-4-6`). Plain effort/`-free` + * variants are derived by `resolveScoresAs` and need no entry here. + */ + scoresAs?: string; } // Reasoning models reject temperature, top_p, penalties, logprobs, n. diff --git a/open-sse/services/autoCombo/__tests__/autoCombo.test.ts b/open-sse/services/autoCombo/__tests__/autoCombo.test.ts index efeae64bcc..96c2dbc42e 100644 --- a/open-sse/services/autoCombo/__tests__/autoCombo.test.ts +++ b/open-sse/services/autoCombo/__tests__/autoCombo.test.ts @@ -101,10 +101,15 @@ describe("Task Fitness", () => { // The fix: getTaskFitnessWithSource strips a trailing "-free" suffix // and re-queries arena_elo with the base id. We seed an arena_elo // row directly via the DB module, look up the free variant, and - // assert the alias path returns the base score with source - // "arena_elo_free_alias". - const baseId = "alias-base-test-4517"; - const freeId = "alias-base-test-4517-free"; + // assert the alias path returns the base score, now tagged + // "arena_elo:inherited" by the shared resolver (#11489). + // + // #11489 also made the base-id lookup CATALOG-ANCHORED, so this uses a + // real catalog pair instead of the synthetic ids it was written with: + // an id that resolves to a base no provider actually ships is a ghost, + // and inheriting a score for it was never meaningful. + const baseId = "mimo-v2.5"; + const freeId = "mimo-v2.5-free"; const { upsertModelIntelligence, deleteModelIntelligence } = await import("../../../../src/lib/db/modelIntelligence.ts"); // Seed arena_elo on the base id only — no row exists for the free id. @@ -121,9 +126,9 @@ describe("Task Fitness", () => { try { const result = getTaskFitnessWithSource(freeId, "coding"); // Without the fix: result.source would be "wildcard_boost" (0.5 default). - // With the fix: result.source is "arena_elo_free_alias" with score 0.42. + // With the fix: result.source is "arena_elo:inherited" with score 0.42. expect(result.score).toBeCloseTo(0.42, 5); - expect(result.source).toBe("arena_elo_free_alias"); + expect(result.source).toBe("arena_elo:inherited"); } finally { deleteModelIntelligence(baseId, "arena_elo", "coding"); invalidateFitnessCache(); diff --git a/open-sse/services/autoCombo/__tests__/scoresAs-11489.test.ts b/open-sse/services/autoCombo/__tests__/scoresAs-11489.test.ts new file mode 100644 index 0000000000..b1d8d31b36 --- /dev/null +++ b/open-sse/services/autoCombo/__tests__/scoresAs-11489.test.ts @@ -0,0 +1,92 @@ +/** + * TDD regression for #11489: auto-combo task fitness scored every catalog id by + * exact string match, while dispatch already resolves `-` ids back + * to a base model. A variant like `gpt-5.6-sol-xhigh` missed every DB layer and + * landed on the wildcard 0.5, while its base model was scored properly. + * + * `resolveScoresAs` is the shared, catalog-anchored resolver the fitness chain + * consults on a miss. It resolves in three tiers and NEVER guesses: + * 1. an explicit `scoresAs` declared on the registry entry (one hop only), + * 2. a trailing reasoning-effort suffix stripped by an EXISTING splitter, + * 3. a trailing `-free` tier marker, + * and tiers 2–3 only accept a base that is itself a routable catalog id — which + * is what keeps `qwen3.7-max` (where `-max` is the model, not an effort) and + * `grok-4.6-fast-high` (whose stripped base is not in the catalog) unresolved. + */ +import { describe, it, expect } from "vitest"; +import { resolveScoresAs } from "../scoresAs"; +import { findRegistryModelById } from "../../../config/providerModels"; + +describe("#11489 resolveScoresAs", () => { + it("strips a reasoning-effort suffix when the base is a catalog id", () => { + expect(resolveScoresAs("gpt-5.6-sol-xhigh")).toEqual({ + base: "gpt-5.6-sol", + via: "effort-suffix", + }); + }); + + it("follows a vendor alias that points FORWARD (gpt-5.6 is an alias of gpt-5.6-sol)", () => { + // No suffix-stripper can produce this direction; it is registry data. + expect(resolveScoresAs("gpt-5.6")).toEqual({ base: "gpt-5.6-sol", via: "explicit" }); + }); + + it("leaves sibling models unresolved (gpt-5.6-luna is its own model)", () => { + const result = resolveScoresAs("gpt-5.6-luna"); + expect(result.via).toBeNull(); + expect(result.base).toBe("gpt-5.6-luna"); + expect(result.base).not.toBe("gpt-5.6"); + expect(result.base).not.toBe("gpt-5.6-sol"); + }); + + it("rejects an effort-stripped base that is not itself a catalog id", () => { + // `grok-4.6-fast-high` strips to `grok-4.6-fast`, a ghost id on today's + // catalog. Asserted against the catalog rather than hardcoded so the test + // stays true if a provider ever ships the base as a routable id. + const result = resolveScoresAs("grok-4.6-fast-high"); + if (findRegistryModelById("grok-4.6-fast")) { + expect(result).toEqual({ base: "grok-4.6-fast", via: "effort-suffix" }); + } else { + expect(result).toEqual({ base: "grok-4.6-fast-high", via: null }); + } + }); + + it("does not treat a trailing '-max' that is part of the model name as an effort", () => { + // `qwen3.7-max` IS the model; `qwen3.7` does not exist. + expect(resolveScoresAs("qwen3.7-max")).toEqual({ base: "qwen3.7-max", via: null }); + }); + + it("never collapses a model onto its family", () => { + expect(resolveScoresAs("claude-sonnet-5")).toEqual({ base: "claude-sonnet-5", via: null }); + }); + + it("resolves the cursor/agy spelling of a Claude model to its canonical id", () => { + // `claude-4.6-opus-high` strips to `claude-4.6-opus`, which is not a catalog + // id — the canonical spelling is `claude-opus-4-6`. Explicit registry data. + expect(resolveScoresAs("claude-4.6-opus-high")).toEqual({ + base: "claude-opus-4-6", + via: "explicit", + }); + expect(resolveScoresAs("claude-4.6-sonnet-medium")).toEqual({ + base: "claude-sonnet-4-6", + via: "explicit", + }); + }); + + it("strips a '-free' tier marker when the paid base is a catalog id", () => { + expect(findRegistryModelById("mimo-v2.5")).toBeTruthy(); + expect(resolveScoresAs("mimo-v2.5-free")).toEqual({ base: "mimo-v2.5", via: "free-suffix" }); + }); + + it("leaves a '-free' id unresolved when no paid base exists in the catalog", () => { + expect(findRegistryModelById("ox-alpha")).toBeFalsy(); + expect(resolveScoresAs("ox-alpha-free")).toEqual({ base: "ox-alpha-free", via: null }); + }); + + it("returns the id unchanged for junk input", () => { + expect(resolveScoresAs("totally-unknown-model")).toEqual({ + base: "totally-unknown-model", + via: null, + }); + expect(resolveScoresAs("")).toEqual({ base: "", via: null }); + }); +}); diff --git a/open-sse/services/autoCombo/scoresAs.ts b/open-sse/services/autoCombo/scoresAs.ts new file mode 100644 index 0000000000..cca584b5ef --- /dev/null +++ b/open-sse/services/autoCombo/scoresAs.ts @@ -0,0 +1,85 @@ +/** + * scoresAs — resolve a catalog id to the id whose QUALITY scores it inherits (#11489). + * + * Dispatch already treats `-` ids as variants of a base model + * (`splitClaudeEffortSuffix`, `splitCodexReasoningSuffix` run on the incoming + * request model). Auto-combo's task fitness did not: it scored every catalog id + * by exact string match, so a variant like `gpt-5.6-sol-xhigh` missed every DB + * layer and landed on the wildcard 0.5 while its base model was scored properly. + * + * This module is the single seam that closes that gap. Three tiers, in order: + * + * 1. `explicit` — the registry entry declares `scoresAs`. For relations + * suffix-stripping cannot express: forward vendor aliases + * (`gpt-5.6` IS an alias of `gpt-5.6-sol`, per OpenAI's + * model reference) and cross-provider spellings of the + * same model (`claude-4.6-opus-high` → `claude-opus-4-6`). + * 2. `effort-suffix` — a trailing reasoning-effort token stripped by one of the + * EXISTING dispatch splitters. No new regex is introduced + * here; a fourth pattern would be a fourth place to get + * the same fact wrong (cf. the #8603 shadowing defect). + * 3. `free-suffix` — a trailing `-free` tier marker, so a free-tier variant + * picks up the benchmark of its paid counterpart (#4517, + * previously a standalone arena_elo-only special case in + * `taskFitness.ts`, now folded in and extended to + * `user_override` too). + * + * Tiers 2 and 3 are CATALOG-ANCHORED: a stripped base is accepted only when it + * is itself a routable catalog id. Without that guard, 57 of the catalog's 201 + * strippable effort ids resolve to a base that does not exist — `qwen3.7-max` + * would inherit from a phantom `qwen3.7` (`-max` is part of the model name + * here, not an effort), `grok-4.6-fast-high` from a phantom `grok-4.6-fast`, + * and `extra-high` from `extra`. Resolution never guesses: anything the three + * tiers cannot justify comes back unresolved. + * + * One hop only — a `scoresAs` target is not itself re-resolved. + */ +import { findRegistryModelById, findRegistryScoresAs } from "../../config/providerModels.ts"; +import { splitClaudeEffortSuffix } from "../../config/providerModels.ts"; +import { splitCodexReasoningSuffix } from "../../executors/codex/reasoningSuffix.ts"; + +/** How a base id was reached; `null` means "not resolved — score the id as given". */ +export type ScoresAsVia = "explicit" | "effort-suffix" | "free-suffix" | null; + +export interface ScoresAsResolution { + /** The id whose quality scores apply. Equals the input when `via` is `null`. */ + base: string; + via: ScoresAsVia; +} + +/** Suffix marking a free-tier variant of a paid model (e.g. `mimo-v2.5-free`). */ +const FREE_SUFFIX = "-free"; + +/** Dispatch-time effort splitters, reused verbatim. Order is not significant: + * both are catalog-anchored below, so a wrong strip is rejected either way. */ +const EFFORT_SPLITTERS = [splitClaudeEffortSuffix, splitCodexReasoningSuffix] as const; + +export function resolveScoresAs(modelId: string): ScoresAsResolution { + const unresolved: ScoresAsResolution = { base: modelId, via: null }; + if (typeof modelId !== "string" || modelId.length === 0) return unresolved; + + // 1. Explicit registry declaration. One hop: the target must itself be a + // catalog id, and its own `scoresAs` (if any) is deliberately not followed. + const declared = findRegistryScoresAs(modelId); + if (declared && declared !== modelId && findRegistryModelById(declared)) { + return { base: declared, via: "explicit" }; + } + + // 2. Reasoning-effort suffix, catalog-anchored. + for (const split of EFFORT_SPLITTERS) { + const base = split(modelId).baseModel; + if (base && base !== modelId && findRegistryModelById(base)) { + return { base, via: "effort-suffix" }; + } + } + + // 3. Free-tier suffix, catalog-anchored. + if (modelId.endsWith(FREE_SUFFIX)) { + const base = modelId.slice(0, -FREE_SUFFIX.length); + if (base.length > 0 && findRegistryModelById(base)) { + return { base, via: "free-suffix" }; + } + } + + return unresolved; +} diff --git a/open-sse/services/autoCombo/taskFitness.ts b/open-sse/services/autoCombo/taskFitness.ts index 6489583ae5..83899365e3 100644 --- a/open-sse/services/autoCombo/taskFitness.ts +++ b/open-sse/services/autoCombo/taskFitness.ts @@ -7,6 +7,9 @@ * Resolution chain (highest → lowest priority): * 1. User override — DB `model_intelligence` where source='user_override' * 2. Arena ELO — DB `model_intelligence` where source='arena_elo' + * 2b. Layers 1-2 retried against the base model this id inherits quality scores + * from, when `resolveScoresAs` resolves one (#11489). Reported as + * `:inherited`. * 3. Models.dev tier — derived from `model_capabilities` table capability data * 4. Static FITNESS_TABLE — existing hardcoded lookup (current behavior) * 5. Wildcard boosts — existing pattern matching boosts (current behavior) @@ -20,6 +23,7 @@ import { setUserFitnessOverrideEntry, deleteUserFitnessOverrideEntry, } from "../../../src/lib/db/modelIntelligence.ts"; +import { resolveScoresAs } from "./scoresAs.ts"; const FITNESS_TABLE: Record> = { coding: { @@ -353,20 +357,23 @@ export function getTaskFitnessWithSource( return { score: userOverride, source: "user_override" }; } - // Try arena_elo with the literal model id first (e.g. "mimo-v2.5"). If that's - // a miss and the model id carries a "-free" suffix (e.g. "mimo-v2.5-free"), - // try the un-suffixed base id so free-tier variants inherit the arena_elo - // score of their paid counterpart. This is what operators expect: the - // upstream's `mimo-v2.5` is benchmarked once, and `mimo-v2.5-free` should - // pick up the same signal rather than falling through to the wildcard 0.5 - // and losing every free-vs-paid comparison. const arenaElo = queryModelIntelligence(normalizedModel, normalizedTask, "arena_elo"); if (arenaElo !== null) { return { score: arenaElo, source: "arena_elo" }; } - const arenaEloBase = lookupFreeAliasArenaElo(normalizedModel, normalizedTask); - if (arenaEloBase !== null) { - return { score: arenaEloBase, source: "arena_elo_free_alias" }; + + // Layers 1-2, retried against the base model this id inherits quality from + // (#11489). Every DB-backed source publishes scores for BASE models only, so + // a variant id — an effort suffix (`gpt-5.6-sol-xhigh`), a vendor alias + // (`gpt-5.6`), a `-free` tier marker (`mimo-v2.5-free`, #4517) — misses both + // literal lookups and used to fall all the way to the wildcard 0.5, losing + // every comparison against a base model that happens to be benchmarked. + // The score is inherited VERBATIM: the 12-factor scoring already prices cost + // and latency per variant, so there is no basis for inventing an effort + // delta. `:inherited` keeps the indirection visible to callers. + const inherited = lookupInheritedFitness(normalizedModel, normalizedTask); + if (inherited !== null) { + return inherited; } const tierScore = getModelsDevTierFitness(normalizedModel, normalizedTask); @@ -382,24 +389,29 @@ export function getTaskFitnessWithSource( return { score: lookupWildcardBoosts(normalizedModel, normalizedTask), source: "wildcard_boost" }; } -/** Suffix used to mark free-tier model variants (e.g. "mimo-v2.5-free"). */ -const FREE_SUFFIX = "-free"; - /** - * Strip a trailing "-free" suffix from the model id and re-query arena_elo. - * Returns `null` when the original id has no "-free" suffix, when the base id - * is identical to the original, or when no arena_elo row exists for the base. + * Re-run the two DB-backed layers against the base model `normalizedModel` + * inherits quality scores from (#11489). Returns `null` when the id resolves to + * itself (nothing to inherit) or when the base has no row either. * - * Examples: - * "mimo-v2.5-free" → look up "mimo-v2.5" - * "deepseek-v4-flash-free" → look up "deepseek-v4-flash" - * "big-pickle" → no "-free" suffix → return null (skip) + * This subsumes the former `-free` arena_elo special case (#4517) and extends + * it: the `-free` base is now consulted for `user_override` too, and the same + * indirection now covers effort suffixes and vendor aliases. `resolveScoresAs` + * is catalog-anchored, so a base that is not a routable id is never queried. */ -function lookupFreeAliasArenaElo(normalizedModel: string, normalizedTask: string): number | null { - if (!normalizedModel.endsWith(FREE_SUFFIX)) return null; - const baseId = normalizedModel.slice(0, -FREE_SUFFIX.length); - if (baseId.length === 0 || baseId === normalizedModel) return null; - return queryModelIntelligence(baseId, normalizedTask, "arena_elo"); +function lookupInheritedFitness( + normalizedModel: string, + normalizedTask: string +): { score: number; source: string } | null { + const { base, via } = resolveScoresAs(normalizedModel); + if (via === null || base === normalizedModel) return null; + const normalizedBase = base.toLowerCase(); + + for (const source of ["user_override", "arena_elo"] as const) { + const score = queryModelIntelligence(normalizedBase, normalizedTask, source); + if (score !== null) return { score, source: `${source}:inherited` }; + } + return null; } export function setUserFitnessOverride(model: string, category: string, score: number): void { diff --git a/tests/unit/task-fitness-scores-as-11489.test.ts b/tests/unit/task-fitness-scores-as-11489.test.ts new file mode 100644 index 0000000000..03f85f24f5 --- /dev/null +++ b/tests/unit/task-fitness-scores-as-11489.test.ts @@ -0,0 +1,53 @@ +import { describe, it, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { + getTaskFitnessWithSource, + setUserFitnessOverride, + clearUserFitnessOverride, + invalidateFitnessCache, +} from "../../open-sse/services/autoCombo/taskFitness.ts"; +import { resetDbInstance } from "../../src/lib/db/core.ts"; + +/** + * TDD regression for #11489: an effort variant must inherit its base model's + * task-fitness score instead of falling through to the wildcard 0.5, and a + * sibling model must NOT inherit it. + */ +describe("taskFitness inherits from the scoresAs base (#11489)", () => { + before(() => { + setUserFitnessOverride("gpt-5.6-sol", "coding", 0.97); + invalidateFitnessCache(); + }); + + after(() => { + clearUserFitnessOverride("gpt-5.6-sol", "coding"); + invalidateFitnessCache(); + resetDbInstance(); + }); + + it("scores the base model from its own user_override row", () => { + const result = getTaskFitnessWithSource("gpt-5.6-sol", "coding"); + assert.equal(result.score, 0.97); + assert.equal(result.source, "user_override"); + }); + + it("inherits the base score for an effort variant, tagged as inherited", () => { + // Before the fix this returned the wildcard 0.5. + const result = getTaskFitnessWithSource("gpt-5.6-sol-xhigh", "coding"); + assert.equal(result.score, 0.97); + assert.equal(result.source, "user_override:inherited"); + }); + + it("inherits through an explicit forward vendor alias (gpt-5.6 -> gpt-5.6-sol)", () => { + const result = getTaskFitnessWithSource("gpt-5.6", "coding"); + assert.equal(result.score, 0.97); + assert.equal(result.source, "user_override:inherited"); + }); + + it("does NOT leak the base score to a sibling model", () => { + const result = getTaskFitnessWithSource("gpt-5.6-luna", "coding"); + assert.notEqual(result.source, "user_override"); + assert.notEqual(result.source, "user_override:inherited"); + assert.notEqual(result.score, 0.97); + }); +});