diff --git a/changelog.d/fixes/11625-retired-pool.md b/changelog.d/fixes/11625-retired-pool.md new file mode 100644 index 0000000000..6d2f9f5be2 --- /dev/null +++ b/changelog.d/fixes/11625-retired-pool.md @@ -0,0 +1 @@ +- **fix(autoCombo,sse):** vendor-retired catalog ids are dropped from the auto-combo candidate pool and no longer win on leftover `arena_elo` / `user_override` rows; `getModelLifecycleDecision` consults `model-lifecycle.json` (prefix-stripped) so aggregator traffic is not `untracked` for ids the snapshot already knows ([#11625](https://github.com/diegosouzapw/OmniRoute/issues/11625)) diff --git a/open-sse/services/autoCombo/taskFitness.ts b/open-sse/services/autoCombo/taskFitness.ts index 5dd0f690a2..b6a160a7a5 100644 --- a/open-sse/services/autoCombo/taskFitness.ts +++ b/open-sse/services/autoCombo/taskFitness.ts @@ -10,6 +10,9 @@ * 2b. Layers 1-2 retried against the base model this id inherits quality scores * from, when `resolveScoresAs` resolves one (#11489). Reported as * `:inherited`. + * 2c. If the request id or its scoresAs base is vendor-retired (#11625), layers + * 1–3 are skipped so a leftover arena row cannot short-circuit the layer-3 + * veto, and a dead *codex id cannot keep the coding wildcard boost. * 3. Models.dev tier — derived from `model_capabilities` table capability data, * with a vendor-lifecycle veto (#11508: a retired id never earns a tier * score) and the same base-model inheritance as layers 1–2 @@ -43,6 +46,7 @@ import { } from "../../../src/lib/db/modelIntelligence.ts"; import { readFileSync } from "node:fs"; import { resolveScoresAs } from "./scoresAs.ts"; +import { isVendorRetiredId } from "../modelLifecycle.ts"; // #11508 — vendor lifecycle snapshot (#11507). An id the vendor has retired // must never earn a capability-derived tier score: models.dev keeps listing @@ -463,48 +467,60 @@ export function getTaskFitness(model: string, taskType: string): number { return getTaskFitnessWithSource(model, taskType).score; } +function isFitnessRetired(modelId: string): boolean { + if (isVendorRetiredId(modelId)) return true; + const { base, via } = resolveScoresAs(modelId); + return via !== null && isVendorRetiredId(base); +} + export function getTaskFitnessWithSource( model: string, taskType: string ): { score: number; source: string } { const normalizedModel = model.toLowerCase(); const normalizedTask = taskType.toLowerCase(); + const fitnessRetired = isFitnessRetired(normalizedModel); - const userOverride = queryModelIntelligence(normalizedModel, normalizedTask, "user_override"); - if (userOverride !== null) { - return { score: userOverride, source: "user_override" }; + if (!fitnessRetired) { + const userOverride = queryModelIntelligence(normalizedModel, normalizedTask, "user_override"); + if (userOverride !== null) { + return { score: userOverride, source: "user_override" }; + } + + const arenaElo = queryModelIntelligence(normalizedModel, normalizedTask, "arena_elo"); + if (arenaElo !== null) { + return { score: arenaElo, source: "arena_elo" }; + } + + // Layers 1-2, retried against the base model this id inherits quality from + // (#11489). Every DB-backed source publishes scores for BASE models only, so + // a variant id — an effort suffix (`gpt-5.6-sol-xhigh`), a vendor alias + // (`gpt-5.6`), a `-free` tier marker (`mimo-v2.5-free`, #4517) — misses both + // literal lookups and used to fall all the way to the wildcard 0.5, losing + // every comparison against a base model that happens to be benchmarked. + // The score is inherited VERBATIM: the 12-factor scoring already prices cost + // and latency per variant, so there is no basis for inventing an effort + // delta. `:inherited` keeps the indirection visible to callers. + const inherited = lookupInheritedFitness(normalizedModel, normalizedTask); + if (inherited !== null) { + return inherited; + } + + const tierScore = getModelsDevTierFitness(normalizedModel, normalizedTask); + if (tierScore !== null) { + return { score: tierScore, source: "models_dev_tier" }; + } + + const staticScore = lookupStaticFitnessTable(normalizedModel, normalizedTask); + if (staticScore !== null) { + return { score: staticScore, source: "fitness_table" }; + } + + return { score: lookupWildcardBoosts(normalizedModel, normalizedTask), source: "wildcard_boost" }; } - const arenaElo = queryModelIntelligence(normalizedModel, normalizedTask, "arena_elo"); - if (arenaElo !== null) { - return { score: arenaElo, source: "arena_elo" }; - } - - // Layers 1-2, retried against the base model this id inherits quality from - // (#11489). Every DB-backed source publishes scores for BASE models only, so - // a variant id — an effort suffix (`gpt-5.6-sol-xhigh`), a vendor alias - // (`gpt-5.6`), a `-free` tier marker (`mimo-v2.5-free`, #4517) — misses both - // literal lookups and used to fall all the way to the wildcard 0.5, losing - // every comparison against a base model that happens to be benchmarked. - // The score is inherited VERBATIM: the 12-factor scoring already prices cost - // and latency per variant, so there is no basis for inventing an effort - // delta. `:inherited` keeps the indirection visible to callers. - const inherited = lookupInheritedFitness(normalizedModel, normalizedTask); - if (inherited !== null) { - return inherited; - } - - const tierHit = getModelsDevTierFitnessWithSource(normalizedModel, normalizedTask); - if (tierHit !== null) { - return tierHit; - } - - const staticScore = lookupStaticFitnessTable(normalizedModel, normalizedTask); - if (staticScore !== null) { - return { score: staticScore, source: "fitness_table" }; - } - - return { score: lookupWildcardBoosts(normalizedModel, normalizedTask), source: "wildcard_boost" }; + // Retired: 0.5 is "no evidence", never a quality claim and never a *codex boost. + return { score: 0.5, source: "wildcard_boost" }; } /** @@ -524,6 +540,7 @@ function lookupInheritedFitness( const { base, via } = resolveScoresAs(normalizedModel); if (via === null || base === normalizedModel) return null; const normalizedBase = base.toLowerCase(); + if (isVendorRetiredId(normalizedBase)) return null; for (const source of ["user_override", "arena_elo"] as const) { const score = queryModelIntelligence(normalizedBase, normalizedTask, source); diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index b2556848b1..0ee416e2f7 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -70,6 +70,7 @@ import { resolveProviderId } from "../../src/shared/constants/providers.ts"; import * as semaphore from "./rateLimitSemaphore.ts"; import { getCircuitBreaker } from "../../src/shared/utils/circuitBreaker"; import { parseModel } from "./model.ts"; +import { rejectRetiredAutoComboCandidates } from "./modelLifecycle.ts"; import { createComboContext } from "./combo/context.ts"; import { phaseComboSetup } from "./combo/comboSetup.ts"; import { checkCredentialGate, logCredentialSkip } from "./credentialGate.ts"; @@ -627,11 +628,14 @@ export async function buildAutoCandidates( }) ); - // Filter out candidates whose model is hidden by the user in the dashboard - return candidates.filter((c) => { - const hiddenModels = hiddenModelsMap.get(c.provider); - return !hiddenModels?.has(c.model); - }); + // Filter out candidates whose model is hidden by the user in the dashboard, + // then drop vendor-retired ids so auto-combo cannot pick them (#11625). + return rejectRetiredAutoComboCandidates( + candidates.filter((c) => { + const hiddenModels = hiddenModelsMap.get(c.provider); + return !hiddenModels?.has(c.model); + }) + ); } // Context-cache pin health gate — moved to combo/dispatchPrelude.ts alongside the diff --git a/open-sse/services/modelLifecycle.ts b/open-sse/services/modelLifecycle.ts index 8efd93db7f..a0557159fd 100644 --- a/open-sse/services/modelLifecycle.ts +++ b/open-sse/services/modelLifecycle.ts @@ -4,7 +4,14 @@ * Replacement model IDs are migration guidance only. This module never rewrites a * request: shutdown models are rejected, deprecated models remain callable until * their shutdown date, and untracked models pass through unchanged. + * + * Dated OpenAI rows in MODEL_LIFECYCLE_RECORDS stay provider-scoped (a warn-before- + * shutdown date on `openai` must not leak onto an aggregator that still serves the + * id). Snapshot `status: "retired"` ids from config/quality/model-lifecycle.json + * are id-scoped and prefix-stripped (#11625): `openai/gpt-5.2-codex` on openrouter + * is the same retired vendor id as `gpt-5.2-codex`. */ +import { readFileSync } from "node:fs"; export const OPENAI_MODEL_DEPRECATIONS_URL = "https://developers.openai.com/api/docs/deprecations"; @@ -128,6 +135,67 @@ function shutdownTimestamp(shutdownAt: string): number { return Date.parse(`${shutdownAt}T00:00:00.000Z`); } +const SNAPSHOT_URL = new URL("../../config/quality/model-lifecycle.json", import.meta.url); +const SNAPSHOT_SOURCE = "config/quality/model-lifecycle.json"; + +type VendorRetiredEntry = { + vendor?: string; + status?: string; + retiredOn?: string | null; + replacement?: string | null; +}; + +let _retiredIds: Set | null = null; +let _retiredEntries: Map | null = null; + +function loadVendorRetiredSnapshot(): { + ids: Set; + entries: Map; +} { + if (_retiredIds && _retiredEntries) return { ids: _retiredIds, entries: _retiredEntries }; + const ids = new Set(); + const entries = new Map(); + try { + const parsed = JSON.parse(readFileSync(SNAPSHOT_URL, "utf8")) as { + retired?: Record; + }; + for (const [id, entry] of Object.entries(parsed.retired ?? {})) { + if (entry?.status !== "retired") continue; + const key = id.toLowerCase(); + ids.add(key); + entries.set(key, entry); + } + } catch { + // Snapshot missing → no id-scoped veto. Dated OpenAI rows still apply. + } + _retiredIds = ids; + _retiredEntries = entries; + return { ids, entries }; +} + +/** True when `modelId` or its last `vendor/` path segment is `status: "retired"` in the snapshot. */ +export function isVendorRetiredId(modelId: string | null | undefined): boolean { + if (typeof modelId !== "string" || modelId.length === 0) return false; + const lower = modelId.toLowerCase(); + const { ids } = loadVendorRetiredSnapshot(); + if (ids.has(lower)) return true; + const slash = lower.lastIndexOf("/"); + return slash !== -1 && ids.has(lower.slice(slash + 1)); +} + +function lookupVendorRetiredEntry(modelId: string): VendorRetiredEntry | null { + const lower = modelId.toLowerCase(); + const { entries } = loadVendorRetiredSnapshot(); + return entries.get(lower) ?? entries.get(lower.slice(lower.lastIndexOf("/") + 1)) ?? null; +} + +/** Drop auto-combo candidates whose model id the vendor has retired (#11625). */ +export function rejectRetiredAutoComboCandidates( + candidates: readonly T[] +): T[] { + return candidates.filter((candidate) => !isVendorRetiredId(candidate.model)); +} + export function getModelLifecycleDecision( provider: string | null | undefined, model: string | null | undefined, @@ -138,6 +206,24 @@ export function getModelLifecycleDecision( const record = RECORDS_BY_KEY.get(lifecycleKey(normalizedProvider, normalizedModel)); if (!record) { + if (isVendorRetiredId(normalizedModel)) { + const entry = lookupVendorRetiredEntry(normalizedModel); + const replacementId = + typeof entry?.replacement === "string" && entry.replacement.length > 0 + ? entry.replacement + : null; + return { + provider: normalizedProvider, + model: normalizedModel, + status: "shutdown", + action: "reject", + shutdownAt: typeof entry?.retiredOn === "string" ? entry.retiredOn : null, + replacement: replacementId + ? { provider: entry?.vendor ?? "", model: replacementId } + : null, + source: SNAPSHOT_SOURCE, + }; + } return { provider: normalizedProvider, model: normalizedModel, @@ -170,7 +256,8 @@ export function formatModelLifecycleMessage(decision: ModelLifecycleDecision): s ? ` Use "${decision.replacement.provider}/${decision.replacement.model}" instead.` : ""; if (decision.status === "shutdown") { - return `Model "${modelRef}" was shut down on ${decision.shutdownAt} and cannot be routed automatically.${replacement}`; + const when = decision.shutdownAt ? ` was shut down on ${decision.shutdownAt}` : " has been retired by its vendor"; + return `Model "${modelRef}"${when} and cannot be routed automatically.${replacement}`; } return `Model "${modelRef}" is deprecated and is scheduled to shut down on ${decision.shutdownAt}.${replacement}`; } diff --git a/tests/unit/model-lifecycle.test.ts b/tests/unit/model-lifecycle.test.ts index f88a18a1e7..5ffc84d3ab 100644 --- a/tests/unit/model-lifecycle.test.ts +++ b/tests/unit/model-lifecycle.test.ts @@ -32,11 +32,23 @@ test("upcoming shutdowns warn before the shutdown date", () => { assert.equal(decision.shutdownAt, "2026-08-10"); }); -test("lifecycle records are provider-scoped", () => { +test("dated OpenAI deprecations stay provider-scoped", () => { + // gpt-3.5-turbo-0125 shuts down 2026-10-23; CURRENT_DATE is 2026-07-26. + // Snapshot marks it retiring (not retired), so an aggregator still allows it. + const openai = getModelLifecycleDecision("openai", "gpt-3.5-turbo-0125", CURRENT_DATE); + const aggregator = getModelLifecycleDecision("opencode-zen", "gpt-3.5-turbo-0125", CURRENT_DATE); + + assert.equal(openai.status, "deprecated"); + assert.equal(openai.action, "warn"); + assert.equal(aggregator.status, "untracked"); + assert.equal(aggregator.action, "allow"); +}); + +test("snapshot-retired ids are rejected on every provider (#11625)", () => { const decision = getModelLifecycleDecision("opencode-zen", "gpt-5.2-codex", CURRENT_DATE); - assert.equal(decision.status, "untracked"); - assert.equal(decision.action, "allow"); + assert.equal(decision.status, "shutdown"); + assert.equal(decision.action, "reject"); }); test("catalog filtering hides deprecated and shutdown models by default", () => { @@ -52,7 +64,7 @@ test("catalog filtering hides deprecated and shutdown models by default", () => ); assert.deepEqual( filterSelectableModels("opencode-zen", models, { asOf: CURRENT_DATE }).map((model) => model.id), - models.map((model) => model.id) + ["gpt-5.6-sol"] ); }); diff --git a/tests/unit/retired-pool-11625.test.ts b/tests/unit/retired-pool-11625.test.ts new file mode 100644 index 0000000000..fdb5bd1768 --- /dev/null +++ b/tests/unit/retired-pool-11625.test.ts @@ -0,0 +1,170 @@ +/** + * TDD for #11625: vendor-retired catalog ids stay selectable and still win on + * arena_elo because auto-combo never consults model-lifecycle.json, and layers + * 1–2 return before the layer-3 veto (#11508 / #11598). + * + * This file was written first and observed failing on the pre-fix chain: + * - isModelSelectable("anthropic", "claude-3-7-sonnet-20250219") === true + * - seeded arena_elo on gpt-5.2-codex leaked through getTaskFitnessWithSource + */ +import { describe, it, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +const { + isVendorRetiredId, + rejectRetiredAutoComboCandidates, + isModelSelectable, + getModelLifecycleDecision, +} = await import("../../open-sse/services/modelLifecycle.ts"); +const { + getTaskFitnessWithSource, + invalidateFitnessCache, +} = await import("../../open-sse/services/autoCombo/taskFitness.ts"); +const { upsertModelIntelligence } = await import("../../src/lib/db/modelIntelligence.ts"); +const { resetDbInstance } = await import("../../src/lib/db/core.ts"); + +const snapshot = JSON.parse( + readFileSync( + fileURLToPath(new URL("../../config/quality/model-lifecycle.json", import.meta.url)), + "utf8" + ) +) as { retired: Record }; + +const snapshotRetired = Object.entries(snapshot.retired) + .filter(([, entry]) => entry.status === "retired") + .map(([id]) => id); + +describe("isVendorRetiredId (#11625) — prefix-stripped snapshot match", () => { + it("treats snapshot retired ids and their vendor-prefixed forms as retired", () => { + assert.equal(isVendorRetiredId("gpt-5.2-codex"), true); + assert.equal(isVendorRetiredId("openai/gpt-5.2-codex"), true); + assert.equal(isVendorRetiredId("claude-3-7-sonnet-20250219"), true); + assert.equal(isVendorRetiredId("GPT-5.2-Codex"), true); + }); + + it("does not mark a live flagship or a retiring-without-date id as retired", () => { + assert.equal(isVendorRetiredId("gpt-5.6-sol"), false); + // Snapshot marks this retiring with retiredOn null — must not auto-promote. + assert.equal(isVendorRetiredId("gpt-4-turbo"), false); + }); +}); + +describe("rejectRetiredAutoComboCandidates (#11625)", () => { + it("drops every snapshot-retired id and keeps a live control", () => { + const live = { model: "gpt-5.6-sol", provider: "openai" }; + const retired = snapshotRetired.slice(0, 8).map((id) => ({ + model: id, + provider: "openrouter", + })); + const prefixed = { model: "openai/gpt-5.2-codex", provider: "openrouter" }; + const kept = rejectRetiredAutoComboCandidates([live, ...retired, prefixed]); + assert.deepEqual( + kept.map((c) => c.model), + ["gpt-5.6-sol"] + ); + for (const dropped of [...retired, prefixed]) { + assert.equal(isVendorRetiredId(dropped.model), true, dropped.model); + } + }); +}); + +describe("isModelSelectable / getModelLifecycleDecision consult the snapshot (#11625)", () => { + it("rejects an Anthropic retired id that MODEL_LIFECYCLE_RECORDS never heard of", () => { + assert.equal(isModelSelectable("anthropic", "claude-3-7-sonnet-20250219"), false); + const decision = getModelLifecycleDecision("anthropic", "claude-3-7-sonnet-20250219"); + assert.equal(decision.status, "shutdown"); + assert.equal(decision.action, "reject"); + }); + + it("rejects a retired OpenAI id on an aggregator that the hardcoded table scopes away", () => { + const decision = getModelLifecycleDecision("openrouter", "gpt-5.2-codex"); + assert.equal(decision.status, "shutdown"); + assert.equal(decision.action, "reject"); + assert.equal(isModelSelectable("openrouter", "openai/gpt-5.2-codex"), false); + }); + + it("still allows a live model", () => { + assert.equal(isModelSelectable("openai", "gpt-5.6-sol"), true); + assert.equal(getModelLifecycleDecision("openai", "gpt-5.6-sol").action, "allow"); + }); +}); + +describe("taskFitness layers 1–2 skip retired ids (#11625)", () => { + before(() => { + upsertModelIntelligence({ + model: "gpt-5.2-codex", + source: "arena_elo", + category: "coding", + score: 0.96, + eloRaw: 1280, + confidence: "high", + expiresAt: "2099-12-31T23:59:59Z", + }); + upsertModelIntelligence({ + model: "openai/gpt-5.2-codex", + source: "arena_elo", + category: "coding", + score: 0.96, + eloRaw: 1280, + confidence: "high", + expiresAt: "2099-12-31T23:59:59Z", + }); + upsertModelIntelligence({ + model: "claude-3-7-sonnet-20250219", + source: "arena_elo", + category: "coding", + score: 0.94, + eloRaw: 1200, + confidence: "high", + expiresAt: "2099-12-31T23:59:59Z", + }); + upsertModelIntelligence({ + model: "gpt-5.6-sol", + source: "arena_elo", + category: "coding", + score: 0.91, + eloRaw: 1500, + confidence: "high", + expiresAt: "2099-12-31T23:59:59Z", + }); + invalidateFitnessCache(); + }); + + after(() => { + resetDbInstance(); + invalidateFitnessCache(); + }); + + it("does not return a seeded arena_elo row for a retired bare id", () => { + const result = getTaskFitnessWithSource("gpt-5.2-codex", "coding"); + assert.notEqual(result.source, "arena_elo"); + assert.notEqual(result.score, 0.96); + }); + + it("does not return a seeded arena_elo row for the vendor-prefixed catalog form", () => { + const result = getTaskFitnessWithSource("openai/gpt-5.2-codex", "coding"); + assert.notEqual(result.source, "arena_elo"); + assert.notEqual(result.score, 0.96); + }); + + it("does not inherit a retired base's arena row onto an effort variant", () => { + const result = getTaskFitnessWithSource("claude-3-7-sonnet-20250219-high", "coding"); + assert.notEqual(result.source, "arena_elo"); + assert.notEqual(result.source, "arena_elo:inherited"); + assert.notEqual(result.score, 0.94); + }); + + it("does not let a retired *codex id keep the coding wildcard boost", () => { + const result = getTaskFitnessWithSource("openai/gpt-5.2-codex", "coding"); + assert.equal(result.score, 0.5); + assert.equal(result.source, "wildcard_boost"); + }); + + it("leaves a live flagship's arena row intact", () => { + const result = getTaskFitnessWithSource("gpt-5.6-sol", "coding"); + assert.equal(result.score, 0.91); + assert.equal(result.source, "arena_elo"); + }); +});