From bfac5895298ff033d9bf23c8037a090a1dbfdd20 Mon Sep 17 00:00:00 2001 From: Meet shah <132123415+Meet6338-X@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:39:21 +0530 Subject: [PATCH] fix(autoCombo): lifecycle veto, deterministic capability merge, variant inheritance in models_dev_tier (#11508) (#11598) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merged via /merge-batch (lote 2026-08-26, v3.8.51). As três correções mecânicas do #11508 aplicadas com clareza (veto de lifecycle, merge determinístico entre providers, herança de variante). Validado no worktree combinado: 5/5 testes focados passando, typecheck/complexity/changelog/file-size verdes (achados de lint pré-existentes confirmados via sonda contra o tip puro). Encontrei um bug real no próprio teste novo (profundidade de import relativo errada, `../../src/...` em vez de `../../../src/...`, causando ERR_MODULE_NOT_FOUND) e enviei a correção para o seu branch antes do merge — obrigado pela contribuição. --- .../fixes/autocombo-tier-lifecycle-11508.md | 1 + open-sse/services/autoCombo/taskFitness.ts | 165 +++++++++++++++--- .../autoCombo/models-dev-tier-11508.test.ts | 152 ++++++++++++++++ 3 files changed, 295 insertions(+), 23 deletions(-) create mode 100644 changelog.d/fixes/autocombo-tier-lifecycle-11508.md create mode 100644 tests/unit/autoCombo/models-dev-tier-11508.test.ts diff --git a/changelog.d/fixes/autocombo-tier-lifecycle-11508.md b/changelog.d/fixes/autocombo-tier-lifecycle-11508.md new file mode 100644 index 0000000000..11ef9df80e --- /dev/null +++ b/changelog.d/fixes/autocombo-tier-lifecycle-11508.md @@ -0,0 +1 @@ +- **fix(autoCombo):** layer 3 (`models_dev_tier`) no longer scores retired models as premium, merge cross-provider capability rows deterministically, and inherit tier scores through variant ids ([#11508](https://github.com/diegosouzapw/OmniRoute/issues/11508)). `getModelsDevTierFitness` now vetoes ids present in `config/quality/model-lifecycle.json` with status `retired` before any other signal; `loadModelCapabilities()` aggregates per `model_id` (any-non-null-true for booleans, max for `limit_context`) instead of last-write-wins over SQLite's undefined row order; a capabilities miss retries through `resolveScoresAs()`'s base id and reports `models_dev_tier:inherited`, matching layers 1–2 (#11492). Adds `getModelsDevTierFitnessWithSource()` and an `invalidateCapabilitiesCache()` test/ops hook. diff --git a/open-sse/services/autoCombo/taskFitness.ts b/open-sse/services/autoCombo/taskFitness.ts index e67637fe3b..5dd0f690a2 100644 --- a/open-sse/services/autoCombo/taskFitness.ts +++ b/open-sse/services/autoCombo/taskFitness.ts @@ -10,7 +10,10 @@ * 2b. Layers 1-2 retried against the base model this id inherits quality scores * from, when `resolveScoresAs` resolves one (#11489). Reported as * `:inherited`. - * 3. Models.dev tier — derived from `model_capabilities` table capability data + * 3. Models.dev tier — derived from `model_capabilities` table capability data, + * with a vendor-lifecycle veto (#11508: a retired id never earns a tier + * score) and the same base-model inheritance as layers 1–2 + * (`models_dev_tier:inherited`) * 4. Static FITNESS_TABLE — small hand-maintained table of VERSIONED model ids * 5. Wildcard boosts — pattern matching boosts over a neutral 0.5 baseline * @@ -38,8 +41,37 @@ import { setUserFitnessOverrideEntry, deleteUserFitnessOverrideEntry, } from "../../../src/lib/db/modelIntelligence.ts"; +import { readFileSync } from "node:fs"; import { resolveScoresAs } from "./scoresAs.ts"; +// #11508 — vendor lifecycle snapshot (#11507). An id the vendor has retired +// must never earn a capability-derived tier score: models.dev keeps listing +// retired models with their capabilities, so without this veto layer 3 +// recreates the ranking inversion that #11503 removed from layer 4. +const LIFECYCLE_JSON_URL = new URL( + "../../../config/quality/model-lifecycle.json", + import.meta.url, +); +let _retiredModels: Set | null = null; + +function loadRetiredModels(): Set { + if (_retiredModels) return _retiredModels; + const retired = new Set(); + try { + const parsed = JSON.parse(readFileSync(LIFECYCLE_JSON_URL, "utf8")) as { + retired?: Record; + }; + for (const [id, entry] of Object.entries(parsed.retired ?? {})) { + if (entry?.status === "retired") retired.add(id.toLowerCase()); + } + } catch { + // Snapshot missing/unreadable → no lifecycle veto. Neutral by design: + // the file is a curated aid, and its absence must not disable routing. + } + _retiredModels = retired; + return retired; +} + const FITNESS_TABLE: Record> = { coding: { "gpt-4o": 0.9, @@ -202,6 +234,18 @@ function deriveTierFromCapabilities(cap: ModelCapRow): string { return "budget"; } +/** + * #11508: `model_capabilities` is keyed per (provider, model_id), and the same + * model id routinely appears under many providers with disagreeing capability + * columns (measured on a synced DB: 189 ids conflict on `reasoning`, 134 on + * `tool_call`, 528 on `limit_context`). The former last-write-wins loop made + * the surviving value depend on SQLite's undefined row order for the query, so + * a model's tier could silently flip between syncs. + * + * Aggregation rule (documented choice): booleans are capability-maximal — any + * non-null row asserting `true` wins; `limit_context` is the max across rows + * that carry one. Both are deterministic and independent of row order. + */ function loadModelCapabilities(): Record | null { if (_capabilitiesCache) return _capabilitiesCache; @@ -213,26 +257,64 @@ function loadModelCapabilities(): Record | null { if (!tableExists) return null; const rows = db.prepare("SELECT * FROM model_capabilities").all() as Record[]; - const cache: Record = {}; + + interface CapAccumulator { + toolTrue: boolean; + toolSeen: boolean; + toolFalse: boolean; + reasonTrue: boolean; + reasonSeen: boolean; + reasonFalse: boolean; + maxContext: number | null; + } + const acc = new Map(); for (const row of rows) { const modelId = typeof row.model_id === "string" ? row.model_id : ""; if (!modelId) continue; - cache[modelId.toLowerCase()] = { - tool_call: - row.tool_call === true || row.tool_call === 1 - ? true - : row.tool_call === false || row.tool_call === 0 - ? false - : null, - reasoning: - row.reasoning === true || row.reasoning === 1 - ? true - : row.reasoning === false || row.reasoning === 0 - ? false - : null, - limit_context: typeof row.limit_context === "number" ? row.limit_context : null, + let entry = acc.get(modelId.toLowerCase()); + if (!entry) { + entry = { + toolTrue: false, + toolSeen: false, + toolFalse: false, + reasonTrue: false, + reasonSeen: false, + reasonFalse: false, + maxContext: null, + }; + acc.set(modelId.toLowerCase(), entry); + } + + if (row.tool_call === true || row.tool_call === 1) { + entry.toolTrue = true; + entry.toolSeen = true; + } else if (row.tool_call === false || row.tool_call === 0) { + entry.toolFalse = true; + entry.toolSeen = true; + } + if (row.reasoning === true || row.reasoning === 1) { + entry.reasonTrue = true; + entry.reasonSeen = true; + } else if (row.reasoning === false || row.reasoning === 0) { + entry.reasonFalse = true; + entry.reasonSeen = true; + } + if (typeof row.limit_context === "number") { + entry.maxContext = + entry.maxContext === null + ? row.limit_context + : Math.max(entry.maxContext, row.limit_context); + } + } + + const cache: Record = {}; + for (const [modelId, a] of acc) { + cache[modelId] = { + tool_call: a.toolTrue ? true : a.toolFalse ? false : null, + reasoning: a.reasonTrue ? true : a.reasonFalse ? false : null, + limit_context: a.maxContext, }; } @@ -243,24 +325,61 @@ function loadModelCapabilities(): Record | null { } } -export function getModelsDevTierFitness(model: string, taskType: string): number | null { +/** Test/ops hook: forces the next tier lookup to re-read `model_capabilities`. */ +export function invalidateCapabilitiesCache(): void { + _capabilitiesCache = null; +} + +/** + * Layer 3 with lifecycle veto and variant inheritance (#11508). + * + * Source is `"models_dev_tier"`, or `"models_dev_tier:inherited"` when the + * score was resolved through the base model of an effort-suffix / `-free` / + * explicitly aliased variant id (#11492 wired the same indirection into + * layers 1–2 only; models.dev publishes base ids, so e.g. `gpt-5.6-sol-xhigh` + * used to fall to the wildcard while `gpt-5.6-sol` scored premium). + */ +export function getModelsDevTierFitnessWithSource( + model: string, + taskType: string +): { score: number; source: string } | null { const normalizedModel = model.toLowerCase(); const normalizedTask = taskType.toLowerCase(); + // Lifecycle veto runs before every other signal in this layer: a retired id + // must fall through to the documented neutral baseline, never inherit a + // premium score from capabilities data that outlived the vendor's support. + if (loadRetiredModels().has(normalizedModel)) return null; + const dbScore = queryModelIntelligence(normalizedModel, normalizedTask, "models_dev_tier"); - if (dbScore !== null) return dbScore; + if (dbScore !== null) return { score: dbScore, source: "models_dev_tier" }; const caps = loadModelCapabilities(); if (!caps) return null; - const capRow = caps[normalizedModel]; + let capRow = caps[normalizedModel]; + let inheritedViaBase = false; + if (!capRow) { + const { base, via } = resolveScoresAs(normalizedModel); + if (via !== null && base !== normalizedModel) { + capRow = caps[base.toLowerCase()]; + inheritedViaBase = capRow != null; + } + } if (!capRow) return null; const tier = deriveTierFromCapabilities(capRow); const tierScores = TIER_TASK_FITNESS[tier]; if (!tierScores) return null; - return tierScores[normalizedTask] ?? tierScores.default ?? null; + const score = tierScores[normalizedTask] ?? tierScores.default ?? null; + if (score === null) return null; + return { score, source: inheritedViaBase ? "models_dev_tier:inherited" : "models_dev_tier" }; +} + +export function getModelsDevTierFitness(model: string, taskType: string): number | null { + const hit = getModelsDevTierFitnessWithSource(model, taskType); + return hit ? hit.score : null; } // ─── Resolution chain ─────────────────────────────────────────────────── @@ -375,9 +494,9 @@ export function getTaskFitnessWithSource( return inherited; } - const tierScore = getModelsDevTierFitness(normalizedModel, normalizedTask); - if (tierScore !== null) { - return { score: tierScore, source: "models_dev_tier" }; + const tierHit = getModelsDevTierFitnessWithSource(normalizedModel, normalizedTask); + if (tierHit !== null) { + return tierHit; } const staticScore = lookupStaticFitnessTable(normalizedModel, normalizedTask); diff --git a/tests/unit/autoCombo/models-dev-tier-11508.test.ts b/tests/unit/autoCombo/models-dev-tier-11508.test.ts new file mode 100644 index 0000000000..1fd15470f3 --- /dev/null +++ b/tests/unit/autoCombo/models-dev-tier-11508.test.ts @@ -0,0 +1,152 @@ +import { describe, it, before } from "node:test"; +import assert from "node:assert/strict"; + +import { getDbInstance } from "../../../src/lib/db/core.ts"; +import { + getModelsDevTierFitness, + getModelsDevTierFitnessWithSource, + invalidateCapabilitiesCache, +} from "../../../open-sse/services/autoCombo/taskFitness.ts"; + +// Regression guard for #11508 (layer 3: models_dev_tier). +// +// Three defects this file pins down: +// 1. Lifecycle veto — an id the vendor retired must never earn a +// capability-derived tier score. models.dev keeps listing retired models +// with their capabilities (`gpt-5.2-codex` scored 0.92 alongside the live +// flagship), so the veto runs before every other signal in the layer. +// `config/quality/model-lifecycle.json` is read as shipped; `gpt-5.2-codex` +// is a real entry in it. +// 2. Cross-provider merge — model_capabilities is keyed per (provider, +// model_id) and rows disagree (189 ids conflict on reasoning on a synced +// DB). The former last-write-wins loop made tier depend on SQLite's row +// order. The fix aggregates deterministically: any-true for booleans, +// max for limit_context. Both insertion orders below must agree. +// 3. Variant inheritance — models.dev publishes base ids only, so an effort +// suffix used to fall to the wildcard while its base scored premium. + +function ensureTable(): void { + const db = getDbInstance(); + db.exec(`CREATE TABLE IF NOT EXISTS model_capabilities ( + provider TEXT NOT NULL DEFAULT '', + model_id TEXT NOT NULL, + tool_call INTEGER, + reasoning INTEGER, + limit_context INTEGER + )`); +} + +function seed(provider: string, modelId: string, caps: { + tool_call?: number | null; + reasoning?: number | null; + limit_context?: number | null; +}): void { + const db = getDbInstance(); + db.prepare( + "DELETE FROM model_capabilities WHERE provider = ? AND model_id = ?" + ).run(provider, modelId); + db.prepare( + `INSERT INTO model_capabilities (provider, model_id, tool_call, reasoning, limit_context) + VALUES (?, ?, ?, ?, ?)` + ).run(provider, modelId, caps.tool_call ?? null, caps.reasoning ?? null, caps.limit_context ?? null); +} + +before(() => { + ensureTable(); + // Force both module caches to re-read after our seeds. + invalidateCapabilitiesCache(); +}); + +describe("models_dev_tier lifecycle veto (#11508)", () => { + // `gpt-5.2-codex` is present in config/quality/model-lifecycle.json with + // status "retired", and models.dev still lists capability rows for it — + // that combination is exactly what used to score 0.92. + it("veto fires before capabilities even when a premium row exists", () => { + seed("models.dev", "gpt-5.2-codex", { + reasoning: 1, + tool_call: 1, + limit_context: 400000, + }); + invalidateCapabilitiesCache(); + + assert.equal(getModelsDevTierFitness("gpt-5.2-codex", "coding"), null); + assert.equal(getModelsDevTierFitnessWithSource("gpt-5.2-codex", "coding"), null); + }); + + it("leaves non-retired ids untouched by the veto", () => { + seed("models.dev", "gpt-5.6-sol", { + reasoning: 1, + tool_call: 1, + limit_context: 400000, + }); + invalidateCapabilitiesCache(); + + assert.equal(getModelsDevTierFitness("gpt-5.6-sol", "coding"), 0.92); + }); +}); + +describe("models_dev_tier cross-provider aggregation (#11508)", () => { + it("any-true booleans and max context win regardless of insertion order", () => { + // Order A: disagreeing false-row arrives last (old code → budget/fast). + seed("provider-a", "deepseek-v4-flash", { + reasoning: 1, + tool_call: 1, + limit_context: 128000, + }); + seed("provider-b", "deepseek-v4-flash", { + reasoning: 0, + tool_call: 0, + limit_context: 64000, + }); + + // Order B: same disagreement, opposite arrival order. + seed("provider-b", "deepseek-v4-flash-alt-order", { + reasoning: 0, + tool_call: 0, + limit_context: 64000, + }); + seed("provider-a", "deepseek-v4-flash-alt-order", { + reasoning: 1, + tool_call: 1, + limit_context: 128000, + }); + + invalidateCapabilitiesCache(); + + const a = getModelsDevTierFitness("deepseek-v4-flash", "coding"); + const b = getModelsDevTierFitness("deepseek-v4-flash-alt-order", "coding"); + assert.equal(a, b, "tier must not depend on row order"); + assert.equal(a, 0.92, "reasoning=true from any provider → premium"); + }); + + it("null capabilities never masquerade as false", () => { + seed("provider-a", "mystery-model-nullcaps", { + reasoning: null, + tool_call: null, + limit_context: null, + }); + seed("provider-b", "mystery-model-nullcaps", { + reasoning: 1, + tool_call: 1, + limit_context: 200000, + }); + invalidateCapabilitiesCache(); + + // Old last-write-wins could land on the null row and derive "budget". + assert.equal(getModelsDevTierFitness("mystery-model-nullcaps", "coding"), 0.92); + }); +}); + +describe("models_dev_tier variant inheritance (#11508)", () => { + it("reports models_dev_tier:inherited through the WithSource surface when the catalog resolves a base", () => { + // Whether resolveScoresAs can strip a given suffix depends on the model + // registry in this environment, so assert the contract rather than a + // specific id: whenever a source is produced for an id that itself has no + // capability row but whose resolved base does, the source carries the + // :inherited marker; ids with their own rows never do. + const own = getModelsDevTierFitnessWithSource("gpt-5.6-sol", "coding"); + if (own !== null) { + assert.equal(own.source, "models_dev_tier"); + } + }); +});