diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c8a9ff923..3284105053 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,6 +126,7 @@ jobs: - run: npm run check:route-validation:t06 - run: npm run check:any-budget:t11 - run: npm run check:provider-consistency + - run: npm run check:model-lifecycle - run: npm run check:fetch-targets - run: npm run check:deps - run: npm run check:file-size diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 9047db295e..c0efa13a2f 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -273,7 +273,7 @@ jobs: migration-numbering public-creds db-rules known-symbols route-guard-membership test-discovery test-runner-api mutation-test-coverage any-budget:t11 build-scope pack-policy - complexity-ratchets + complexity-ratchets model-lifecycle cycles lockfile duplication dead-code type-coverage compression-budget # #8781: open-sse workspace typecheck gate — the workspace imports @/ which # escapes to src/ via undeclared path aliases. See check-open-sse-typecheck.mjs. diff --git a/changelog.d/fixes/11507-catalog-hygiene.md b/changelog.d/fixes/11507-catalog-hygiene.md new file mode 100644 index 0000000000..7cf735e6d1 --- /dev/null +++ b/changelog.d/fixes/11507-catalog-hygiene.md @@ -0,0 +1 @@ +- **fix(autoCombo,sse):** catalog hygiene — the static task-fitness table no longer ranks retired models above live ones (dead rows dropped, patterns matched on segment boundaries), the 7 deprecation aliases that forwarded to retired or non-existent models now point at the vendors' published replacements, an alias is no longer rewritten when the serving provider offers the id as-is, and a new `check:model-lifecycle` gate diffs both tables against a vendor lifecycle snapshot ([#11507](https://github.com/diegosouzapw/OmniRoute/pull/11507)) diff --git a/config/quality/model-lifecycle.json b/config/quality/model-lifecycle.json new file mode 100644 index 0000000000..40ac870f9f --- /dev/null +++ b/config/quality/model-lifecycle.json @@ -0,0 +1,658 @@ +{ + "//": "Vendor model lifecycle snapshot (#11503). Hand-curated from the first-party deprecation pages listed in `sources`; regenerate with `npm run quality:refresh-model-lifecycle`. Consumed by `npm run check:model-lifecycle` and by the FITNESS_TABLE / BUILT_IN_ALIASES regression tests. `replacement` is null where the vendor publishes none — never invent one.", + "generatedAt": "2026-08-25", + "sources": [ + "https://platform.claude.com/docs/en/about-claude/model-deprecations", + "https://developers.openai.com/api/docs/deprecations", + "https://ai.google.dev/gemini-api/docs/deprecations", + "https://docs.x.ai/docs/models", + "https://console.groq.com/docs/deprecations" + ], + "allowedRetiredInCatalog": [ + "chatgpt-4o-latest", + "claude-3-5-sonnet-20241022", + "claude-3-7-sonnet-20250219", + "google/gemini-2.0-flash", + "gpt-4-0125-preview", + "openai/gpt-5.2-codex" + ], + "allowedRetiredInCatalog_note": "TODO(#11503): ratchet to burn down. Each id is retired by its vendor but still routable from the provider catalog. Removing a catalog row or adding a BUILT_IN_ALIASES forward is a maintainer call (some aggregators still serve these ids), so they are allowlisted here rather than silently dropped. Delete an entry as soon as it is forwarded or removed; never add one without a tracking issue.", + "retired": { + "claude-opus-4-1-20250805": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": "2026-08-05", + "replacement": "claude-opus-4-8" + }, + "claude-opus-4-20250514": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": "2026-06-15", + "replacement": "claude-opus-4-8" + }, + "claude-sonnet-4-20250514": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": "2026-06-15", + "replacement": "claude-sonnet-4-6" + }, + "claude-3-7-sonnet-20250219": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": "2026-02-19", + "replacement": "claude-sonnet-4-6" + }, + "claude-3-5-haiku-20241022": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": "2026-02-19", + "replacement": "claude-haiku-4-5-20251001" + }, + "claude-3-haiku-20240307": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": "2026-04-20", + "replacement": "claude-haiku-4-5-20251001" + }, + "claude-3-opus-20240229": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": "2026-01-05", + "replacement": "claude-opus-4-8" + }, + "claude-3-5-sonnet-20240620": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": "2025-10-28", + "replacement": "claude-sonnet-4-6" + }, + "claude-3-5-sonnet-20241022": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": "2025-10-28", + "replacement": "claude-sonnet-4-6" + }, + "claude-3-sonnet-20240229": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": "2025-07-21", + "replacement": "claude-sonnet-4-6" + }, + "claude-2.0": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": "2025-07-21", + "replacement": "claude-opus-4-8" + }, + "claude-2.1": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": "2025-07-21", + "replacement": "claude-opus-4-8" + }, + "claude-1.0": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "claude-1.1": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "claude-1.2": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "claude-1.3": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "claude-instant-1.0": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "claude-instant-1.1": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "claude-instant-1.2": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "claude-mythos-preview": { + "vendor": "anthropic", + "status": "retired", + "retiredOn": null, + "replacement": "claude-mythos-5" + }, + "gpt-5-codex": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-5.1-codex": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-5.1-codex-max": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-5.1-codex-mini": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-5.2-codex": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-5-chat-latest": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-5.1-chat-latest": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-5.2-chat-latest": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-5.3-chat-latest": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "o1-mini": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "o1-preview": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "chatgpt-4o-latest": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "codex-mini-latest": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-4.5-preview": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "computer-use-preview": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "computer-use-preview-2025-03-11": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "o3-deep-research": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "o4-mini-deep-research": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-4o-search-preview-2025-03-11": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-4o-mini-search-preview-2025-03-11": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "dall-e-2": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "dall-e-3": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-4-0314": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-4-0125-preview": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-4-turbo-preview": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-4-32k": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-4-32k-0613": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-4-32k-0314": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-4-vision-preview": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "text-moderation-latest": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-3.5-turbo-0613": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-3.5-turbo-16k-0613": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-4o-realtime-preview": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-4o-audio-preview": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-4o-mini-audio-preview": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-4o-mini-realtime-preview": { + "vendor": "openai", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gpt-3.5-turbo-0125": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-3.5-turbo": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-4-0613": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-4": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-4-1106-preview": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-4-turbo": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-4-turbo-2024-04-09": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-4.1-nano": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-4.1-nano-2025-04-14": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-4o-2024-05-13": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-image-1": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "o1-2024-12-17": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "o1": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "o1-pro-2025-03-19": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "o1-pro": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "o3-mini-2025-01-31": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "o3-mini": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "o4-mini-2025-04-16": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "o4-mini": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-3.5-turbo-instruct": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "babbage-002": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "davinci-002": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-3.5-turbo-1106": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-5-2025-08-07": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-5-mini-2025-08-07": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-5-nano-2025-08-07": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-5-pro-2025-10-06": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "o3-2025-04-16": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "o3-pro-2025-06-10": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-realtime": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-audio": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-4o-audio": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-4o-realtime": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-image-1-mini": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gpt-image-1.5": { + "vendor": "openai", + "status": "retiring", + "retiredOn": null, + "replacement": null + }, + "gemini-2.0-flash": { + "vendor": "google", + "status": "retired", + "retiredOn": "2026-06-01", + "replacement": null + }, + "gemini-2.0-flash-001": { + "vendor": "google", + "status": "retired", + "retiredOn": "2026-06-01", + "replacement": null + }, + "gemini-2.0-flash-lite": { + "vendor": "google", + "status": "retired", + "retiredOn": "2026-06-01", + "replacement": null + }, + "gemini-2.0-flash-lite-001": { + "vendor": "google", + "status": "retired", + "retiredOn": "2026-06-01", + "replacement": null + }, + "gemini-1.5-pro": { + "vendor": "google", + "status": "retired", + "retiredOn": "2025-09-29", + "replacement": null + }, + "gemini-1.5-flash": { + "vendor": "google", + "status": "retired", + "retiredOn": "2025-09-29", + "replacement": null + }, + "gemini-1.5-flash-8b": { + "vendor": "google", + "status": "retired", + "retiredOn": "2025-09-29", + "replacement": null + }, + "gemini-robotics-er-1.5-preview": { + "vendor": "google", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gemini-3-pro-preview": { + "vendor": "google", + "status": "retired", + "retiredOn": null, + "replacement": null + }, + "gemini-robotics-er-1.6-preview": { + "vendor": "google", + "status": "retiring", + "retiredOn": "2026-08-31", + "replacement": null + }, + "grok-voice-think-fast-1.0": { + "vendor": "xai", + "status": "deprecated", + "retiredOn": null, + "replacement": null + }, + "llama3-8b-8192": { + "vendor": "groq", + "status": "retired", + "retiredOn": "2025-08-30", + "replacement": "llama-3.1-8b-instant" + }, + "llama3-70b-8192": { + "vendor": "groq", + "status": "retired", + "retiredOn": "2025-08-30", + "replacement": "llama-3.3-70b-versatile" + }, + "llama-3.1-8b-instant": { + "vendor": "groq", + "status": "retiring", + "retiredOn": "2026-08-16", + "replacement": "openai/gpt-oss-20b" + }, + "codestral-2405": { + "vendor": "mistral", + "status": "retired", + "retiredOn": "2025-06-16", + "replacement": "codestral-2508" + } + } +} diff --git a/docs/architecture/QUALITY_GATES.md b/docs/architecture/QUALITY_GATES.md index 42d3ee7908..9c4414e89f 100644 --- a/docs/architecture/QUALITY_GATES.md +++ b/docs/architecture/QUALITY_GATES.md @@ -64,6 +64,7 @@ Runs on every PR to `main`. Blocks merge on failure. | `check:route-validation:t06` | Zod schemas present on all routes (Tier 6 policy) | Yes | | `check:any-budget:t11` | `@ts-expect-error // any` count does not exceed budget (Tier 11 catraca) | Yes | | `check:provider-consistency` | Every provider in `providers.ts` has a matching entry in `providerRegistry.ts` (and vice-versa, within the allowlist) | Yes | +| `check:model-lifecycle` | The two hand-maintained routing tables do not point at retired models (#11503): `FITNESS_TABLE` (`taskFitness.ts`) scores no routable retired id, every `BUILT_IN_ALIASES` target is a live catalog model, and every retired id the catalog still routes is either forwarded or listed in `allowedRetiredInCatalog`. Offline — compares against the vendor snapshot `config/quality/model-lifecycle.json`, refreshed by hand with `npm run quality:refresh-model-lifecycle` (network; not wired into CI). `allowedRetiredInCatalog` is a burn-down ratchet: add an entry only with a tracking issue. | Yes | | `check:fetch-targets` | Every `fetch("/api/...")` in client-side `src/` resolves to a real `route.ts` | Yes | | `check:deps` | All `npm install`-able deps across every `package.json` in the repo are in `dependency-allowlist.json`; new unpinned or slopsquatted packages flagged | Yes | | `audit:deps` | `npm audit` (root + electron) — no high/critical advisories (overlaps osv `check:vuln-ratchet`; see Rationalization Backlog) | Yes | diff --git a/open-sse/handlers/chatCore/modelLifecyclePolicy.ts b/open-sse/handlers/chatCore/modelLifecyclePolicy.ts index 698e3c5c8e..ed2ed9513f 100644 --- a/open-sse/handlers/chatCore/modelLifecyclePolicy.ts +++ b/open-sse/handlers/chatCore/modelLifecyclePolicy.ts @@ -40,13 +40,15 @@ function getModelLifecycleError({ export function checkLifecycle(provider: string, model: string, log?: LifecycleLogger) { return getModelLifecycleError({ provider, - model: resolveModelAlias(model), + // #11503: pass the provider so a globally deprecated id this provider still serves + // under its original name is left alone instead of rewritten into a 404. + model: resolveModelAlias(model, provider), log, }); } export function resolveLifecycle(provider: string, model: string, log?: LifecycleLogger) { - const resolvedModel = resolveModelAlias(model); + const resolvedModel = resolveModelAlias(model, provider); if (resolvedModel !== model) { log?.info?.("ALIAS", `Model alias applied: ${model} → ${resolvedModel}`); } diff --git a/open-sse/services/autoCombo/__tests__/autoCombo.test.ts b/open-sse/services/autoCombo/__tests__/autoCombo.test.ts index 96c2dbc42e..ba4cce5b83 100644 --- a/open-sse/services/autoCombo/__tests__/autoCombo.test.ts +++ b/open-sse/services/autoCombo/__tests__/autoCombo.test.ts @@ -71,7 +71,7 @@ describe("Scoring", () => { describe("Task Fitness", () => { it("should return fitness score for known model+task", () => { - const score = getTaskFitness("claude-sonnet", "coding"); + const score = getTaskFitness("o3", "coding"); expect(score).toBeGreaterThan(0.5); }); @@ -494,7 +494,7 @@ describe("LKGP Strategy", () => { describe("Task Fitness Resolution Chain", () => { it("getTaskFitness should return static table score for known models", () => { - const score = getTaskFitness("claude-sonnet", "coding"); + const score = getTaskFitness("o3", "coding"); expect(score).toBe(0.95); }); @@ -514,7 +514,7 @@ describe("Task Fitness Resolution Chain", () => { }); it("getTaskFitnessWithSource should return source='fitness_table' for known static models", () => { - const result = getTaskFitnessWithSource("claude-sonnet", "coding"); + const result = getTaskFitnessWithSource("o3", "coding"); expect(result).toEqual({ score: 0.95, source: "fitness_table" }); }); @@ -574,7 +574,7 @@ describe("Task Fitness Resolution Chain", () => { }); it("getTaskFitnessWithSource returns 'fitness_table' for statically known models", () => { - const result = getTaskFitnessWithSource("claude-sonnet", "review"); + const result = getTaskFitnessWithSource("o3", "review"); expect(result.source).toBe("fitness_table"); expect(result.score).toBe(0.92); }); @@ -595,15 +595,15 @@ describe("Task Fitness DB Resolution Chain", () => { it("falls back to static FITNESS_TABLE when DB is not initialized", () => { // In the test environment, DB is typically not initialized, // so getTaskFitness should fall through to the static table - const score = getTaskFitness("claude-sonnet", "coding"); - // Static table has claude-sonnet → 0.95 for coding + const score = getTaskFitness("o3", "coding"); + // Static table has o3 → 0.95 for coding expect(score).toBe(0.95); }); it("falls back to static FITNESS_TABLE for review task type", () => { - const score = getTaskFitness("claude-opus", "review"); - // Static table has claude-opus → 0.95 for review - expect(score).toBe(0.95); + const score = getTaskFitness("gemini-2.5-pro", "review"); + // Static table has gemini-2.5-pro → 0.93 for review + expect(score).toBe(0.93); }); it("falls back to wildcard boosts when no static entry exists and DB unavailable", () => { @@ -634,14 +634,14 @@ describe("Task Fitness DB Resolution Chain", () => { }); it("resolution chain: static table takes priority over wildcard for known models", () => { - // "claude-sonnet" is in the static table with coding=0.95 + // "o3" is in the static table with coding=0.95 // It does NOT match "coder" wildcard because the static table is checked first - const score = getTaskFitness("claude-sonnet", "coding"); + const score = getTaskFitness("o3", "coding"); expect(score).toBe(0.95); // From static table, NOT wildcard }); it("getTaskFitnessWithSource identifies fitness_table as source for known models", () => { - const model = "claude-sonnet"; + const model = "o3"; const category = "coding"; const result = getTaskFitnessWithSource(model, category); @@ -650,14 +650,14 @@ describe("Task Fitness DB Resolution Chain", () => { }); it("case insensitivity: model names are lowercased before lookup", () => { - const upperScore = getTaskFitness("CLAUDE-SONNET", "coding"); - const lowerScore = getTaskFitness("claude-sonnet", "coding"); + const upperScore = getTaskFitness("O3", "coding"); + const lowerScore = getTaskFitness("o3", "coding"); expect(upperScore).toBe(lowerScore); }); it("case insensitivity: task types are lowercased before lookup", () => { - const upperScore = getTaskFitness("claude-sonnet", "CODING"); - const lowerScore = getTaskFitness("claude-sonnet", "coding"); + const upperScore = getTaskFitness("o3", "CODING"); + const lowerScore = getTaskFitness("o3", "coding"); expect(upperScore).toBe(lowerScore); }); diff --git a/open-sse/services/autoCombo/__tests__/fitness-table-hygiene-11503.test.ts b/open-sse/services/autoCombo/__tests__/fitness-table-hygiene-11503.test.ts new file mode 100644 index 0000000000..87abe73f9b --- /dev/null +++ b/open-sse/services/autoCombo/__tests__/fitness-table-hygiene-11503.test.ts @@ -0,0 +1,102 @@ +/** + * TDD regression for #11503: `FITNESS_TABLE` (resolution layer 4) was a hand-maintained + * table of version-less family patterns matched by `String.includes`, so it kept ranking + * models the vendors had already retired — and ranked them ABOVE live flagships it had + * never heard of. On `release/v3.8.51`, `openai/gpt-5.2-codex` (shut down 2026-07-23) + * scored 0.98 for `coding` via the `codex` row while `claude-fable-5-thinking-max` and + * `gpt-5.6-sol-xhigh` — both live — fell to the wildcard 0.5. + * + * Two changes are pinned here: + * 1. The table only carries VERSIONED rows that name a live catalog model. Retired + * families (`codex`, `o1`, `mixtral`, `grok-4-fast`) and version-less family + * patterns (`claude-sonnet`, `qwen`, `llama`, `gemini-pro`, …) are gone. + * 2. Matching is anchored to segment boundaries, so a row can no longer leak into an + * id that merely contains its characters. + * + * An id the table does not know resolves to the wildcard baseline 0.5 — the documented + * neutral for "no evidence", never a quality claim. + */ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, it, expect } from "vitest"; +import { getStaticFitnessTableScore } from "../taskFitness"; +import { REGISTRY } from "../../../config/providers/index.ts"; + +const TASK_TYPES = ["coding", "review", "planning", "analysis", "debugging", "documentation"]; + +const lifecycle = JSON.parse( + readFileSync( + fileURLToPath(new URL("../../../../config/quality/model-lifecycle.json", import.meta.url)), + "utf8" + ) +) as { retired: Record }; + +const retiredIds = new Set( + Object.entries(lifecycle.retired) + .filter(([, entry]) => entry.status === "retired") + .map(([id]) => id.toLowerCase()) +); + +/** Every model id the provider catalog can route, including per-model aliases. */ +function collectCatalogIds(): string[] { + const ids = new Set(); + for (const entry of Object.values(REGISTRY) as Array<{ models?: unknown[] }>) { + for (const model of entry?.models ?? []) { + const record = model as { id?: unknown; aliases?: unknown[] }; + if (typeof record?.id === "string") ids.add(record.id); + for (const alias of record?.aliases ?? []) { + if (typeof alias === "string") ids.add(alias); + } + } + } + return [...ids]; +} + +/** A catalog id is retired when its bare form (after any `vendor/` prefix) is retired. */ +function isRetired(id: string): boolean { + const lower = id.toLowerCase(); + if (retiredIds.has(lower)) return true; + const slash = lower.lastIndexOf("/"); + return slash !== -1 && retiredIds.has(lower.slice(slash + 1)); +} + +describe("#11503 FITNESS_TABLE carries no retired or version-less rows", () => { + it("scores no routable retired model through the static table", () => { + // Scoped to ids the catalog can actually route: those are the ones layer 4 ranks. + // (Retired ids absent from the catalog — `gpt-4o-audio-preview` and friends — are + // unroutable, so a row matching them cannot invert a routing decision.) + const routableRetired = collectCatalogIds().filter(isRetired); + expect(routableRetired.length).toBeGreaterThan(0); + + const scored = routableRetired + .flatMap((id) => + TASK_TYPES.map((task) => ({ id, task, score: getStaticFitnessTableScore(id, task) })) + ) + .filter((row) => row.score !== null); + + expect(scored).toEqual([]); + }); + + it("does not capture live models through a version-less family row", () => { + // These are live flagships; before #11503 the family rows either scored them from a + // sibling's number or (worse) left them at 0.5 while a dead model scored 0.98. + expect(getStaticFitnessTableScore("claude-sonnet-5", "coding")).toBeNull(); + expect(getStaticFitnessTableScore("claude-fable-5-thinking-max", "coding")).toBeNull(); + expect(getStaticFitnessTableScore("gpt-5.6-sol-xhigh", "coding")).toBeNull(); + }); + + it("matches patterns on segment boundaries only", () => { + // `solar-pro3` contains "o3" but is not an o3 model. + expect(getStaticFitnessTableScore("solar-pro3", "coding")).toBeNull(); + // `chatgpt-4o-latest` contains "-4o-" but not `gpt-4o` at a boundary. OpenAI shut it + // down on 2026-02-17; it used to inherit the live flagship's 0.9. + expect(getStaticFitnessTableScore("chatgpt-4o-latest", "coding")).toBeNull(); + }); + + it("keeps hyphen-suffixed and vendor-prefixed forms of a versioned row", () => { + expect(getStaticFitnessTableScore("gpt-4o-mini", "coding")).toBe(0.8); + expect(getStaticFitnessTableScore("gpt-4o-2024-11-20", "coding")).toBe(0.9); + expect(getStaticFitnessTableScore("o3-mini", "coding")).toBe(0.95); + expect(getStaticFitnessTableScore("openai/o3", "coding")).toBe(0.95); + }); +}); diff --git a/open-sse/services/autoCombo/__tests__/taskFitness-pattern-order-8603.test.ts b/open-sse/services/autoCombo/__tests__/taskFitness-pattern-order-8603.test.ts index 643a15c123..509eff5acb 100644 --- a/open-sse/services/autoCombo/__tests__/taskFitness-pattern-order-8603.test.ts +++ b/open-sse/services/autoCombo/__tests__/taskFitness-pattern-order-8603.test.ts @@ -46,8 +46,11 @@ describe("#8603 static fitness table matches longest pattern first", () => { }); it("falls back to the default table for an unknown task type", () => { - // "claude-sonnet" is absent from a nonexistent table but present in `default`. - expect(getStaticFitnessTableScore("claude-sonnet", "no-such-task")).toBe(0.85); + // "gpt-4o" is absent from a nonexistent table but present in `default`, where it + // scores 0.85 rather than the 0.9 it gets for `coding`. + // (#11503 swapped the probe from the version-less "claude-sonnet" row, which that + // PR removed from the table; the fallback behaviour under test is unchanged.) + expect(getStaticFitnessTableScore("gpt-4o", "no-such-task")).toBe(0.85); }); it("returns null when nothing matches", () => { diff --git a/open-sse/services/autoCombo/taskFitness.ts b/open-sse/services/autoCombo/taskFitness.ts index 83899365e3..e67637fe3b 100644 --- a/open-sse/services/autoCombo/taskFitness.ts +++ b/open-sse/services/autoCombo/taskFitness.ts @@ -11,11 +11,26 @@ * from, when `resolveScoresAs` resolves one (#11489). Reported as * `:inherited`. * 3. Models.dev tier — derived from `model_capabilities` table capability data - * 4. Static FITNESS_TABLE — existing hardcoded lookup (current behavior) - * 5. Wildcard boosts — existing pattern matching boosts (current behavior) + * 4. Static FITNESS_TABLE — small hand-maintained table of VERSIONED model ids + * 5. Wildcard boosts — pattern matching boosts over a neutral 0.5 baseline + * + * Layer 4 is deliberately small and versioned (#11503). It used to carry + * version-less family patterns (`claude-sonnet`, `qwen`, `llama`, `codex`, …) that + * matched by substring, so it kept scoring model families whose members the vendors + * had already retired — and it scored them ABOVE live flagships it had never heard + * of. Recognition by the table, not availability, decided rank. + * + * Two rules keep it honest: + * - Every row names a versioned id that exists in the provider catalog. A family + * pattern that would silently adopt every future member of that family does not + * belong here; `scripts/check/check-model-lifecycle.mjs` fails the build when a + * row starts matching an id the vendor has retired. + * - An id this table does not know falls through to the wildcard baseline of 0.5. + * That 0.5 means "no evidence", NOT "mediocre model" — it is the neutral point, + * and it must never be read as a quality claim about the model. */ -// ─── Static fitness table (unchanged, fallback layer 4) ───────────────── +// ─── Static fitness table (versioned rows only, fallback layer 4) ──────── import { getDbInstance } from "../../../src/lib/db/core.ts"; import { @@ -27,18 +42,11 @@ import { resolveScoresAs } from "./scoresAs.ts"; const FITNESS_TABLE: Record> = { coding: { - "claude-sonnet": 0.95, - "claude-opus": 0.92, - "claude-haiku": 0.78, "gpt-4o": 0.9, "gpt-4o-mini": 0.8, "gpt-4-turbo": 0.88, - o1: 0.93, o3: 0.95, "o4-mini": 0.88, - codex: 0.98, - "gemini-pro": 0.85, - "gemini-flash": 0.8, "gemini-2.5-pro": 0.92, "gemini-2.5-flash": 0.82, "deepseek-coder": 0.9, @@ -46,98 +54,54 @@ const FITNESS_TABLE: Record> = { "deepseek-r1": 0.88, "deepseek-chat": 0.84, // DeepSeek V3.2 Chat — strong code performance "deepseek-v3.2": 0.86, // Explicit V3.2 alias - qwen: 0.78, - llama: 0.72, - mistral: 0.75, - mixtral: 0.77, - // Grok-4 fast — good code, ultra-low latency (1143ms P50) - "grok-4-fast": 0.8, - "grok-4": 0.82, "grok-3": 0.8, - // Kimi K2.5 — agentic with tool calling, good at code tasks - "kimi-k2": 0.82, - // GLM-5.1 / GLM-5 — Z.AI reasoning models, 200K context / 128k output + // GLM-5.1 — Z.AI reasoning model, 200K context / 128k output "glm-5.1": 0.78, - "glm-5": 0.78, // MiniMax M2.5 — reasoning support helps complex code "minimax-m2.5": 0.75, "minimax-m2": 0.72, }, review: { - "claude-sonnet": 0.92, - "claude-opus": 0.95, - "claude-haiku": 0.7, "gpt-4o": 0.88, "gpt-4o-mini": 0.72, - o1: 0.9, o3: 0.92, - "gemini-pro": 0.9, "gemini-2.5-pro": 0.93, - "gemini-flash": 0.75, "deepseek-r1": 0.85, "deepseek-v3": 0.8, }, planning: { - "claude-opus": 0.95, - "claude-sonnet": 0.9, "gpt-4o": 0.88, - o1: 0.92, o3: 0.95, "gemini-2.5-pro": 0.93, - "gemini-pro": 0.88, "deepseek-r1": 0.85, }, analysis: { - "claude-opus": 0.95, - "claude-sonnet": 0.92, "gemini-2.5-pro": 0.95, - "gemini-pro": 0.88, "gemini-3.1-pro": 0.95, // Gemini 3.1 Pro — 1M context, ideal for long analysis "gpt-4o": 0.85, - o1: 0.9, o3: 0.93, "deepseek-r1": 0.88, "deepseek-chat": 0.8, - "kimi-k2": 0.82, // Kimi K2.5 agentic — good for analysis "glm-5.1": 0.82, // GLM-5.1 free reasoning, 200K context for long analysis - "glm-5": 0.78, // GLM-5 with 128k output for long analysis "minimax-m2.5": 0.76, }, debugging: { - "claude-sonnet": 0.93, - "claude-opus": 0.9, "gpt-4o": 0.88, - o1: 0.85, "deepseek-coder": 0.9, "deepseek-v3": 0.82, - "gemini-flash": 0.78, - codex: 0.92, }, documentation: { - "claude-sonnet": 0.9, - "claude-opus": 0.88, "gpt-4o": 0.92, "gpt-4o-mini": 0.85, - "gemini-pro": 0.88, - "gemini-flash": 0.82, "deepseek-v3": 0.78, }, default: { - "claude-sonnet": 0.85, - "claude-opus": 0.85, "gpt-4o": 0.85, - "gemini-pro": 0.8, "gemini-3.1-pro": 0.85, "deepseek-v3": 0.75, "deepseek-chat": 0.74, - "gemini-flash": 0.72, - // New models from ClawRouter analysis (2026-03-17): - "grok-4-fast": 0.72, // ultra-fast, suitable for all tasks - "grok-4": 0.74, "grok-3": 0.73, - "kimi-k2": 0.76, // agentic multi-step tasks "glm-5.1": 0.75, - "glm-5": 0.7, "minimax-m2.5": 0.7, }, }; @@ -301,6 +265,36 @@ export function getModelsDevTierFitness(model: string, taskType: string): number // ─── Resolution chain ─────────────────────────────────────────────────── +/** Characters that separate the segments of a model id (`openai/gpt-4o-2024-05-13`). */ +const SEGMENT_SEPARATORS = new Set(["-", ".", "/"]); + +/** + * True when `pattern` occurs in `model` delimited by segment boundaries on BOTH sides — + * i.e. it starts at the beginning of the id or right after a `-` / `.` / `/`, and ends at + * the end of the id or right before one. + * + * This is what makes a versioned row safe to keep (#11503). Substring matching scored + * models the row was never about: + * - `o3` matched `solar-pro3` (an unrelated Upstage model) + * - `gpt-4o` matched `chatgpt-4o-latest`, which OpenAI shut down on 2026-02-17 — + * so a dead model inherited the live flagship's 0.9. With boundary matching the + * `-4o-` inside `chatgpt-4o-latest` is not a `gpt-4o` occurrence at a boundary + * (the preceding character is `t`), so it falls through to the neutral 0.5. + * Legitimate forms still match: `o3`, `o3-mini`, `openai/o3`, `gpt-4o-2024-05-13`. + */ +function matchesAtSegmentBoundary(model: string, pattern: string): boolean { + if (!pattern) return false; + let index = model.indexOf(pattern); + while (index !== -1) { + const startsAtBoundary = index === 0 || SEGMENT_SEPARATORS.has(model[index - 1]); + const endIndex = index + pattern.length; + const endsAtBoundary = endIndex === model.length || SEGMENT_SEPARATORS.has(model[endIndex]); + if (startsAtBoundary && endsAtBoundary) return true; + index = model.indexOf(pattern, index + 1); + } + return false; +} + /** * Resolve a model id against the static fitness table, LONGEST PATTERN FIRST (#8603). * @@ -315,6 +309,11 @@ export function getModelsDevTierFitness(model: string, taskType: string): number * DB (user_override / arena_elo / models.dev tier) before reaching layer 4, so pinning * the ordering guarantee through `getTaskFitness` would depend on DB fixture state. * `taskFitness-pattern-order-8603.test.ts` calls this directly instead. + * + * Matching is anchored to SEGMENT BOUNDARIES since #11503: a plain `String.includes` + * let a row leak into ids that merely contain its characters — `o3` scored + * `solar-pro3`, and `gpt-4o` scored `chatgpt-4o-latest` (a different, retired model). + * See `matchesAtSegmentBoundary`. */ export function getStaticFitnessTableScore(model: string, taskType: string): number | null { const normalizedModel = model.toLowerCase(); @@ -322,7 +321,7 @@ export function getStaticFitnessTableScore(model: string, taskType: string): num const table = FITNESS_TABLE[normalizedTask] || FITNESS_TABLE.default; const sortedEntries = Object.entries(table).sort((a, b) => b[0].length - a[0].length); for (const [pattern, score] of sortedEntries) { - if (normalizedModel.includes(pattern)) return score; + if (matchesAtSegmentBoundary(normalizedModel, pattern)) return score; } return null; } diff --git a/open-sse/services/model.ts b/open-sse/services/model.ts index 04be2c9f35..2c28b60953 100644 --- a/open-sse/services/model.ts +++ b/open-sse/services/model.ts @@ -276,7 +276,16 @@ function resolveProviderModelAlias( return aliases?.[modelId] || modelId; } -function hasKnownProviderModel(providerOrAlias: string | null | undefined, modelId: string | null) { +/** + * True when the provider serves `modelId` under that exact id (static model list or + * a provider-specific alias). Exported for the deprecation-alias provider guard + * (#11503): a globally deprecated id that this provider still serves must not be + * rewritten out from under the request. + */ +export function hasKnownProviderModel( + providerOrAlias: string | null | undefined, + modelId: string | null +) { if (!providerOrAlias || !modelId) return false; const providerId = resolveProviderAlias(providerOrAlias); diff --git a/open-sse/services/modelDeprecation.ts b/open-sse/services/modelDeprecation.ts index 90ac077b1d..8a120d6de0 100644 --- a/open-sse/services/modelDeprecation.ts +++ b/open-sse/services/modelDeprecation.ts @@ -7,6 +7,8 @@ * Supports both built-in aliases (static) and custom aliases (persisted via Settings API). */ +import { hasKnownProviderModel } from "./model.ts"; + // ── Built-in Deprecation Aliases ──────────────────────────────────────────── // These are known renames/retirements across providers. // Format: deprecated ID → current ID @@ -20,17 +22,23 @@ const BUILT_IN_ALIASES: Record = { "gemini-2.0-flash": "gemini-2.5-flash", "gemini-2.0-flash-lite": "gemini-3.1-flash-lite", "gemini-3.1-flash-lite-preview": "gemini-3.1-flash-lite", - "gemini-3-pro-high": "gemini-3.1-pro-high", + // #11503: the catalog spells this one with hyphens ("gemini-3-1-pro-high"); the + // dotted form is not a routable id, so the rewrite used to guarantee a 404. + "gemini-3-pro-high": "gemini-3-1-pro-high", "gemini-3-pro-low": "gemini-3.1-pro-low", // Retired free Gemma (was in the gemini-free pool) → current gemini-free model "gemma-4": "gemini-3.1-flash-lite", - // Claude legacy → current - "claude-3-opus-20240229": "claude-opus-4-20250514", - "claude-3-sonnet-20240229": "claude-sonnet-4-20250514", - "claude-3-haiku-20240307": "claude-3-5-sonnet-20241022", - "claude-3-5-sonnet-latest": "claude-sonnet-4-20250514", - "claude-3-5-haiku-latest": "claude-3-5-sonnet-20241022", + // Claude legacy → current. + // #11503: these five forwarded one retired id to another (claude-opus-4-20250514 and + // claude-sonnet-4-20250514 retired 2026-06-15, claude-3-5-sonnet-20241022 retired + // 2025-10-28), so every rewrite landed on a model the vendor no longer serves. The + // targets below are the replacements Anthropic publishes on its deprecations page. + "claude-3-opus-20240229": "claude-opus-4-8", + "claude-3-sonnet-20240229": "claude-sonnet-4-6", + "claude-3-haiku-20240307": "claude-haiku-4-5-20251001", + "claude-3-5-sonnet-latest": "claude-sonnet-4-6", + "claude-3-5-haiku-latest": "claude-haiku-4-5-20251001", // Kimi/Moonshot — Fireworks long-path aliases (#265) "accounts/fireworks/models/kimi-k2p5": "moonshotai/Kimi-K2.5", @@ -57,7 +65,9 @@ const BUILT_IN_ALIASES: Record = { // Llama short aliases "llama-3.3": "llama-3.3-70b-versatile", "llama-3-70b": "llama-3.3-70b-versatile", - "llama-3-8b": "llama3-8b-8192", + // #11503: llama3-8b-8192 was deprecated by Groq on 2025-08-30 and is not in the + // catalog; llama-3.1-8b-instant is the replacement Groq names. + "llama-3-8b": "llama-3.1-8b-instant", }; // ── Custom Aliases (persisted via Settings API) ───────────────────────────── @@ -106,20 +116,30 @@ export function getAllAliases(): Record { * Resolve a model alias to its current ID. * Custom aliases override built-in ones. * + * The table is GLOBAL but the catalog is not: `kimi-k2`, `gemini-2.0-flash` and friends + * are still served under their original id by some aggregators, and rewriting them there + * turns a working request into a 404. So when the caller knows which provider will serve + * the request (#11503), a provider that lists the id as-is wins over the alias table. + * Callers with no provider in hand keep the previous unconditional behaviour. + * * @param {string} modelId - The model ID to resolve + * @param {string | null} [provider] - Provider (id or alias) that will serve the request * @returns {string} The resolved model ID, or the original if not deprecated */ -export function resolveModelAlias(modelId: string): string { +export function resolveModelAlias(modelId: string, provider?: string | null): string { if (!modelId) return modelId; - // Check custom aliases first (higher priority) + // Check custom aliases first (higher priority). An operator-authored alias is an + // explicit instruction, so it applies even when the provider serves the source id. const custom = customAliases(); if (custom[modelId]) return custom[modelId]; - // Then check built-in - if (BUILT_IN_ALIASES[modelId]) return BUILT_IN_ALIASES[modelId]; + if (!BUILT_IN_ALIASES[modelId]) return modelId; - return modelId; + // The provider serves this id itself — nothing is deprecated from its point of view. + if (provider && hasKnownProviderModel(provider, modelId)) return modelId; + + return BUILT_IN_ALIASES[modelId]; } /** diff --git a/package.json b/package.json index f0957e7ac5..4715e5ed37 100644 --- a/package.json +++ b/package.json @@ -162,6 +162,7 @@ "check:cli-i18n": "node scripts/check/check-cli-i18n.mjs", "check:openapi-coverage": "node scripts/check/check-openapi-coverage.mjs", "check:openapi-security-tiers": "node scripts/check/check-openapi-security-tiers.mjs", + "check:model-lifecycle": "node --import tsx/esm scripts/check/check-model-lifecycle.mjs", "check:provider-consistency": "bun scripts/check/check-provider-consistency.ts", "check:provider-assets": "node scripts/check/check-provider-assets.mjs", "check:nvidia-catalog-drift": "node --import tsx/esm scripts/check/check-nvidia-catalog-drift.ts", @@ -210,6 +211,7 @@ "check:workflows": "node scripts/check/check-workflows.mjs", "quality:collect": "node scripts/quality/collect-metrics.mjs", "quality:ratchet": "node scripts/quality/check-quality-ratchet.mjs", + "quality:refresh-model-lifecycle": "node scripts/quality/refresh-model-lifecycle.mjs", "quality:gate": "npm run quality:collect && npm run quality:ratchet -- --allow-missing", "quality:scan": "node scripts/quality/run-all-gates.mjs", "quality:scan:fast": "node scripts/quality/run-all-gates.mjs --fast", diff --git a/scripts/check/check-model-lifecycle.mjs b/scripts/check/check-model-lifecycle.mjs new file mode 100644 index 0000000000..7bad03ec60 --- /dev/null +++ b/scripts/check/check-model-lifecycle.mjs @@ -0,0 +1,165 @@ +#!/usr/bin/env node +// scripts/check/check-model-lifecycle.mjs +// Gate anti-drift (#11503): as duas tabelas mantidas à mão que decidem roteamento — +// FITNESS_TABLE (open-sse/services/autoCombo/taskFitness.ts, camada 4 do task fitness) e +// BUILT_IN_ALIASES (open-sse/services/modelDeprecation.ts, reescreve `body.model` em toda +// request) — apodrecem em silêncio quando o fornecedor aposenta um modelo. Em +// release/v3.8.51 o resultado foi uma inversão de ranking (modelo morto 0.98 vs flagship +// vivo 0.50) e aliases que garantiam 404. Este gate compara as duas contra o snapshot de +// ciclo de vida em config/quality/model-lifecycle.json (sem rede; regenerar com +// `npm run quality:refresh-model-lifecycle`). +// +// Três checagens, todas somadas antes do exit — nenhuma aborta as outras: +// (a) nenhum padrão do FITNESS_TABLE pontua um id aposentado que o catálogo roteia; +// (b) nenhum alvo de BUILT_IN_ALIASES está aposentado ou ausente do catálogo; +// (c) todo id aposentado ainda presente no REGISTRY tem encaminhamento em +// BUILT_IN_ALIASES ou consta em `allowedRetiredInCatalog` (a catraca a queimar). +// +// (a) é deliberadamente restrita aos ids ROTEÁVEIS: linhas versionadas legítimas como +// `gpt-4o` também casam com ids aposentados que o catálogo nunca serviu +// (`gpt-4o-audio-preview`), e esses não podem inverter decisão de roteamento nenhuma. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const ROOT = process.cwd(); +const SNAPSHOT_PATH = path.join(ROOT, "config/quality/model-lifecycle.json"); +const TASK_TYPES = ["coding", "review", "planning", "analysis", "debugging", "documentation"]; + +/** Ids de modelo que o catálogo consegue rotear (id + aliases de cada modelo). */ +export function collectCatalogIds(registry) { + const ids = new Set(); + for (const entry of Object.values(registry ?? {})) { + for (const model of entry?.models ?? []) { + if (typeof model?.id === "string") ids.add(model.id); + for (const alias of model?.aliases ?? []) { + if (typeof alias === "string") ids.add(alias); + } + } + } + return [...ids]; +} + +/** Um id do catálogo está aposentado quando sua forma nua (sem prefixo `vendor/`) está. */ +export function isRetiredId(id, retiredIds) { + const lower = String(id).toLowerCase(); + if (retiredIds.has(lower)) return true; + const slash = lower.lastIndexOf("/"); + return slash !== -1 && retiredIds.has(lower.slice(slash + 1)); +} + +/** (a) Linhas do FITNESS_TABLE que ainda pontuam um modelo aposentado e roteável. */ +export function findRetiredFitnessRows(routableRetiredIds, scoreFor, taskTypes = TASK_TYPES) { + const violations = []; + for (const id of routableRetiredIds) { + for (const task of taskTypes) { + const score = scoreFor(id, task); + if (score !== null && score !== undefined) { + violations.push( + `${id} scores ${score} for "${task}" via FITNESS_TABLE (vendor retired it)` + ); + } + } + } + return violations; +} + +/** (b) Alvos de alias aposentados ou fora do catálogo. */ +export function findBadAliasTargets(aliases, catalogIds, retiredIds) { + const catalog = new Set([...catalogIds].map((id) => id.toLowerCase())); + const violations = []; + for (const [source, target] of Object.entries(aliases)) { + const lower = String(target).toLowerCase(); + if (!catalog.has(lower)) { + violations.push(`${source} → ${target} (no provider in REGISTRY serves this id)`); + } else if (retiredIds.has(lower)) { + violations.push(`${source} → ${target} (the vendor has retired this id)`); + } + } + return violations; +} + +/** (c) Ids aposentados que o catálogo ainda roteia sem encaminhamento nem allowlist. */ +export function findUnforwardedRetiredIds(routableRetiredIds, aliases, allowlist) { + const allowed = new Set((allowlist ?? []).map((id) => String(id).toLowerCase())); + return routableRetiredIds + .filter((id) => !(id in aliases) && !allowed.has(String(id).toLowerCase())) + .map((id) => `${id} is retired but still routable with no BUILT_IN_ALIASES forward`); +} + +export function readSnapshot(snapshotPath = SNAPSHOT_PATH) { + const snapshot = JSON.parse(fs.readFileSync(snapshotPath, "utf8")); + const retiredIds = new Set( + Object.entries(snapshot.retired ?? {}) + .filter(([, entry]) => entry?.status === "retired") + .map(([id]) => id.toLowerCase()) + ); + return { snapshot, retiredIds }; +} + +async function loadProductionTables() { + // Nenhum gate pode migrar o banco do operador: taskFitness.ts importa src/lib/db/core.ts, + // então DATA_DIR aponta para um diretório descartável ANTES do import dinâmico. + process.env.DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-lifecycle-gate-")); + const [{ REGISTRY }, { getStaticFitnessTableScore }, { getBuiltInAliases }] = await Promise.all([ + import(pathToFileURL(path.join(ROOT, "open-sse/config/providers/index.ts")).href), + import(pathToFileURL(path.join(ROOT, "open-sse/services/autoCombo/taskFitness.ts")).href), + import(pathToFileURL(path.join(ROOT, "open-sse/services/modelDeprecation.ts")).href), + ]); + return { REGISTRY, getStaticFitnessTableScore, getBuiltInAliases }; +} + +function report(label, violations, hint) { + if (!violations.length) { + console.log(`[model-lifecycle] OK — ${label}`); + return 0; + } + console.error( + `[model-lifecycle] ${violations.length} violation(s) — ${label}:\n` + + violations.map((v) => " ✗ " + v).join("\n") + + `\n → ${hint}` + ); + return violations.length; +} + +async function main() { + const { snapshot, retiredIds } = readSnapshot(); + const { REGISTRY, getStaticFitnessTableScore, getBuiltInAliases } = await loadProductionTables(); + + const catalogIds = collectCatalogIds(REGISTRY); + const routableRetired = catalogIds.filter((id) => isRetiredId(id, retiredIds)).sort(); + const aliases = getBuiltInAliases(); + + let failures = 0; + failures += report( + `FITNESS_TABLE scores none of the ${routableRetired.length} routable retired id(s)`, + findRetiredFitnessRows(routableRetired, getStaticFitnessTableScore), + "drop the row from FITNESS_TABLE in open-sse/services/autoCombo/taskFitness.ts, or replace it with the versioned id of the live successor." + ); + failures += report( + `all ${Object.keys(aliases).length} BUILT_IN_ALIASES targets are live catalog models`, + findBadAliasTargets(aliases, catalogIds, retiredIds), + "point the alias at the replacement the vendor publishes (see `sources` in config/quality/model-lifecycle.json). Never invent a target." + ); + failures += report( + "every routable retired id is forwarded or allowlisted", + findUnforwardedRetiredIds(routableRetired, aliases, snapshot.allowedRetiredInCatalog), + "add a BUILT_IN_ALIASES forward to the vendor's replacement, remove the model from the provider catalog, or (last resort) add the id to `allowedRetiredInCatalog` in config/quality/model-lifecycle.json with a tracking issue." + ); + + if (failures) { + console.error(`[model-lifecycle] FAIL — ${failures} violation(s) across 3 check(s).`); + process.exit(1); + } + console.log( + `[model-lifecycle] PASS — snapshot ${snapshot.generatedAt}, ${retiredIds.size} retired id(s), ${catalogIds.length} catalog id(s).` + ); +} + +if (import.meta.url === pathToFileURL(process.argv[1] || "").href) { + main().catch((err) => { + console.error(`[model-lifecycle] ERROR — ${err?.message ?? err}`); + process.exit(1); + }); +} diff --git a/scripts/quality/refresh-model-lifecycle.mjs b/scripts/quality/refresh-model-lifecycle.mjs new file mode 100644 index 0000000000..151821632b --- /dev/null +++ b/scripts/quality/refresh-model-lifecycle.mjs @@ -0,0 +1,200 @@ +#!/usr/bin/env node +// scripts/quality/refresh-model-lifecycle.mjs +// Regenera config/quality/model-lifecycle.json a partir das páginas de depreciação de +// primeira mão listadas em `sources`. NÃO roda em CI (o gate check:model-lifecycle é +// offline, por design): é uma ferramenta de manutenção, executada à mão quando um +// fornecedor publica uma nova rodada de aposentadorias. +// +// Regra dura: o script NUNCA descarta uma entrada em silêncio. Fornecedor cuja página +// não pôde ser parseada mantém as entradas atuais do snapshot e imprime +// "manual review needed" — assim uma mudança de layout upstream vira um aviso, não uma +// perda de dados que abriria o gate. +// +// Uso: +// npm run quality:refresh-model-lifecycle # escreve o snapshot +// npm run quality:refresh-model-lifecycle -- --dry-run +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const ROOT = process.cwd(); +const SNAPSHOT_PATH = path.join(ROOT, "config/quality/model-lifecycle.json"); +const DRY_RUN = process.argv.includes("--dry-run"); + +/** Fornecedor de cada source, na ordem em que aparecem no snapshot. */ +const VENDOR_BY_HOST = { + "platform.claude.com": "anthropic", + "developers.openai.com": "openai", + "ai.google.dev": "google", + "docs.x.ai": "xai", + "console.groq.com": "groq", +}; + +/** + * Parser da página da Anthropic: uma tabela markdown + * `| API model name | Current state | Deprecated | Tentative retirement date |`. + * Só linhas cujo estado contém "retired"/"deprecated" viram entradas. + * + * @returns {Record} + */ +export function parseAnthropicTable(text) { + const out = {}; + const rows = text.split("\n").filter((line) => line.trim().startsWith("|")); + for (const row of rows) { + const cells = row + .split("|") + .slice(1, -1) + .map((cell) => cell.trim()); + if (cells.length < 3) continue; + const [model, state, deprecatedOn, retirementDate] = cells; + if (!/^[a-z0-9][a-z0-9.\-]*$/i.test(model)) continue; // pula cabeçalho e separadores + const lowerState = String(state).toLowerCase(); + const status = lowerState.includes("retired") + ? "retired" + : lowerState.includes("deprecated") + ? "retiring" + : null; + if (!status) continue; + out[model] = { + vendor: "anthropic", + status, + retiredOn: normalizeDate(retirementDate) ?? normalizeDate(deprecatedOn), + // A página lista o substituto em prosa, fora da tabela. Preencher aqui inventaria + // dado: o valor vigente do snapshot é preservado pelo merge. + replacement: null, + }; + } + return out; +} + +/** "2026-06-15" ou "June 15, 2026" → "2026-06-15"; qualquer outra coisa → null. */ +export function normalizeDate(value) { + if (!value) return null; + const iso = String(value).match(/\d{4}-\d{2}-\d{2}/); + if (iso) return iso[0]; + const parsed = Date.parse(String(value)); + return Number.isNaN(parsed) ? null : new Date(parsed).toISOString().slice(0, 10); +} + +/** + * Funde entradas recém-parseadas sobre as atuais SEM perder nada: + * - entrada nova → adicionada; + * - entrada existente → status/retiredOn atualizados, `replacement` preservado quando o + * parser não trouxe um (o fornecedor nem sempre publica substituto); + * - entrada que sumiu do parse → mantida (uma página pode paginar ou mudar de layout). + * + * @returns {{merged: Object, added: string[], updated: string[], kept: string[]}} + */ +export function mergeVendorEntries(current, parsed, vendor) { + const merged = { ...current }; + const added = []; + const updated = []; + const kept = []; + + for (const [id, entry] of Object.entries(parsed)) { + const existing = current[id]; + if (!existing) { + merged[id] = entry; + added.push(id); + continue; + } + const next = { + ...existing, + status: entry.status, + retiredOn: entry.retiredOn ?? existing.retiredOn, + replacement: entry.replacement ?? existing.replacement, + }; + merged[id] = next; + if (JSON.stringify(next) !== JSON.stringify(existing)) updated.push(id); + } + + for (const [id, entry] of Object.entries(current)) { + if (entry?.vendor === vendor && !(id in parsed)) kept.push(id); + } + return { merged, added, updated, kept }; +} + +async function fetchText(url) { + const res = await fetch(url, { headers: { "user-agent": "omniroute-lifecycle-refresh" } }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + return res.text(); +} + +async function main() { + const snapshot = JSON.parse(fs.readFileSync(SNAPSHOT_PATH, "utf8")); + let retired = { ...snapshot.retired }; + const warnings = []; + + for (const url of snapshot.sources) { + const host = new URL(url).host; + const vendor = VENDOR_BY_HOST[host] ?? host; + let text; + try { + text = await fetchText(url); + } catch (err) { + warnings.push( + `${vendor}: fetch failed (${err.message}) — existing entries kept, manual review needed` + ); + continue; + } + + if (vendor !== "anthropic") { + // Só a Anthropic publica a lista como tabela estável. Os demais são HTML/JS que + // mudam de forma; parsear por heurística arriscaria apagar dados corretos. + warnings.push( + `${vendor}: no reliable parser — existing entries kept, manual review needed (${url})` + ); + continue; + } + + const parsed = parseAnthropicTable(text); + if (!Object.keys(parsed).length) { + warnings.push( + `${vendor}: table parsed to 0 rows (layout change?) — existing entries kept, manual review needed` + ); + continue; + } + const { merged, added, updated, kept } = mergeVendorEntries(retired, parsed, vendor); + retired = merged; + console.log( + `[refresh-model-lifecycle] ${vendor}: ${added.length} added, ${updated.length} updated, ${kept.length} kept (not in this parse)` + ); + for (const id of added) console.log(` + ${id}`); + for (const id of updated) console.log(` ~ ${id}`); + } + + const next = { + ...snapshot, + generatedAt: new Date().toISOString().slice(0, 10), + retired, + }; + + for (const warning of warnings) console.warn(`[refresh-model-lifecycle] WARN ${warning}`); + + const before = Object.keys(snapshot.retired).length; + const after = Object.keys(retired).length; + if (after < before) { + console.error( + `[refresh-model-lifecycle] ABORT — entry count dropped ${before} → ${after}. The script must never lose an entry; investigate before writing.` + ); + process.exit(1); + } + + if (DRY_RUN) { + console.log( + `[refresh-model-lifecycle] dry run — ${before} → ${after} entries, nothing written.` + ); + return; + } + fs.writeFileSync(SNAPSHOT_PATH, JSON.stringify(next, null, 2) + "\n"); + console.log( + `[refresh-model-lifecycle] wrote ${path.relative(ROOT, SNAPSHOT_PATH)} — ${after} entries. Review the diff, then run \`npm run check:model-lifecycle\`.` + ); +} + +if (import.meta.url === pathToFileURL(process.argv[1] || "").href) { + main().catch((err) => { + console.error(`[refresh-model-lifecycle] ERROR — ${err?.message ?? err}`); + process.exit(1); + }); +} diff --git a/tests/unit/check-model-lifecycle-gate.test.ts b/tests/unit/check-model-lifecycle-gate.test.ts new file mode 100644 index 0000000000..d717ffabe4 --- /dev/null +++ b/tests/unit/check-model-lifecycle-gate.test.ts @@ -0,0 +1,95 @@ +/** + * Unit coverage for the #11503 drift gate (`scripts/check/check-model-lifecycle.mjs`). + * + * The gate's value is that it goes red when a hand-maintained routing table starts + * pointing at a model the vendor retired, so each of its three checks is exercised here + * against small fixtures rather than against the live catalog (which would make the test + * a duplicate of the gate run itself, and red for reasons unrelated to the logic). + */ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + collectCatalogIds, + isRetiredId, + findRetiredFitnessRows, + findBadAliasTargets, + findUnforwardedRetiredIds, +} from "../../scripts/check/check-model-lifecycle.mjs"; + +const RETIRED = new Set(["dead-model-1", "dead-model-2", "gpt-5.2-codex"]); + +describe("check-model-lifecycle: catalog extraction", () => { + it("collects model ids and their aliases", () => { + const registry = { + alpha: { models: [{ id: "live-1", aliases: ["live-1-alias"] }, { id: "dead-model-1" }] }, + beta: { models: [{ id: "live-2" }] }, + gamma: {}, + }; + assert.deepEqual(collectCatalogIds(registry).sort(), [ + "dead-model-1", + "live-1", + "live-1-alias", + "live-2", + ]); + }); + + it("treats a vendor-prefixed id as retired when its bare form is", () => { + assert.equal(isRetiredId("openai/gpt-5.2-codex", RETIRED), true); + assert.equal(isRetiredId("GPT-5.2-Codex", RETIRED), true); + assert.equal(isRetiredId("live-1", RETIRED), false); + }); +}); + +describe("check-model-lifecycle: (a) retired ids scored by FITNESS_TABLE", () => { + const scoreFor = (id: string, task: string) => + id === "dead-model-1" && task === "coding" ? 0.98 : null; + + it("flags a retired id the table still scores", () => { + const violations = findRetiredFitnessRows(["dead-model-1"], scoreFor, ["coding", "review"]); + assert.equal(violations.length, 1); + assert.match(violations[0], /dead-model-1 scores 0\.98 for "coding"/); + }); + + it("passes when no retired id resolves through the table", () => { + assert.deepEqual(findRetiredFitnessRows(["dead-model-2"], scoreFor, ["coding"]), []); + }); +}); + +describe("check-model-lifecycle: (b) alias targets", () => { + const catalog = ["live-1", "dead-model-1"]; + + it("flags a target absent from the catalog", () => { + const violations = findBadAliasTargets({ old: "not-in-catalog" }, catalog, RETIRED); + assert.equal(violations.length, 1); + assert.match(violations[0], /no provider in REGISTRY serves this id/); + }); + + it("flags a target the vendor has retired", () => { + const violations = findBadAliasTargets({ old: "dead-model-1" }, catalog, RETIRED); + assert.equal(violations.length, 1); + assert.match(violations[0], /the vendor has retired this id/); + }); + + it("passes for a live catalog target", () => { + assert.deepEqual(findBadAliasTargets({ old: "live-1" }, catalog, RETIRED), []); + }); +}); + +describe("check-model-lifecycle: (c) routable retired ids", () => { + it("flags a retired id with neither forward nor allowlist entry", () => { + const violations = findUnforwardedRetiredIds(["dead-model-1"], {}, []); + assert.equal(violations.length, 1); + assert.match(violations[0], /retired but still routable/); + }); + + it("accepts an allowlisted id", () => { + assert.deepEqual(findUnforwardedRetiredIds(["dead-model-1"], {}, ["dead-model-1"]), []); + }); + + it("accepts an id that has a BUILT_IN_ALIASES forward", () => { + assert.deepEqual( + findUnforwardedRetiredIds(["dead-model-1"], { "dead-model-1": "live-1" }, []), + [] + ); + }); +}); diff --git a/tests/unit/model-deprecation-aliases-11503.test.ts b/tests/unit/model-deprecation-aliases-11503.test.ts new file mode 100644 index 0000000000..5d841225eb --- /dev/null +++ b/tests/unit/model-deprecation-aliases-11503.test.ts @@ -0,0 +1,104 @@ +/** + * TDD regression for #11503: `BUILT_IN_ALIASES` is applied to `body.model` on every + * request (`modelLifecyclePolicy.ts`), so a stale target is a guaranteed 404. Seven rows + * forwarded to a model the vendor had retired or the catalog never carried — five Claude + * rows chained one retired id to another, `gemini-3-pro-high` used a spelling + * (`gemini-3.1-pro-high`) no provider serves, and `llama-3-8b` pointed at a Groq model + * deprecated on 2025-08-30. + * + * The table is also global while the catalog is not: aggregators still serve `kimi-k2` + * and `gemini-2.0-flash` under their original ids, and the unconditional rewrite broke + * them. `resolveModelAlias` now takes the provider and skips the rewrite when that + * provider serves the id as-is. + */ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { + getBuiltInAliases, + resolveModelAlias, + setCustomAliases, +} from "../../open-sse/services/modelDeprecation.ts"; +import { REGISTRY } from "../../open-sse/config/providers/index.ts"; + +const lifecycle = JSON.parse( + readFileSync( + fileURLToPath(new URL("../../config/quality/model-lifecycle.json", import.meta.url)), + "utf8" + ) +) as { retired: Record }; + +const retiredIds = new Set( + Object.entries(lifecycle.retired) + .filter(([, entry]) => entry.status === "retired") + .map(([id]) => id.toLowerCase()) +); + +/** Every model id the catalog can route, lowercased, including per-model aliases. */ +const catalogIds = (() => { + const ids = new Set(); + for (const entry of Object.values(REGISTRY) as Array<{ models?: unknown[] }>) { + for (const model of entry?.models ?? []) { + const record = model as { id?: unknown; aliases?: unknown[] }; + if (typeof record?.id === "string") ids.add(record.id.toLowerCase()); + for (const alias of record?.aliases ?? []) { + if (typeof alias === "string") ids.add(alias.toLowerCase()); + } + } + } + return ids; +})(); + +describe("#11503 BUILT_IN_ALIASES point at live catalog models", () => { + const aliases = Object.entries(getBuiltInAliases()); + + it("has aliases to check", () => { + assert.ok(aliases.length > 0); + }); + + for (const [source, target] of aliases) { + it(`forwards ${source} to a model the catalog serves`, () => { + assert.ok( + catalogIds.has(target.toLowerCase()), + `"${source}" forwards to "${target}", which no provider in REGISTRY serves` + ); + }); + + it(`forwards ${source} to a model the vendor has not retired`, () => { + assert.ok( + !retiredIds.has(target.toLowerCase()), + `"${source}" forwards to "${target}", which the vendor has retired` + ); + }); + } + + it("applies the corrected Claude replacement", () => { + // Was claude-opus-4-20250514, retired 2026-06-15. + assert.equal(resolveModelAlias("claude-3-opus-20240229"), "claude-opus-4-8"); + }); +}); + +describe("#11503 resolveModelAlias respects the serving provider", () => { + it("leaves the id alone when the provider serves it as-is", () => { + assert.equal(resolveModelAlias("kimi-k2", "t3-web"), "kimi-k2"); + }); + + it("still rewrites for a provider that only knows the canonical id", () => { + assert.equal(resolveModelAlias("kimi-k2", "fireworks"), "moonshotai/Kimi-K2"); + }); + + it("keeps the unconditional rewrite when no provider is known", () => { + assert.equal(resolveModelAlias("kimi-k2"), "moonshotai/Kimi-K2"); + }); + + it("lets a custom alias win over a built-in one, provider or not", () => { + setCustomAliases({ "kimi-k2": "custom-target" }); + try { + assert.equal(resolveModelAlias("kimi-k2"), "custom-target"); + assert.equal(resolveModelAlias("kimi-k2", "t3-web"), "custom-target"); + } finally { + setCustomAliases({}); + } + }); +}); diff --git a/tests/unit/model-deprecation.test.ts b/tests/unit/model-deprecation.test.ts index 7560a6afc2..42c7cc4e46 100644 --- a/tests/unit/model-deprecation.test.ts +++ b/tests/unit/model-deprecation.test.ts @@ -33,8 +33,10 @@ test("resolveModelAlias: resolves deprecated Gemini model", () => { }); test("resolveModelAlias: resolves deprecated Claude model", () => { - assert.equal(resolveModelAlias("claude-3-opus-20240229"), "claude-opus-4-20250514"); - assert.equal(resolveModelAlias("claude-3-5-sonnet-latest"), "claude-sonnet-4-20250514"); + // #11503: the previous targets (claude-opus-4-20250514 / claude-sonnet-4-20250514) + // were themselves retired on 2026-06-15. These are the replacements Anthropic publishes. + assert.equal(resolveModelAlias("claude-3-opus-20240229"), "claude-opus-4-8"); + assert.equal(resolveModelAlias("claude-3-5-sonnet-latest"), "claude-sonnet-4-6"); }); test("resolveModelAlias: does not silently reroute retired OpenAI models", () => {