#!/usr/bin/env node // scripts/check/check-model-lifecycle.mjs // Gate anti-drift (#11503): as duas tabelas mantidas à mão que decidem roteamento — // FITNESS_TABLE (open-sse/services/autoCombo/taskFitness.ts, camada 4 do task fitness) e // BUILT_IN_ALIASES (open-sse/services/modelDeprecation.ts, reescreve `body.model` em toda // request) — apodrecem em silêncio quando o fornecedor aposenta um modelo. Em // release/v3.8.51 o resultado foi uma inversão de ranking (modelo morto 0.98 vs flagship // vivo 0.50) e aliases que garantiam 404. Este gate compara as duas contra o snapshot de // ciclo de vida em config/quality/model-lifecycle.json (sem rede; regenerar com // `npm run quality:refresh-model-lifecycle`). // // Três checagens, todas somadas antes do exit — nenhuma aborta as outras: // (a) nenhum padrão do FITNESS_TABLE pontua um id aposentado que o catálogo roteia; // (b) nenhum alvo de BUILT_IN_ALIASES está aposentado ou ausente do catálogo; // (c) todo id aposentado ainda presente no REGISTRY tem encaminhamento em // BUILT_IN_ALIASES ou consta em `allowedRetiredInCatalog` (a catraca a queimar). // // (a) é deliberadamente restrita aos ids ROTEÁVEIS: linhas versionadas legítimas como // `gpt-4o` também casam com ids aposentados que o catálogo nunca serviu // (`gpt-4o-audio-preview`), e esses não podem inverter decisão de roteamento nenhuma. import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; const ROOT = process.cwd(); const SNAPSHOT_PATH = path.join(ROOT, "config/quality/model-lifecycle.json"); const TASK_TYPES = ["coding", "review", "planning", "analysis", "debugging", "documentation"]; /** Ids de modelo que o catálogo consegue rotear (id + aliases de cada modelo). */ export function collectCatalogIds(registry) { const ids = new Set(); for (const entry of Object.values(registry ?? {})) { for (const model of entry?.models ?? []) { if (typeof model?.id === "string") ids.add(model.id); for (const alias of model?.aliases ?? []) { if (typeof alias === "string") ids.add(alias); } } } return [...ids]; } /** Um id do catálogo está aposentado quando sua forma nua (sem prefixo `vendor/`) está. */ export function isRetiredId(id, retiredIds) { const lower = String(id).toLowerCase(); if (retiredIds.has(lower)) return true; const slash = lower.lastIndexOf("/"); return slash !== -1 && retiredIds.has(lower.slice(slash + 1)); } /** (a) Linhas do FITNESS_TABLE que ainda pontuam um modelo aposentado e roteável. */ export function findRetiredFitnessRows(routableRetiredIds, scoreFor, taskTypes = TASK_TYPES) { const violations = []; for (const id of routableRetiredIds) { for (const task of taskTypes) { const score = scoreFor(id, task); if (score !== null && score !== undefined) { violations.push( `${id} scores ${score} for "${task}" via FITNESS_TABLE (vendor retired it)` ); } } } return violations; } /** (b) Alvos de alias aposentados ou fora do catálogo. */ export function findBadAliasTargets(aliases, catalogIds, retiredIds) { const catalog = new Set([...catalogIds].map((id) => id.toLowerCase())); const violations = []; for (const [source, target] of Object.entries(aliases)) { const lower = String(target).toLowerCase(); if (!catalog.has(lower)) { violations.push(`${source} → ${target} (no provider in REGISTRY serves this id)`); } else if (retiredIds.has(lower)) { violations.push(`${source} → ${target} (the vendor has retired this id)`); } } return violations; } /** (c) Ids aposentados que o catálogo ainda roteia sem encaminhamento nem allowlist. */ export function findUnforwardedRetiredIds(routableRetiredIds, aliases, allowlist) { const allowed = new Set((allowlist ?? []).map((id) => String(id).toLowerCase())); return routableRetiredIds .filter((id) => !(id in aliases) && !allowed.has(String(id).toLowerCase())) .map((id) => `${id} is retired but still routable with no BUILT_IN_ALIASES forward`); } export function readSnapshot(snapshotPath = SNAPSHOT_PATH) { const snapshot = JSON.parse(fs.readFileSync(snapshotPath, "utf8")); const retiredIds = new Set( Object.entries(snapshot.retired ?? {}) .filter(([, entry]) => entry?.status === "retired") .map(([id]) => id.toLowerCase()) ); return { snapshot, retiredIds }; } async function loadProductionTables() { // Nenhum gate pode migrar o banco do operador: taskFitness.ts importa src/lib/db/core.ts, // então DATA_DIR aponta para um diretório descartável ANTES do import dinâmico. process.env.DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-lifecycle-gate-")); const [{ REGISTRY }, { getStaticFitnessTableScore }, { getBuiltInAliases }] = await Promise.all([ import(pathToFileURL(path.join(ROOT, "open-sse/config/providers/index.ts")).href), import(pathToFileURL(path.join(ROOT, "open-sse/services/autoCombo/taskFitness.ts")).href), import(pathToFileURL(path.join(ROOT, "open-sse/services/modelDeprecation.ts")).href), ]); return { REGISTRY, getStaticFitnessTableScore, getBuiltInAliases }; } function report(label, violations, hint) { if (!violations.length) { console.log(`[model-lifecycle] OK — ${label}`); return 0; } console.error( `[model-lifecycle] ${violations.length} violation(s) — ${label}:\n` + violations.map((v) => " ✗ " + v).join("\n") + `\n → ${hint}` ); return violations.length; } async function main() { const { snapshot, retiredIds } = readSnapshot(); const { REGISTRY, getStaticFitnessTableScore, getBuiltInAliases } = await loadProductionTables(); const catalogIds = collectCatalogIds(REGISTRY); const routableRetired = catalogIds.filter((id) => isRetiredId(id, retiredIds)).sort(); const aliases = getBuiltInAliases(); let failures = 0; failures += report( `FITNESS_TABLE scores none of the ${routableRetired.length} routable retired id(s)`, findRetiredFitnessRows(routableRetired, getStaticFitnessTableScore), "drop the row from FITNESS_TABLE in open-sse/services/autoCombo/taskFitness.ts, or replace it with the versioned id of the live successor." ); failures += report( `all ${Object.keys(aliases).length} BUILT_IN_ALIASES targets are live catalog models`, findBadAliasTargets(aliases, catalogIds, retiredIds), "point the alias at the replacement the vendor publishes (see `sources` in config/quality/model-lifecycle.json). Never invent a target." ); failures += report( "every routable retired id is forwarded or allowlisted", findUnforwardedRetiredIds(routableRetired, aliases, snapshot.allowedRetiredInCatalog), "add a BUILT_IN_ALIASES forward to the vendor's replacement, remove the model from the provider catalog, or (last resort) add the id to `allowedRetiredInCatalog` in config/quality/model-lifecycle.json with a tracking issue." ); if (failures) { console.error(`[model-lifecycle] FAIL — ${failures} violation(s) across 3 check(s).`); process.exit(1); } console.log( `[model-lifecycle] PASS — snapshot ${snapshot.generatedAt}, ${retiredIds.size} retired id(s), ${catalogIds.length} catalog id(s).` ); } if (import.meta.url === pathToFileURL(process.argv[1] || "").href) { main().catch((err) => { console.error(`[model-lifecycle] ERROR — ${err?.message ?? err}`); process.exit(1); }); }