diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9eb7202791..2f2a97b390 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,7 @@ jobs: - run: npm run check:cycles - run: npm run check:route-validation:t06 - run: npm run check:any-budget:t11 + - run: npm run check:provider-consistency - run: npm run check:docs-sync - run: npm run typecheck:core # typecheck:noimplicit:core is a forward-looking gate (noImplicitAny). @@ -45,6 +46,33 @@ jobs: - run: npm run typecheck:noimplicit:core continue-on-error: true + quality-gate: + name: Quality Ratchet + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.CI_NODE_VERSION }} + cache: npm + - run: npm ci + - run: npm run quality:collect + # Catraca: falha se qualquer métrica regredir vs quality-baseline.json (commitado). + # Hoje: contagem de warnings do ESLint. Fases 3/4 estendem com duplicação, + # tamanho de arquivo e cobertura (esta última lida do coverage mergeado). + - name: Ratchet check + run: node scripts/quality/check-quality-ratchet.mjs --summary .artifacts/quality-ratchet.md + - name: Append summary + if: always() + run: cat .artifacts/quality-ratchet.md >> "$GITHUB_STEP_SUMMARY" + - name: Upload ratchet report + if: always() + uses: actions/upload-artifact@v7 + with: + name: quality-ratchet + path: .artifacts/quality-ratchet.md + if-no-files-found: warn + docs-sync-strict: name: Docs Sync (Strict) runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 656b3a6f82..1f267ffbf9 100644 --- a/.gitignore +++ b/.gitignore @@ -201,4 +201,6 @@ pr_reviews*.json .data-dev/ # internal setup prompts with personal credentials — never commit -CODEX-SETUP-PROMPT.md \ No newline at end of file +CODEX-SETUP-PROMPT.md +# Quality ratchet — métricas efêmeras (baseline é commitado, métricas não) +quality-metrics.json diff --git a/package.json b/package.json index ca90c89911..7f9e625e0a 100644 --- a/package.json +++ b/package.json @@ -112,6 +112,7 @@ "check:cli-i18n": "node scripts/check/check-cli-i18n.mjs", "check:openapi-coverage": "node scripts/check/check-openapi-coverage.mjs", "check:openapi-security-tiers": "node scripts/check/check-openapi-security-tiers.mjs", + "check:provider-consistency": "node --import tsx scripts/check/check-provider-consistency.ts", "quality:collect": "node scripts/quality/collect-metrics.mjs", "quality:ratchet": "node scripts/quality/check-quality-ratchet.mjs", "quality:gate": "npm run quality:collect && npm run quality:ratchet", diff --git a/quality-baseline.json b/quality-baseline.json new file mode 100644 index 0000000000..347dc1aaf7 --- /dev/null +++ b/quality-baseline.json @@ -0,0 +1,6 @@ +{ + "_comment": "Catraca de qualidade. 'down' = nao pode aumentar; 'up' = nao pode cair. Atualize via 'npm run quality:ratchet -- --update' (somente quando melhora). Cada valor e um numero REAL medido, nunca um chute. Cobertura entra na Fase 4 a partir de um run de cobertura mergeada no CI.", + "metrics": { + "eslintWarnings": { "value": 3482, "direction": "down" } + } +} diff --git a/scripts/check/check-provider-consistency.ts b/scripts/check/check-provider-consistency.ts new file mode 100644 index 0000000000..ab08643815 --- /dev/null +++ b/scripts/check/check-provider-consistency.ts @@ -0,0 +1,45 @@ +#!/usr/bin/env node +// scripts/check/check-provider-consistency.ts +// Gate anti-alucinação nº1: toda entrada em REGISTRY (open-sse/config/providerRegistry.ts) +// deve corresponder a um provider canônico em src/shared/constants/providers.ts. +// Pega entradas de registry inventadas/meia-registradas (provider com baseUrl+models +// mas ausente da lista canônica → não selecionável pela máquina normal de providers). +// Catraca: exceções pré-existentes ficam em KNOWN_REGISTRY_ONLY; só NOVOS órfãos falham. +import { pathToFileURL } from "node:url"; +import { AI_PROVIDERS, getProviderById } from "@/shared/constants/providers.ts"; +import { REGISTRY } from "@omniroute/open-sse/config/providerRegistry.ts"; + +// Entradas registry-only conhecidas (meia-registro pré-existente). Cada uma com +// justificativa. Remover daqui ao registrar o provider em providers.ts. +export const KNOWN_REGISTRY_ONLY: Record = { + krutrim: + "Registry-only (baseUrl + krutrim-2-7b-instruct presentes) mas ausente de providers.ts — meia-registro pré-existente; triar em follow-up (registrar em APIKEY_PROVIDERS ou remover a entrada).", +}; + +/** Ids do REGISTRY que não são providers canônicos e não estão na allowlist. */ +export function findOrphanRegistryIds( + registryIds: string[], + isKnownProvider: (id: string) => boolean, + allowlist: Record +): string[] { + return registryIds.filter((id) => !isKnownProvider(id) && !(id in allowlist)); +} + +function main(): void { + const canonical = new Set(Object.keys(AI_PROVIDERS)); + const isKnown = (id: string) => canonical.has(id) || Boolean(getProviderById(id)); + const orphans = findOrphanRegistryIds(Object.keys(REGISTRY), isKnown, KNOWN_REGISTRY_ONLY); + if (orphans.length) { + console.error( + `[provider-consistency] ${orphans.length} entrada(s) no REGISTRY sem provider canônico em providers.ts:\n` + + orphans.map((id) => ` ✗ ${id}`).join("\n") + + `\n → registre o provider em src/shared/constants/providers.ts ou adicione a KNOWN_REGISTRY_ONLY (scripts/check/check-provider-consistency.ts) com justificativa.` + ); + process.exit(1); + } + console.log( + `[provider-consistency] OK — ${Object.keys(REGISTRY).length} entradas REGISTRY, ${canonical.size} providers canônicos, ${Object.keys(KNOWN_REGISTRY_ONLY).length} exceção(ões) conhecida(s)` + ); +} + +if (import.meta.url === pathToFileURL(process.argv[1] || "").href) main(); diff --git a/scripts/quality/collect-metrics.mjs b/scripts/quality/collect-metrics.mjs new file mode 100644 index 0000000000..471094e99b --- /dev/null +++ b/scripts/quality/collect-metrics.mjs @@ -0,0 +1,43 @@ +#!/usr/bin/env node +// scripts/quality/collect-metrics.mjs — emite quality-metrics.json +// Coletores incrementais: Fase 1 traz ESLint warnings + cobertura. +// Fases 3/4 estendem com duplicação (jscpd), tamanho de arquivo e cobertura por módulo. +import fs from "node:fs"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; + +const cwd = process.cwd(); +const out = {}; + +// 1) ESLint: contagem de warnings (errors devem ser 0; o lint já gata isso) +function eslintCounts() { + let stdout; + try { + stdout = execFileSync("npx", ["eslint", ".", "--format", "json"], { + encoding: "utf8", + maxBuffer: 256 * 1024 * 1024, + }); + } catch (e) { + // eslint sai com código != 0 quando há errors; o JSON ainda vem no stdout + stdout = e.stdout?.toString() || "[]"; + } + const results = JSON.parse(stdout); + out.eslintWarnings = results.reduce((n, r) => n + (r.warningCount || 0), 0); + out.eslintErrors = results.reduce((n, r) => n + (r.errorCount || 0), 0); +} + +// 2) Cobertura: lê coverage/coverage-summary.json se existir (gerado por c8) +function coverage() { + const p = path.join(cwd, "coverage", "coverage-summary.json"); + if (!fs.existsSync(p)) return; + const t = JSON.parse(fs.readFileSync(p, "utf8")).total; + out["coverage.statements"] = t.statements.pct; + out["coverage.lines"] = t.lines.pct; + out["coverage.functions"] = t.functions.pct; + out["coverage.branches"] = t.branches.pct; +} + +eslintCounts(); +coverage(); +fs.writeFileSync(path.join(cwd, "quality-metrics.json"), JSON.stringify(out, null, 2) + "\n"); +console.log("[collect-metrics]", JSON.stringify(out)); diff --git a/tests/unit/check-provider-consistency.test.ts b/tests/unit/check-provider-consistency.test.ts new file mode 100644 index 0000000000..7f269e234f --- /dev/null +++ b/tests/unit/check-provider-consistency.test.ts @@ -0,0 +1,25 @@ +import { test } from "node:test"; +import assert from "node:assert"; +import { findOrphanRegistryIds } from "../../scripts/check/check-provider-consistency.ts"; + +const known = new Set(["openai", "anthropic", "gemini"]); +const isKnown = (id: string) => known.has(id); + +test("no orphans when every registry id is a known provider", () => { + assert.deepEqual(findOrphanRegistryIds(["openai", "anthropic"], isKnown, {}), []); +}); + +test("flags a registry id that is not a canonical provider (hallucinated/half-registered)", () => { + assert.deepEqual(findOrphanRegistryIds(["openai", "ghostprovider"], isKnown, {}), ["ghostprovider"]); +}); + +test("allowlisted ids are not flagged", () => { + assert.deepEqual( + findOrphanRegistryIds(["openai", "krutrim"], isKnown, { krutrim: "pré-existente" }), + [] + ); +}); + +test("flags multiple orphans, preserves order", () => { + assert.deepEqual(findOrphanRegistryIds(["a", "openai", "b"], isKnown, {}), ["a", "b"]); +}); diff --git a/tests/unit/quality-ratchet.test.ts b/tests/unit/quality-ratchet.test.ts index edca607b93..6c94c4ddc7 100644 --- a/tests/unit/quality-ratchet.test.ts +++ b/tests/unit/quality-ratchet.test.ts @@ -18,8 +18,9 @@ function run(baseline: unknown, metrics: unknown, extraArgs: string[] = []) { encoding: "utf8", }); return { code: 0, out, dir, bPath }; - } catch (e: any) { - return { code: e.status as number, out: (e.stdout || "") + (e.stderr || ""), dir, bPath }; + } catch (e) { + const err = e as { status?: number; stdout?: string; stderr?: string }; + return { code: err.status as number, out: (err.stdout || "") + (err.stderr || ""), dir, bPath }; } }