diff --git a/scripts/check/check-docs-counts-sync.mjs b/scripts/check/check-docs-counts-sync.mjs index c0e4cf83f6..61077b8cf9 100644 --- a/scripts/check/check-docs-counts-sync.mjs +++ b/scripts/check/check-docs-counts-sync.mjs @@ -18,9 +18,13 @@ // Exits 0 on success, 1 on STRICT drift (or any drift with --strict). // Run: node scripts/check/check-docs-counts-sync.mjs // -// NOTE: the provider check trusts PROVIDER_REFERENCE.md as the canonical total. If a -// provider is added to the code but the reference is not regenerated, this guard will -// not catch it — regenerate with `npm run gen:provider-reference` before relying on it. +// NOTE: PROVIDER_REFERENCE.md is no longer blindly trusted — a STRICT check compares +// the doc's `Total providers` against the live provider modules (the same collections +// the generator reads), so a hand-stale doc is a red, not a silently propagated total. +// Fix by running `npm run gen:provider-reference`. Additional STRICT coverage added in +// the 2026-08-12 hardening: llm.txt + package.json description (providers), migration +// count (README/AGENTS/llm.txt), and canonical numbers inside the README SVG diagrams +// (providers / MCP tools / routing strategies / free-tier pools). import fs from "node:fs"; import { spawnSync } from "node:child_process"; @@ -76,6 +80,13 @@ export function readProviderTotal() { return parseProviderTotal(fs.readFileSync(abs, "utf8")); } +// STRICT: number of SQL migration files shipped with the app. +export function countMigrations() { + const abs = path.join(ROOT, "src", "lib", "db", "migrations"); + if (!fs.existsSync(abs)) return 0; + return fs.readdirSync(abs).filter((f) => f.endsWith(".sql")).length; +} + // STRICT: canonical i18n locale count, read from the shared config. export function countLocales() { const abs = path.join(ROOT, "config", "i18n.json"); @@ -148,6 +159,15 @@ function readCodeFacts() { 'import {notionTools} from "./open-sse/mcp-server/tools/notionTools.ts";', 'import {obsidianTools} from "./open-sse/mcp-server/tools/obsidianTools.ts";', 'import {compressionTools} from "./open-sse/mcp-server/tools/compressionTools.ts";', + // Live provider total — the SAME collections gen-provider-reference.ts unions, so the + // doc-vs-live check below cannot drift from the generator's definition of "provider". + 'import * as PROV from "./src/shared/constants/providers.ts";', + "const provCols=[PROV.FREE_PROVIDERS,PROV.NOAUTH_PROVIDERS,PROV.OAUTH_PROVIDERS,", + "PROV.WEB_COOKIE_PROVIDERS,PROV.APIKEY_PROVIDERS,PROV.LOCAL_PROVIDERS,PROV.SEARCH_PROVIDERS,", + "PROV.AUDIO_ONLY_PROVIDERS,PROV.UPSTREAM_PROXY_PROVIDERS,PROV.CLOUD_AGENT_PROVIDERS,", + "PROV.SYSTEM_PROVIDERS];", + "const pids=new Set();", + "for(const c of provCols)for(const p of Object.values(c||{}))if(p&&p.id)pids.add(p.id);", "const cols={MCP_TOOLS,memoryTools,skillTools,agentSkillTools,githubSkillTools,poolTools,", "gamificationTools,pluginTools,notionTools,obsidianTools,compressionTools};", "const sc=new Set();", @@ -158,7 +178,7 @@ function readCodeFacts() { 'console.log("@@"+JSON.stringify({freeSteady:t.steadyRecurringTokens,', "freeFirst:t.firstMonthRealisticTokens,freePools:t.poolCount,engines:ENGINE_IDS.length,", "cliTotal:cli.length,cliCode:by('code'),cliAgent:by('agent'),", - "mcpTools:countUniqueMcpTools(cols),mcpScopes:sc.size}));", + "mcpTools:countUniqueMcpTools(cols),mcpScopes:sc.size,providers:pids.size}));", ].join(""); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "docs-counts-")); try { @@ -252,6 +272,82 @@ export function makeNumberClaimValidator(expected, opts) { }; } +// --- v3.8.50 hardening validators -------------------------------------------- +// PURE: doc total must equal the live provider-module total (closes the falso-verde +// found in the 2026-08-12 audit: the doc sat hand-stale at 291 while the modules +// defined 338, and every downstream check inherited the stale total). +export function makeProviderReferenceValidator(expected) { + return (content) => { + const total = parseProviderTotal(content); + if (!total) return { ok: false, detail: "no `Total providers: **N**` marker found" }; + if (total === expected) + return { ok: true, detail: `doc total ${total} matches the live provider modules` }; + return { + ok: false, + detail: + `doc total ${total} is stale — the live provider modules define ${expected} ` + + `(run npm run gen:provider-reference)`, + }; + }; +} + +// PURE: the npm package description must carry the live provider count. +export function makePackageDescriptionValidator(expected) { + return (content) => { + let desc = ""; + try { + desc = String(JSON.parse(content).description || ""); + } catch { + return { ok: false, detail: "package.json could not be parsed" }; + } + if (desc.includes(String(expected))) + return { ok: true, detail: `description mentions the live provider count ${expected}` }; + return { + ok: false, + detail: `description does not mention the live provider count ${expected}: "${desc}"`, + }; + }; +} + +// PURE: sweep an SVG's text/aria content for the canonical numbers. Patterns are +// deliberately narrow — they anchor on the surrounding words so path coordinates, +// width/font-size attributes and small unrelated counts ("15 providers ToS-flagged", +// "100+ providers") can never register as claims. Providers require 3+ digits for the +// same reason. +const SVG_CANONICAL_PATTERNS = [ + { key: "providers", what: "providers", pattern: /(\d{3,4}) (?:AI )?providers\b/g }, + { key: "mcpTools", what: "MCP tools", pattern: /MCP (?:server with |with |\()(\d+)/g }, + { key: "strategies", what: "routing strategies", pattern: /(\d+) routing strategies\b/g }, + { key: "pools", what: "free-tier pools", pattern: /(\d+) provider pools\b/g }, +]; + +export function checkSvgCanonicalNumbers(content, expected) { + const stale = []; + let claims = 0; + for (const { key, what, pattern } of SVG_CANONICAL_PATTERNS) { + if (expected[key] == null) continue; + for (const m of content.matchAll(pattern)) { + claims++; + const value = Number(m[1]); + if (value !== expected[key]) stale.push(`"${m[0]}" (${what} — code has ${expected[key]})`); + } + } + if (!claims) return { ok: true, detail: "no canonical-number claims in this SVG" }; + if (!stale.length) return { ok: true, detail: `${claims} canonical claim(s) match the code` }; + return { ok: false, detail: `stale: ${[...new Set(stale)].join(", ")}` }; +} + +// The README-embedded diagrams that historically rotted because no gate read them +// (the alt-text in README.md is checked, the SVG text nodes never were). +const SVG_DIAGRAM_FILES = [ + "docs/diagrams/readme-hero.svg", + "docs/diagrams/free-tier-budget.svg", + "docs/diagrams/promise-pillars.svg", + "docs/diagrams/comparison-table.svg", + "docs/diagrams/cli-terminal.svg", + "docs/diagrams/tier-cascade.svg", +]; + export function buildChecks() { return [ { @@ -259,7 +355,26 @@ export function buildChecks() { actual: readProviderTotal(), docKey: "providers", strict: true, - files: ["README.md", "AGENTS.md"], + files: ["README.md", "AGENTS.md", "llm.txt"], + }, + { + label: "Provider count (package.json description)", + actual: readProviderTotal(), + docKey: "providers", + strict: true, + files: ["package.json"], + validate: makePackageDescriptionValidator(readProviderTotal()), + }, + { + label: "DB migrations count", + actual: countMigrations(), + docKey: "migrations", + strict: true, + files: ["README.md", "AGENTS.md", "llm.txt"], + validate: makeNumberClaimValidator(countMigrations(), { + what: "migrations", + pattern: /(\d+)\+? migrations?\b/gi, + }), }, { label: "i18n locales count", @@ -289,6 +404,30 @@ export function buildChecks() { validate: makeNumberClaimValidator(expected, { what, ...opts }), }); return [ + { + label: "Provider reference total (doc vs live modules)", + actual: f.providers, + docKey: "providers (live)", + strict: true, + files: ["docs/reference/PROVIDER_REFERENCE.md"], + validate: makeProviderReferenceValidator(f.providers), + }, + { + label: "SVG canonical numbers (live code)", + actual: + `${f.providers} providers / ${f.mcpTools} MCP tools / ` + + `${countRoutingStrategies()} strategies / ${f.freePools} pools`, + docKey: "SVG canonical numbers", + strict: true, + files: SVG_DIAGRAM_FILES, + validate: (content) => + checkSvgCanonicalNumbers(content, { + providers: f.providers, + mcpTools: f.mcpTools, + strategies: countRoutingStrategies(), + pools: f.freePools, + }), + }, { label: "Free-tier headline (live catalog)", actual: `~${(f.freeSteady / 1e9).toFixed(2)}B steady / ${f.freePools} pools`, diff --git a/tests/unit/check-docs-counts-sync.test.ts b/tests/unit/check-docs-counts-sync.test.ts index 27deae3057..9e2afca449 100644 --- a/tests/unit/check-docs-counts-sync.test.ts +++ b/tests/unit/check-docs-counts-sync.test.ts @@ -196,3 +196,87 @@ test("compression-engines and CLI-tools gates catch their v3.8.49 drift", () => assert.equal(cli("all 33 tools (25 CLI Code's)").ok, true); assert.equal(cli("all 26 tools (25 CLI Code's)").ok, false); }); + +// --- v3.8.50 hardening: live provider source, llm.txt/package.json, migrations, SVGs -- +// Regression guards for the 2026-08-12 audit: PROVIDER_REFERENCE.md was hand-stale at +// 291 while the live provider modules defined 338, and the gate trusted the doc — so +// README/AGENTS validated against a stale total and the gate stayed falsely green. +import { + countMigrations, + makeProviderReferenceValidator, + makePackageDescriptionValidator, + checkSvgCanonicalNumbers, +} from "../../scripts/check/check-docs-counts-sync.mjs"; + +const countMigs = countMigrations as () => number; +const makeRefValidator = makeProviderReferenceValidator as ( + expected: number +) => (content: string) => { ok: boolean; detail: string }; +const makePkgValidator = makePackageDescriptionValidator as ( + expected: number +) => (content: string) => { ok: boolean; detail: string }; +const checkSvg = checkSvgCanonicalNumbers as ( + content: string, + expected: { providers?: number; mcpTools?: number; strategies?: number; pools?: number } +) => { ok: boolean; detail: string }; + +test("countMigrations reads a real, positive migration count", () => { + assert.ok(countMigs() > 100, "migrations dir should hold > 100 .sql files"); +}); + +test("provider-reference validator accepts the live total and rejects a stale doc", () => { + const v = makeRefValidator(338); + assert.equal(v("Total providers: **338**. See category breakdown below.").ok, true); + const stale = v("Total providers: **291**. See category breakdown below."); + assert.equal(stale.ok, false, "a hand-stale doc total must be a red, not a silent pass"); + assert.match(stale.detail, /gen:provider-reference/); + assert.equal(v("# Provider Reference\n\nNo total marker.").ok, false); +}); + +test("package.json description validator catches a stale provider count", () => { + const v = makePkgValidator(338); + assert.equal(v(JSON.stringify({ description: "Unified AI router with 338 providers" })).ok, true); + assert.equal( + v(JSON.stringify({ description: "Unified AI router with 291 providers" })).ok, + false + ); + assert.equal(v("not json at all {").ok, false); +}); + +test("migrations claim validator accepts the real count and rejects stale styles", () => { + const v = makeValidator(144, { what: "migrations", pattern: /(\d+)\+? migrations?\b/gi }); + assert.equal(v("SQLite domain modules (144 migrations)").ok, true); + assert.equal(v("local, zero-config, 110+ migrations").ok, false); + assert.equal(v("(130 migrations)").ok, false); +}); + +const SVG_EXPECTED = { providers: 338, mcpTools: 105, strategies: 19, pools: 42 }; + +test("SVG gate accepts canonical numbers in text and aria-label claims", () => { + const good = + 'aria-label="338 AI providers, 19 routing strategies, MCP with 105 tools, ' + + '42 provider pools" 338 providersMCP (105'; + assert.equal(checkSvg(good, SVG_EXPECTED).ok, true); +}); + +test("SVG gate flags each stale canonical number the audit found", () => { + for (const stale of [ + "290 providers", + 'aria-label="MCP server with 104 tools"', + "MCP (104", + "18 routing strategies", + 'aria-label="43 provider pools and 460+ models"', + ]) { + const r = checkSvg(stale, SVG_EXPECTED); + assert.equal(r.ok, false, `expected stale claim to be flagged: ${stale}`); + } +}); + +test("SVG gate ignores coordinates, sizes and unrelated small counts", () => { + const noise = + '' + + '15 providers ToS-flagged' + + "100+ providers90+ free"; + const r = checkSvg(noise, SVG_EXPECTED); + assert.equal(r.ok, true, `coordinates/attrs must never register claims: ${r.detail}`); +});