feat(check): harden docs-counts gate - live provider source, llm.txt, migrations, SVGs

The gate trusted PROVIDER_REFERENCE.md as the provider total, so a hand-stale
doc (291 vs 338 live) kept it falsely green. New STRICT checks: doc total vs
the live provider modules (same collections the generator unions), provider
count in llm.txt and package.json description, migration count vs
README/AGENTS/llm.txt, and a canonical-number sweep (providers/MCP
tools/strategies/pools) over the six README SVG diagrams with
coordinate/attribute-safe patterns. TDD: 9 new unit tests (red first on the
missing exports, green after) in tests/unit/check-docs-counts-sync.test.ts.

Refs #9985

scripts/check/check-docs-counts-sync.mjs

@@ -18,9 +18,13 @@
 // Exits 0 on success, 1 on STRICT drift (or any drift with --strict).
 // Run: node scripts/check/check-docs-counts-sync.mjs
 //
-// NOTE: the provider check trusts PROVIDER_REFERENCE.md as the canonical total. If a
-// provider is added to the code but the reference is not regenerated, this guard will
-// not catch it — regenerate with `npm run gen:provider-reference` before relying on it.
+// NOTE: PROVIDER_REFERENCE.md is no longer blindly trusted — a STRICT check compares
+// the doc's `Total providers` against the live provider modules (the same collections
+// the generator reads), so a hand-stale doc is a red, not a silently propagated total.
+// Fix by running `npm run gen:provider-reference`. Additional STRICT coverage added in
+// the 2026-08-12 hardening: llm.txt + package.json description (providers), migration
+// count (README/AGENTS/llm.txt), and canonical numbers inside the README SVG diagrams
+// (providers / MCP tools / routing strategies / free-tier pools).
 
 import fs from "node:fs";
 import { spawnSync } from "node:child_process";
@@ -76,6 +80,13 @@ export function readProviderTotal() {
   return parseProviderTotal(fs.readFileSync(abs, "utf8"));
 }
 
+// STRICT: number of SQL migration files shipped with the app.
+export function countMigrations() {
+  const abs = path.join(ROOT, "src", "lib", "db", "migrations");
+  if (!fs.existsSync(abs)) return 0;
+  return fs.readdirSync(abs).filter((f) => f.endsWith(".sql")).length;
+}
+
 // STRICT: canonical i18n locale count, read from the shared config.
 export function countLocales() {
   const abs = path.join(ROOT, "config", "i18n.json");
@@ -148,6 +159,15 @@ function readCodeFacts() {
     'import {notionTools} from "./open-sse/mcp-server/tools/notionTools.ts";',
     'import {obsidianTools} from "./open-sse/mcp-server/tools/obsidianTools.ts";',
     'import {compressionTools} from "./open-sse/mcp-server/tools/compressionTools.ts";',
+    // Live provider total — the SAME collections gen-provider-reference.ts unions, so the
+    // doc-vs-live check below cannot drift from the generator's definition of "provider".
+    'import * as PROV from "./src/shared/constants/providers.ts";',
+    "const provCols=[PROV.FREE_PROVIDERS,PROV.NOAUTH_PROVIDERS,PROV.OAUTH_PROVIDERS,",
+    "PROV.WEB_COOKIE_PROVIDERS,PROV.APIKEY_PROVIDERS,PROV.LOCAL_PROVIDERS,PROV.SEARCH_PROVIDERS,",
+    "PROV.AUDIO_ONLY_PROVIDERS,PROV.UPSTREAM_PROXY_PROVIDERS,PROV.CLOUD_AGENT_PROVIDERS,",
+    "PROV.SYSTEM_PROVIDERS];",
+    "const pids=new Set();",
+    "for(const c of provCols)for(const p of Object.values(c||{}))if(p&&p.id)pids.add(p.id);",
     "const cols={MCP_TOOLS,memoryTools,skillTools,agentSkillTools,githubSkillTools,poolTools,",
     "gamificationTools,pluginTools,notionTools,obsidianTools,compressionTools};",
     "const sc=new Set();",
@@ -158,7 +178,7 @@ function readCodeFacts() {
     'console.log("@@"+JSON.stringify({freeSteady:t.steadyRecurringTokens,',
     "freeFirst:t.firstMonthRealisticTokens,freePools:t.poolCount,engines:ENGINE_IDS.length,",
     "cliTotal:cli.length,cliCode:by('code'),cliAgent:by('agent'),",
-    "mcpTools:countUniqueMcpTools(cols),mcpScopes:sc.size}));",
+    "mcpTools:countUniqueMcpTools(cols),mcpScopes:sc.size,providers:pids.size}));",
   ].join("");
   const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "docs-counts-"));
   try {
@@ -252,6 +272,82 @@ export function makeNumberClaimValidator(expected, opts) {
   };
 }
 
+// --- v3.8.50 hardening validators --------------------------------------------
+// PURE: doc total must equal the live provider-module total (closes the falso-verde
+// found in the 2026-08-12 audit: the doc sat hand-stale at 291 while the modules
+// defined 338, and every downstream check inherited the stale total).
+export function makeProviderReferenceValidator(expected) {
+  return (content) => {
+    const total = parseProviderTotal(content);
+    if (!total) return { ok: false, detail: "no `Total providers: **N**` marker found" };
+    if (total === expected)
+      return { ok: true, detail: `doc total ${total} matches the live provider modules` };
+    return {
+      ok: false,
+      detail:
+        `doc total ${total} is stale — the live provider modules define ${expected} ` +
+        `(run npm run gen:provider-reference)`,
+    };
+  };
+}
+
+// PURE: the npm package description must carry the live provider count.
+export function makePackageDescriptionValidator(expected) {
+  return (content) => {
+    let desc = "";
+    try {
+      desc = String(JSON.parse(content).description || "");
+    } catch {
+      return { ok: false, detail: "package.json could not be parsed" };
+    }
+    if (desc.includes(String(expected)))
+      return { ok: true, detail: `description mentions the live provider count ${expected}` };
+    return {
+      ok: false,
+      detail: `description does not mention the live provider count ${expected}: "${desc}"`,
+    };
+  };
+}
+
+// PURE: sweep an SVG's text/aria content for the canonical numbers. Patterns are
+// deliberately narrow — they anchor on the surrounding words so path coordinates,
+// width/font-size attributes and small unrelated counts ("15 providers ToS-flagged",
+// "100+ providers") can never register as claims. Providers require 3+ digits for the
+// same reason.
+const SVG_CANONICAL_PATTERNS = [
+  { key: "providers", what: "providers", pattern: /(\d{3,4}) (?:AI )?providers\b/g },
+  { key: "mcpTools", what: "MCP tools", pattern: /MCP (?:server with |with |\()(\d+)/g },
+  { key: "strategies", what: "routing strategies", pattern: /(\d+) routing strategies\b/g },
+  { key: "pools", what: "free-tier pools", pattern: /(\d+) provider pools\b/g },
+];
+
+export function checkSvgCanonicalNumbers(content, expected) {
+  const stale = [];
+  let claims = 0;
+  for (const { key, what, pattern } of SVG_CANONICAL_PATTERNS) {
+    if (expected[key] == null) continue;
+    for (const m of content.matchAll(pattern)) {
+      claims++;
+      const value = Number(m[1]);
+      if (value !== expected[key]) stale.push(`"${m[0]}" (${what} — code has ${expected[key]})`);
+    }
+  }
+  if (!claims) return { ok: true, detail: "no canonical-number claims in this SVG" };
+  if (!stale.length) return { ok: true, detail: `${claims} canonical claim(s) match the code` };
+  return { ok: false, detail: `stale: ${[...new Set(stale)].join(", ")}` };
+}
+
+// The README-embedded diagrams that historically rotted because no gate read them
+// (the alt-text in README.md is checked, the SVG text nodes never were).
+const SVG_DIAGRAM_FILES = [
+  "docs/diagrams/readme-hero.svg",
+  "docs/diagrams/free-tier-budget.svg",
+  "docs/diagrams/promise-pillars.svg",
+  "docs/diagrams/comparison-table.svg",
+  "docs/diagrams/cli-terminal.svg",
+  "docs/diagrams/tier-cascade.svg",
+];
+
 export function buildChecks() {
   return [
     {
@@ -259,7 +355,26 @@ export function buildChecks() {
       actual: readProviderTotal(),
       docKey: "providers",
       strict: true,
-      files: ["README.md", "AGENTS.md"],
+      files: ["README.md", "AGENTS.md", "llm.txt"],
+    },
+    {
+      label: "Provider count (package.json description)",
+      actual: readProviderTotal(),
+      docKey: "providers",
+      strict: true,
+      files: ["package.json"],
+      validate: makePackageDescriptionValidator(readProviderTotal()),
+    },
+    {
+      label: "DB migrations count",
+      actual: countMigrations(),
+      docKey: "migrations",
+      strict: true,
+      files: ["README.md", "AGENTS.md", "llm.txt"],
+      validate: makeNumberClaimValidator(countMigrations(), {
+        what: "migrations",
+        pattern: /(\d+)\+? migrations?\b/gi,
+      }),
     },
     {
       label: "i18n locales count",
@@ -289,6 +404,30 @@ export function buildChecks() {
         validate: makeNumberClaimValidator(expected, { what, ...opts }),
       });
       return [
+        {
+          label: "Provider reference total (doc vs live modules)",
+          actual: f.providers,
+          docKey: "providers (live)",
+          strict: true,
+          files: ["docs/reference/PROVIDER_REFERENCE.md"],
+          validate: makeProviderReferenceValidator(f.providers),
+        },
+        {
+          label: "SVG canonical numbers (live code)",
+          actual:
+            `${f.providers} providers / ${f.mcpTools} MCP tools / ` +
+            `${countRoutingStrategies()} strategies / ${f.freePools} pools`,
+          docKey: "SVG canonical numbers",
+          strict: true,
+          files: SVG_DIAGRAM_FILES,
+          validate: (content) =>
+            checkSvgCanonicalNumbers(content, {
+              providers: f.providers,
+              mcpTools: f.mcpTools,
+              strategies: countRoutingStrategies(),
+              pools: f.freePools,
+            }),
+        },
         {
           label: "Free-tier headline (live catalog)",
           actual: `~${(f.freeSteady / 1e9).toFixed(2)}B steady / ${f.freePools} pools`,

tests/unit/check-docs-counts-sync.test.ts

@@ -196,3 +196,87 @@ test("compression-engines and CLI-tools gates catch their v3.8.49 drift", () =>
   assert.equal(cli("all 33 tools (25 CLI Code's)").ok, true);
   assert.equal(cli("all 26 tools (25 CLI Code's)").ok, false);
 });
+
+// --- v3.8.50 hardening: live provider source, llm.txt/package.json, migrations, SVGs --
+// Regression guards for the 2026-08-12 audit: PROVIDER_REFERENCE.md was hand-stale at
+// 291 while the live provider modules defined 338, and the gate trusted the doc — so
+// README/AGENTS validated against a stale total and the gate stayed falsely green.
+import {
+  countMigrations,
+  makeProviderReferenceValidator,
+  makePackageDescriptionValidator,
+  checkSvgCanonicalNumbers,
+} from "../../scripts/check/check-docs-counts-sync.mjs";
+
+const countMigs = countMigrations as () => number;
+const makeRefValidator = makeProviderReferenceValidator as (
+  expected: number
+) => (content: string) => { ok: boolean; detail: string };
+const makePkgValidator = makePackageDescriptionValidator as (
+  expected: number
+) => (content: string) => { ok: boolean; detail: string };
+const checkSvg = checkSvgCanonicalNumbers as (
+  content: string,
+  expected: { providers?: number; mcpTools?: number; strategies?: number; pools?: number }
+) => { ok: boolean; detail: string };
+
+test("countMigrations reads a real, positive migration count", () => {
+  assert.ok(countMigs() > 100, "migrations dir should hold > 100 .sql files");
+});
+
+test("provider-reference validator accepts the live total and rejects a stale doc", () => {
+  const v = makeRefValidator(338);
+  assert.equal(v("Total providers: **338**. See category breakdown below.").ok, true);
+  const stale = v("Total providers: **291**. See category breakdown below.");
+  assert.equal(stale.ok, false, "a hand-stale doc total must be a red, not a silent pass");
+  assert.match(stale.detail, /gen:provider-reference/);
+  assert.equal(v("# Provider Reference\n\nNo total marker.").ok, false);
+});
+
+test("package.json description validator catches a stale provider count", () => {
+  const v = makePkgValidator(338);
+  assert.equal(v(JSON.stringify({ description: "Unified AI router with 338 providers" })).ok, true);
+  assert.equal(
+    v(JSON.stringify({ description: "Unified AI router with 291 providers" })).ok,
+    false
+  );
+  assert.equal(v("not json at all {").ok, false);
+});
+
+test("migrations claim validator accepts the real count and rejects stale styles", () => {
+  const v = makeValidator(144, { what: "migrations", pattern: /(\d+)\+? migrations?\b/gi });
+  assert.equal(v("SQLite domain modules (144 migrations)").ok, true);
+  assert.equal(v("local, zero-config, 110+ migrations").ok, false);
+  assert.equal(v("(130 migrations)").ok, false);
+});
+
+const SVG_EXPECTED = { providers: 338, mcpTools: 105, strategies: 19, pools: 42 };
+
+test("SVG gate accepts canonical numbers in text and aria-label claims", () => {
+  const good =
+    'aria-label="338 AI providers, 19 routing strategies, MCP with 105 tools, ' +
+    '42 provider pools" <text>338 providers</text><text>MCP (105</text>';
+  assert.equal(checkSvg(good, SVG_EXPECTED).ok, true);
+});
+
+test("SVG gate flags each stale canonical number the audit found", () => {
+  for (const stale of [
+    "<text>290 providers</text>",
+    'aria-label="MCP server with 104 tools"',
+    "<text>MCP (104</text>",
+    "<text>18 routing strategies</text>",
+    'aria-label="43 provider pools and 460+ models"',
+  ]) {
+    const r = checkSvg(stale, SVG_EXPECTED);
+    assert.equal(r.ok, false, `expected stale claim to be flagged: ${stale}`);
+  }
+});
+
+test("SVG gate ignores coordinates, sizes and unrelated small counts", () => {
+  const noise =
+    '<path d="M 30,310 C 136,290 176,240 296,206"/><rect width="104" height="24"/>' +
+    '<animate values="250;290;250"/><text font-size="104">15 providers ToS-flagged</text>' +
+    "<text>100+ providers</text><text>90+ free</text>";
+  const r = checkSvg(noise, SVG_EXPECTED);
+  assert.equal(r.ok, true, `coordinates/attrs must never register claims: ${r.detail}`);
+});