OmniRoute/scripts/check/check-docs-counts-sync.mjs

#!/usr/bin/env node
// Validates that count-based assertions in docs match the actual code state.
//
// Two tiers of checks:
//   • STRICT (always blocking — exit 1 on drift): high-confidence, slow-moving counts
//     that historically caused the worst drift across README / AGENTS / docs.
//       - provider count (source of truth: docs/reference/PROVIDER_REFERENCE.md total,
//         which is auto-generated from src/shared/constants/providers.ts)
//       - i18n locale count (source of truth: config/i18n.json `locales`)
//   • SOFT (heuristic — only fails with --strict): file-count based assertions that can
//     false-positive.
//       - executors count in open-sse/executors/
//       - routing strategies in src/shared/constants/routingStrategies.ts
//       - OAuth providers in src/lib/oauth/providers/
//       - A2A skills in src/lib/a2a/skills/
//       - Cloud agents in src/lib/cloudAgent/agents/
//
// Exits 0 on success, 1 on STRICT drift (or any drift with --strict).
// Run: node scripts/check/check-docs-counts-sync.mjs
//
// NOTE: the provider check trusts PROVIDER_REFERENCE.md as the canonical total. If a
// provider is added to the code but the reference is not regenerated, this guard will
// not catch it — regenerate with `npm run gen:provider-reference` before relying on it.

import fs from "node:fs";
import { spawnSync } from "node:child_process";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.resolve(__dirname, "..", "..");

const COMMON_NON_IMPL_BASENAMES = new Set([
  "index.ts",
  "index.mts",
  "types.ts",
  "base.ts",
  "constants.ts",
]);

function countFiles(dir, suffix = ".ts") {
  const abs = path.join(ROOT, dir);
  if (!fs.existsSync(abs)) return 0;
  return fs
    .readdirSync(abs)
    .filter(
      (f) =>
        f.endsWith(suffix) &&
        !f.endsWith(".test.ts") &&
        !f.startsWith("__") &&
        !COMMON_NON_IMPL_BASENAMES.has(f)
    ).length;
}

function countRoutingStrategies() {
  const file = path.join(ROOT, "src", "shared", "constants", "routingStrategies.ts");
  if (!fs.existsSync(file)) return 0;
  const txt = fs.readFileSync(file, "utf8");
  const m = txt.match(/ROUTING_STRATEGY_VALUES\s*=\s*\[([^\]]*)\]/);
  if (!m) return 0;
  return (m[1].match(/"[^"]+"/g) || []).length;
}

// PURE: parse the canonical provider total out of the auto-generated catalog text.
export function parseProviderTotal(referenceText) {
  if (!referenceText) return 0;
  const m = referenceText.match(/Total providers:\s*\*\*(\d+)\*\*/);
  return m ? Number(m[1]) : 0;
}

// STRICT: canonical provider total, read from the auto-generated catalog.
export function readProviderTotal() {
  const abs = path.join(ROOT, "docs", "reference", "PROVIDER_REFERENCE.md");
  if (!fs.existsSync(abs)) return 0;
  return parseProviderTotal(fs.readFileSync(abs, "utf8"));
}

// STRICT: canonical i18n locale count, read from the shared config.
export function countLocales() {
  const abs = path.join(ROOT, "config", "i18n.json");
  if (!fs.existsSync(abs)) return 0;
  try {
    const cfg = JSON.parse(fs.readFileSync(abs, "utf8"));
    return Array.isArray(cfg.locales) ? cfg.locales.length : 0;
  } catch {
    return 0;
  }
}

// PURE: tally STRICT vs SOFT drift for a list of checks, given a content lookup.
// `getContent(file) -> string | null`. A check whose `actual` is 0 is skipped (the
// source count could not be determined). Returns { strict, soft, lines }.
export function tallyDrift(checks, getContent) {
  let strict = 0;
  let soft = 0;
  const lines = [];
  for (const c of checks) {
    const tier = c.strict ? "STRICT" : "soft";
    lines.push(`\n• ${c.label}: ${c.actual} (real) [${tier}]`);
    if (!c.actual) {
      lines.push(`  ⚠ could not determine ${c.docKey} count from source — skipping`);
      continue;
    }
    for (const f of c.files) {
      const content = getContent(f);
      if (c.validate) {
        if (content == null) continue;
        const v = c.validate(content);
        lines.push(`  ${v.ok ? "✓" : c.strict ? "✗" : "⚠"} ${f} — ${v.detail}`);
        if (!v.ok) {
          if (c.strict) strict++;
          else soft++;
        }
        continue;
      }
      const found = content != null && content.includes(String(c.actual));
      if (found) {
        lines.push(`  ✓ ${f} mentions "${c.actual}"`);
      } else {
        lines.push(`  ${c.strict ? "✗" : "⚠"} ${f} does NOT mention "${c.actual}" for ${c.docKey}`);
        if (c.strict) strict++;
        else soft++;
      }
    }
  }
  return { strict, soft, lines };
}

// Reads every code-derived fact in ONE tsx subprocess — the same functions the app
// serves at runtime, never a hardcoded copy. DATA_DIR is redirected to a throwaway dir
// so importing the MCP tool modules cannot touch the operator's real SQLite file.
// Returns null when tsx is unavailable so the gate degrades to a skip, not a false red.
function readCodeFacts() {
  const script = [
    'import {computeFreeModelTotals} from "./open-sse/config/freeModelCatalog.ts";',
    'import {ENGINE_IDS} from "./open-sse/services/compression/engineCatalog.ts";',
    'import {CLI_TOOLS} from "./src/shared/constants/cliTools.ts";',
    'import {countUniqueMcpTools} from "./open-sse/mcp-server/toolCount.ts";',
    'import {MCP_TOOLS} from "./open-sse/mcp-server/schemas/tools.ts";',
    'import {memoryTools} from "./open-sse/mcp-server/tools/memoryTools.ts";',
    'import {skillTools} from "./open-sse/mcp-server/tools/skillTools.ts";',
    'import {agentSkillTools} from "./open-sse/mcp-server/tools/agentSkillTools.ts";',
    'import {githubSkillTools} from "./open-sse/mcp-server/tools/githubSkillTools.ts";',
    'import {poolTools} from "./open-sse/mcp-server/tools/poolTools.ts";',
    'import {gamificationTools} from "./open-sse/mcp-server/tools/gamificationTools.ts";',
    'import {pluginTools} from "./open-sse/mcp-server/tools/pluginTools.ts";',
    'import {notionTools} from "./open-sse/mcp-server/tools/notionTools.ts";',
    'import {obsidianTools} from "./open-sse/mcp-server/tools/obsidianTools.ts";',
    'import {compressionTools} from "./open-sse/mcp-server/tools/compressionTools.ts";',
    "const cols={MCP_TOOLS,memoryTools,skillTools,agentSkillTools,githubSkillTools,poolTools,",
    "gamificationTools,pluginTools,notionTools,obsidianTools,compressionTools};",
    "const sc=new Set();",
    "for(const col of Object.values(cols))for(const t of Object.values(col))",
    "for(const x of (t?.scopes||[]))sc.add(x);",
    "const t=computeFreeModelTotals();const cli=Object.values(CLI_TOOLS);",
    "const by=(c)=>cli.filter(x=>x.category===c).length;",
    'console.log("@@"+JSON.stringify({freeSteady:t.steadyRecurringTokens,',
    "freeFirst:t.firstMonthRealisticTokens,freePools:t.poolCount,engines:ENGINE_IDS.length,",
    "cliTotal:cli.length,cliCode:by('code'),cliAgent:by('agent'),",
    "mcpTools:countUniqueMcpTools(cols),mcpScopes:sc.size}));",
  ].join("");
  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "docs-counts-"));
  try {
    const r = spawnSync(process.execPath, ["--import", "tsx/esm", "-e", script], {
      cwd: ROOT,
      encoding: "utf8",
      timeout: 180000,
      env: { ...process.env, DATA_DIR: tmp, APP_LOG_LEVEL: "silent" },
    });
    if (r.status !== 0 || !r.stdout) return null;
    const line = r.stdout.split("\n").find((l) => l.startsWith("@@"));
    return line ? JSON.parse(line.slice(2)) : null;
  } catch {
    return null;
  } finally {
    fs.rmSync(tmp, { recursive: true, force: true });
  }
}

// The docs publish a rounded aggregate headline ("~1.4B"). Accept a claim that rounds to
// the live value at one decimal place; reject a stale one. Tolerance is tight on purpose:
// this gate exists so the headline cannot drift upward unnoticed.
//
// Only the AGGREGATE headline is validated, via an explicit whitelist. These files also
// carry figures that are legitimately not the headline and must never trip the gate:
// the theoretical ceiling ("would read ~10B; not published"), the historical "previous
// ~1.94B", and per-model rows ("mistral … ~1.00B"). A whitelist keeps those safe without
// having to enumerate every contrastive phrasing.
const HEADLINE_AFTER = /^\s*(?:documented\s+)?free tokens|^\s*in (?:your|the) first month/i;
const HEADLINE_BEFORE = /(recurring grant[^|]*\|\s*\**|signup credits[^|]*\|\s*\**|up to\s*)$/i;

export function extractHeadlineClaims(content) {
  const claims = [];
  for (const m of content.matchAll(/~(\d+(?:\.\d+)?)B/g)) {
    const after = content.slice(m.index + m[0].length, m.index + m[0].length + 80);
    const before = content.slice(Math.max(0, m.index - 80), m.index);
    if (!HEADLINE_AFTER.test(after) && !HEADLINE_BEFORE.test(before)) continue;
    claims.push({ value: Number(m[1]), text: m[0] });
  }
  return claims;
}

export function checkFreeTierHeadline(content, totals) {
  const claims = extractHeadlineClaims(content);
  if (!claims.length) return { ok: true, detail: "no aggregate free-tier headline in this file" };
  const steady = totals.s / 1e9;
  const first = totals.m / 1e9;
  const stale = claims.filter(
    (c) => Math.abs(c.value - steady) >= 0.05 && Math.abs(c.value - first) >= 0.05
  );
  if (!stale.length)
    return { ok: true, detail: `${claims.length} headline claim(s) match the live catalog` };
  return {
    ok: false,
    detail:
      `stale headline ${[...new Set(stale.map((c) => c.text))].join(", ")} — live catalog ` +
      `computes ~${steady.toFixed(2)}B steady / ~${first.toFixed(2)}B first month`,
  };
}

// --- Generic numeric-claim gate ---------------------------------------------
// Same principle as the free-tier headline: docs legitimately carry numbers that are
// NOT the aggregate being gated (per-module tool counts like "Memory tool definitions
// (3 tools)", the CLI catalog's "33 tools (25 CLI Code's ...)" next to the MCP total).
// So every check declares what to skip rather than assuming any "N tools" is the claim.
export function extractNumberClaims(content, { pattern, skipBefore, skipAfter }) {
  const claims = [];
  for (const m of content.matchAll(pattern)) {
    const before = content.slice(Math.max(0, m.index - 40), m.index);
    const after = content.slice(m.index + m[0].length, m.index + m[0].length + 40);
    if (skipBefore && skipBefore.test(before)) continue;
    if (skipAfter && skipAfter.test(after)) continue;
    claims.push({ value: Number(m[1]), text: m[0].trim() });
  }
  return claims;
}

export function makeNumberClaimValidator(expected, opts) {
  return (content) => {
    const claims = extractNumberClaims(content, opts);
    if (!claims.length) return { ok: true, detail: `no ${opts.what} claim in this file` };
    const stale = claims.filter((c) => c.value !== expected);
    if (!stale.length)
      return { ok: true, detail: `${claims.length} ${opts.what} claim(s) match the code` };
    return {
      ok: false,
      detail:
        `stale ${opts.what}: ${[...new Set(stale.map((c) => `"${c.text}"`))].join(", ")} — ` +
        `code has ${expected}`,
    };
  };
}

export function buildChecks() {
  return [
    {
      label: "Provider count",
      actual: readProviderTotal(),
      docKey: "providers",
      strict: true,
      files: ["README.md", "AGENTS.md", "CLAUDE.md"],
    },
    {
      label: "i18n locales count",
      actual: countLocales(),
      docKey: "i18n locales",
      strict: true,
      files: ["docs/README.md", "docs/guides/I18N.md", "AGENTS.md"],
    },
    ...(() => {
      const f = readCodeFacts();
      if (!f)
        return [
          {
            label: "Code-derived counts",
            actual: 0,
            docKey: "code facts",
            strict: false,
            files: [],
          },
        ];
      const claim = (expected, what, opts, files) => ({
        label: `${what} (live code)`,
        actual: expected,
        docKey: what,
        strict: true,
        files,
        validate: makeNumberClaimValidator(expected, { what, ...opts }),
      });
      return [
        {
          label: "Free-tier headline (live catalog)",
          actual: `~${(f.freeSteady / 1e9).toFixed(2)}B steady / ${f.freePools} pools`,
          docKey: "free-tier headline",
          strict: true,
          files: ["README.md", "docs/reference/FREE_TIERS.md"],
          validate: (content) =>
            checkFreeTierHeadline(content, { s: f.freeSteady, m: f.freeFirst }),
        },
        claim(
          f.engines,
          "compression engines",
          { pattern: /(\d+)[-\s](?:engine stack|composable engines|stacked engines)/gi },
          ["README.md"]
        ),
        claim(
          f.mcpTools,
          "MCP tools",
          {
            pattern: /(\d+) tools/gi,
            // per-module rows ("Memory tool definitions (3 tools)") and the CLI catalog
            // total ("33 tools (25 CLI Code's …)") are not the MCP aggregate
            // per-module rows read "… tool definitions (N tools" / "… management tools
            // (N tools" — the word tool(s)/definitions sits right before the paren. The
            // aggregate ("MCP Server (104 tools", "all 104 tools") never does.
            skipBefore: /(tools?|definitions?)\s*\(\s*$/i,
            skipAfter: /^\s*\(\d+ CLI/,
          },
          ["README.md", "CLAUDE.md", "AGENTS.md", "docs/frameworks/MCP-SERVER.md"]
        ),
        claim(f.mcpScopes, "MCP scopes", { pattern: /(\d+) scopes/gi }, [
          "README.md",
          "CLAUDE.md",
          "AGENTS.md",
        ]),
        claim(
          f.cliTotal,
          "CLI tools",
          { pattern: /(\d+) tools(?=\s*\(\d+ CLI)/gi },
          ["README.md"]
        ),
      ];
    })(),
    {
      label: "Executors count",
      actual: countFiles("open-sse/executors"),
      docKey: "executors",
      strict: false,
      files: ["docs/architecture/ARCHITECTURE.md", "docs/architecture/CODEBASE_DOCUMENTATION.md"],
    },
    {
      label: "Routing strategies count",
      actual: countRoutingStrategies(),
      docKey: "strategies",
      strict: false,
      files: ["docs/routing/AUTO-COMBO.md", "docs/architecture/RESILIENCE_GUIDE.md"],
    },
    {
      label: "OAuth providers count",
      actual: countFiles("src/lib/oauth/providers"),
      docKey: "OAuth providers",
      strict: false,
      files: ["docs/architecture/ARCHITECTURE.md"],
    },
    {
      label: "A2A skills count",
      actual: countFiles("src/lib/a2a/skills"),
      docKey: "A2A skills",
      strict: false,
      files: ["docs/frameworks/A2A-SERVER.md"],
    },
    {
      label: "Cloud agents count",
      actual: countFiles("src/lib/cloudAgent/agents"),
      docKey: "cloud agents",
      strict: false,
      files: ["docs/frameworks/CLOUD_AGENT.md", "docs/frameworks/AGENT_PROTOCOLS_GUIDE.md"],
    },
  ];
}

function main() {
  const checks = buildChecks();
  const getContent = (relPath) => {
    const abs = path.join(ROOT, relPath);
    return fs.existsSync(abs) ? fs.readFileSync(abs, "utf8") : null;
  };

  console.log("Docs counts sync report");
  console.log("=======================");
  const { strict, soft, lines } = tallyDrift(checks, getContent);
  for (const l of lines) console.log(l);

  console.log();
  if (strict > 0) {
    console.error(
      `✗ ${strict} STRICT drift(s) detected. ` +
        `Update the docs above to the real counts, or regenerate auto-generated sources ` +
        `(npm run gen:provider-reference).`
    );
    process.exit(1);
  }
  if (soft > 0) {
    console.warn(`⚠ ${soft} potential (soft) drift(s) detected. Review the docs above.`);
    if (process.argv.includes("--strict")) process.exit(1);
  } else {
    console.log("✓ All checks pass.");
  }
}

const invokedDirectly =
  process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
if (invokedDirectly) main();