docs(checks): keep doc counts honest — headings, rankings, catalog, weights, quality gate and scoring diagram now covered (#12507)

Estender o gate de contagens para headings, rankings, catálogo, pesos, quality gate e o diagrama de scoring é exatamente o tipo de trabalho que evita a classe inteira em vez de um caso.

Falo por experiência desta campanha: o `check:docs-counts` caiu **duas vezes** hoje pela mesma causa — contagem de migration escrita à mão em três arquivos mais 41 mirrors, desatualizando a cada migration nova (#12970 e #13209). Cada superfície que este PR passa a cobrir é uma que deixa de virar base-red na mão de quem vier depois.

Revalidei sobre o tip: **19/19**, `check:docs-counts-sync` com 0 drifts, `check:docs-all` PASS, `check:doc-links` PASS.

**Integração:** dois conflitos.

1. `scripts/check/check-docs-counts-sync.mjs` — o bloco de leitura de fatos conflitou com os imports de free-tier que entraram pelo #12786/#12744 nesta campanha. Aditivo, os dois conjuntos ficaram.
2. `docs/diagrams/auto-combo-scoring.mmd` — o seu rótulo dizia `reliability (0.0000)`, mas o #12731 mergeou horas antes e passou a dar peso de reliability a todo mode pack. Ficou o rótulo do tip, `reliability (0.0000 DEFAULT, 0.03 packs, 0.04 reliable)`, que é o número real agora.
This commit is contained in:
Dizzle
2026-09-10 15:54:38 +02:00
committed by GitHub
parent 955b28ef5c
commit 81bf3cc36e
8 changed files with 403 additions and 12 deletions

View File

@@ -0,0 +1 @@
- **docs(checks):** keep doc counts honest — headings, rankings, catalog, weights, quality gate and scoring diagram now covered ([#12507](https://github.com/diegosouzapw/OmniRoute/pull/12507)) — thanks @maxmad64bis

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 26 KiB

After

Width:  |  Height:  |  Size: 27 KiB

View File

@@ -27,11 +27,19 @@
// (providers / MCP tools / routing strategies / free-tier pools).
import fs from "node:fs";
import { spawnSync } from "node:child_process";
import { spawnSync as _spawnSync } from "node:child_process";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
let _spawnSyncImpl = _spawnSync;
export function __setSpawnSyncForTest(fn) {
_spawnSyncImpl = fn;
}
export function __resetSpawnSyncForTest() {
_spawnSyncImpl = _spawnSync;
}
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.resolve(__dirname, "..", "..");
@@ -141,7 +149,8 @@ export function countLocales() {
// PURE: tally STRICT vs SOFT drift for a list of checks, given a content lookup.
// `getContent(file) -> string | null`. A check whose `actual` is 0 is skipped (the
// source count could not be determined). Returns { strict, soft, lines }.
// source count could not be determined). `actual==="ERR"` is a STRICT failure, not a skip.
// Returns { strict, soft, lines }.
export function tallyDrift(checks, getContent) {
let strict = 0;
let soft = 0;
@@ -149,7 +158,22 @@ export function tallyDrift(checks, getContent) {
for (const c of checks) {
const tier = c.strict ? "STRICT" : "soft";
lines.push(`\n${c.label}: ${c.actual} (real) [${tier}]`);
if (!c.actual) {
if (c.actual === "ERR") {
if (c.validate) {
const v = c.validate("", `code facts:${c.actual}`);
lines.push(` ${v.ok ? "✓" : c.strict ? "✗" : "⚠"} ${c.label}${v.detail}`);
if (!v.ok) {
if (c.strict) strict++;
else soft++;
}
} else {
lines.push(` ${c.strict ? "✗" : "⚠"} ${c.label} — readCodeFacts unavailable`);
if (c.strict) strict++;
else soft++;
}
continue;
}
if (c.actual === 0 || c.actual === "0") {
lines.push(` ⚠ could not determine ${c.docKey} count from source — skipping`);
continue;
}
@@ -178,16 +202,35 @@ export function tallyDrift(checks, getContent) {
return { strict, soft, lines };
}
// Lightweight literal claim helper — checks that the expected string appears in the file.
function makeLiteralClaimValidator(expected, opts) {
return (content) =>
content.includes(String(expected))
? { ok: true, detail: `literal "${expected}" present — ${opts?.what ?? "literal"}` }
: {
ok: false,
detail: `expected literal "${expected}" not found — ${opts?.what ?? "literal"}`,
};
}
// Reads every code-derived fact in ONE tsx subprocess — the same functions the app
// serves at runtime, never a hardcoded copy. DATA_DIR is redirected to a throwaway dir
// so importing the MCP tool modules cannot touch the operator's real SQLite file.
// Returns null when tsx is unavailable so the gate degrades to a skip, not a false red.
// Returns null when tsx is unavailable — caller must treat it as a failing check, not a skip.
function readCodeFacts() {
const script = [
'import {computeFreeModelTotals,FREE_MODEL_BUDGETS} from "./open-sse/config/freeModelCatalog.ts";',
'import {FREE_TIER_PROVIDER_SET} from "./open-sse/config/freeTierProviders.ts";',
'import {generateProviderPluginManifest} from "./open-sse/config/providerPluginManifestRegistry.ts";',
'import {REGISTRY} from "./open-sse/config/providers/index.ts";',
'import fs2 from "node:fs";',
'import path2 from "node:path";',
'const __rtxt=fs2.readFileSync(path2.join(process.cwd(),"src/lib/freeProviderRankings.ts"),"utf8");',
'const __dtxt=fs2.readFileSync(path2.join(process.cwd(),"open-sse/config/freeModelCatalog.data.ts"),"utf8");',
'const __itxt=fs2.readFileSync(path2.join(process.cwd(),"src/lib/combos/intelligentRouting.ts"),"utf8");',
'const __cat=__dtxt.match(/FREE_CATALOG_CURATED_AT\\s*=\\s*"([^"]+)"/)?.[1]??null;',
'const __sb=(__rtxt.match(/sortBy\\?\\s*:\\s*"elo"\\s*\\|\\s*"reliability"/)?"reliability":null);',
'const __ik=__itxt.match(/DEFAULT_INTELLIGENT_WEIGHTS[^=]*=\\s*\\{([\\s\\S]*?)\\n\\};/)?.[1]?.split("\\n").filter(l=>l.includes(":")).length??0;',
'import {MODE_PACKS} from "./open-sse/services/autoCombo/modePacks.ts";',
'import {ENGINE_IDS} from "./open-sse/services/compression/engineCatalog.ts";',
'import {CLI_TOOLS} from "./src/shared/constants/cliTools.ts";',
@@ -233,7 +276,7 @@ function readCodeFacts() {
"freePools:t.poolCount,engines:ENGINE_IDS.length,",
"cliTotal:cli.length,cliCode:by('code'),cliAgent:by('agent'),",
"mcpTools:countUniqueMcpTools(cols),mcpScopes:sc.size,providers:pids.size,freeForever:ff.size,",
"modePacks:Object.keys(MODE_PACKS),",
"modePacks:Object.keys(MODE_PACKS),catalogDate:__cat,sortBy:__sb,intelligentKeys:__ik,",
"hardStop:FREE_MODEL_BUDGETS.filter(e=>e.hardStopGuaranteed===true).length,",
"trainsOnPrompts:FREE_MODEL_BUDGETS.filter(e=>e.trainsOnPrompts===true).length,",
"freeTierCount:FREE_TIER_PROVIDER_SET.size,",
@@ -242,7 +285,7 @@ function readCodeFacts() {
].join("");
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "docs-counts-"));
try {
const r = spawnSync(process.execPath, ["--import", "tsx/esm", "-e", script], {
const r = _spawnSyncImpl(process.execPath, ["--import", "tsx/esm", "-e", script], {
cwd: ROOT,
encoding: "utf8",
timeout: 180000,
@@ -550,10 +593,14 @@ export function buildChecks() {
return [
{
label: "Code-derived counts",
actual: 0,
actual: "ERR",
docKey: "code facts",
strict: false,
strict: true,
files: [],
validate: () => ({
ok: false,
detail: "readCodeFacts unavailable — tsx/spawnSync failed",
}),
},
];
const claim = (expected, what, opts, files) => ({
@@ -761,17 +808,82 @@ export function buildChecks() {
f.trainsOnPrompts,
"training-disclosure entries",
{
// `requireClaim`: this page is the one place that states the number,
// so a reworded or deleted sentence must fail rather than pass as
// "no claim in this file" — otherwise the gate is one edit from silent.
requireClaim: true,
pattern:
/(\d+) entr(?:y|ies) (?:that )?(?:carry|carries) a (?:prompt-)?training disclosure/gi,
},
["docs/reference/FREE_TIERS.md"]
),
{
label: "Free provider rankings sortBy (live code)",
actual: f.sortBy ?? "reliability",
docKey: "rankings sortBy",
strict: true,
files: ["src/lib/freeProviderRankings.ts", "src/app/api/free-provider-rankings/route.ts"],
validate: (content) => {
// Disjunctive: the gate loops over two files with different shapes —
// freeProviderRankings.ts carries the union + branch, route.ts the z.enum.
const hasUnion = /sortBy\?\s*:\s*"elo"\s*\|\s*"reliability"/.test(content);
const hasReliabilityBranch =
/sortBy\s*===\s*"reliability"|sortBy\s*!==\s*"reliability"/.test(content);
const hasZEnum = /z\.enum\(\["elo",\s*"reliability"\]\)/.test(content);
return (hasUnion && hasReliabilityBranch) || hasZEnum
? { ok: true, detail: "union+branch (rankings) or z.enum (route) present" }
: { ok: false, detail: "ELO-only regression: union+branch and z.enum both missing" };
},
},
{
label: "FREE_CATALOG_CURATED_AT (live code)",
actual: f.catalogDate ?? 0,
docKey: "FREE_CATALOG_CURATED_AT",
strict: false,
files: ["open-sse/config/freeModelCatalog.data.ts"],
validate: makeLiteralClaimValidator(f.catalogDate, { what: "FREE_CATALOG_CURATED_AT" }),
},
// Duplicate coverage with combo-scoring-weights-schema-coverage.test.ts — soft gate only.
{
label: "INTELLIGENT vs DEFAULT (live code)",
actual: f.intelligentKeys ?? 16,
docKey: "INTELLIGENT vs DEFAULT",
strict: false,
files: ["src/lib/combos/intelligentRouting.ts", "open-sse/services/autoCombo/scoring.ts"],
validate: (content) =>
content.includes("DEFAULT_INTELLIGENT_WEIGHTS") || content.includes("DEFAULT_WEIGHTS")
? {
ok: true,
detail:
"weight constant present (strict coverage in combo-scoring-weights-schema-coverage:66)",
}
: { ok: false, detail: "no weight constant found" },
},
];
})(),
{
label: "ToS caution (16) (live docs)",
actual: 16,
docKey: "ToS caution (16)",
strict: false,
files: ["docs/reference/FREE_TIERS.md"],
validate: makeNumberClaimValidator(16, {
what: "ToS caution (16)",
pattern: /Caution[^\n]*\(\s*(16)\s*\)/gi,
requireClaim: true,
}),
},
{
label: "quality neutral prose (live docs)",
actual: "quality neutral 0.5",
docKey: "quality neutral",
strict: false,
files: ["docs/routing/AUTO-COMBO.md"],
validate: (content) =>
/quality.*neutral.*0\.5/is.test(content)
? { ok: true, detail: "quality neutral 0.5 mentioned" }
: {
ok: false,
detail: "quality neutral 0.5 not found — prose must carry quality neutral 0.5",
},
},
{
label: "Executors count",
actual: countFiles("open-sse/executors"),

View File

@@ -0,0 +1,73 @@
import { afterEach, describe, it } from "node:test";
import assert from "node:assert/strict";
import {
buildChecks,
tallyDrift,
__setSpawnSyncForTest,
__resetSpawnSyncForTest,
} from "../../scripts/check/check-docs-counts-sync.mjs";
type Check = {
label?: string;
docKey?: string;
actual?: unknown;
strict?: boolean;
files?: string[];
validate?: (content: string, claim?: string) => { ok: boolean; detail: string };
};
// inject a failing spawnSync so we can test the failure path (node:child_process.spawnSync is non-configurable in ESM)
afterEach(() => {
try {
__resetSpawnSyncForTest();
} catch {}
});
describe("readCodeFacts failure is reported as a strict failure", () => {
it("reports a strict error when the code facts cannot be loaded", () => {
__setSpawnSyncForTest(
() =>
({
status: 1,
stdout: "",
stderr: "tsx not found",
pid: 1,
output: [],
signal: null,
}) as never
);
const checks = buildChecks() as Check[];
const errCheck = checks.find((c) => c.actual === "ERR");
assert.ok(
errCheck,
"should include a check with actual ERR when the facts loader returns null"
);
assert.equal(errCheck.strict, true, "ERR must be strict");
assert.ok(errCheck.validate, "validate must be defined");
const v = errCheck.validate!("", "code facts:ERR");
assert.equal(v.ok, false, "validate must return ok:false");
assert.match(String(v.detail), /readCodeFacts|tsx|spawnSync/i);
});
it("tallyDrift does not skip actual ERR", () => {
const checks = [
{
label: "Code-derived counts",
actual: "ERR",
strict: true,
files: ["docs/README.md"],
validate: () => ({ ok: false, detail: "ERR" }),
},
] as never;
const { strict } = tallyDrift(checks, () => "");
assert.equal(strict, 1);
});
it("tallyDrift skips actual 0 (source unknown)", () => {
const checks = [
{ label: "Code-derived counts", actual: 0, strict: true, files: ["README.md"] },
] as never;
const { strict } = tallyDrift(checks, () => "content");
assert.equal(strict, 0);
});
});

View File

@@ -0,0 +1,44 @@
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { buildChecks } from "../../scripts/check/check-docs-counts-sync.mjs";
type Check = {
label?: string;
docKey?: string;
actual?: unknown;
strict?: boolean;
files?: string[];
validate?: (content: string, claim?: string) => { ok: boolean; detail: string };
};
describe("quality neutral prose", () => {
it("AUTO-COMBO.md prose carries quality neutral 0.5", () => {
const doc = readFileSync("docs/routing/AUTO-COMBO.md", "utf8");
assert.match(doc, /quality.*neutral.*0\.5/i);
assert.match(doc, /quality.*0\.5/);
});
it("buildChecks exposes a soft entry for quality neutral prose", () => {
const checks = buildChecks() as Check[];
const entry = checks.find(
(c) =>
String(c.docKey ?? "").includes("quality neutral") ||
String(c.label ?? "")
.toLowerCase()
.includes("quality neutral")
);
assert.ok(entry, "quality neutral entry missing from buildChecks");
assert.equal(entry.strict, false, "quality neutral entry must be soft");
assert.ok(
(entry.files ?? []).includes("docs/routing/AUTO-COMBO.md"),
"must gate docs/routing/AUTO-COMBO.md"
);
const doc = readFileSync("docs/routing/AUTO-COMBO.md", "utf8");
const res = entry.validate!(doc);
assert.equal(res.ok, true, `expected soft gate to pass on current doc: ${res.detail}`);
const silent = "no mention of quality at all";
const r2 = entry.validate!(silent);
assert.equal(r2.ok, false, "reworded-away prose must fail");
});
});

View File

@@ -0,0 +1,71 @@
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { buildChecks } from "../../scripts/check/check-docs-counts-sync.mjs";
type Check = {
label?: string;
docKey?: string;
actual?: unknown;
strict?: boolean;
files?: string[];
validate?: (content: string, claim?: string) => { ok: boolean; detail: string };
};
describe("free provider rankings sort key + catalog date", () => {
it("freeProviderRankings.ts declares sortBy elo|reliability and MIN_USAGE_REQUESTS=5", () => {
const txt = readFileSync("src/lib/freeProviderRankings.ts", "utf8");
assert.match(txt, /MIN_USAGE_REQUESTS\s*=\s*5/);
assert.match(txt, /sortBy\?\s*:\s*"elo"\s*\|\s*"reliability"/);
});
it("free-provider-rankings route carries z.enum elo/reliability", () => {
const txt = readFileSync("src/app/api/free-provider-rankings/route.ts", "utf8");
assert.match(txt, /z\.enum\(\["elo",\s*"reliability"\]\)/);
});
it("freeModelCatalog.data.ts carries FREE_CATALOG_CURATED_AT literal", () => {
const txt = readFileSync("open-sse/config/freeModelCatalog.data.ts", "utf8");
assert.match(txt, /export const FREE_CATALOG_CURATED_AT\s*=\s*"\d{4}-\d{2}-\d{2}"/);
});
it("buildChecks exposes rankings sortBy strict + FREE_CATALOG_CURATED_AT soft", () => {
const checks = buildChecks() as Check[];
const rankings = checks.find(
(c) =>
String(c.docKey ?? "").includes("rankings sortBy") ||
String(c.label ?? "").includes("rankings sortBy")
);
const curatedAt = checks.find(
(c) =>
String(c.docKey ?? "").includes("FREE_CATALOG_CURATED_AT") ||
String(c.label ?? "").includes("FREE_CATALOG_CURATED_AT")
);
assert.ok(rankings, "rankings sortBy entry missing");
assert.equal(rankings.strict, true, "rankings sortBy must be strict");
assert.ok(curatedAt, "FREE_CATALOG_CURATED_AT entry missing");
assert.equal(curatedAt.strict, false, "catalog date must be soft");
});
it("summary route references FREE_CATALOG_CURATED_AT with slice(0, 10)", () => {
const txt = readFileSync("src/app/api/free-tier/summary/route.ts", "utf8");
assert.match(txt, /FREE_CATALOG_CURATED_AT/);
assert.match(txt, /slice\(0,\s*10\)/);
});
it("rankings sortBy gate fails closed on ELO-only comparator", () => {
const checks = buildChecks() as Check[];
const rankings = checks.find((c) => String(c.docKey ?? "").includes("rankings sortBy"));
// ELO-only code with "reliability" only in a comment: a validator that merely looks for
// the literal would pass, so this must fail.
const content = 'sortBy?: "elo"; // reliability removed, ELO-only regression';
const v = rankings!.validate!(content);
assert.equal(v.ok, false, "gate must fail on ELO-only content");
});
it("rankings sortBy gate passes on live reliability code (both files)", () => {
const checks = buildChecks() as Check[];
const rankings = checks.find((c) => String(c.docKey ?? "").includes("rankings sortBy"));
for (const f of [
"src/lib/freeProviderRankings.ts",
"src/app/api/free-provider-rankings/route.ts",
]) {
const v = rankings!.validate!(readFileSync(f, "utf8"));
assert.equal(v.ok, true, `gate must pass on live ${f}`);
}
});
});

View File

@@ -0,0 +1,44 @@
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { buildChecks } from "../../scripts/check/check-docs-counts-sync.mjs";
type Check = {
label?: string;
docKey?: string;
actual?: unknown;
strict?: boolean;
files?: string[];
validate?: (content: string, claim?: string) => { ok: boolean; detail: string };
};
describe("ToS caution heading count", () => {
it("the Caution heading carries the count the gate checks", () => {
const txt = readFileSync(join(process.cwd(), "docs/reference/FREE_TIERS.md"), "utf8");
const heading = txt.split("\n").find((line) => /^#+ .*Caution/.test(line));
assert.ok(heading, "FREE_TIERS.md must keep its Caution heading");
const tos = (buildChecks() as Check[]).find((c) =>
String(c.docKey ?? "").includes("ToS caution")
);
assert.match(heading, new RegExp(`\(\s*${String(tos?.actual)}\s*\)`));
});
it("buildChecks exposes a soft ToS entry on FREE_TIERS.md with requireClaim", () => {
const checks = buildChecks() as Check[];
const tos = checks.find(
(c) =>
String(c.docKey ?? "").includes("ToS caution") ||
String(c.label ?? "").includes("ToS caution")
);
assert.ok(tos, "missing ToS caution entry");
assert.equal(tos.strict, false);
assert.ok((tos.files ?? []).includes("docs/reference/FREE_TIERS.md"));
const doc = readFileSync("docs/reference/FREE_TIERS.md", "utf8");
const v = tos.validate!(doc, "ToS caution (16):(16)");
assert.equal(v.ok, true);
});
it("FREE_TIERS.md no longer uses legacy providers ToS-flagged phrasing", () => {
const txt = readFileSync("docs/reference/FREE_TIERS.md", "utf8");
assert.equal(/providers ToS-flagged/i.test(txt), false);
});
});

View File

@@ -0,0 +1,46 @@
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { buildChecks } from "../../scripts/check/check-docs-counts-sync.mjs";
type Check = {
label?: string;
docKey?: string;
actual?: unknown;
strict?: boolean;
files?: string[];
validate?: (content: string, claim?: string) => { ok: boolean; detail: string };
};
describe("routing weights cross-check (soft)", () => {
it("scoring.ts DEFAULT_WEIGHTS and intelligentRouting.ts DEFAULT_INTELLIGENT_WEIGHTS exist", () => {
assert.match(readFileSync("open-sse/services/autoCombo/scoring.ts", "utf8"), /DEFAULT_WEIGHTS/);
assert.match(
readFileSync("src/lib/combos/intelligentRouting.ts", "utf8"),
/DEFAULT_INTELLIGENT_WEIGHTS/
);
});
it("buildChecks exposes a soft INTELLIGENT keys vs DEFAULT entry", () => {
const checks = buildChecks() as Check[];
const w = checks.find(
(c) =>
String(c.docKey ?? "").includes("INTELLIGENT vs DEFAULT") ||
String(c.label ?? "").includes("INTELLIGENT vs DEFAULT")
);
assert.ok(w, "INTELLIGENT vs DEFAULT entry missing");
assert.equal(w.strict, false);
assert.ok(Array.isArray(w.files) && w.files.length > 0);
});
it("combo-scoring-weights-schema-coverage still references DEFAULT_INTELLIGENT_WEIGHTS", () => {
const txt = readFileSync("tests/unit/combo-scoring-weights-schema-coverage.test.ts", "utf8");
assert.match(txt, /DEFAULT_INTELLIGENT_WEIGHTS/);
});
it("no MIN_USAGE gate exists (documented as not gated separately)", () => {
const checks = buildChecks() as Check[];
const m = checks.find(
(c) =>
String(c.docKey ?? "").includes("MIN_USAGE") || String(c.label ?? "").includes("MIN_USAGE")
);
assert.equal(m, undefined);
});
});