mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-19 05:12:16 +03:00
docs(checks): keep doc counts honest — headings, rankings, catalog, weights, quality gate and scoring diagram now covered (#12507)
Estender o gate de contagens para headings, rankings, catálogo, pesos, quality gate e o diagrama de scoring é exatamente o tipo de trabalho que evita a classe inteira em vez de um caso. Falo por experiência desta campanha: o `check:docs-counts` caiu **duas vezes** hoje pela mesma causa — contagem de migration escrita à mão em três arquivos mais 41 mirrors, desatualizando a cada migration nova (#12970 e #13209). Cada superfície que este PR passa a cobrir é uma que deixa de virar base-red na mão de quem vier depois. Revalidei sobre o tip: **19/19**, `check:docs-counts-sync` com 0 drifts, `check:docs-all` PASS, `check:doc-links` PASS. **Integração:** dois conflitos. 1. `scripts/check/check-docs-counts-sync.mjs` — o bloco de leitura de fatos conflitou com os imports de free-tier que entraram pelo #12786/#12744 nesta campanha. Aditivo, os dois conjuntos ficaram. 2. `docs/diagrams/auto-combo-scoring.mmd` — o seu rótulo dizia `reliability (0.0000)`, mas o #12731 mergeou horas antes e passou a dar peso de reliability a todo mode pack. Ficou o rótulo do tip, `reliability (0.0000 DEFAULT, 0.03 packs, 0.04 reliable)`, que é o número real agora.
This commit is contained in:
1
changelog.d/maintenance/12507-docs-counts-gates.md
Normal file
1
changelog.d/maintenance/12507-docs-counts-gates.md
Normal file
@@ -0,0 +1 @@
|
||||
- **docs(checks):** keep doc counts honest — headings, rankings, catalog, weights, quality gate and scoring diagram now covered ([#12507](https://github.com/diegosouzapw/OmniRoute/pull/12507)) — thanks @maxmad64bis
|
||||
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 26 KiB After Width: | Height: | Size: 27 KiB |
@@ -27,11 +27,19 @@
|
||||
// (providers / MCP tools / routing strategies / free-tier pools).
|
||||
|
||||
import fs from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { spawnSync as _spawnSync } from "node:child_process";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
let _spawnSyncImpl = _spawnSync;
|
||||
export function __setSpawnSyncForTest(fn) {
|
||||
_spawnSyncImpl = fn;
|
||||
}
|
||||
export function __resetSpawnSyncForTest() {
|
||||
_spawnSyncImpl = _spawnSync;
|
||||
}
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(__dirname, "..", "..");
|
||||
|
||||
@@ -141,7 +149,8 @@ export function countLocales() {
|
||||
|
||||
// PURE: tally STRICT vs SOFT drift for a list of checks, given a content lookup.
|
||||
// `getContent(file) -> string | null`. A check whose `actual` is 0 is skipped (the
|
||||
// source count could not be determined). Returns { strict, soft, lines }.
|
||||
// source count could not be determined). `actual==="ERR"` is a STRICT failure, not a skip.
|
||||
// Returns { strict, soft, lines }.
|
||||
export function tallyDrift(checks, getContent) {
|
||||
let strict = 0;
|
||||
let soft = 0;
|
||||
@@ -149,7 +158,22 @@ export function tallyDrift(checks, getContent) {
|
||||
for (const c of checks) {
|
||||
const tier = c.strict ? "STRICT" : "soft";
|
||||
lines.push(`\n• ${c.label}: ${c.actual} (real) [${tier}]`);
|
||||
if (!c.actual) {
|
||||
if (c.actual === "ERR") {
|
||||
if (c.validate) {
|
||||
const v = c.validate("", `code facts:${c.actual}`);
|
||||
lines.push(` ${v.ok ? "✓" : c.strict ? "✗" : "⚠"} ${c.label} — ${v.detail}`);
|
||||
if (!v.ok) {
|
||||
if (c.strict) strict++;
|
||||
else soft++;
|
||||
}
|
||||
} else {
|
||||
lines.push(` ${c.strict ? "✗" : "⚠"} ${c.label} — readCodeFacts unavailable`);
|
||||
if (c.strict) strict++;
|
||||
else soft++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (c.actual === 0 || c.actual === "0") {
|
||||
lines.push(` ⚠ could not determine ${c.docKey} count from source — skipping`);
|
||||
continue;
|
||||
}
|
||||
@@ -178,16 +202,35 @@ export function tallyDrift(checks, getContent) {
|
||||
return { strict, soft, lines };
|
||||
}
|
||||
|
||||
// Lightweight literal claim helper — checks that the expected string appears in the file.
|
||||
function makeLiteralClaimValidator(expected, opts) {
|
||||
return (content) =>
|
||||
content.includes(String(expected))
|
||||
? { ok: true, detail: `literal "${expected}" present — ${opts?.what ?? "literal"}` }
|
||||
: {
|
||||
ok: false,
|
||||
detail: `expected literal "${expected}" not found — ${opts?.what ?? "literal"}`,
|
||||
};
|
||||
}
|
||||
|
||||
// Reads every code-derived fact in ONE tsx subprocess — the same functions the app
|
||||
// serves at runtime, never a hardcoded copy. DATA_DIR is redirected to a throwaway dir
|
||||
// so importing the MCP tool modules cannot touch the operator's real SQLite file.
|
||||
// Returns null when tsx is unavailable so the gate degrades to a skip, not a false red.
|
||||
// Returns null when tsx is unavailable — caller must treat it as a failing check, not a skip.
|
||||
function readCodeFacts() {
|
||||
const script = [
|
||||
'import {computeFreeModelTotals,FREE_MODEL_BUDGETS} from "./open-sse/config/freeModelCatalog.ts";',
|
||||
'import {FREE_TIER_PROVIDER_SET} from "./open-sse/config/freeTierProviders.ts";',
|
||||
'import {generateProviderPluginManifest} from "./open-sse/config/providerPluginManifestRegistry.ts";',
|
||||
'import {REGISTRY} from "./open-sse/config/providers/index.ts";',
|
||||
'import fs2 from "node:fs";',
|
||||
'import path2 from "node:path";',
|
||||
'const __rtxt=fs2.readFileSync(path2.join(process.cwd(),"src/lib/freeProviderRankings.ts"),"utf8");',
|
||||
'const __dtxt=fs2.readFileSync(path2.join(process.cwd(),"open-sse/config/freeModelCatalog.data.ts"),"utf8");',
|
||||
'const __itxt=fs2.readFileSync(path2.join(process.cwd(),"src/lib/combos/intelligentRouting.ts"),"utf8");',
|
||||
'const __cat=__dtxt.match(/FREE_CATALOG_CURATED_AT\\s*=\\s*"([^"]+)"/)?.[1]??null;',
|
||||
'const __sb=(__rtxt.match(/sortBy\\?\\s*:\\s*"elo"\\s*\\|\\s*"reliability"/)?"reliability":null);',
|
||||
'const __ik=__itxt.match(/DEFAULT_INTELLIGENT_WEIGHTS[^=]*=\\s*\\{([\\s\\S]*?)\\n\\};/)?.[1]?.split("\\n").filter(l=>l.includes(":")).length??0;',
|
||||
'import {MODE_PACKS} from "./open-sse/services/autoCombo/modePacks.ts";',
|
||||
'import {ENGINE_IDS} from "./open-sse/services/compression/engineCatalog.ts";',
|
||||
'import {CLI_TOOLS} from "./src/shared/constants/cliTools.ts";',
|
||||
@@ -233,7 +276,7 @@ function readCodeFacts() {
|
||||
"freePools:t.poolCount,engines:ENGINE_IDS.length,",
|
||||
"cliTotal:cli.length,cliCode:by('code'),cliAgent:by('agent'),",
|
||||
"mcpTools:countUniqueMcpTools(cols),mcpScopes:sc.size,providers:pids.size,freeForever:ff.size,",
|
||||
"modePacks:Object.keys(MODE_PACKS),",
|
||||
"modePacks:Object.keys(MODE_PACKS),catalogDate:__cat,sortBy:__sb,intelligentKeys:__ik,",
|
||||
"hardStop:FREE_MODEL_BUDGETS.filter(e=>e.hardStopGuaranteed===true).length,",
|
||||
"trainsOnPrompts:FREE_MODEL_BUDGETS.filter(e=>e.trainsOnPrompts===true).length,",
|
||||
"freeTierCount:FREE_TIER_PROVIDER_SET.size,",
|
||||
@@ -242,7 +285,7 @@ function readCodeFacts() {
|
||||
].join("");
|
||||
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "docs-counts-"));
|
||||
try {
|
||||
const r = spawnSync(process.execPath, ["--import", "tsx/esm", "-e", script], {
|
||||
const r = _spawnSyncImpl(process.execPath, ["--import", "tsx/esm", "-e", script], {
|
||||
cwd: ROOT,
|
||||
encoding: "utf8",
|
||||
timeout: 180000,
|
||||
@@ -550,10 +593,14 @@ export function buildChecks() {
|
||||
return [
|
||||
{
|
||||
label: "Code-derived counts",
|
||||
actual: 0,
|
||||
actual: "ERR",
|
||||
docKey: "code facts",
|
||||
strict: false,
|
||||
strict: true,
|
||||
files: [],
|
||||
validate: () => ({
|
||||
ok: false,
|
||||
detail: "readCodeFacts unavailable — tsx/spawnSync failed",
|
||||
}),
|
||||
},
|
||||
];
|
||||
const claim = (expected, what, opts, files) => ({
|
||||
@@ -761,17 +808,82 @@ export function buildChecks() {
|
||||
f.trainsOnPrompts,
|
||||
"training-disclosure entries",
|
||||
{
|
||||
// `requireClaim`: this page is the one place that states the number,
|
||||
// so a reworded or deleted sentence must fail rather than pass as
|
||||
// "no claim in this file" — otherwise the gate is one edit from silent.
|
||||
requireClaim: true,
|
||||
pattern:
|
||||
/(\d+) entr(?:y|ies) (?:that )?(?:carry|carries) a (?:prompt-)?training disclosure/gi,
|
||||
},
|
||||
["docs/reference/FREE_TIERS.md"]
|
||||
),
|
||||
{
|
||||
label: "Free provider rankings sortBy (live code)",
|
||||
actual: f.sortBy ?? "reliability",
|
||||
docKey: "rankings sortBy",
|
||||
strict: true,
|
||||
files: ["src/lib/freeProviderRankings.ts", "src/app/api/free-provider-rankings/route.ts"],
|
||||
validate: (content) => {
|
||||
// Disjunctive: the gate loops over two files with different shapes —
|
||||
// freeProviderRankings.ts carries the union + branch, route.ts the z.enum.
|
||||
const hasUnion = /sortBy\?\s*:\s*"elo"\s*\|\s*"reliability"/.test(content);
|
||||
const hasReliabilityBranch =
|
||||
/sortBy\s*===\s*"reliability"|sortBy\s*!==\s*"reliability"/.test(content);
|
||||
const hasZEnum = /z\.enum\(\["elo",\s*"reliability"\]\)/.test(content);
|
||||
return (hasUnion && hasReliabilityBranch) || hasZEnum
|
||||
? { ok: true, detail: "union+branch (rankings) or z.enum (route) present" }
|
||||
: { ok: false, detail: "ELO-only regression: union+branch and z.enum both missing" };
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "FREE_CATALOG_CURATED_AT (live code)",
|
||||
actual: f.catalogDate ?? 0,
|
||||
docKey: "FREE_CATALOG_CURATED_AT",
|
||||
strict: false,
|
||||
files: ["open-sse/config/freeModelCatalog.data.ts"],
|
||||
validate: makeLiteralClaimValidator(f.catalogDate, { what: "FREE_CATALOG_CURATED_AT" }),
|
||||
},
|
||||
// Duplicate coverage with combo-scoring-weights-schema-coverage.test.ts — soft gate only.
|
||||
{
|
||||
label: "INTELLIGENT vs DEFAULT (live code)",
|
||||
actual: f.intelligentKeys ?? 16,
|
||||
docKey: "INTELLIGENT vs DEFAULT",
|
||||
strict: false,
|
||||
files: ["src/lib/combos/intelligentRouting.ts", "open-sse/services/autoCombo/scoring.ts"],
|
||||
validate: (content) =>
|
||||
content.includes("DEFAULT_INTELLIGENT_WEIGHTS") || content.includes("DEFAULT_WEIGHTS")
|
||||
? {
|
||||
ok: true,
|
||||
detail:
|
||||
"weight constant present (strict coverage in combo-scoring-weights-schema-coverage:66)",
|
||||
}
|
||||
: { ok: false, detail: "no weight constant found" },
|
||||
},
|
||||
];
|
||||
})(),
|
||||
{
|
||||
label: "ToS caution (16) (live docs)",
|
||||
actual: 16,
|
||||
docKey: "ToS caution (16)",
|
||||
strict: false,
|
||||
files: ["docs/reference/FREE_TIERS.md"],
|
||||
validate: makeNumberClaimValidator(16, {
|
||||
what: "ToS caution (16)",
|
||||
pattern: /Caution[^\n]*\(\s*(16)\s*\)/gi,
|
||||
requireClaim: true,
|
||||
}),
|
||||
},
|
||||
{
|
||||
label: "quality neutral prose (live docs)",
|
||||
actual: "quality neutral 0.5",
|
||||
docKey: "quality neutral",
|
||||
strict: false,
|
||||
files: ["docs/routing/AUTO-COMBO.md"],
|
||||
validate: (content) =>
|
||||
/quality.*neutral.*0\.5/is.test(content)
|
||||
? { ok: true, detail: "quality neutral 0.5 mentioned" }
|
||||
: {
|
||||
ok: false,
|
||||
detail: "quality neutral 0.5 not found — prose must carry quality neutral 0.5",
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Executors count",
|
||||
actual: countFiles("open-sse/executors"),
|
||||
|
||||
73
tests/unit/check-docs-counts-err-seam.test.ts
Normal file
73
tests/unit/check-docs-counts-err-seam.test.ts
Normal file
@@ -0,0 +1,73 @@
|
||||
import { afterEach, describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
buildChecks,
|
||||
tallyDrift,
|
||||
__setSpawnSyncForTest,
|
||||
__resetSpawnSyncForTest,
|
||||
} from "../../scripts/check/check-docs-counts-sync.mjs";
|
||||
|
||||
type Check = {
|
||||
label?: string;
|
||||
docKey?: string;
|
||||
actual?: unknown;
|
||||
strict?: boolean;
|
||||
files?: string[];
|
||||
validate?: (content: string, claim?: string) => { ok: boolean; detail: string };
|
||||
};
|
||||
|
||||
// inject a failing spawnSync so we can test the failure path (node:child_process.spawnSync is non-configurable in ESM)
|
||||
afterEach(() => {
|
||||
try {
|
||||
__resetSpawnSyncForTest();
|
||||
} catch {}
|
||||
});
|
||||
|
||||
describe("readCodeFacts failure is reported as a strict failure", () => {
|
||||
it("reports a strict error when the code facts cannot be loaded", () => {
|
||||
__setSpawnSyncForTest(
|
||||
() =>
|
||||
({
|
||||
status: 1,
|
||||
stdout: "",
|
||||
stderr: "tsx not found",
|
||||
pid: 1,
|
||||
output: [],
|
||||
signal: null,
|
||||
}) as never
|
||||
);
|
||||
const checks = buildChecks() as Check[];
|
||||
const errCheck = checks.find((c) => c.actual === "ERR");
|
||||
assert.ok(
|
||||
errCheck,
|
||||
"should include a check with actual ERR when the facts loader returns null"
|
||||
);
|
||||
assert.equal(errCheck.strict, true, "ERR must be strict");
|
||||
assert.ok(errCheck.validate, "validate must be defined");
|
||||
const v = errCheck.validate!("", "code facts:ERR");
|
||||
assert.equal(v.ok, false, "validate must return ok:false");
|
||||
assert.match(String(v.detail), /readCodeFacts|tsx|spawnSync/i);
|
||||
});
|
||||
|
||||
it("tallyDrift does not skip actual ERR", () => {
|
||||
const checks = [
|
||||
{
|
||||
label: "Code-derived counts",
|
||||
actual: "ERR",
|
||||
strict: true,
|
||||
files: ["docs/README.md"],
|
||||
validate: () => ({ ok: false, detail: "ERR" }),
|
||||
},
|
||||
] as never;
|
||||
const { strict } = tallyDrift(checks, () => "");
|
||||
assert.equal(strict, 1);
|
||||
});
|
||||
|
||||
it("tallyDrift skips actual 0 (source unknown)", () => {
|
||||
const checks = [
|
||||
{ label: "Code-derived counts", actual: 0, strict: true, files: ["README.md"] },
|
||||
] as never;
|
||||
const { strict } = tallyDrift(checks, () => "content");
|
||||
assert.equal(strict, 0);
|
||||
});
|
||||
});
|
||||
44
tests/unit/check-docs-counts-quality-neutral.test.ts
Normal file
44
tests/unit/check-docs-counts-quality-neutral.test.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { buildChecks } from "../../scripts/check/check-docs-counts-sync.mjs";
|
||||
|
||||
type Check = {
|
||||
label?: string;
|
||||
docKey?: string;
|
||||
actual?: unknown;
|
||||
strict?: boolean;
|
||||
files?: string[];
|
||||
validate?: (content: string, claim?: string) => { ok: boolean; detail: string };
|
||||
};
|
||||
|
||||
describe("quality neutral prose", () => {
|
||||
it("AUTO-COMBO.md prose carries quality neutral 0.5", () => {
|
||||
const doc = readFileSync("docs/routing/AUTO-COMBO.md", "utf8");
|
||||
assert.match(doc, /quality.*neutral.*0\.5/i);
|
||||
assert.match(doc, /quality.*0\.5/);
|
||||
});
|
||||
|
||||
it("buildChecks exposes a soft entry for quality neutral prose", () => {
|
||||
const checks = buildChecks() as Check[];
|
||||
const entry = checks.find(
|
||||
(c) =>
|
||||
String(c.docKey ?? "").includes("quality neutral") ||
|
||||
String(c.label ?? "")
|
||||
.toLowerCase()
|
||||
.includes("quality neutral")
|
||||
);
|
||||
assert.ok(entry, "quality neutral entry missing from buildChecks");
|
||||
assert.equal(entry.strict, false, "quality neutral entry must be soft");
|
||||
assert.ok(
|
||||
(entry.files ?? []).includes("docs/routing/AUTO-COMBO.md"),
|
||||
"must gate docs/routing/AUTO-COMBO.md"
|
||||
);
|
||||
const doc = readFileSync("docs/routing/AUTO-COMBO.md", "utf8");
|
||||
const res = entry.validate!(doc);
|
||||
assert.equal(res.ok, true, `expected soft gate to pass on current doc: ${res.detail}`);
|
||||
const silent = "no mention of quality at all";
|
||||
const r2 = entry.validate!(silent);
|
||||
assert.equal(r2.ok, false, "reworded-away prose must fail");
|
||||
});
|
||||
});
|
||||
71
tests/unit/check-docs-counts-rankings-catalog.test.ts
Normal file
71
tests/unit/check-docs-counts-rankings-catalog.test.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { buildChecks } from "../../scripts/check/check-docs-counts-sync.mjs";
|
||||
|
||||
type Check = {
|
||||
label?: string;
|
||||
docKey?: string;
|
||||
actual?: unknown;
|
||||
strict?: boolean;
|
||||
files?: string[];
|
||||
validate?: (content: string, claim?: string) => { ok: boolean; detail: string };
|
||||
};
|
||||
|
||||
describe("free provider rankings sort key + catalog date", () => {
|
||||
it("freeProviderRankings.ts declares sortBy elo|reliability and MIN_USAGE_REQUESTS=5", () => {
|
||||
const txt = readFileSync("src/lib/freeProviderRankings.ts", "utf8");
|
||||
assert.match(txt, /MIN_USAGE_REQUESTS\s*=\s*5/);
|
||||
assert.match(txt, /sortBy\?\s*:\s*"elo"\s*\|\s*"reliability"/);
|
||||
});
|
||||
it("free-provider-rankings route carries z.enum elo/reliability", () => {
|
||||
const txt = readFileSync("src/app/api/free-provider-rankings/route.ts", "utf8");
|
||||
assert.match(txt, /z\.enum\(\["elo",\s*"reliability"\]\)/);
|
||||
});
|
||||
it("freeModelCatalog.data.ts carries FREE_CATALOG_CURATED_AT literal", () => {
|
||||
const txt = readFileSync("open-sse/config/freeModelCatalog.data.ts", "utf8");
|
||||
assert.match(txt, /export const FREE_CATALOG_CURATED_AT\s*=\s*"\d{4}-\d{2}-\d{2}"/);
|
||||
});
|
||||
it("buildChecks exposes rankings sortBy strict + FREE_CATALOG_CURATED_AT soft", () => {
|
||||
const checks = buildChecks() as Check[];
|
||||
const rankings = checks.find(
|
||||
(c) =>
|
||||
String(c.docKey ?? "").includes("rankings sortBy") ||
|
||||
String(c.label ?? "").includes("rankings sortBy")
|
||||
);
|
||||
const curatedAt = checks.find(
|
||||
(c) =>
|
||||
String(c.docKey ?? "").includes("FREE_CATALOG_CURATED_AT") ||
|
||||
String(c.label ?? "").includes("FREE_CATALOG_CURATED_AT")
|
||||
);
|
||||
assert.ok(rankings, "rankings sortBy entry missing");
|
||||
assert.equal(rankings.strict, true, "rankings sortBy must be strict");
|
||||
assert.ok(curatedAt, "FREE_CATALOG_CURATED_AT entry missing");
|
||||
assert.equal(curatedAt.strict, false, "catalog date must be soft");
|
||||
});
|
||||
it("summary route references FREE_CATALOG_CURATED_AT with slice(0, 10)", () => {
|
||||
const txt = readFileSync("src/app/api/free-tier/summary/route.ts", "utf8");
|
||||
assert.match(txt, /FREE_CATALOG_CURATED_AT/);
|
||||
assert.match(txt, /slice\(0,\s*10\)/);
|
||||
});
|
||||
it("rankings sortBy gate fails closed on ELO-only comparator", () => {
|
||||
const checks = buildChecks() as Check[];
|
||||
const rankings = checks.find((c) => String(c.docKey ?? "").includes("rankings sortBy"));
|
||||
// ELO-only code with "reliability" only in a comment: a validator that merely looks for
|
||||
// the literal would pass, so this must fail.
|
||||
const content = 'sortBy?: "elo"; // reliability removed, ELO-only regression';
|
||||
const v = rankings!.validate!(content);
|
||||
assert.equal(v.ok, false, "gate must fail on ELO-only content");
|
||||
});
|
||||
it("rankings sortBy gate passes on live reliability code (both files)", () => {
|
||||
const checks = buildChecks() as Check[];
|
||||
const rankings = checks.find((c) => String(c.docKey ?? "").includes("rankings sortBy"));
|
||||
for (const f of [
|
||||
"src/lib/freeProviderRankings.ts",
|
||||
"src/app/api/free-provider-rankings/route.ts",
|
||||
]) {
|
||||
const v = rankings!.validate!(readFileSync(f, "utf8"));
|
||||
assert.equal(v.ok, true, `gate must pass on live ${f}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
44
tests/unit/check-docs-counts-tos-heading.test.ts
Normal file
44
tests/unit/check-docs-counts-tos-heading.test.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { buildChecks } from "../../scripts/check/check-docs-counts-sync.mjs";
|
||||
|
||||
type Check = {
|
||||
label?: string;
|
||||
docKey?: string;
|
||||
actual?: unknown;
|
||||
strict?: boolean;
|
||||
files?: string[];
|
||||
validate?: (content: string, claim?: string) => { ok: boolean; detail: string };
|
||||
};
|
||||
|
||||
describe("ToS caution heading count", () => {
|
||||
it("the Caution heading carries the count the gate checks", () => {
|
||||
const txt = readFileSync(join(process.cwd(), "docs/reference/FREE_TIERS.md"), "utf8");
|
||||
const heading = txt.split("\n").find((line) => /^#+ .*Caution/.test(line));
|
||||
assert.ok(heading, "FREE_TIERS.md must keep its Caution heading");
|
||||
const tos = (buildChecks() as Check[]).find((c) =>
|
||||
String(c.docKey ?? "").includes("ToS caution")
|
||||
);
|
||||
assert.match(heading, new RegExp(`\(\s*${String(tos?.actual)}\s*\)`));
|
||||
});
|
||||
it("buildChecks exposes a soft ToS entry on FREE_TIERS.md with requireClaim", () => {
|
||||
const checks = buildChecks() as Check[];
|
||||
const tos = checks.find(
|
||||
(c) =>
|
||||
String(c.docKey ?? "").includes("ToS caution") ||
|
||||
String(c.label ?? "").includes("ToS caution")
|
||||
);
|
||||
assert.ok(tos, "missing ToS caution entry");
|
||||
assert.equal(tos.strict, false);
|
||||
assert.ok((tos.files ?? []).includes("docs/reference/FREE_TIERS.md"));
|
||||
const doc = readFileSync("docs/reference/FREE_TIERS.md", "utf8");
|
||||
const v = tos.validate!(doc, "ToS caution (16):(16)");
|
||||
assert.equal(v.ok, true);
|
||||
});
|
||||
it("FREE_TIERS.md no longer uses legacy providers ToS-flagged phrasing", () => {
|
||||
const txt = readFileSync("docs/reference/FREE_TIERS.md", "utf8");
|
||||
assert.equal(/providers ToS-flagged/i.test(txt), false);
|
||||
});
|
||||
});
|
||||
46
tests/unit/check-docs-counts-weights-cross-check.test.ts
Normal file
46
tests/unit/check-docs-counts-weights-cross-check.test.ts
Normal file
@@ -0,0 +1,46 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { buildChecks } from "../../scripts/check/check-docs-counts-sync.mjs";
|
||||
|
||||
type Check = {
|
||||
label?: string;
|
||||
docKey?: string;
|
||||
actual?: unknown;
|
||||
strict?: boolean;
|
||||
files?: string[];
|
||||
validate?: (content: string, claim?: string) => { ok: boolean; detail: string };
|
||||
};
|
||||
|
||||
describe("routing weights cross-check (soft)", () => {
|
||||
it("scoring.ts DEFAULT_WEIGHTS and intelligentRouting.ts DEFAULT_INTELLIGENT_WEIGHTS exist", () => {
|
||||
assert.match(readFileSync("open-sse/services/autoCombo/scoring.ts", "utf8"), /DEFAULT_WEIGHTS/);
|
||||
assert.match(
|
||||
readFileSync("src/lib/combos/intelligentRouting.ts", "utf8"),
|
||||
/DEFAULT_INTELLIGENT_WEIGHTS/
|
||||
);
|
||||
});
|
||||
it("buildChecks exposes a soft INTELLIGENT keys vs DEFAULT entry", () => {
|
||||
const checks = buildChecks() as Check[];
|
||||
const w = checks.find(
|
||||
(c) =>
|
||||
String(c.docKey ?? "").includes("INTELLIGENT vs DEFAULT") ||
|
||||
String(c.label ?? "").includes("INTELLIGENT vs DEFAULT")
|
||||
);
|
||||
assert.ok(w, "INTELLIGENT vs DEFAULT entry missing");
|
||||
assert.equal(w.strict, false);
|
||||
assert.ok(Array.isArray(w.files) && w.files.length > 0);
|
||||
});
|
||||
it("combo-scoring-weights-schema-coverage still references DEFAULT_INTELLIGENT_WEIGHTS", () => {
|
||||
const txt = readFileSync("tests/unit/combo-scoring-weights-schema-coverage.test.ts", "utf8");
|
||||
assert.match(txt, /DEFAULT_INTELLIGENT_WEIGHTS/);
|
||||
});
|
||||
it("no MIN_USAGE gate exists (documented as not gated separately)", () => {
|
||||
const checks = buildChecks() as Check[];
|
||||
const m = checks.find(
|
||||
(c) =>
|
||||
String(c.docKey ?? "").includes("MIN_USAGE") || String(c.label ?? "").includes("MIN_USAGE")
|
||||
);
|
||||
assert.equal(m, undefined);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user