feat(check): docs-counts gate validates the eligibility-gated free-tier figure

This commit is contained in:
diegosouzapw
2026-09-02 20:24:24 -03:00
parent dc95ef8e25
commit 635bf47cd1
2 changed files with 86 additions and 11 deletions

View File

@@ -223,7 +223,8 @@ function readCodeFacts() {
"'recurring-credit','keyless']);",
"const ff=new Set();for(const m of t.perModel)if(FOREVER.has(m.freeType))ff.add(m.provider);",
'console.log("@@"+JSON.stringify({freeSteady:t.steadyRecurringTokens,entries:t.perModel.length,',
"freeFirst:t.firstMonthRealisticTokens,freePools:t.poolCount,engines:ENGINE_IDS.length,",
"freeFirst:t.firstMonthRealisticTokens,freeGated:t.gatedRecurringTokens,",
"freePools:t.poolCount,engines:ENGINE_IDS.length,",
"cliTotal:cli.length,cliCode:by('code'),cliAgent:by('agent'),",
"mcpTools:countUniqueMcpTools(cols),mcpScopes:sc.size,providers:pids.size,freeForever:ff.size,",
"modePacks:Object.keys(MODE_PACKS),",
@@ -271,6 +272,20 @@ export function extractHeadlineClaims(content) {
return claims;
}
// The eligibility-gated figure ("+~6M behind regional identity verification") is validated
// with its own anchor so it can neither drift nor be silently dropped once it exists.
const GATED_ANCHOR = /^\s*behind regional identity verification/i;
export function extractGatedClaims(content) {
const claims = [];
for (const m of content.matchAll(/\+?~?(\d+(?:\.\d+)?)([BM])\b/g)) {
const after = content.slice(m.index + m[0].length, m.index + m[0].length + 60);
if (!GATED_ANCHOR.test(after)) continue;
claims.push({ tokens: Number(m[1]) * (m[2] === "B" ? 1e9 : 1e6), unit: m[2], text: m[0] });
}
return claims;
}
export function checkFreeTierHeadline(content, totals) {
const claims = extractHeadlineClaims(content);
if (!claims.length) return { ok: true, detail: "no aggregate free-tier headline in this file" };
@@ -279,14 +294,31 @@ export function checkFreeTierHeadline(content, totals) {
const stale = claims.filter(
(c) => Math.abs(c.value - steady) >= 0.05 && Math.abs(c.value - first) >= 0.05
);
if (!stale.length)
return { ok: true, detail: `${claims.length} headline claim(s) match the live catalog` };
return {
ok: false,
detail:
const problems = [];
if (stale.length) {
problems.push(
`stale headline ${[...new Set(stale.map((c) => c.text))].join(", ")} — live catalog ` +
`computes ~${steady.toFixed(2)}B steady / ~${first.toFixed(2)}B first month`,
};
`computes ~${steady.toFixed(2)}B steady / ~${first.toFixed(2)}B first month`
);
}
if (totals.g != null && totals.g > 0) {
const gated = extractGatedClaims(content);
const tol = (c) => (c.unit === "B" ? 0.05e9 : 0.5e6);
const gatedStale = gated.filter((c) => Math.abs(c.tokens - totals.g) >= tol(c));
if (!gated.length) {
problems.push(
`missing gated figure — live catalog computes ${Math.round(totals.g / 1e6)}M behind regional identity verification`
);
} else if (gatedStale.length) {
problems.push(
`stale gated figure ${[...new Set(gatedStale.map((c) => c.text))].join(", ")} — live catalog ` +
`computes ${Math.round(totals.g / 1e6)}M behind regional identity verification`
);
}
}
if (!problems.length)
return { ok: true, detail: `${claims.length} headline claim(s) match the live catalog` };
return { ok: false, detail: problems.join("; ") };
}
// PURE: docs prose that names the product version ("OmniRoute v3.8.50 ·",
@@ -599,12 +631,12 @@ export function buildChecks() {
},
{
label: "Free-tier headline (live catalog)",
actual: `~${(f.freeSteady / 1e9).toFixed(2)}B steady / ${f.freePools} pools`,
actual: `~${(f.freeSteady / 1e9).toFixed(2)}B steady / ${f.freePools} pools / ${Math.round(f.freeGated / 1e6)}M gated`,
docKey: "free-tier headline",
strict: true,
files: ["README.md", "docs/reference/FREE_TIERS.md"],
validate: (content) =>
checkFreeTierHeadline(content, { s: f.freeSteady, m: f.freeFirst }),
checkFreeTierHeadline(content, { s: f.freeSteady, m: f.freeFirst, g: f.freeGated }),
},
claim(
f.engines,

View File

@@ -106,16 +106,20 @@ test("the gate exits 0 against the current (synced) repo state", () => {
// down to 1.37B, because no gate watched that number.
import {
checkFreeTierHeadline,
extractGatedClaims,
extractHeadlineClaims,
} from "../../scripts/check/check-docs-counts-sync.mjs";
const checkHeadline = checkFreeTierHeadline as (
content: string,
totals: { s: number; m: number; p: number }
totals: { s: number; m: number; p: number; g?: number }
) => { ok: boolean; detail: string };
const extractClaims = extractHeadlineClaims as (
content: string
) => { value: number; text: string }[];
const extractGated = extractGatedClaims as (
content: string
) => { tokens: number; unit: "B" | "M"; text: string }[];
const TOTALS = { s: 1_371_725_000, m: 1_998_225_000, p: 39 };
@@ -147,6 +151,45 @@ test("free-tier gate passes when a file carries no headline at all", () => {
assert.equal(checkHeadline("no figures here", TOTALS).ok, true);
});
// --- Eligibility-gated bucket ("+~6M behind regional identity verification") --
// The gated figure sits next to the headline and is validated with its own anchor,
// so it can neither drift nor be silently dropped once the catalog reports one.
const TOTALS_G = { s: 1_503_225_000, m: 2_129_725_000, p: 35, g: 6_000_000 };
test("free-tier gate validates the gated figure that sits next to the headline", () => {
const ok =
"~1.5B free tokens per month … +~6M behind regional identity verification (ModelScope)";
assert.equal(checkHeadline(ok, TOTALS_G).ok, true);
const stale = "~1.5B free tokens per month … +~60M behind regional identity verification";
assert.equal(checkHeadline(stale, TOTALS_G).ok, false);
assert.match(checkHeadline(stale, TOTALS_G).detail, /gated/);
});
test("free-tier gate rejects a file that carries the headline but omits the gated line", () => {
assert.equal(checkHeadline("~1.5B free tokens per month", TOTALS_G).ok, false);
// a file with no headline at all is still fine (per-provider tables, changelogs)
assert.equal(checkHeadline("no figures here", TOTALS_G).ok, true);
// and nothing changes for callers that pass no gated total
assert.equal(
checkHeadline("~1.5B free tokens per month", { s: TOTALS_G.s, m: TOTALS_G.m, p: 35 }).ok,
true
);
});
test("gated claims are read in M or B and need the anchor phrase", () => {
assert.deepEqual(extractGated("~6M of unrelated text"), []);
assert.deepEqual(extractGated("+~6M behind regional identity verification"), [
{ tokens: 6_000_000, unit: "M", text: "+~6M" },
]);
assert.equal(
checkHeadline("~1.5B free tokens per month · ~1.2B behind regional identity verification", {
...TOTALS_G,
g: 1_230_000_000,
}).ok,
true
);
});
// --- Generic numeric-claim gate (engines / MCP tools / scopes / CLI) --------
// Extends the same drift guard to the counts that silently drifted in v3.8.49:
// 11→12 engines, 94→109 MCP tools, 30→33 scopes, 26→33 CLI tools.