diff --git a/scripts/check/check-docs-counts-sync.mjs b/scripts/check/check-docs-counts-sync.mjs index 29a42fca2a..07833beed3 100644 --- a/scripts/check/check-docs-counts-sync.mjs +++ b/scripts/check/check-docs-counts-sync.mjs @@ -223,7 +223,8 @@ function readCodeFacts() { "'recurring-credit','keyless']);", "const ff=new Set();for(const m of t.perModel)if(FOREVER.has(m.freeType))ff.add(m.provider);", 'console.log("@@"+JSON.stringify({freeSteady:t.steadyRecurringTokens,entries:t.perModel.length,', - "freeFirst:t.firstMonthRealisticTokens,freePools:t.poolCount,engines:ENGINE_IDS.length,", + "freeFirst:t.firstMonthRealisticTokens,freeGated:t.gatedRecurringTokens,", + "freePools:t.poolCount,engines:ENGINE_IDS.length,", "cliTotal:cli.length,cliCode:by('code'),cliAgent:by('agent'),", "mcpTools:countUniqueMcpTools(cols),mcpScopes:sc.size,providers:pids.size,freeForever:ff.size,", "modePacks:Object.keys(MODE_PACKS),", @@ -271,6 +272,20 @@ export function extractHeadlineClaims(content) { return claims; } +// The eligibility-gated figure ("+~6M behind regional identity verification") is validated +// with its own anchor so it can neither drift nor be silently dropped once it exists. +const GATED_ANCHOR = /^\s*behind regional identity verification/i; + +export function extractGatedClaims(content) { + const claims = []; + for (const m of content.matchAll(/\+?~?(\d+(?:\.\d+)?)([BM])\b/g)) { + const after = content.slice(m.index + m[0].length, m.index + m[0].length + 60); + if (!GATED_ANCHOR.test(after)) continue; + claims.push({ tokens: Number(m[1]) * (m[2] === "B" ? 1e9 : 1e6), unit: m[2], text: m[0] }); + } + return claims; +} + export function checkFreeTierHeadline(content, totals) { const claims = extractHeadlineClaims(content); if (!claims.length) return { ok: true, detail: "no aggregate free-tier headline in this file" }; @@ -279,14 +294,31 @@ export function checkFreeTierHeadline(content, totals) { const stale = claims.filter( (c) => Math.abs(c.value - steady) >= 0.05 && Math.abs(c.value - first) >= 0.05 ); - if (!stale.length) - return { ok: true, detail: `${claims.length} headline claim(s) match the live catalog` }; - return { - ok: false, - detail: + const problems = []; + if (stale.length) { + problems.push( `stale headline ${[...new Set(stale.map((c) => c.text))].join(", ")} — live catalog ` + - `computes ~${steady.toFixed(2)}B steady / ~${first.toFixed(2)}B first month`, - }; + `computes ~${steady.toFixed(2)}B steady / ~${first.toFixed(2)}B first month` + ); + } + if (totals.g != null && totals.g > 0) { + const gated = extractGatedClaims(content); + const tol = (c) => (c.unit === "B" ? 0.05e9 : 0.5e6); + const gatedStale = gated.filter((c) => Math.abs(c.tokens - totals.g) >= tol(c)); + if (!gated.length) { + problems.push( + `missing gated figure — live catalog computes ${Math.round(totals.g / 1e6)}M behind regional identity verification` + ); + } else if (gatedStale.length) { + problems.push( + `stale gated figure ${[...new Set(gatedStale.map((c) => c.text))].join(", ")} — live catalog ` + + `computes ${Math.round(totals.g / 1e6)}M behind regional identity verification` + ); + } + } + if (!problems.length) + return { ok: true, detail: `${claims.length} headline claim(s) match the live catalog` }; + return { ok: false, detail: problems.join("; ") }; } // PURE: docs prose that names the product version ("OmniRoute v3.8.50 ·", @@ -599,12 +631,12 @@ export function buildChecks() { }, { label: "Free-tier headline (live catalog)", - actual: `~${(f.freeSteady / 1e9).toFixed(2)}B steady / ${f.freePools} pools`, + actual: `~${(f.freeSteady / 1e9).toFixed(2)}B steady / ${f.freePools} pools / ${Math.round(f.freeGated / 1e6)}M gated`, docKey: "free-tier headline", strict: true, files: ["README.md", "docs/reference/FREE_TIERS.md"], validate: (content) => - checkFreeTierHeadline(content, { s: f.freeSteady, m: f.freeFirst }), + checkFreeTierHeadline(content, { s: f.freeSteady, m: f.freeFirst, g: f.freeGated }), }, claim( f.engines, diff --git a/tests/unit/check-docs-counts-sync.test.ts b/tests/unit/check-docs-counts-sync.test.ts index 5f786f327a..3b27f5448c 100644 --- a/tests/unit/check-docs-counts-sync.test.ts +++ b/tests/unit/check-docs-counts-sync.test.ts @@ -106,16 +106,20 @@ test("the gate exits 0 against the current (synced) repo state", () => { // down to 1.37B, because no gate watched that number. import { checkFreeTierHeadline, + extractGatedClaims, extractHeadlineClaims, } from "../../scripts/check/check-docs-counts-sync.mjs"; const checkHeadline = checkFreeTierHeadline as ( content: string, - totals: { s: number; m: number; p: number } + totals: { s: number; m: number; p: number; g?: number } ) => { ok: boolean; detail: string }; const extractClaims = extractHeadlineClaims as ( content: string ) => { value: number; text: string }[]; +const extractGated = extractGatedClaims as ( + content: string +) => { tokens: number; unit: "B" | "M"; text: string }[]; const TOTALS = { s: 1_371_725_000, m: 1_998_225_000, p: 39 }; @@ -147,6 +151,45 @@ test("free-tier gate passes when a file carries no headline at all", () => { assert.equal(checkHeadline("no figures here", TOTALS).ok, true); }); +// --- Eligibility-gated bucket ("+~6M behind regional identity verification") -- +// The gated figure sits next to the headline and is validated with its own anchor, +// so it can neither drift nor be silently dropped once the catalog reports one. +const TOTALS_G = { s: 1_503_225_000, m: 2_129_725_000, p: 35, g: 6_000_000 }; + +test("free-tier gate validates the gated figure that sits next to the headline", () => { + const ok = + "~1.5B free tokens per month … +~6M behind regional identity verification (ModelScope)"; + assert.equal(checkHeadline(ok, TOTALS_G).ok, true); + const stale = "~1.5B free tokens per month … +~60M behind regional identity verification"; + assert.equal(checkHeadline(stale, TOTALS_G).ok, false); + assert.match(checkHeadline(stale, TOTALS_G).detail, /gated/); +}); + +test("free-tier gate rejects a file that carries the headline but omits the gated line", () => { + assert.equal(checkHeadline("~1.5B free tokens per month", TOTALS_G).ok, false); + // a file with no headline at all is still fine (per-provider tables, changelogs) + assert.equal(checkHeadline("no figures here", TOTALS_G).ok, true); + // and nothing changes for callers that pass no gated total + assert.equal( + checkHeadline("~1.5B free tokens per month", { s: TOTALS_G.s, m: TOTALS_G.m, p: 35 }).ok, + true + ); +}); + +test("gated claims are read in M or B and need the anchor phrase", () => { + assert.deepEqual(extractGated("~6M of unrelated text"), []); + assert.deepEqual(extractGated("+~6M behind regional identity verification"), [ + { tokens: 6_000_000, unit: "M", text: "+~6M" }, + ]); + assert.equal( + checkHeadline("~1.5B free tokens per month · ~1.2B behind regional identity verification", { + ...TOTALS_G, + g: 1_230_000_000, + }).ok, + true + ); +}); + // --- Generic numeric-claim gate (engines / MCP tools / scopes / CLI) -------- // Extends the same drift guard to the counts that silently drifted in v3.8.49: // 11→12 engines, 94→109 MCP tools, 30→33 scopes, 26→33 CLI tools.