diff --git a/changelog.d/fixes/11866-alibaba-allowlist-test-timebomb.md b/changelog.d/fixes/11866-alibaba-allowlist-test-timebomb.md new file mode 100644 index 0000000000..e448bb8ed8 --- /dev/null +++ b/changelog.d/fixes/11866-alibaba-allowlist-test-timebomb.md @@ -0,0 +1,5 @@ +- Fixed the Alibaba free-tier allowlist test that went red on its own once the + shipped catalog's `validUntil` (2026-08-27) passed, leaving every PR and `main` + with a failing `Unit Tests (1/8)`. The test now builds its own packs with dates + it controls, and covers the expired-pack fallback that production has actually + been serving. diff --git a/tests/unit/alibaba-free-tier-allowlist.test.ts b/tests/unit/alibaba-free-tier-allowlist.test.ts index 1fa87dc5e1..df468d82e5 100644 --- a/tests/unit/alibaba-free-tier-allowlist.test.ts +++ b/tests/unit/alibaba-free-tier-allowlist.test.ts @@ -7,6 +7,9 @@ */ import { test } from "node:test"; import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { ALIBABA_FREE_TIER_TEXT_CAPABLE_MODELS, ALIBABA_NO_FREE_TIER_TEXT_MODELS, @@ -27,18 +30,90 @@ test("built-in allowlist includes operator free models and excludes paid blockli assert.equal(isAlibabaBuiltinFreeTierTextModel("qwen3.7-max"), false); }); -test("allowlist JSON pack overrides embedded lists when valid", () => { +/** + * The shipped `config/alibaba-free-tier-allowlist.json` carries a `validUntil`, + * so asserting against it made this test a time bomb: it went red on its own on + * 2026-08-28, the day after the pack expired, and stayed red on every PR and on + * `main` (#11866). Nothing had changed — the clock moved. + * + * Production was never affected: an expired pack falls back to the embedded + * list by design. So the contract worth pinning is the BEHAVIOR on both sides of + * the expiry, with packs this test owns and dates it controls — never the + * freshness of the catalog that ships in the repo. + */ +function withAllowlistPack( + pack: Record, + assertions: () => void +): void { + const dir = mkdtempSync(join(tmpdir(), "alibaba-allowlist-")); + const packPath = join(dir, "allowlist.json"); + writeFileSync(packPath, JSON.stringify(pack), "utf8"); + const previousPath = process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH; - const packPath = `${process.cwd()}/config/alibaba-free-tier-allowlist.json`; process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH = packPath; resetAlibabaFreeTierAllowlistCache(); + try { + assertions(); + } finally { + if (previousPath) process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH = previousPath; + else delete process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH; + resetAlibabaFreeTierAllowlistCache(); + rmSync(dir, { recursive: true, force: true }); + } +} - const pack = loadAlibabaFreeTierAllowlistPack(); - assert.ok(pack); - assert.ok(isAlibabaFreeTierAllowlistPackValid(pack!)); - assert.ok(pack!.capable.includes("qwen3.6-plus")); +test("allowlist JSON pack overrides embedded lists while it is still valid", () => { + withAllowlistPack( + { + asOf: "2026-07-28", + validUntil: "2999-01-01", + capable: ["pack-only-capable-model", "qwen3.6-plus"], + noFreeTier: ["pack-only-paid-model"], + }, + () => { + const pack = loadAlibabaFreeTierAllowlistPack(); + assert.ok(pack, "a pack inside its validity window must load"); + assert.ok(isAlibabaFreeTierAllowlistPackValid(pack!)); + assert.ok(pack!.capable.includes("qwen3.6-plus")); + // Positive anchor: the pack must actually REPLACE the embedded list, not + // merely load. `pack-only-capable-model` exists nowhere else. + assert.equal(isAlibabaBuiltinFreeTierTextModel("pack-only-capable-model"), true); + assert.equal(isAlibabaBuiltinNoFreeTierTextModel("pack-only-paid-model"), true); + } + ); +}); - if (previousPath) process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH = previousPath; - else delete process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH; - resetAlibabaFreeTierAllowlistCache(); +test("an expired allowlist pack is ignored and the embedded list serves instead", () => { + // This is the path production has actually been on since 2026-08-27, and it + // had no coverage at all — which is why the expiry surfaced as a red test + // rather than as a deliberate, understood fallback. + withAllowlistPack( + { + asOf: "2026-07-28", + validUntil: "2026-08-27", + capable: ["pack-only-capable-model"], + noFreeTier: ["pack-only-paid-model"], + }, + () => { + assert.equal(loadAlibabaFreeTierAllowlistPack(), null, "expired pack must not load"); + assert.equal(isAlibabaBuiltinFreeTierTextModel("pack-only-capable-model"), false); + // The embedded list must be what answers once the pack is rejected. + assert.equal(isAlibabaBuiltinFreeTierTextModel("qwen3.6-plus"), true); + assert.equal(isAlibabaBuiltinNoFreeTierTextModel("qwen3.7-max"), true); + } + ); +}); + +test("isAlibabaFreeTierAllowlistPackValid compares against the instant it is given", () => { + const pack = { asOf: "2026-07-28", validUntil: "2026-08-27", capable: ["x"], noFreeTier: [] }; + assert.equal(isAlibabaFreeTierAllowlistPackValid(pack, Date.parse("2026-08-26")), true); + assert.equal(isAlibabaFreeTierAllowlistPackValid(pack, Date.parse("2026-08-28")), false); + // No expiry declared means the pack never goes stale on its own. + assert.equal( + isAlibabaFreeTierAllowlistPackValid( + { asOf: "2026-07-28", capable: ["x"], noFreeTier: [] }, + Date.parse("2999-01-01") + ), + true + ); });