From 142aba78e983dc0b05d5dc7c4df9e4a16d14899b Mon Sep 17 00:00:00 2001 From: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> Date: Fri, 28 Aug 2026 03:34:06 -0300 Subject: [PATCH] fix(test): stop the Alibaba allowlist test from expiring with the catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Unit Tests (1/8)` went red on 2026-08-28 across every PR and on main, with nothing changed — the clock had moved past the shipped catalog's expiry: config/alibaba-free-tier-allowlist.json → "validUntil": "2026-08-27" isAlibabaFreeTierAllowlistPackValid() compares that against Date.now(), so from 28/08 loadAlibabaFreeTierAllowlistPack() returns null and the old assert.ok(pack) could never pass again. Refreshing the date would only reschedule the same break. Production was never affected: resolveActiveAllowlistPack() falls back to the embedded list when a pack expires, which is the intended design. The defect was the test asserting the shipped catalog is currently fresh — a data property, not a behavioral contract. The test now writes its own packs to a temp dir with dates it controls, and pins both halves of the contract: - inside the validity window, the pack REPLACES the embedded list (anchored on a model that exists nowhere else, so loading alone cannot satisfy it); - once expired, the pack is ignored and the embedded list serves. That second path is what production has been running since 27/08 and had no coverage at all, which is why the expiry surfaced as a red test rather than as understood behavior. A third case pins the comparison against an injected instant, including the no-expiry pack that never goes stale. Whether the curated free-tier catalog still matches reality — and so deserves a freshly dated pack — is a data question left to the operator in #11866. Closes #11866 --- .../11866-alibaba-allowlist-test-timebomb.md | 5 + .../unit/alibaba-free-tier-allowlist.test.ts | 93 +++++++++++++++++-- 2 files changed, 89 insertions(+), 9 deletions(-) create mode 100644 changelog.d/fixes/11866-alibaba-allowlist-test-timebomb.md diff --git a/changelog.d/fixes/11866-alibaba-allowlist-test-timebomb.md b/changelog.d/fixes/11866-alibaba-allowlist-test-timebomb.md new file mode 100644 index 0000000000..e448bb8ed8 --- /dev/null +++ b/changelog.d/fixes/11866-alibaba-allowlist-test-timebomb.md @@ -0,0 +1,5 @@ +- Fixed the Alibaba free-tier allowlist test that went red on its own once the + shipped catalog's `validUntil` (2026-08-27) passed, leaving every PR and `main` + with a failing `Unit Tests (1/8)`. The test now builds its own packs with dates + it controls, and covers the expired-pack fallback that production has actually + been serving. diff --git a/tests/unit/alibaba-free-tier-allowlist.test.ts b/tests/unit/alibaba-free-tier-allowlist.test.ts index 1fa87dc5e1..df468d82e5 100644 --- a/tests/unit/alibaba-free-tier-allowlist.test.ts +++ b/tests/unit/alibaba-free-tier-allowlist.test.ts @@ -7,6 +7,9 @@ */ import { test } from "node:test"; import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { ALIBABA_FREE_TIER_TEXT_CAPABLE_MODELS, ALIBABA_NO_FREE_TIER_TEXT_MODELS, @@ -27,18 +30,90 @@ test("built-in allowlist includes operator free models and excludes paid blockli assert.equal(isAlibabaBuiltinFreeTierTextModel("qwen3.7-max"), false); }); -test("allowlist JSON pack overrides embedded lists when valid", () => { +/** + * The shipped `config/alibaba-free-tier-allowlist.json` carries a `validUntil`, + * so asserting against it made this test a time bomb: it went red on its own on + * 2026-08-28, the day after the pack expired, and stayed red on every PR and on + * `main` (#11866). Nothing had changed — the clock moved. + * + * Production was never affected: an expired pack falls back to the embedded + * list by design. So the contract worth pinning is the BEHAVIOR on both sides of + * the expiry, with packs this test owns and dates it controls — never the + * freshness of the catalog that ships in the repo. + */ +function withAllowlistPack( + pack: Record, + assertions: () => void +): void { + const dir = mkdtempSync(join(tmpdir(), "alibaba-allowlist-")); + const packPath = join(dir, "allowlist.json"); + writeFileSync(packPath, JSON.stringify(pack), "utf8"); + const previousPath = process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH; - const packPath = `${process.cwd()}/config/alibaba-free-tier-allowlist.json`; process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH = packPath; resetAlibabaFreeTierAllowlistCache(); + try { + assertions(); + } finally { + if (previousPath) process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH = previousPath; + else delete process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH; + resetAlibabaFreeTierAllowlistCache(); + rmSync(dir, { recursive: true, force: true }); + } +} - const pack = loadAlibabaFreeTierAllowlistPack(); - assert.ok(pack); - assert.ok(isAlibabaFreeTierAllowlistPackValid(pack!)); - assert.ok(pack!.capable.includes("qwen3.6-plus")); +test("allowlist JSON pack overrides embedded lists while it is still valid", () => { + withAllowlistPack( + { + asOf: "2026-07-28", + validUntil: "2999-01-01", + capable: ["pack-only-capable-model", "qwen3.6-plus"], + noFreeTier: ["pack-only-paid-model"], + }, + () => { + const pack = loadAlibabaFreeTierAllowlistPack(); + assert.ok(pack, "a pack inside its validity window must load"); + assert.ok(isAlibabaFreeTierAllowlistPackValid(pack!)); + assert.ok(pack!.capable.includes("qwen3.6-plus")); + // Positive anchor: the pack must actually REPLACE the embedded list, not + // merely load. `pack-only-capable-model` exists nowhere else. + assert.equal(isAlibabaBuiltinFreeTierTextModel("pack-only-capable-model"), true); + assert.equal(isAlibabaBuiltinNoFreeTierTextModel("pack-only-paid-model"), true); + } + ); +}); - if (previousPath) process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH = previousPath; - else delete process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH; - resetAlibabaFreeTierAllowlistCache(); +test("an expired allowlist pack is ignored and the embedded list serves instead", () => { + // This is the path production has actually been on since 2026-08-27, and it + // had no coverage at all — which is why the expiry surfaced as a red test + // rather than as a deliberate, understood fallback. + withAllowlistPack( + { + asOf: "2026-07-28", + validUntil: "2026-08-27", + capable: ["pack-only-capable-model"], + noFreeTier: ["pack-only-paid-model"], + }, + () => { + assert.equal(loadAlibabaFreeTierAllowlistPack(), null, "expired pack must not load"); + assert.equal(isAlibabaBuiltinFreeTierTextModel("pack-only-capable-model"), false); + // The embedded list must be what answers once the pack is rejected. + assert.equal(isAlibabaBuiltinFreeTierTextModel("qwen3.6-plus"), true); + assert.equal(isAlibabaBuiltinNoFreeTierTextModel("qwen3.7-max"), true); + } + ); +}); + +test("isAlibabaFreeTierAllowlistPackValid compares against the instant it is given", () => { + const pack = { asOf: "2026-07-28", validUntil: "2026-08-27", capable: ["x"], noFreeTier: [] }; + assert.equal(isAlibabaFreeTierAllowlistPackValid(pack, Date.parse("2026-08-26")), true); + assert.equal(isAlibabaFreeTierAllowlistPackValid(pack, Date.parse("2026-08-28")), false); + // No expiry declared means the pack never goes stale on its own. + assert.equal( + isAlibabaFreeTierAllowlistPackValid( + { asOf: "2026-07-28", capable: ["x"], noFreeTier: [] }, + Date.parse("2999-01-01") + ), + true + ); });