From bfbd090a96cb6f4e2d56e3dc4321607e6ce2e9c3 Mon Sep 17 00:00:00 2001 From: "Bob.Hou" Date: Thu, 10 Sep 2026 07:15:57 -0400 Subject: [PATCH] fix(codex): lift nested child cooldowns on parent clear and on snapshot headroom (#12951) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validado numa worktree combinada com a onda de persistência desta leva sobre `release/v3.8.51`: check-file-size e check-changelog-integrity OK, typecheck:core limpo, check-api-typecheck OK (289). Revalidei o head atual mergeado com o tip: typecheck:core limpo e **36/36** entre `db-rate-limit-guard` e as suítes desta PR. Levantar o cooldown do escopo pai sem deixar os filhos aninhados presos é o miolo — um cooldown órfão em filho é invisível no dashboard e mantém a conexão fora de rota sem explicação. **Nota de coordenação:** o `setConnectionRateLimitUntil` colidiu com o #12788 (guard contra timestamp não-finito ou já expirado), que mergeei nesta mesma onda. Eu tinha resolvido a integração na minha worktree, mas ao empurrar o push foi rejeitado — você já tinha empurrado `441fd44`, `f853ba5` e `af2ed01` com a integração feita, e a sua ordenação é equivalente à minha. Descartei a minha e mantive a sua; o crédito é seu inteiro. Fica o registro de que push rejeitado não é erro leve: se eu tivesse mergeado sem reler, teria levado a branch errada. --- .../fixes/12951-codex-scope-cooldown-clear.md | 1 + open-sse/services/codexAccount/quota.ts | 4 +- src/lib/db/providers.ts | 77 +-- src/lib/db/providers/codexAccountState.ts | 247 +++++++++- src/lib/db/providers/rateLimit.ts | 73 +++ src/lib/db/quotaSnapshots.ts | 60 +++ tests/unit/codex-reset-credits.test.ts | 153 ++++++ .../codex-scope-cooldown-clear-12817.test.ts | 449 ++++++++++++++++++ 8 files changed, 979 insertions(+), 85 deletions(-) create mode 100644 changelog.d/fixes/12951-codex-scope-cooldown-clear.md create mode 100644 tests/unit/codex-scope-cooldown-clear-12817.test.ts diff --git a/changelog.d/fixes/12951-codex-scope-cooldown-clear.md b/changelog.d/fixes/12951-codex-scope-cooldown-clear.md new file mode 100644 index 0000000000..edc1d4a64c --- /dev/null +++ b/changelog.d/fixes/12951-codex-scope-cooldown-clear.md @@ -0,0 +1 @@ +- **fix(codex):** dashboard "clear cooldown" and the CAS recovery path now drop nested `codexScopeRateLimitedUntil` maps in the same write that nulls `rate_limited_until`, and fresh quota snapshots with headroom lift fallback-sourced scope cooldowns parked by quota preflight ([#12817](https://github.com/diegosouzapw/OmniRoute/issues/12817), [#12860](https://github.com/diegosouzapw/OmniRoute/issues/12860), [#12951](https://github.com/diegosouzapw/OmniRoute/pull/12951)) The dashboard reset-credit button (`consumeCodexResetCredit`) now exercises that same snapshot path after a successful redeem, so a filled quota bar is enough to unpark a leftover fallback Codex child without another manual clear. diff --git a/open-sse/services/codexAccount/quota.ts b/open-sse/services/codexAccount/quota.ts index 3317ebd5be..80b15c8f0f 100644 --- a/open-sse/services/codexAccount/quota.ts +++ b/open-sse/services/codexAccount/quota.ts @@ -56,7 +56,9 @@ export async function persistCodexChildQuotaResponse(params: { rateLimitedUntil, rateLimitSource: exhaustedWindow ? ("quota_reset" as const) : ("fallback" as const), } - : {}), + : params.status === 200 + ? { rateLimitedUntil: null } + : {}), }); if (!providerSpecificData) return null; diff --git a/src/lib/db/providers.ts b/src/lib/db/providers.ts index 04c4de9b6f..1cb3f5a73a 100644 --- a/src/lib/db/providers.ts +++ b/src/lib/db/providers.ts @@ -39,6 +39,7 @@ import { pickCodexConnectionForUser } from "@/lib/oauth/utils/codexConnectionSel import { isMicrosoftDesignerWebRetiredProviderId } from "@/shared/constants/designerWebRetirement"; import { reconcileCodexUsageHistory } from "./providers/usageIdentityReconciliation"; import { isRuntimeRetiredProviderId } from "@/shared/constants/providerRetirement"; +import { applyCodexChildCooldownClearOnUpdate } from "./providers/codexAccountState"; /** * normalizeProviderSpecificData + the Codex fingerprint-seed invariant: Codex @@ -951,11 +952,14 @@ export async function updateProviderConnection(id: string, data: JsonRecord) { ...data, updatedAt: new Date().toISOString(), }; - merged.providerSpecificData = normalizeConnectionProviderSpecificData( - toStringOrNull(merged.provider), - merged.providerSpecificData, - merged, - existingCamel.providerSpecificData + merged.providerSpecificData = applyCodexChildCooldownClearOnUpdate( + data, + normalizeConnectionProviderSpecificData( + toStringOrNull(merged.provider), + merged.providerSpecificData, + merged, + existingCamel.providerSpecificData + ) ); // Mirror the sanitization the create path applies — keep the returned // object in lockstep with what we persist. @@ -1024,65 +1028,13 @@ export async function updateProviderConnection(id: string, data: JsonRecord) { export { updateCodexScopedQuotaState, updateCodexScopeCooldown, + applyCodexChildCooldownClearOnUpdate, + stripCodexChildCooldownFields, + stripCodexChildCooldownsFromConnection, + hasCodexScopeCooldown, + liftCodexScopeCooldownOnHeadroom, } from "./providers/codexAccountState"; -/** - * Atomic conditional clear of recoverable error state on a connection row. - * - * Returns true when the row was cleared, false when a concurrent writer - * (markAccountUnavailable, connectionRecovery tick, test, etc.) changed the - * row between the caller's snapshot read and this UPDATE — in which case the - * clear is skipped to preserve the freshest error state. Closes the TOCTOU - * window in the quota-recovery path. - * - * CAS token = (test_status, last_error_at, rate_limited_until). - * markAccountUnavailable always bumps last_error_at on every cooldown/error - * write, so an unchanged last_error_at reliably indicates no concurrent write. - */ -export async function clearConnectionErrorIfUnchanged( - id: string, - expected: { - testStatus: string | null | undefined; - lastErrorAt: string | null | undefined; - rateLimitedUntil: string | null | undefined; - } -): Promise { - const db = getDbInstance() as unknown as DbLike; - const result = db - .prepare( - ` - UPDATE provider_connections SET - test_status = 'active', - last_error = NULL, - last_error_at = NULL, - last_error_type = NULL, - last_error_source = NULL, - error_code = NULL, - rate_limited_until = NULL, - backoff_level = 0, - updated_at = ? - WHERE id = ? - AND IFNULL(test_status, '') = ? - AND IFNULL(last_error_at, '') = ? - AND IFNULL(rate_limited_until, '') = ? - ` - ) - .run( - new Date().toISOString(), - id, - expected.testStatus ?? "", - expected.lastErrorAt ?? "", - expected.rateLimitedUntil ?? "" - ); - const applied = (result.changes ?? 0) > 0; - if (applied) { - backupDbFile("pre-write"); - invalidateDbCache("connections"); - bumpProxyConfigGeneration(); - } - return applied; -} - /** * Lightweight stat bump — updates lastUsedAt and consecutiveUseCount without * SELECT, re-encrypt, cache invalidation, or file backup. @@ -1183,4 +1135,5 @@ export { formatResetCountdown, isConnectionRateLimited, getRateLimitedConnections, + clearConnectionErrorIfUnchanged, } from "./providers/rateLimit"; diff --git a/src/lib/db/providers/codexAccountState.ts b/src/lib/db/providers/codexAccountState.ts index 77600de3bd..ab3c0fc57b 100644 --- a/src/lib/db/providers/codexAccountState.ts +++ b/src/lib/db/providers/codexAccountState.ts @@ -18,10 +18,105 @@ interface DbLike { type CodexScopedQuotaPatch = { quotaState?: JsonRecord; exhaustedWindow?: "5h" | "7d" | null; - rateLimitedUntil?: string; + rateLimitedUntil?: string | null; rateLimitSource?: "fallback" | "quota_reset"; }; +const CODEX_CHILD_COOLDOWN_KEYS = [ + "codexScopeRateLimitedUntil", + "codexScopeRateLimitSource", +] as const; + +function omitEmptyRecord(record: JsonRecord): JsonRecord | undefined { + return Object.keys(record).length > 0 ? record : undefined; +} + +/** Drop nested Codex child cooldowns; keep quota snapshots and unrelated keys. */ +export function stripCodexChildCooldownFields(psd: JsonRecord): JsonRecord { + if (!connectionHasCodexChildCooldown(psd)) return psd; + const next = { ...psd }; + for (const key of CODEX_CHILD_COOLDOWN_KEYS) delete next[key]; + return next; +} + +/** PUT/CAS payload that clears the parent column must also drop nested maps. */ +export function applyCodexChildCooldownClearOnUpdate( + data: JsonRecord, + psd: T +): T { + if (psd == null) return psd; + if (!Object.hasOwn(data, "rateLimitedUntil")) return psd; + if (data.rateLimitedUntil != null && data.rateLimitedUntil !== "") return psd; + return stripCodexChildCooldownFields(psd) as T; +} + +function connectionHasCodexChildCooldown(psd: JsonRecord): boolean { + return "codexScopeRateLimitedUntil" in psd || "codexScopeRateLimitSource" in psd; +} + +/** + * Persist a full-parent cooldown lift into the nested Codex child maps. + * When `alsoClearTopLevel` is set, the parent `rate_limited_until` column is + * nulled in the same transaction so a crash between the two writes cannot + * leave a nested child map behind a cleared parent column. + */ +export function stripCodexChildCooldownsFromConnection( + id: string, + options?: { alsoClearTopLevel?: boolean } +): void { + if (typeof id !== "string" || id.length === 0) return; + const db = getDbInstance() as unknown as DbLike; + const alsoClearTopLevel = options?.alsoClearTopLevel === true; + const candidate = db + .prepare("SELECT provider FROM provider_connections WHERE id = ?") + .get(id); + const isCodex = toRecord(candidate).provider === "codex"; + if (!alsoClearTopLevel && !isCodex) return; + + backupDbFile("pre-write"); + const wrote = db.transaction(() => { + const existing = db + .prepare( + "SELECT provider, provider_specific_data FROM provider_connections WHERE id = ?" + ) + .get(id); + if (!existing) return false; + const existingRecord = toRecord(rowToCamel(existing)); + const providerSpecificData = toRecord(existingRecord.providerSpecificData); + const stripNested = + existingRecord.provider === "codex" && + connectionHasCodexChildCooldown(providerSpecificData); + if (!alsoClearTopLevel && !stripNested) return false; + + const now = new Date().toISOString(); + if (alsoClearTopLevel && stripNested) { + db.prepare( + `UPDATE provider_connections + SET rate_limited_until = NULL, + provider_specific_data = ?, + updated_at = ? + WHERE id = ?` + ).run(JSON.stringify(stripCodexChildCooldownFields(providerSpecificData)), now, id); + return true; + } + if (alsoClearTopLevel) { + db.prepare( + `UPDATE provider_connections + SET rate_limited_until = NULL, updated_at = ? + WHERE id = ?` + ).run(now, id); + return true; + } + db.prepare( + `UPDATE provider_connections + SET provider_specific_data = ?, updated_at = ? + WHERE id = ?` + ).run(JSON.stringify(stripCodexChildCooldownFields(providerSpecificData)), now, id); + return true; + })(); + if (wrote) invalidateDbCache("connections"); +} + /** * Atomically merge one virtual Codex child's quota evidence into its persisted parent. * The transaction reads the latest row so sibling child state cannot be lost. @@ -70,28 +165,35 @@ export async function updateCodexScopedQuotaState( } } - if (patch.rateLimitedUntil) { - const scopeCooldowns = toRecord(providerSpecificData.codexScopeRateLimitedUntil); - const sourceByScope = toRecord(providerSpecificData.codexScopeRateLimitSource); - const existingCooldownMs = - typeof scopeCooldowns[scope] === "string" - ? new Date(scopeCooldowns[scope] as string).getTime() - : NaN; - const existingIsAuthoritative = - sourceByScope[scope] === "quota_reset" && - patch.rateLimitSource !== "quota_reset" && - Number.isFinite(existingCooldownMs) && - existingCooldownMs > Date.now(); - nextProviderSpecificData.codexScopeRateLimitedUntil = { - ...scopeCooldowns, - [scope]: existingIsAuthoritative ? scopeCooldowns[scope] : patch.rateLimitedUntil, - }; - nextProviderSpecificData.codexScopeRateLimitSource = { - ...sourceByScope, - [scope]: existingIsAuthoritative + if (patch.rateLimitedUntil !== undefined) { + const scopeCooldowns = { ...toRecord(providerSpecificData.codexScopeRateLimitedUntil) }; + const sourceByScope = { ...toRecord(providerSpecificData.codexScopeRateLimitSource) }; + if (patch.rateLimitedUntil) { + const existingCooldownMs = + typeof scopeCooldowns[scope] === "string" + ? new Date(scopeCooldowns[scope] as string).getTime() + : NaN; + const existingIsAuthoritative = + sourceByScope[scope] === "quota_reset" && + patch.rateLimitSource !== "quota_reset" && + Number.isFinite(existingCooldownMs) && + existingCooldownMs > Date.now(); + scopeCooldowns[scope] = existingIsAuthoritative + ? scopeCooldowns[scope] + : patch.rateLimitedUntil; + sourceByScope[scope] = existingIsAuthoritative ? sourceByScope[scope] - : (patch.rateLimitSource ?? "fallback"), - }; + : (patch.rateLimitSource ?? "fallback"); + } else { + delete scopeCooldowns[scope]; + delete sourceByScope[scope]; + } + const nextCooldowns = omitEmptyRecord(scopeCooldowns); + const nextSources = omitEmptyRecord(sourceByScope); + if (nextCooldowns) nextProviderSpecificData.codexScopeRateLimitedUntil = nextCooldowns; + else delete nextProviderSpecificData.codexScopeRateLimitedUntil; + if (nextSources) nextProviderSpecificData.codexScopeRateLimitSource = nextSources; + else delete nextProviderSpecificData.codexScopeRateLimitSource; } db.prepare( @@ -106,6 +208,107 @@ export async function updateCodexScopedQuotaState( return persisted; } +/** Grace window absorbing clock skew against the upstream quota server. */ +const QUOTA_RESET_CLOCK_SKEW_GRACE_MS = 30_000; + +/** Cheap probe: does this connection+scope currently carry a child cooldown? */ +export function hasCodexScopeCooldown(id: string, scope: "codex" | "spark"): boolean { + if (typeof id !== "string" || id.length === 0) return false; + const db = getDbInstance() as unknown as DbLike; + const row = db + .prepare("SELECT provider, provider_specific_data FROM provider_connections WHERE id = ?") + .get(id); + if (!row) return false; + const record = toRecord(rowToCamel(row)); + if (record.provider !== "codex") return false; + const psd = toRecord(record.providerSpecificData); + return Boolean(toRecord(psd.codexScopeRateLimitedUntil)[scope]); +} + +/** + * #12860: When fresh quota snapshot data demonstrates headroom on a scope, + * lift any fallback-sourced cooldown (e.g. parked by quota preflight). + * Cooldowns sourced from upstream 429 quota_reset retain their authority + * until their reset timestamp has elapsed. + */ +export function liftCodexScopeCooldownOnHeadroom( + id: string, + scope: "codex" | "spark" +): boolean { + if (typeof id !== "string" || id.length === 0) return false; + const db = getDbInstance() as unknown as DbLike; + + // Cheap eligibility probe so the backup only runs when a write is plausible. + // The transaction below re-reads under lock and remains the authority. + if (!hasCodexScopeCooldown(id, scope)) return false; + + backupDbFile("pre-write"); + const wrote = db.transaction(() => { + const existing = db + .prepare( + "SELECT provider, provider_specific_data FROM provider_connections WHERE id = ?" + ) + .get(id); + if (!existing) return false; + const existingRecord = toRecord(rowToCamel(existing)); + if (existingRecord.provider !== "codex") return false; + const currentPsd = toRecord(existingRecord.providerSpecificData); + const currentCooldowns = { ...toRecord(currentPsd.codexScopeRateLimitedUntil) }; + if (!currentCooldowns[scope]) return false; + + const currentSources = { ...toRecord(currentPsd.codexScopeRateLimitSource) }; + const curSource = currentSources[scope]; + const curUntilMs = + typeof currentCooldowns[scope] === "string" + ? new Date(currentCooldowns[scope] as string).getTime() + : NaN; + // An upstream-authoritative `quota_reset` deadline outranks a local snapshot: + // the grace window absorbs clock skew against the quota server, so a fast + // local clock cannot lift a cooldown upstream still considers active. + if ( + curSource === "quota_reset" && + Number.isFinite(curUntilMs) && + curUntilMs > Date.now() - QUOTA_RESET_CLOCK_SKEW_GRACE_MS + ) { + return false; + } + + delete currentCooldowns[scope]; + delete currentSources[scope]; + + const nextPsd: JsonRecord = { ...currentPsd }; + const nextCooldowns = omitEmptyRecord(currentCooldowns); + const nextSources = omitEmptyRecord(currentSources); + if (nextCooldowns) nextPsd.codexScopeRateLimitedUntil = nextCooldowns; + else delete nextPsd.codexScopeRateLimitedUntil; + if (nextSources) nextPsd.codexScopeRateLimitSource = nextSources; + else delete nextPsd.codexScopeRateLimitSource; + + const exhaustedByScope = { ...toRecord(currentPsd.codexExhaustedWindowByScope) }; + if (exhaustedByScope[scope]) { + delete exhaustedByScope[scope]; + const nextExhausted = omitEmptyRecord(exhaustedByScope); + if (nextExhausted) { + nextPsd.codexExhaustedWindowByScope = nextExhausted; + } else { + delete nextPsd.codexExhaustedWindowByScope; + delete nextPsd.codexExhaustedWindow; + } + } + + const now = new Date().toISOString(); + db.prepare( + `UPDATE provider_connections + SET provider_specific_data = ?, updated_at = ? + WHERE id = ?` + ).run(JSON.stringify(nextPsd), now, id); + return true; + })(); + + if (wrote) invalidateDbCache("connections"); + return wrote; +} + /** Persist one child cooldown through the shared scoped quota-state transaction. */ export async function updateCodexScopeCooldown( id: string, diff --git a/src/lib/db/providers/rateLimit.ts b/src/lib/db/providers/rateLimit.ts index e89199c3b6..c15dd3b2ee 100644 --- a/src/lib/db/providers/rateLimit.ts +++ b/src/lib/db/providers/rateLimit.ts @@ -4,6 +4,9 @@ import { getDbInstance } from "../core"; import { invalidateDbCache } from "../readCache"; +import { backupDbFile } from "../backup"; +import { bumpProxyConfigGeneration } from "../settings"; +import { stripCodexChildCooldownsFromConnection } from "./codexAccountState"; interface StatementLike { all: (...params: unknown[]) => TRow[]; @@ -33,6 +36,10 @@ export function setConnectionRateLimitUntil(connectionId: string, until: number // is the only clear path (via clearConnectionRateLimit); past/zero // timestamps are noops so an expired write cannot overwrite a live row. if (until !== null && (!Number.isFinite(until) || until <= Date.now())) return; + if (until == null) { + stripCodexChildCooldownsFromConnection(connectionId, { alsoClearTopLevel: true }); + return; + } const db = getDbInstance() as unknown as DbLike; db.prepare( "UPDATE provider_connections SET rate_limited_until = ?, updated_at = ? WHERE id = ?" @@ -234,6 +241,72 @@ export function clearStaleCrashCooldowns(): { cleared: number } { return { cleared: toReset.length }; } +/** + * Atomic conditional clear of recoverable error state on a connection row. + * + * Returns true when the row was cleared, false when a concurrent writer + * (markAccountUnavailable, connectionRecovery tick, test, etc.) changed the + * row between the caller's snapshot read and this UPDATE — in which case the + * clear is skipped to preserve the freshest error state. Closes the TOCTOU + * window in the quota-recovery path. + * + * CAS token = (test_status, last_error_at, rate_limited_until). + * Nested Codex child cooldown maps are stripped in the same UPDATE so a + * concurrent writer cannot re-persist them between two statements. + */ +export async function clearConnectionErrorIfUnchanged( + id: string, + expected: { + testStatus: string | null | undefined; + lastErrorAt: string | null | undefined; + rateLimitedUntil: string | null | undefined; + } +): Promise { + const db = getDbInstance() as unknown as DbLike; + backupDbFile("pre-write"); + const result = db + .prepare( + ` + UPDATE provider_connections SET + test_status = 'active', + last_error = NULL, + last_error_at = NULL, + last_error_type = NULL, + last_error_source = NULL, + error_code = NULL, + rate_limited_until = NULL, + backoff_level = 0, + provider_specific_data = CASE + WHEN provider = 'codex' AND json_valid(provider_specific_data) + THEN json_remove( + provider_specific_data, + '$.codexScopeRateLimitedUntil', + '$.codexScopeRateLimitSource' + ) + ELSE provider_specific_data + END, + updated_at = ? + WHERE id = ? + AND IFNULL(test_status, '') = ? + AND IFNULL(last_error_at, '') = ? + AND IFNULL(rate_limited_until, '') = ? + ` + ) + .run( + new Date().toISOString(), + id, + expected.testStatus ?? "", + expected.lastErrorAt ?? "", + expected.rateLimitedUntil ?? "" + ); + const applied = (result.changes ?? 0) > 0; + if (applied) { + invalidateDbCache("connections"); + bumpProxyConfigGeneration(); + } + return applied; +} + // T13: Format a reset countdown as a human-readable string ("2h 35m" / "4m 30s"). // The implementation lives in the client-safe formatting utils so client // components (e.g. CoolingConnectionsPanel) can import it without pulling this diff --git a/src/lib/db/quotaSnapshots.ts b/src/lib/db/quotaSnapshots.ts index 1e77cc47fb..5a2391130e 100644 --- a/src/lib/db/quotaSnapshots.ts +++ b/src/lib/db/quotaSnapshots.ts @@ -1,5 +1,10 @@ import { getDbInstance, rowToCamel } from "./core"; import type { QuotaSnapshotRow, ProviderUtilizationPoint } from "@/shared/types/utilization"; +import { + hasCodexScopeCooldown, + liftCodexScopeCooldownOnHeadroom, +} from "./providers/codexAccountState"; +import { isCodexSparkQuotaKey } from "@omniroute/open-sse/config/codexQuotaScopes"; type JsonRecord = Record; @@ -45,6 +50,61 @@ export function saveQuotaSnapshot(snapshot: Omit & { + windowKey?: string; + remainingPercentage?: number; + isExhausted?: number; +}; + +function snapshotHasHeadroom(s: SnapshotShape): boolean { + const pct = s.remainingPercentage ?? s.remaining_percentage ?? 0; + const exhausted = s.isExhausted ?? s.is_exhausted ?? 0; + return pct > 0 && exhausted !== 1; +} + +function maybeLiftCodexCooldownOnHeadroom( + snapshot: Omit +): void { + if ( + snapshot.provider?.toLowerCase() !== "codex" || + typeof snapshot.connection_id !== "string" || + snapshot.connection_id.length === 0 || + (snapshot.remaining_percentage ?? 0) <= 0 || + snapshot.is_exhausted === 1 + ) { + return; + } + + try { + const scope = isCodexSparkQuotaKey(snapshot.window_key) ? "spark" : "codex"; + // The scope-wide read is only worth paying for when a cooldown is + // actually parked on this connection+scope; the common case is clean. + if (!hasCodexScopeCooldown(snapshot.connection_id, scope)) return; + + const scopeWindows = getLatestQuotaSnapshotsForConnection(snapshot.connection_id).filter( + (s: SnapshotShape) => { + const key = s.windowKey ?? s.window_key; + return scope === "spark" ? isCodexSparkQuotaKey(key) : !isCodexSparkQuotaKey(key); + } + ); + // Every window in the scope must be healthy: one exhausted window still + // justifies the cooldown even when a sibling reports full headroom. + if (scopeWindows.length > 0 && scopeWindows.every(snapshotHasHeadroom)) { + liftCodexScopeCooldownOnHeadroom(snapshot.connection_id, scope); + } + } catch (err) { + console.debug("[QuotaSnapshots] Headroom evaluation skipped:", err); + } } export function getQuotaSnapshots(opts: { diff --git a/tests/unit/codex-reset-credits.test.ts b/tests/unit/codex-reset-credits.test.ts index 2a6ecccee7..b1278a5d11 100644 --- a/tests/unit/codex-reset-credits.test.ts +++ b/tests/unit/codex-reset-credits.test.ts @@ -11,6 +11,7 @@ process.env.API_KEY_SECRET = "test-codex-reset-credits-secret"; const core = await import("../../src/lib/db/core.ts"); const providersDb = await import("../../src/lib/db/providers.ts"); const resetCredits = await import("../../src/lib/usage/codexResetCredits.ts"); +const codexAccount = await import("../../open-sse/services/codexAccount/index.ts"); const originalFetch = globalThis.fetch; type QuotaUsageRecord = Record; @@ -45,6 +46,69 @@ test.after(async () => { fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); }); +async function persistBothChildCooldowns(id: string) { + const codexUntil = new Date(Date.now() + 60_000).toISOString(); + const sparkUntil = new Date(Date.now() + 120_000).toISOString(); + await codexAccount.persistCodexChildCooldown({ + connectionId: id, + model: "gpt-5.5", + rateLimitedUntil: codexUntil, + }); + await codexAccount.persistCodexChildCooldown({ + connectionId: id, + model: "gpt-5.3-codex-spark", + rateLimitedUntil: sparkUntil, + }); + return { codexUntil, sparkUntil }; +} + +async function readConnection(id: string) { + const connection = await providersDb.getProviderConnectionById(id); + assert.ok(connection); + return connection as Record; +} + +function mockResetThenUsage(opts: { + consumeBody: unknown; + consumeStatus?: number; + usageBody?: unknown; + usageStatus?: number; +}) { + globalThis.fetch = (async (url) => { + const href = String(url); + if (href.endsWith("/rate-limit-reset-credits")) { + return new Response( + JSON.stringify({ credits: [{ id: "credit-123", status: "available" }] }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + if (href.includes("/rate-limit-reset-credits/consume")) { + return new Response(JSON.stringify(opts.consumeBody), { + status: opts.consumeStatus ?? 200, + headers: { "content-type": "application/json" }, + }); + } + if (href.includes("/backend-api/wham/usage")) { + if (opts.usageStatus && opts.usageStatus >= 500) { + return new Response("upstream down", { status: opts.usageStatus }); + } + return new Response( + JSON.stringify( + opts.usageBody ?? { + plan_type: "plus", + rate_limit: { + primary_window: { used_percent: 0 }, + secondary_window: { used_percent: 0 }, + }, + } + ), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + return new Response("unexpected", { status: 500 }); + }) as typeof fetch; +} + test("consumeCodexResetCredit fetches a credit id, posts it, then refreshes usage", async () => { const connection = (await createCodexConnection()) as { id: string }; const calls: Array<{ url: string; init: RequestInit }> = []; @@ -355,3 +419,92 @@ test("consumeCodexResetCredit rejects non-Codex and missing connections", async error.code === "codex_provider_required" ); }); + +test("#12951 consumeCodexResetCredit lifts fallback Codex child cooldown after healthy usage refresh", async () => { + const connection = (await createCodexConnection()) as { id: string }; + const parked = await persistBothChildCooldowns(connection.id); + const before = await readConnection(connection.id); + assert.equal(codexAccount.getCodexChildCooldown(before as never, "gpt-5.5"), parked.codexUntil); + assert.equal( + codexAccount.getCodexChildCooldown(before as never, "gpt-5.3-codex-spark"), + parked.sparkUntil + ); + + mockResetThenUsage({ consumeBody: { code: "reset" } }); + + const result = await resetCredits.consumeCodexResetCredit(connection.id, "redeem-button-path"); + assert.equal(result.outcome, "reset"); + + const after = await readConnection(connection.id); + assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), null); + assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false); + assert.equal( + codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"), + parked.sparkUntil + ); +}); + +test("#12951 consumeCodexResetCredit alreadyRedeemed still needs healthy usage to lift fallback cooldown", async () => { + const connection = (await createCodexConnection()) as { id: string }; + await persistBothChildCooldowns(connection.id); + + mockResetThenUsage({ + consumeBody: { code: "alreadyRedeemed" }, + usageBody: { + plan_type: "plus", + rate_limit: { + primary_window: { used_percent: 100 }, + secondary_window: { used_percent: 100 }, + }, + }, + }); + + const result = await resetCredits.consumeCodexResetCredit(connection.id, "redeem-already"); + assert.equal(result.outcome, "alreadyRedeemed"); + + const after = await readConnection(connection.id); + assert.ok(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5")); + assert.ok(codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark")); +}); + +test("#12951 consumeCodexResetCredit failed redeem keeps nested child cooldowns", async () => { + const connection = (await createCodexConnection()) as { id: string }; + const parked = await persistBothChildCooldowns(connection.id); + + mockResetThenUsage({ + consumeBody: { code: "noCredit" }, + consumeStatus: 409, + }); + + await assert.rejects( + () => resetCredits.consumeCodexResetCredit(connection.id, "redeem-fail"), + (error: unknown) => + error instanceof resetCredits.CodexResetCreditError && + error.status === 409 && + error.code === "no_credit" + ); + + const after = await readConnection(connection.id); + assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), parked.codexUntil); + assert.equal( + codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"), + parked.sparkUntil + ); +}); + +test("#12951 consumeCodexResetCredit incomplete usage refresh keeps nested child cooldowns", async () => { + const connection = (await createCodexConnection()) as { id: string }; + const parked = await persistBothChildCooldowns(connection.id); + + mockResetThenUsage({ consumeBody: { code: "reset" }, usageStatus: 500 }); + + const result = await resetCredits.consumeCodexResetCredit(connection.id, "redeem-incomplete"); + assert.equal(result.outcome, "reset"); + + const after = await readConnection(connection.id); + assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), parked.codexUntil); + assert.equal( + codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"), + parked.sparkUntil + ); +}); diff --git a/tests/unit/codex-scope-cooldown-clear-12817.test.ts b/tests/unit/codex-scope-cooldown-clear-12817.test.ts new file mode 100644 index 0000000000..10a538cb98 --- /dev/null +++ b/tests/unit/codex-scope-cooldown-clear-12817.test.ts @@ -0,0 +1,449 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-codex-clear-12817-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "codex-clear-12817-secret"; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const rateLimit = await import("../../src/lib/db/providers/rateLimit.ts"); +const codexAccount = await import("../../open-sse/services/codexAccount/index.ts"); +const quotaSnapshots = await import("../../src/lib/db/quotaSnapshots.ts"); +const quotaCache = await import("../../src/domain/quotaCache.ts"); + +async function resetStorage(): Promise { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +interface SeededConnection { + id: string; + providerSpecificData: Record; +} + +async function seedCodexConnection(): Promise { + return providersDb.createProviderConnection({ + provider: "codex", + authType: "oauth", + name: "codex-clear-12817", + email: "codex-clear-12817@example.com", + apiKey: "codex-clear-12817-key", + accessToken: "codex-clear-12817-access", + refreshToken: "codex-clear-12817-refresh", + providerSpecificData: { + unrelated: { retained: true }, + }, + }) as unknown as Promise; +} + +async function seedGlmConnection(): Promise { + return providersDb.createProviderConnection({ + provider: "glm", + authType: "apikey", + name: "glm-clear-12817", + apiKey: "glm-clear-12817-key", + providerSpecificData: { + leftover: "keep-me", + }, + }) as unknown as Promise; +} + +async function readConnection(id: string): Promise> { + const connection = await providersDb.getProviderConnectionById(id); + assert.ok(connection); + return connection as unknown as Record; +} + +function psd(connection: Record): Record { + return (connection.providerSpecificData ?? {}) as Record; +} + +function futureIso(ms: number): string { + return new Date(Date.now() + ms).toISOString(); +} + +function quotaHeaders(resetAt5h: string, resetAt7d: string, usage5h = "10") { + return { + "x-codex-5h-usage": usage5h, + "x-codex-5h-limit": "100", + "x-codex-5h-reset-at": resetAt5h, + "x-codex-7d-usage": "10", + "x-codex-7d-limit": "100", + "x-codex-7d-reset-at": resetAt7d, + }; +} + +async function persistBothChildCooldowns(id: string): Promise<{ + codexUntil: string; + sparkUntil: string; +}> { + const codexUntil = futureIso(60_000); + const sparkUntil = futureIso(120_000); + await codexAccount.persistCodexChildCooldown({ + connectionId: id, + model: "gpt-5.5", + rateLimitedUntil: codexUntil, + }); + await codexAccount.persistCodexChildCooldown({ + connectionId: id, + model: "gpt-5.3-codex-spark", + rateLimitedUntil: sparkUntil, + }); + return { codexUntil, sparkUntil }; +} + +test.beforeEach(resetStorage); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("#12817 PUT rateLimitedUntil:null also drops nested Codex child cooldowns", async () => { + const connection = await seedCodexConnection(); + const { sparkUntil } = await persistBothChildCooldowns(connection.id); + await providersDb.updateProviderConnection(connection.id, { + rateLimitedUntil: futureIso(90_000), + }); + + const before = await readConnection(connection.id); + assert.equal( + codexAccount.getCodexChildCooldown(before as never, "gpt-5.3-codex-spark"), + sparkUntil + ); + + await providersDb.updateProviderConnection(connection.id, { rateLimitedUntil: null }); + + const after = await readConnection(connection.id); + const data = psd(after); + assert.equal(after.rateLimitedUntil, undefined); + assert.equal(data.codexScopeRateLimitedUntil, undefined); + assert.equal(data.codexScopeRateLimitSource, undefined); + assert.deepEqual(data.unrelated, { retained: true }); + assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), null); + assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"), null); + assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false); +}); + +test("#12817 PUT rateLimitedUntil:\"\" also drops nested Codex child cooldowns", async () => { + const connection = await seedCodexConnection(); + await persistBothChildCooldowns(connection.id); + await providersDb.updateProviderConnection(connection.id, { + rateLimitedUntil: futureIso(90_000), + }); + + await providersDb.updateProviderConnection(connection.id, { rateLimitedUntil: "" }); + + const after = await readConnection(connection.id); + const data = psd(after); + assert.equal(after.rateLimitedUntil, undefined); + assert.equal(data.codexScopeRateLimitedUntil, undefined); + assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false); +}); + +test("#12817 clearConnectionRateLimit strips nested Codex child cooldowns", async () => { + const connection = await seedCodexConnection(); + await persistBothChildCooldowns(connection.id); + rateLimit.setConnectionRateLimitUntil(connection.id, Date.now() + 90_000); + + rateLimit.clearConnectionRateLimit(connection.id); + + const after = await readConnection(connection.id); + const data = psd(after); + assert.equal(after.rateLimitedUntil, undefined); + assert.equal(data.codexScopeRateLimitedUntil, undefined); + assert.equal(data.codexScopeRateLimitSource, undefined); + assert.deepEqual(data.unrelated, { retained: true }); + assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false); +}); + +test("#12817 CAS error-clear also strips nested Codex child cooldowns", async () => { + const connection = await seedCodexConnection(); + await persistBothChildCooldowns(connection.id); + const until = futureIso(90_000); + await providersDb.updateProviderConnection(connection.id, { + testStatus: "unavailable", + lastError: "429", + lastErrorAt: new Date().toISOString(), + lastErrorType: "rate_limit_exceeded", + rateLimitedUntil: until, + }); + const before = await readConnection(connection.id); + + const applied = await providersDb.clearConnectionErrorIfUnchanged(connection.id, { + testStatus: (before.testStatus as string) ?? null, + lastErrorAt: (before.lastErrorAt as string) ?? null, + rateLimitedUntil: (before.rateLimitedUntil as string) ?? null, + }); + assert.equal(applied, true); + + const after = await readConnection(connection.id); + assert.equal(after.testStatus, "active"); + assert.equal(psd(after).codexScopeRateLimitedUntil, undefined); + assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false); +}); + +test("#12817 a successful quota observation clears that child's leftover cooldown", async () => { + const connection = await seedCodexConnection(); + const reset5h = futureIso(60_000); + const reset7d = futureIso(600_000); + + await persistBothChildCooldowns(connection.id); + await codexAccount.persistCodexChildQuotaResponse({ + connectionId: connection.id, + model: "gpt-5.5", + headers: quotaHeaders(reset5h, reset7d, "95"), + status: 429, + }); + await codexAccount.persistCodexChildQuotaResponse({ + connectionId: connection.id, + model: "gpt-5.5", + headers: quotaHeaders(reset5h, reset7d, "10"), + status: 200, + }); + + const after = await readConnection(connection.id); + const data = psd(after); + const until = data.codexScopeRateLimitedUntil as Record | undefined; + const exhausted = data.codexExhaustedWindowByScope as Record | undefined; + assert.equal(until?.codex, undefined); + assert.equal(typeof until?.spark, "string"); + assert.equal(exhausted?.codex, undefined); + assert.deepEqual(data.unrelated, { retained: true }); + assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false); + assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.3-codex-spark"), true); +}); + +test("#12817 clearing a non-Codex cooldown leaves providerSpecificData alone", async () => { + const connection = await seedGlmConnection(); + await providersDb.updateProviderConnection(connection.id, { + rateLimitedUntil: futureIso(90_000), + }); + await providersDb.updateProviderConnection(connection.id, { rateLimitedUntil: null }); + const after = await readConnection(connection.id); + assert.equal(after.rateLimitedUntil, undefined); + assert.equal(psd(after).leftover, "keep-me"); +}); + +test("#12860 saveQuotaSnapshot with headroom lifts fallback-sourced scope cooldown", async () => { + const connection = await seedCodexConnection(); + await persistBothChildCooldowns(connection.id); + + const before = await readConnection(connection.id); + assert.ok(codexAccount.getCodexChildCooldown(before as never, "gpt-5.5")); + + quotaSnapshots.saveQuotaSnapshot({ + provider: "codex", + connection_id: connection.id, + window_key: "primary", + remaining_percentage: 100, + is_exhausted: 0, + next_reset_at: futureIso(3600_000), + window_duration_ms: 18_000_000, + raw_data: null, + }); + + const after = await readConnection(connection.id); + const data = psd(after); + const until = data.codexScopeRateLimitedUntil as Record | undefined; + assert.equal(until?.codex, undefined); + assert.equal(typeof until?.spark, "string"); + assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), null); + assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false); +}); + +test("#12860 saveQuotaSnapshot with headroom does NOT lift authoritative quota_reset cooldown", async () => { + const connection = await seedCodexConnection(); + const reset5h = futureIso(120_000); + const reset7d = futureIso(600_000); + + await codexAccount.persistCodexChildQuotaResponse({ + connectionId: connection.id, + model: "gpt-5.5", + headers: quotaHeaders(reset5h, reset7d, "95"), + status: 429, + }); + + const before = await readConnection(connection.id); + assert.ok(codexAccount.getCodexChildCooldown(before as never, "gpt-5.5")); + + quotaSnapshots.saveQuotaSnapshot({ + provider: "codex", + connection_id: connection.id, + window_key: "session", + remaining_percentage: 100, + is_exhausted: 0, + next_reset_at: futureIso(3600_000), + window_duration_ms: 18_000_000, + raw_data: null, + }); + + const after = await readConnection(connection.id); + const data = psd(after); + const until = data.codexScopeRateLimitedUntil as Record | undefined; + assert.equal(typeof until?.codex, "string"); + assert.ok(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5")); +}); + +test("#12860 saveQuotaSnapshot does NOT lift scope cooldown if another window for that scope is still exhausted", async () => { + const connection = await seedCodexConnection(); + await codexAccount.persistCodexChildCooldown({ + connectionId: connection.id, + model: "gpt-5.5", + rateLimitedUntil: futureIso(120_000), + }); + + // Weekly window is currently exhausted (0% remaining) + quotaSnapshots.saveQuotaSnapshot({ + provider: "codex", + connection_id: connection.id, + window_key: "weekly", + remaining_percentage: 0, + is_exhausted: 1, + next_reset_at: futureIso(7200_000), + window_duration_ms: 604_800_000, + raw_data: null, + }); + + // Session window reports 100% headroom, but weekly is still exhausted + quotaSnapshots.saveQuotaSnapshot({ + provider: "codex", + connection_id: connection.id, + window_key: "session", + remaining_percentage: 100, + is_exhausted: 0, + next_reset_at: futureIso(3600_000), + window_duration_ms: 18_000_000, + raw_data: null, + }); + + let mid = await readConnection(connection.id); + assert.ok(codexAccount.getCodexChildCooldown(mid as never, "gpt-5.5")); + + // Now weekly also recovers to 100% headroom + quotaSnapshots.saveQuotaSnapshot({ + provider: "codex", + connection_id: connection.id, + window_key: "weekly", + remaining_percentage: 100, + is_exhausted: 0, + next_reset_at: futureIso(7200_000), + window_duration_ms: 604_800_000, + raw_data: null, + }); + + let after = await readConnection(connection.id); + assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), null); + assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false); +}); + +test("#12860 saveQuotaSnapshot for spark scope with headroom lifts only spark cooldown", async () => { + const connection = await seedCodexConnection(); + await persistBothChildCooldowns(connection.id); + + quotaSnapshots.saveQuotaSnapshot({ + provider: "codex", + connection_id: connection.id, + window_key: "gpt_5_3_codex_spark_session", + remaining_percentage: 100, + is_exhausted: 0, + next_reset_at: futureIso(3600_000), + window_duration_ms: 18_000_000, + raw_data: null, + }); + + const after = await readConnection(connection.id); + const data = psd(after); + const until = data.codexScopeRateLimitedUntil as Record | undefined; + assert.equal(typeof until?.codex, "string"); + assert.equal(until?.spark, undefined); + assert.ok(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5")); + assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"), null); +}); + +test("#12860 setQuotaCache with fresh usage headroom lifts fallback-sourced scope cooldown", async () => { + const connection = await seedCodexConnection(); + await persistBothChildCooldowns(connection.id); + + const before = await readConnection(connection.id); + assert.ok(codexAccount.getCodexChildCooldown(before as never, "gpt-5.5")); + + quotaCache.setQuotaCache(connection.id, "codex", { + session: { + used: 0, + total: 100, + remainingPercentage: 100, + resetAt: futureIso(3600_000), + }, + }); + + const after = await readConnection(connection.id); + const data = psd(after); + const until = data.codexScopeRateLimitedUntil as Record | undefined; + assert.equal(until?.codex, undefined); + assert.equal(typeof until?.spark, "string"); + assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), null); + assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false); +}); + +test("#12860 quota_reset cooldown that just elapsed is still held by the skew grace window", async () => { + const connection = await seedCodexConnection(); + // Reset deadline sits 5s in the past — inside the 30s clock-skew grace, so a + // fast local clock must not lift an upstream-authoritative cooldown early. + const justElapsed = new Date(Date.now() - 5_000).toISOString(); + + await providersDb.updateProviderConnection(connection.id, { + providerSpecificData: { + codexScopeRateLimitedUntil: { codex: justElapsed }, + codexScopeRateLimitSource: { codex: "quota_reset" }, + }, + }); + + const lifted = providersDb.liftCodexScopeCooldownOnHeadroom(connection.id, "codex"); + assert.equal(lifted, false); + + const after = await readConnection(connection.id); + const until = psd(after).codexScopeRateLimitedUntil as Record | undefined; + assert.equal(until?.codex, justElapsed); +}); + +test("#12860 quota_reset cooldown past the skew grace window is lifted", async () => { + const connection = await seedCodexConnection(); + const wellElapsed = new Date(Date.now() - 120_000).toISOString(); + + await providersDb.updateProviderConnection(connection.id, { + providerSpecificData: { + codexScopeRateLimitedUntil: { codex: wellElapsed }, + codexScopeRateLimitSource: { codex: "quota_reset" }, + }, + }); + + const lifted = providersDb.liftCodexScopeCooldownOnHeadroom(connection.id, "codex"); + assert.equal(lifted, true); + + const after = await readConnection(connection.id); + assert.equal(psd(after).codexScopeRateLimitedUntil, undefined); +}); + +test("#12860 hasCodexScopeCooldown short-circuits the snapshot scan when nothing is parked", async () => { + const connection = await seedCodexConnection(); + assert.equal(providersDb.hasCodexScopeCooldown(connection.id, "codex"), false); + + await codexAccount.persistCodexChildCooldown({ + connectionId: connection.id, + model: "gpt-5.5", + rateLimitedUntil: futureIso(120_000), + }); + + assert.equal(providersDb.hasCodexScopeCooldown(connection.id, "codex"), true); + assert.equal(providersDb.hasCodexScopeCooldown(connection.id, "spark"), false); + + const glm = await seedGlmConnection(); + assert.equal(providersDb.hasCodexScopeCooldown(glm.id, "codex"), false); + assert.equal(providersDb.hasCodexScopeCooldown("does-not-exist", "codex"), false); +});