fix(codex): lift nested child cooldowns on parent clear and on snapshot headroom (#12951)

Validado numa worktree combinada com a onda de persistência desta leva sobre `release/v3.8.51`: check-file-size e check-changelog-integrity OK, typecheck:core limpo, check-api-typecheck OK (289). Revalidei o head atual mergeado com o tip: typecheck:core limpo e **36/36** entre `db-rate-limit-guard` e as suítes desta PR.

Levantar o cooldown do escopo pai sem deixar os filhos aninhados presos é o miolo — um cooldown órfão em filho é invisível no dashboard e mantém a conexão fora de rota sem explicação.

**Nota de coordenação:** o `setConnectionRateLimitUntil` colidiu com o #12788 (guard contra timestamp não-finito ou já expirado), que mergeei nesta mesma onda. Eu tinha resolvido a integração na minha worktree, mas ao empurrar o push foi rejeitado — você já tinha empurrado `441fd44`, `f853ba5` e `af2ed01` com a integração feita, e a sua ordenação é equivalente à minha. Descartei a minha e mantive a sua; o crédito é seu inteiro. Fica o registro de que push rejeitado não é erro leve: se eu tivesse mergeado sem reler, teria levado a branch errada.
This commit is contained in:
Bob.Hou
2026-09-10 07:15:57 -04:00
committed by GitHub
parent 9492357360
commit bfbd090a96
8 changed files with 979 additions and 85 deletions

View File

@@ -0,0 +1 @@
- **fix(codex):** dashboard "clear cooldown" and the CAS recovery path now drop nested `codexScopeRateLimitedUntil` maps in the same write that nulls `rate_limited_until`, and fresh quota snapshots with headroom lift fallback-sourced scope cooldowns parked by quota preflight ([#12817](https://github.com/diegosouzapw/OmniRoute/issues/12817), [#12860](https://github.com/diegosouzapw/OmniRoute/issues/12860), [#12951](https://github.com/diegosouzapw/OmniRoute/pull/12951)) The dashboard reset-credit button (`consumeCodexResetCredit`) now exercises that same snapshot path after a successful redeem, so a filled quota bar is enough to unpark a leftover fallback Codex child without another manual clear.

View File

@@ -56,7 +56,9 @@ export async function persistCodexChildQuotaResponse(params: {
rateLimitedUntil,
rateLimitSource: exhaustedWindow ? ("quota_reset" as const) : ("fallback" as const),
}
: {}),
: params.status === 200
? { rateLimitedUntil: null }
: {}),
});
if (!providerSpecificData) return null;

View File

@@ -39,6 +39,7 @@ import { pickCodexConnectionForUser } from "@/lib/oauth/utils/codexConnectionSel
import { isMicrosoftDesignerWebRetiredProviderId } from "@/shared/constants/designerWebRetirement";
import { reconcileCodexUsageHistory } from "./providers/usageIdentityReconciliation";
import { isRuntimeRetiredProviderId } from "@/shared/constants/providerRetirement";
import { applyCodexChildCooldownClearOnUpdate } from "./providers/codexAccountState";
/**
* normalizeProviderSpecificData + the Codex fingerprint-seed invariant: Codex
@@ -951,11 +952,14 @@ export async function updateProviderConnection(id: string, data: JsonRecord) {
...data,
updatedAt: new Date().toISOString(),
};
merged.providerSpecificData = normalizeConnectionProviderSpecificData(
toStringOrNull(merged.provider),
merged.providerSpecificData,
merged,
existingCamel.providerSpecificData
merged.providerSpecificData = applyCodexChildCooldownClearOnUpdate(
data,
normalizeConnectionProviderSpecificData(
toStringOrNull(merged.provider),
merged.providerSpecificData,
merged,
existingCamel.providerSpecificData
)
);
// Mirror the sanitization the create path applies — keep the returned
// object in lockstep with what we persist.
@@ -1024,65 +1028,13 @@ export async function updateProviderConnection(id: string, data: JsonRecord) {
export {
updateCodexScopedQuotaState,
updateCodexScopeCooldown,
applyCodexChildCooldownClearOnUpdate,
stripCodexChildCooldownFields,
stripCodexChildCooldownsFromConnection,
hasCodexScopeCooldown,
liftCodexScopeCooldownOnHeadroom,
} from "./providers/codexAccountState";
/**
* Atomic conditional clear of recoverable error state on a connection row.
*
* Returns true when the row was cleared, false when a concurrent writer
* (markAccountUnavailable, connectionRecovery tick, test, etc.) changed the
* row between the caller's snapshot read and this UPDATE — in which case the
* clear is skipped to preserve the freshest error state. Closes the TOCTOU
* window in the quota-recovery path.
*
* CAS token = (test_status, last_error_at, rate_limited_until).
* markAccountUnavailable always bumps last_error_at on every cooldown/error
* write, so an unchanged last_error_at reliably indicates no concurrent write.
*/
export async function clearConnectionErrorIfUnchanged(
id: string,
expected: {
testStatus: string | null | undefined;
lastErrorAt: string | null | undefined;
rateLimitedUntil: string | null | undefined;
}
): Promise<boolean> {
const db = getDbInstance() as unknown as DbLike;
const result = db
.prepare(
`
UPDATE provider_connections SET
test_status = 'active',
last_error = NULL,
last_error_at = NULL,
last_error_type = NULL,
last_error_source = NULL,
error_code = NULL,
rate_limited_until = NULL,
backoff_level = 0,
updated_at = ?
WHERE id = ?
AND IFNULL(test_status, '') = ?
AND IFNULL(last_error_at, '') = ?
AND IFNULL(rate_limited_until, '') = ?
`
)
.run(
new Date().toISOString(),
id,
expected.testStatus ?? "",
expected.lastErrorAt ?? "",
expected.rateLimitedUntil ?? ""
);
const applied = (result.changes ?? 0) > 0;
if (applied) {
backupDbFile("pre-write");
invalidateDbCache("connections");
bumpProxyConfigGeneration();
}
return applied;
}
/**
* Lightweight stat bump — updates lastUsedAt and consecutiveUseCount without
* SELECT, re-encrypt, cache invalidation, or file backup.
@@ -1183,4 +1135,5 @@ export {
formatResetCountdown,
isConnectionRateLimited,
getRateLimitedConnections,
clearConnectionErrorIfUnchanged,
} from "./providers/rateLimit";

View File

@@ -18,10 +18,105 @@ interface DbLike {
type CodexScopedQuotaPatch = {
quotaState?: JsonRecord;
exhaustedWindow?: "5h" | "7d" | null;
rateLimitedUntil?: string;
rateLimitedUntil?: string | null;
rateLimitSource?: "fallback" | "quota_reset";
};
const CODEX_CHILD_COOLDOWN_KEYS = [
"codexScopeRateLimitedUntil",
"codexScopeRateLimitSource",
] as const;
function omitEmptyRecord(record: JsonRecord): JsonRecord | undefined {
return Object.keys(record).length > 0 ? record : undefined;
}
/** Drop nested Codex child cooldowns; keep quota snapshots and unrelated keys. */
export function stripCodexChildCooldownFields(psd: JsonRecord): JsonRecord {
if (!connectionHasCodexChildCooldown(psd)) return psd;
const next = { ...psd };
for (const key of CODEX_CHILD_COOLDOWN_KEYS) delete next[key];
return next;
}
/** PUT/CAS payload that clears the parent column must also drop nested maps. */
export function applyCodexChildCooldownClearOnUpdate<T extends JsonRecord | undefined>(
data: JsonRecord,
psd: T
): T {
if (psd == null) return psd;
if (!Object.hasOwn(data, "rateLimitedUntil")) return psd;
if (data.rateLimitedUntil != null && data.rateLimitedUntil !== "") return psd;
return stripCodexChildCooldownFields(psd) as T;
}
function connectionHasCodexChildCooldown(psd: JsonRecord): boolean {
return "codexScopeRateLimitedUntil" in psd || "codexScopeRateLimitSource" in psd;
}
/**
* Persist a full-parent cooldown lift into the nested Codex child maps.
* When `alsoClearTopLevel` is set, the parent `rate_limited_until` column is
* nulled in the same transaction so a crash between the two writes cannot
* leave a nested child map behind a cleared parent column.
*/
export function stripCodexChildCooldownsFromConnection(
id: string,
options?: { alsoClearTopLevel?: boolean }
): void {
if (typeof id !== "string" || id.length === 0) return;
const db = getDbInstance() as unknown as DbLike;
const alsoClearTopLevel = options?.alsoClearTopLevel === true;
const candidate = db
.prepare("SELECT provider FROM provider_connections WHERE id = ?")
.get(id);
const isCodex = toRecord(candidate).provider === "codex";
if (!alsoClearTopLevel && !isCodex) return;
backupDbFile("pre-write");
const wrote = db.transaction(() => {
const existing = db
.prepare(
"SELECT provider, provider_specific_data FROM provider_connections WHERE id = ?"
)
.get(id);
if (!existing) return false;
const existingRecord = toRecord(rowToCamel(existing));
const providerSpecificData = toRecord(existingRecord.providerSpecificData);
const stripNested =
existingRecord.provider === "codex" &&
connectionHasCodexChildCooldown(providerSpecificData);
if (!alsoClearTopLevel && !stripNested) return false;
const now = new Date().toISOString();
if (alsoClearTopLevel && stripNested) {
db.prepare(
`UPDATE provider_connections
SET rate_limited_until = NULL,
provider_specific_data = ?,
updated_at = ?
WHERE id = ?`
).run(JSON.stringify(stripCodexChildCooldownFields(providerSpecificData)), now, id);
return true;
}
if (alsoClearTopLevel) {
db.prepare(
`UPDATE provider_connections
SET rate_limited_until = NULL, updated_at = ?
WHERE id = ?`
).run(now, id);
return true;
}
db.prepare(
`UPDATE provider_connections
SET provider_specific_data = ?, updated_at = ?
WHERE id = ?`
).run(JSON.stringify(stripCodexChildCooldownFields(providerSpecificData)), now, id);
return true;
})();
if (wrote) invalidateDbCache("connections");
}
/**
* Atomically merge one virtual Codex child's quota evidence into its persisted parent.
* The transaction reads the latest row so sibling child state cannot be lost.
@@ -70,28 +165,35 @@ export async function updateCodexScopedQuotaState(
}
}
if (patch.rateLimitedUntil) {
const scopeCooldowns = toRecord(providerSpecificData.codexScopeRateLimitedUntil);
const sourceByScope = toRecord(providerSpecificData.codexScopeRateLimitSource);
const existingCooldownMs =
typeof scopeCooldowns[scope] === "string"
? new Date(scopeCooldowns[scope] as string).getTime()
: NaN;
const existingIsAuthoritative =
sourceByScope[scope] === "quota_reset" &&
patch.rateLimitSource !== "quota_reset" &&
Number.isFinite(existingCooldownMs) &&
existingCooldownMs > Date.now();
nextProviderSpecificData.codexScopeRateLimitedUntil = {
...scopeCooldowns,
[scope]: existingIsAuthoritative ? scopeCooldowns[scope] : patch.rateLimitedUntil,
};
nextProviderSpecificData.codexScopeRateLimitSource = {
...sourceByScope,
[scope]: existingIsAuthoritative
if (patch.rateLimitedUntil !== undefined) {
const scopeCooldowns = { ...toRecord(providerSpecificData.codexScopeRateLimitedUntil) };
const sourceByScope = { ...toRecord(providerSpecificData.codexScopeRateLimitSource) };
if (patch.rateLimitedUntil) {
const existingCooldownMs =
typeof scopeCooldowns[scope] === "string"
? new Date(scopeCooldowns[scope] as string).getTime()
: NaN;
const existingIsAuthoritative =
sourceByScope[scope] === "quota_reset" &&
patch.rateLimitSource !== "quota_reset" &&
Number.isFinite(existingCooldownMs) &&
existingCooldownMs > Date.now();
scopeCooldowns[scope] = existingIsAuthoritative
? scopeCooldowns[scope]
: patch.rateLimitedUntil;
sourceByScope[scope] = existingIsAuthoritative
? sourceByScope[scope]
: (patch.rateLimitSource ?? "fallback"),
};
: (patch.rateLimitSource ?? "fallback");
} else {
delete scopeCooldowns[scope];
delete sourceByScope[scope];
}
const nextCooldowns = omitEmptyRecord(scopeCooldowns);
const nextSources = omitEmptyRecord(sourceByScope);
if (nextCooldowns) nextProviderSpecificData.codexScopeRateLimitedUntil = nextCooldowns;
else delete nextProviderSpecificData.codexScopeRateLimitedUntil;
if (nextSources) nextProviderSpecificData.codexScopeRateLimitSource = nextSources;
else delete nextProviderSpecificData.codexScopeRateLimitSource;
}
db.prepare(
@@ -106,6 +208,107 @@ export async function updateCodexScopedQuotaState(
return persisted;
}
/** Grace window absorbing clock skew against the upstream quota server. */
const QUOTA_RESET_CLOCK_SKEW_GRACE_MS = 30_000;
/** Cheap probe: does this connection+scope currently carry a child cooldown? */
export function hasCodexScopeCooldown(id: string, scope: "codex" | "spark"): boolean {
if (typeof id !== "string" || id.length === 0) return false;
const db = getDbInstance() as unknown as DbLike;
const row = db
.prepare("SELECT provider, provider_specific_data FROM provider_connections WHERE id = ?")
.get(id);
if (!row) return false;
const record = toRecord(rowToCamel(row));
if (record.provider !== "codex") return false;
const psd = toRecord(record.providerSpecificData);
return Boolean(toRecord(psd.codexScopeRateLimitedUntil)[scope]);
}
/**
* #12860: When fresh quota snapshot data demonstrates headroom on a scope,
* lift any fallback-sourced cooldown (e.g. parked by quota preflight).
* Cooldowns sourced from upstream 429 quota_reset retain their authority
* until their reset timestamp has elapsed.
*/
export function liftCodexScopeCooldownOnHeadroom(
id: string,
scope: "codex" | "spark"
): boolean {
if (typeof id !== "string" || id.length === 0) return false;
const db = getDbInstance() as unknown as DbLike;
// Cheap eligibility probe so the backup only runs when a write is plausible.
// The transaction below re-reads under lock and remains the authority.
if (!hasCodexScopeCooldown(id, scope)) return false;
backupDbFile("pre-write");
const wrote = db.transaction(() => {
const existing = db
.prepare(
"SELECT provider, provider_specific_data FROM provider_connections WHERE id = ?"
)
.get(id);
if (!existing) return false;
const existingRecord = toRecord(rowToCamel(existing));
if (existingRecord.provider !== "codex") return false;
const currentPsd = toRecord(existingRecord.providerSpecificData);
const currentCooldowns = { ...toRecord(currentPsd.codexScopeRateLimitedUntil) };
if (!currentCooldowns[scope]) return false;
const currentSources = { ...toRecord(currentPsd.codexScopeRateLimitSource) };
const curSource = currentSources[scope];
const curUntilMs =
typeof currentCooldowns[scope] === "string"
? new Date(currentCooldowns[scope] as string).getTime()
: NaN;
// An upstream-authoritative `quota_reset` deadline outranks a local snapshot:
// the grace window absorbs clock skew against the quota server, so a fast
// local clock cannot lift a cooldown upstream still considers active.
if (
curSource === "quota_reset" &&
Number.isFinite(curUntilMs) &&
curUntilMs > Date.now() - QUOTA_RESET_CLOCK_SKEW_GRACE_MS
) {
return false;
}
delete currentCooldowns[scope];
delete currentSources[scope];
const nextPsd: JsonRecord = { ...currentPsd };
const nextCooldowns = omitEmptyRecord(currentCooldowns);
const nextSources = omitEmptyRecord(currentSources);
if (nextCooldowns) nextPsd.codexScopeRateLimitedUntil = nextCooldowns;
else delete nextPsd.codexScopeRateLimitedUntil;
if (nextSources) nextPsd.codexScopeRateLimitSource = nextSources;
else delete nextPsd.codexScopeRateLimitSource;
const exhaustedByScope = { ...toRecord(currentPsd.codexExhaustedWindowByScope) };
if (exhaustedByScope[scope]) {
delete exhaustedByScope[scope];
const nextExhausted = omitEmptyRecord(exhaustedByScope);
if (nextExhausted) {
nextPsd.codexExhaustedWindowByScope = nextExhausted;
} else {
delete nextPsd.codexExhaustedWindowByScope;
delete nextPsd.codexExhaustedWindow;
}
}
const now = new Date().toISOString();
db.prepare(
`UPDATE provider_connections
SET provider_specific_data = ?, updated_at = ?
WHERE id = ?`
).run(JSON.stringify(nextPsd), now, id);
return true;
})();
if (wrote) invalidateDbCache("connections");
return wrote;
}
/** Persist one child cooldown through the shared scoped quota-state transaction. */
export async function updateCodexScopeCooldown(
id: string,

View File

@@ -4,6 +4,9 @@
import { getDbInstance } from "../core";
import { invalidateDbCache } from "../readCache";
import { backupDbFile } from "../backup";
import { bumpProxyConfigGeneration } from "../settings";
import { stripCodexChildCooldownsFromConnection } from "./codexAccountState";
interface StatementLike<TRow = unknown> {
all: (...params: unknown[]) => TRow[];
@@ -33,6 +36,10 @@ export function setConnectionRateLimitUntil(connectionId: string, until: number
// is the only clear path (via clearConnectionRateLimit); past/zero
// timestamps are noops so an expired write cannot overwrite a live row.
if (until !== null && (!Number.isFinite(until) || until <= Date.now())) return;
if (until == null) {
stripCodexChildCooldownsFromConnection(connectionId, { alsoClearTopLevel: true });
return;
}
const db = getDbInstance() as unknown as DbLike;
db.prepare(
"UPDATE provider_connections SET rate_limited_until = ?, updated_at = ? WHERE id = ?"
@@ -234,6 +241,72 @@ export function clearStaleCrashCooldowns(): { cleared: number } {
return { cleared: toReset.length };
}
/**
* Atomic conditional clear of recoverable error state on a connection row.
*
* Returns true when the row was cleared, false when a concurrent writer
* (markAccountUnavailable, connectionRecovery tick, test, etc.) changed the
* row between the caller's snapshot read and this UPDATE — in which case the
* clear is skipped to preserve the freshest error state. Closes the TOCTOU
* window in the quota-recovery path.
*
* CAS token = (test_status, last_error_at, rate_limited_until).
* Nested Codex child cooldown maps are stripped in the same UPDATE so a
* concurrent writer cannot re-persist them between two statements.
*/
export async function clearConnectionErrorIfUnchanged(
id: string,
expected: {
testStatus: string | null | undefined;
lastErrorAt: string | null | undefined;
rateLimitedUntil: string | null | undefined;
}
): Promise<boolean> {
const db = getDbInstance() as unknown as DbLike;
backupDbFile("pre-write");
const result = db
.prepare(
`
UPDATE provider_connections SET
test_status = 'active',
last_error = NULL,
last_error_at = NULL,
last_error_type = NULL,
last_error_source = NULL,
error_code = NULL,
rate_limited_until = NULL,
backoff_level = 0,
provider_specific_data = CASE
WHEN provider = 'codex' AND json_valid(provider_specific_data)
THEN json_remove(
provider_specific_data,
'$.codexScopeRateLimitedUntil',
'$.codexScopeRateLimitSource'
)
ELSE provider_specific_data
END,
updated_at = ?
WHERE id = ?
AND IFNULL(test_status, '') = ?
AND IFNULL(last_error_at, '') = ?
AND IFNULL(rate_limited_until, '') = ?
`
)
.run(
new Date().toISOString(),
id,
expected.testStatus ?? "",
expected.lastErrorAt ?? "",
expected.rateLimitedUntil ?? ""
);
const applied = (result.changes ?? 0) > 0;
if (applied) {
invalidateDbCache("connections");
bumpProxyConfigGeneration();
}
return applied;
}
// T13: Format a reset countdown as a human-readable string ("2h 35m" / "4m 30s").
// The implementation lives in the client-safe formatting utils so client
// components (e.g. CoolingConnectionsPanel) can import it without pulling this

View File

@@ -1,5 +1,10 @@
import { getDbInstance, rowToCamel } from "./core";
import type { QuotaSnapshotRow, ProviderUtilizationPoint } from "@/shared/types/utilization";
import {
hasCodexScopeCooldown,
liftCodexScopeCooldownOnHeadroom,
} from "./providers/codexAccountState";
import { isCodexSparkQuotaKey } from "@omniroute/open-sse/config/codexQuotaScopes";
type JsonRecord = Record<string, unknown>;
@@ -45,6 +50,61 @@ export function saveQuotaSnapshot(snapshot: Omit<QuotaSnapshotRow, "id" | "creat
}
throw err;
}
// #12860: When a new snapshot demonstrates headroom on a Codex connection,
// lift any fallback-sourced scope cooldown parked by quota preflight.
maybeLiftCodexCooldownOnHeadroom(snapshot);
}
/**
* `rowToCamel` leaves snake_case keys intact on some driver paths, so accept
* either casing rather than trusting one shape.
*/
type SnapshotShape = Partial<QuotaSnapshotRow> & {
windowKey?: string;
remainingPercentage?: number;
isExhausted?: number;
};
function snapshotHasHeadroom(s: SnapshotShape): boolean {
const pct = s.remainingPercentage ?? s.remaining_percentage ?? 0;
const exhausted = s.isExhausted ?? s.is_exhausted ?? 0;
return pct > 0 && exhausted !== 1;
}
function maybeLiftCodexCooldownOnHeadroom(
snapshot: Omit<QuotaSnapshotRow, "id" | "created_at">
): void {
if (
snapshot.provider?.toLowerCase() !== "codex" ||
typeof snapshot.connection_id !== "string" ||
snapshot.connection_id.length === 0 ||
(snapshot.remaining_percentage ?? 0) <= 0 ||
snapshot.is_exhausted === 1
) {
return;
}
try {
const scope = isCodexSparkQuotaKey(snapshot.window_key) ? "spark" : "codex";
// The scope-wide read is only worth paying for when a cooldown is
// actually parked on this connection+scope; the common case is clean.
if (!hasCodexScopeCooldown(snapshot.connection_id, scope)) return;
const scopeWindows = getLatestQuotaSnapshotsForConnection(snapshot.connection_id).filter(
(s: SnapshotShape) => {
const key = s.windowKey ?? s.window_key;
return scope === "spark" ? isCodexSparkQuotaKey(key) : !isCodexSparkQuotaKey(key);
}
);
// Every window in the scope must be healthy: one exhausted window still
// justifies the cooldown even when a sibling reports full headroom.
if (scopeWindows.length > 0 && scopeWindows.every(snapshotHasHeadroom)) {
liftCodexScopeCooldownOnHeadroom(snapshot.connection_id, scope);
}
} catch (err) {
console.debug("[QuotaSnapshots] Headroom evaluation skipped:", err);
}
}
export function getQuotaSnapshots(opts: {

View File

@@ -11,6 +11,7 @@ process.env.API_KEY_SECRET = "test-codex-reset-credits-secret";
const core = await import("../../src/lib/db/core.ts");
const providersDb = await import("../../src/lib/db/providers.ts");
const resetCredits = await import("../../src/lib/usage/codexResetCredits.ts");
const codexAccount = await import("../../open-sse/services/codexAccount/index.ts");
const originalFetch = globalThis.fetch;
type QuotaUsageRecord = Record<string, { used?: unknown } | undefined>;
@@ -45,6 +46,69 @@ test.after(async () => {
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
});
async function persistBothChildCooldowns(id: string) {
const codexUntil = new Date(Date.now() + 60_000).toISOString();
const sparkUntil = new Date(Date.now() + 120_000).toISOString();
await codexAccount.persistCodexChildCooldown({
connectionId: id,
model: "gpt-5.5",
rateLimitedUntil: codexUntil,
});
await codexAccount.persistCodexChildCooldown({
connectionId: id,
model: "gpt-5.3-codex-spark",
rateLimitedUntil: sparkUntil,
});
return { codexUntil, sparkUntil };
}
async function readConnection(id: string) {
const connection = await providersDb.getProviderConnectionById(id);
assert.ok(connection);
return connection as Record<string, unknown>;
}
function mockResetThenUsage(opts: {
consumeBody: unknown;
consumeStatus?: number;
usageBody?: unknown;
usageStatus?: number;
}) {
globalThis.fetch = (async (url) => {
const href = String(url);
if (href.endsWith("/rate-limit-reset-credits")) {
return new Response(
JSON.stringify({ credits: [{ id: "credit-123", status: "available" }] }),
{ status: 200, headers: { "content-type": "application/json" } }
);
}
if (href.includes("/rate-limit-reset-credits/consume")) {
return new Response(JSON.stringify(opts.consumeBody), {
status: opts.consumeStatus ?? 200,
headers: { "content-type": "application/json" },
});
}
if (href.includes("/backend-api/wham/usage")) {
if (opts.usageStatus && opts.usageStatus >= 500) {
return new Response("upstream down", { status: opts.usageStatus });
}
return new Response(
JSON.stringify(
opts.usageBody ?? {
plan_type: "plus",
rate_limit: {
primary_window: { used_percent: 0 },
secondary_window: { used_percent: 0 },
},
}
),
{ status: 200, headers: { "content-type": "application/json" } }
);
}
return new Response("unexpected", { status: 500 });
}) as typeof fetch;
}
test("consumeCodexResetCredit fetches a credit id, posts it, then refreshes usage", async () => {
const connection = (await createCodexConnection()) as { id: string };
const calls: Array<{ url: string; init: RequestInit }> = [];
@@ -355,3 +419,92 @@ test("consumeCodexResetCredit rejects non-Codex and missing connections", async
error.code === "codex_provider_required"
);
});
test("#12951 consumeCodexResetCredit lifts fallback Codex child cooldown after healthy usage refresh", async () => {
const connection = (await createCodexConnection()) as { id: string };
const parked = await persistBothChildCooldowns(connection.id);
const before = await readConnection(connection.id);
assert.equal(codexAccount.getCodexChildCooldown(before as never, "gpt-5.5"), parked.codexUntil);
assert.equal(
codexAccount.getCodexChildCooldown(before as never, "gpt-5.3-codex-spark"),
parked.sparkUntil
);
mockResetThenUsage({ consumeBody: { code: "reset" } });
const result = await resetCredits.consumeCodexResetCredit(connection.id, "redeem-button-path");
assert.equal(result.outcome, "reset");
const after = await readConnection(connection.id);
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), null);
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
assert.equal(
codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"),
parked.sparkUntil
);
});
test("#12951 consumeCodexResetCredit alreadyRedeemed still needs healthy usage to lift fallback cooldown", async () => {
const connection = (await createCodexConnection()) as { id: string };
await persistBothChildCooldowns(connection.id);
mockResetThenUsage({
consumeBody: { code: "alreadyRedeemed" },
usageBody: {
plan_type: "plus",
rate_limit: {
primary_window: { used_percent: 100 },
secondary_window: { used_percent: 100 },
},
},
});
const result = await resetCredits.consumeCodexResetCredit(connection.id, "redeem-already");
assert.equal(result.outcome, "alreadyRedeemed");
const after = await readConnection(connection.id);
assert.ok(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"));
assert.ok(codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"));
});
test("#12951 consumeCodexResetCredit failed redeem keeps nested child cooldowns", async () => {
const connection = (await createCodexConnection()) as { id: string };
const parked = await persistBothChildCooldowns(connection.id);
mockResetThenUsage({
consumeBody: { code: "noCredit" },
consumeStatus: 409,
});
await assert.rejects(
() => resetCredits.consumeCodexResetCredit(connection.id, "redeem-fail"),
(error: unknown) =>
error instanceof resetCredits.CodexResetCreditError &&
error.status === 409 &&
error.code === "no_credit"
);
const after = await readConnection(connection.id);
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), parked.codexUntil);
assert.equal(
codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"),
parked.sparkUntil
);
});
test("#12951 consumeCodexResetCredit incomplete usage refresh keeps nested child cooldowns", async () => {
const connection = (await createCodexConnection()) as { id: string };
const parked = await persistBothChildCooldowns(connection.id);
mockResetThenUsage({ consumeBody: { code: "reset" }, usageStatus: 500 });
const result = await resetCredits.consumeCodexResetCredit(connection.id, "redeem-incomplete");
assert.equal(result.outcome, "reset");
const after = await readConnection(connection.id);
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), parked.codexUntil);
assert.equal(
codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"),
parked.sparkUntil
);
});

View File

@@ -0,0 +1,449 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "path";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-codex-clear-12817-"));
process.env.DATA_DIR = TEST_DATA_DIR;
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "codex-clear-12817-secret";
const core = await import("../../src/lib/db/core.ts");
const providersDb = await import("../../src/lib/db/providers.ts");
const rateLimit = await import("../../src/lib/db/providers/rateLimit.ts");
const codexAccount = await import("../../open-sse/services/codexAccount/index.ts");
const quotaSnapshots = await import("../../src/lib/db/quotaSnapshots.ts");
const quotaCache = await import("../../src/domain/quotaCache.ts");
async function resetStorage(): Promise<void> {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
}
interface SeededConnection {
id: string;
providerSpecificData: Record<string, unknown>;
}
async function seedCodexConnection(): Promise<SeededConnection> {
return providersDb.createProviderConnection({
provider: "codex",
authType: "oauth",
name: "codex-clear-12817",
email: "codex-clear-12817@example.com",
apiKey: "codex-clear-12817-key",
accessToken: "codex-clear-12817-access",
refreshToken: "codex-clear-12817-refresh",
providerSpecificData: {
unrelated: { retained: true },
},
}) as unknown as Promise<SeededConnection>;
}
async function seedGlmConnection(): Promise<SeededConnection> {
return providersDb.createProviderConnection({
provider: "glm",
authType: "apikey",
name: "glm-clear-12817",
apiKey: "glm-clear-12817-key",
providerSpecificData: {
leftover: "keep-me",
},
}) as unknown as Promise<SeededConnection>;
}
async function readConnection(id: string): Promise<Record<string, unknown>> {
const connection = await providersDb.getProviderConnectionById(id);
assert.ok(connection);
return connection as unknown as Record<string, unknown>;
}
function psd(connection: Record<string, unknown>): Record<string, unknown> {
return (connection.providerSpecificData ?? {}) as Record<string, unknown>;
}
function futureIso(ms: number): string {
return new Date(Date.now() + ms).toISOString();
}
function quotaHeaders(resetAt5h: string, resetAt7d: string, usage5h = "10") {
return {
"x-codex-5h-usage": usage5h,
"x-codex-5h-limit": "100",
"x-codex-5h-reset-at": resetAt5h,
"x-codex-7d-usage": "10",
"x-codex-7d-limit": "100",
"x-codex-7d-reset-at": resetAt7d,
};
}
async function persistBothChildCooldowns(id: string): Promise<{
codexUntil: string;
sparkUntil: string;
}> {
const codexUntil = futureIso(60_000);
const sparkUntil = futureIso(120_000);
await codexAccount.persistCodexChildCooldown({
connectionId: id,
model: "gpt-5.5",
rateLimitedUntil: codexUntil,
});
await codexAccount.persistCodexChildCooldown({
connectionId: id,
model: "gpt-5.3-codex-spark",
rateLimitedUntil: sparkUntil,
});
return { codexUntil, sparkUntil };
}
test.beforeEach(resetStorage);
test.after(() => {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
});
test("#12817 PUT rateLimitedUntil:null also drops nested Codex child cooldowns", async () => {
const connection = await seedCodexConnection();
const { sparkUntil } = await persistBothChildCooldowns(connection.id);
await providersDb.updateProviderConnection(connection.id, {
rateLimitedUntil: futureIso(90_000),
});
const before = await readConnection(connection.id);
assert.equal(
codexAccount.getCodexChildCooldown(before as never, "gpt-5.3-codex-spark"),
sparkUntil
);
await providersDb.updateProviderConnection(connection.id, { rateLimitedUntil: null });
const after = await readConnection(connection.id);
const data = psd(after);
assert.equal(after.rateLimitedUntil, undefined);
assert.equal(data.codexScopeRateLimitedUntil, undefined);
assert.equal(data.codexScopeRateLimitSource, undefined);
assert.deepEqual(data.unrelated, { retained: true });
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), null);
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"), null);
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
});
test("#12817 PUT rateLimitedUntil:\"\" also drops nested Codex child cooldowns", async () => {
const connection = await seedCodexConnection();
await persistBothChildCooldowns(connection.id);
await providersDb.updateProviderConnection(connection.id, {
rateLimitedUntil: futureIso(90_000),
});
await providersDb.updateProviderConnection(connection.id, { rateLimitedUntil: "" });
const after = await readConnection(connection.id);
const data = psd(after);
assert.equal(after.rateLimitedUntil, undefined);
assert.equal(data.codexScopeRateLimitedUntil, undefined);
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
});
test("#12817 clearConnectionRateLimit strips nested Codex child cooldowns", async () => {
const connection = await seedCodexConnection();
await persistBothChildCooldowns(connection.id);
rateLimit.setConnectionRateLimitUntil(connection.id, Date.now() + 90_000);
rateLimit.clearConnectionRateLimit(connection.id);
const after = await readConnection(connection.id);
const data = psd(after);
assert.equal(after.rateLimitedUntil, undefined);
assert.equal(data.codexScopeRateLimitedUntil, undefined);
assert.equal(data.codexScopeRateLimitSource, undefined);
assert.deepEqual(data.unrelated, { retained: true });
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
});
test("#12817 CAS error-clear also strips nested Codex child cooldowns", async () => {
const connection = await seedCodexConnection();
await persistBothChildCooldowns(connection.id);
const until = futureIso(90_000);
await providersDb.updateProviderConnection(connection.id, {
testStatus: "unavailable",
lastError: "429",
lastErrorAt: new Date().toISOString(),
lastErrorType: "rate_limit_exceeded",
rateLimitedUntil: until,
});
const before = await readConnection(connection.id);
const applied = await providersDb.clearConnectionErrorIfUnchanged(connection.id, {
testStatus: (before.testStatus as string) ?? null,
lastErrorAt: (before.lastErrorAt as string) ?? null,
rateLimitedUntil: (before.rateLimitedUntil as string) ?? null,
});
assert.equal(applied, true);
const after = await readConnection(connection.id);
assert.equal(after.testStatus, "active");
assert.equal(psd(after).codexScopeRateLimitedUntil, undefined);
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
});
test("#12817 a successful quota observation clears that child's leftover cooldown", async () => {
const connection = await seedCodexConnection();
const reset5h = futureIso(60_000);
const reset7d = futureIso(600_000);
await persistBothChildCooldowns(connection.id);
await codexAccount.persistCodexChildQuotaResponse({
connectionId: connection.id,
model: "gpt-5.5",
headers: quotaHeaders(reset5h, reset7d, "95"),
status: 429,
});
await codexAccount.persistCodexChildQuotaResponse({
connectionId: connection.id,
model: "gpt-5.5",
headers: quotaHeaders(reset5h, reset7d, "10"),
status: 200,
});
const after = await readConnection(connection.id);
const data = psd(after);
const until = data.codexScopeRateLimitedUntil as Record<string, unknown> | undefined;
const exhausted = data.codexExhaustedWindowByScope as Record<string, unknown> | undefined;
assert.equal(until?.codex, undefined);
assert.equal(typeof until?.spark, "string");
assert.equal(exhausted?.codex, undefined);
assert.deepEqual(data.unrelated, { retained: true });
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.3-codex-spark"), true);
});
test("#12817 clearing a non-Codex cooldown leaves providerSpecificData alone", async () => {
const connection = await seedGlmConnection();
await providersDb.updateProviderConnection(connection.id, {
rateLimitedUntil: futureIso(90_000),
});
await providersDb.updateProviderConnection(connection.id, { rateLimitedUntil: null });
const after = await readConnection(connection.id);
assert.equal(after.rateLimitedUntil, undefined);
assert.equal(psd(after).leftover, "keep-me");
});
test("#12860 saveQuotaSnapshot with headroom lifts fallback-sourced scope cooldown", async () => {
const connection = await seedCodexConnection();
await persistBothChildCooldowns(connection.id);
const before = await readConnection(connection.id);
assert.ok(codexAccount.getCodexChildCooldown(before as never, "gpt-5.5"));
quotaSnapshots.saveQuotaSnapshot({
provider: "codex",
connection_id: connection.id,
window_key: "primary",
remaining_percentage: 100,
is_exhausted: 0,
next_reset_at: futureIso(3600_000),
window_duration_ms: 18_000_000,
raw_data: null,
});
const after = await readConnection(connection.id);
const data = psd(after);
const until = data.codexScopeRateLimitedUntil as Record<string, unknown> | undefined;
assert.equal(until?.codex, undefined);
assert.equal(typeof until?.spark, "string");
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), null);
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
});
test("#12860 saveQuotaSnapshot with headroom does NOT lift authoritative quota_reset cooldown", async () => {
const connection = await seedCodexConnection();
const reset5h = futureIso(120_000);
const reset7d = futureIso(600_000);
await codexAccount.persistCodexChildQuotaResponse({
connectionId: connection.id,
model: "gpt-5.5",
headers: quotaHeaders(reset5h, reset7d, "95"),
status: 429,
});
const before = await readConnection(connection.id);
assert.ok(codexAccount.getCodexChildCooldown(before as never, "gpt-5.5"));
quotaSnapshots.saveQuotaSnapshot({
provider: "codex",
connection_id: connection.id,
window_key: "session",
remaining_percentage: 100,
is_exhausted: 0,
next_reset_at: futureIso(3600_000),
window_duration_ms: 18_000_000,
raw_data: null,
});
const after = await readConnection(connection.id);
const data = psd(after);
const until = data.codexScopeRateLimitedUntil as Record<string, unknown> | undefined;
assert.equal(typeof until?.codex, "string");
assert.ok(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"));
});
test("#12860 saveQuotaSnapshot does NOT lift scope cooldown if another window for that scope is still exhausted", async () => {
const connection = await seedCodexConnection();
await codexAccount.persistCodexChildCooldown({
connectionId: connection.id,
model: "gpt-5.5",
rateLimitedUntil: futureIso(120_000),
});
// Weekly window is currently exhausted (0% remaining)
quotaSnapshots.saveQuotaSnapshot({
provider: "codex",
connection_id: connection.id,
window_key: "weekly",
remaining_percentage: 0,
is_exhausted: 1,
next_reset_at: futureIso(7200_000),
window_duration_ms: 604_800_000,
raw_data: null,
});
// Session window reports 100% headroom, but weekly is still exhausted
quotaSnapshots.saveQuotaSnapshot({
provider: "codex",
connection_id: connection.id,
window_key: "session",
remaining_percentage: 100,
is_exhausted: 0,
next_reset_at: futureIso(3600_000),
window_duration_ms: 18_000_000,
raw_data: null,
});
let mid = await readConnection(connection.id);
assert.ok(codexAccount.getCodexChildCooldown(mid as never, "gpt-5.5"));
// Now weekly also recovers to 100% headroom
quotaSnapshots.saveQuotaSnapshot({
provider: "codex",
connection_id: connection.id,
window_key: "weekly",
remaining_percentage: 100,
is_exhausted: 0,
next_reset_at: futureIso(7200_000),
window_duration_ms: 604_800_000,
raw_data: null,
});
let after = await readConnection(connection.id);
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), null);
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
});
test("#12860 saveQuotaSnapshot for spark scope with headroom lifts only spark cooldown", async () => {
const connection = await seedCodexConnection();
await persistBothChildCooldowns(connection.id);
quotaSnapshots.saveQuotaSnapshot({
provider: "codex",
connection_id: connection.id,
window_key: "gpt_5_3_codex_spark_session",
remaining_percentage: 100,
is_exhausted: 0,
next_reset_at: futureIso(3600_000),
window_duration_ms: 18_000_000,
raw_data: null,
});
const after = await readConnection(connection.id);
const data = psd(after);
const until = data.codexScopeRateLimitedUntil as Record<string, unknown> | undefined;
assert.equal(typeof until?.codex, "string");
assert.equal(until?.spark, undefined);
assert.ok(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"));
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"), null);
});
test("#12860 setQuotaCache with fresh usage headroom lifts fallback-sourced scope cooldown", async () => {
const connection = await seedCodexConnection();
await persistBothChildCooldowns(connection.id);
const before = await readConnection(connection.id);
assert.ok(codexAccount.getCodexChildCooldown(before as never, "gpt-5.5"));
quotaCache.setQuotaCache(connection.id, "codex", {
session: {
used: 0,
total: 100,
remainingPercentage: 100,
resetAt: futureIso(3600_000),
},
});
const after = await readConnection(connection.id);
const data = psd(after);
const until = data.codexScopeRateLimitedUntil as Record<string, unknown> | undefined;
assert.equal(until?.codex, undefined);
assert.equal(typeof until?.spark, "string");
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), null);
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
});
test("#12860 quota_reset cooldown that just elapsed is still held by the skew grace window", async () => {
const connection = await seedCodexConnection();
// Reset deadline sits 5s in the past — inside the 30s clock-skew grace, so a
// fast local clock must not lift an upstream-authoritative cooldown early.
const justElapsed = new Date(Date.now() - 5_000).toISOString();
await providersDb.updateProviderConnection(connection.id, {
providerSpecificData: {
codexScopeRateLimitedUntil: { codex: justElapsed },
codexScopeRateLimitSource: { codex: "quota_reset" },
},
});
const lifted = providersDb.liftCodexScopeCooldownOnHeadroom(connection.id, "codex");
assert.equal(lifted, false);
const after = await readConnection(connection.id);
const until = psd(after).codexScopeRateLimitedUntil as Record<string, unknown> | undefined;
assert.equal(until?.codex, justElapsed);
});
test("#12860 quota_reset cooldown past the skew grace window is lifted", async () => {
const connection = await seedCodexConnection();
const wellElapsed = new Date(Date.now() - 120_000).toISOString();
await providersDb.updateProviderConnection(connection.id, {
providerSpecificData: {
codexScopeRateLimitedUntil: { codex: wellElapsed },
codexScopeRateLimitSource: { codex: "quota_reset" },
},
});
const lifted = providersDb.liftCodexScopeCooldownOnHeadroom(connection.id, "codex");
assert.equal(lifted, true);
const after = await readConnection(connection.id);
assert.equal(psd(after).codexScopeRateLimitedUntil, undefined);
});
test("#12860 hasCodexScopeCooldown short-circuits the snapshot scan when nothing is parked", async () => {
const connection = await seedCodexConnection();
assert.equal(providersDb.hasCodexScopeCooldown(connection.id, "codex"), false);
await codexAccount.persistCodexChildCooldown({
connectionId: connection.id,
model: "gpt-5.5",
rateLimitedUntil: futureIso(120_000),
});
assert.equal(providersDb.hasCodexScopeCooldown(connection.id, "codex"), true);
assert.equal(providersDb.hasCodexScopeCooldown(connection.id, "spark"), false);
const glm = await seedGlmConnection();
assert.equal(providersDb.hasCodexScopeCooldown(glm.id, "codex"), false);
assert.equal(providersDb.hasCodexScopeCooldown("does-not-exist", "codex"), false);
});