mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-13 18:32:12 +03:00
fix(codex): lift nested child cooldowns on parent clear and on snapshot headroom (#12951)
Validado numa worktree combinada com a onda de persistência desta leva sobre `release/v3.8.51`: check-file-size e check-changelog-integrity OK, typecheck:core limpo, check-api-typecheck OK (289). Revalidei o head atual mergeado com o tip: typecheck:core limpo e **36/36** entre `db-rate-limit-guard` e as suítes desta PR. Levantar o cooldown do escopo pai sem deixar os filhos aninhados presos é o miolo — um cooldown órfão em filho é invisível no dashboard e mantém a conexão fora de rota sem explicação. **Nota de coordenação:** o `setConnectionRateLimitUntil` colidiu com o #12788 (guard contra timestamp não-finito ou já expirado), que mergeei nesta mesma onda. Eu tinha resolvido a integração na minha worktree, mas ao empurrar o push foi rejeitado — você já tinha empurrado `441fd44`, `f853ba5` e `af2ed01` com a integração feita, e a sua ordenação é equivalente à minha. Descartei a minha e mantive a sua; o crédito é seu inteiro. Fica o registro de que push rejeitado não é erro leve: se eu tivesse mergeado sem reler, teria levado a branch errada.
This commit is contained in:
1
changelog.d/fixes/12951-codex-scope-cooldown-clear.md
Normal file
1
changelog.d/fixes/12951-codex-scope-cooldown-clear.md
Normal file
@@ -0,0 +1 @@
|
||||
- **fix(codex):** dashboard "clear cooldown" and the CAS recovery path now drop nested `codexScopeRateLimitedUntil` maps in the same write that nulls `rate_limited_until`, and fresh quota snapshots with headroom lift fallback-sourced scope cooldowns parked by quota preflight ([#12817](https://github.com/diegosouzapw/OmniRoute/issues/12817), [#12860](https://github.com/diegosouzapw/OmniRoute/issues/12860), [#12951](https://github.com/diegosouzapw/OmniRoute/pull/12951)) The dashboard reset-credit button (`consumeCodexResetCredit`) now exercises that same snapshot path after a successful redeem, so a filled quota bar is enough to unpark a leftover fallback Codex child without another manual clear.
|
||||
@@ -56,7 +56,9 @@ export async function persistCodexChildQuotaResponse(params: {
|
||||
rateLimitedUntil,
|
||||
rateLimitSource: exhaustedWindow ? ("quota_reset" as const) : ("fallback" as const),
|
||||
}
|
||||
: {}),
|
||||
: params.status === 200
|
||||
? { rateLimitedUntil: null }
|
||||
: {}),
|
||||
});
|
||||
if (!providerSpecificData) return null;
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ import { pickCodexConnectionForUser } from "@/lib/oauth/utils/codexConnectionSel
|
||||
import { isMicrosoftDesignerWebRetiredProviderId } from "@/shared/constants/designerWebRetirement";
|
||||
import { reconcileCodexUsageHistory } from "./providers/usageIdentityReconciliation";
|
||||
import { isRuntimeRetiredProviderId } from "@/shared/constants/providerRetirement";
|
||||
import { applyCodexChildCooldownClearOnUpdate } from "./providers/codexAccountState";
|
||||
|
||||
/**
|
||||
* normalizeProviderSpecificData + the Codex fingerprint-seed invariant: Codex
|
||||
@@ -951,11 +952,14 @@ export async function updateProviderConnection(id: string, data: JsonRecord) {
|
||||
...data,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
merged.providerSpecificData = normalizeConnectionProviderSpecificData(
|
||||
toStringOrNull(merged.provider),
|
||||
merged.providerSpecificData,
|
||||
merged,
|
||||
existingCamel.providerSpecificData
|
||||
merged.providerSpecificData = applyCodexChildCooldownClearOnUpdate(
|
||||
data,
|
||||
normalizeConnectionProviderSpecificData(
|
||||
toStringOrNull(merged.provider),
|
||||
merged.providerSpecificData,
|
||||
merged,
|
||||
existingCamel.providerSpecificData
|
||||
)
|
||||
);
|
||||
// Mirror the sanitization the create path applies — keep the returned
|
||||
// object in lockstep with what we persist.
|
||||
@@ -1024,65 +1028,13 @@ export async function updateProviderConnection(id: string, data: JsonRecord) {
|
||||
export {
|
||||
updateCodexScopedQuotaState,
|
||||
updateCodexScopeCooldown,
|
||||
applyCodexChildCooldownClearOnUpdate,
|
||||
stripCodexChildCooldownFields,
|
||||
stripCodexChildCooldownsFromConnection,
|
||||
hasCodexScopeCooldown,
|
||||
liftCodexScopeCooldownOnHeadroom,
|
||||
} from "./providers/codexAccountState";
|
||||
|
||||
/**
|
||||
* Atomic conditional clear of recoverable error state on a connection row.
|
||||
*
|
||||
* Returns true when the row was cleared, false when a concurrent writer
|
||||
* (markAccountUnavailable, connectionRecovery tick, test, etc.) changed the
|
||||
* row between the caller's snapshot read and this UPDATE — in which case the
|
||||
* clear is skipped to preserve the freshest error state. Closes the TOCTOU
|
||||
* window in the quota-recovery path.
|
||||
*
|
||||
* CAS token = (test_status, last_error_at, rate_limited_until).
|
||||
* markAccountUnavailable always bumps last_error_at on every cooldown/error
|
||||
* write, so an unchanged last_error_at reliably indicates no concurrent write.
|
||||
*/
|
||||
export async function clearConnectionErrorIfUnchanged(
|
||||
id: string,
|
||||
expected: {
|
||||
testStatus: string | null | undefined;
|
||||
lastErrorAt: string | null | undefined;
|
||||
rateLimitedUntil: string | null | undefined;
|
||||
}
|
||||
): Promise<boolean> {
|
||||
const db = getDbInstance() as unknown as DbLike;
|
||||
const result = db
|
||||
.prepare(
|
||||
`
|
||||
UPDATE provider_connections SET
|
||||
test_status = 'active',
|
||||
last_error = NULL,
|
||||
last_error_at = NULL,
|
||||
last_error_type = NULL,
|
||||
last_error_source = NULL,
|
||||
error_code = NULL,
|
||||
rate_limited_until = NULL,
|
||||
backoff_level = 0,
|
||||
updated_at = ?
|
||||
WHERE id = ?
|
||||
AND IFNULL(test_status, '') = ?
|
||||
AND IFNULL(last_error_at, '') = ?
|
||||
AND IFNULL(rate_limited_until, '') = ?
|
||||
`
|
||||
)
|
||||
.run(
|
||||
new Date().toISOString(),
|
||||
id,
|
||||
expected.testStatus ?? "",
|
||||
expected.lastErrorAt ?? "",
|
||||
expected.rateLimitedUntil ?? ""
|
||||
);
|
||||
const applied = (result.changes ?? 0) > 0;
|
||||
if (applied) {
|
||||
backupDbFile("pre-write");
|
||||
invalidateDbCache("connections");
|
||||
bumpProxyConfigGeneration();
|
||||
}
|
||||
return applied;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lightweight stat bump — updates lastUsedAt and consecutiveUseCount without
|
||||
* SELECT, re-encrypt, cache invalidation, or file backup.
|
||||
@@ -1183,4 +1135,5 @@ export {
|
||||
formatResetCountdown,
|
||||
isConnectionRateLimited,
|
||||
getRateLimitedConnections,
|
||||
clearConnectionErrorIfUnchanged,
|
||||
} from "./providers/rateLimit";
|
||||
|
||||
@@ -18,10 +18,105 @@ interface DbLike {
|
||||
type CodexScopedQuotaPatch = {
|
||||
quotaState?: JsonRecord;
|
||||
exhaustedWindow?: "5h" | "7d" | null;
|
||||
rateLimitedUntil?: string;
|
||||
rateLimitedUntil?: string | null;
|
||||
rateLimitSource?: "fallback" | "quota_reset";
|
||||
};
|
||||
|
||||
const CODEX_CHILD_COOLDOWN_KEYS = [
|
||||
"codexScopeRateLimitedUntil",
|
||||
"codexScopeRateLimitSource",
|
||||
] as const;
|
||||
|
||||
function omitEmptyRecord(record: JsonRecord): JsonRecord | undefined {
|
||||
return Object.keys(record).length > 0 ? record : undefined;
|
||||
}
|
||||
|
||||
/** Drop nested Codex child cooldowns; keep quota snapshots and unrelated keys. */
|
||||
export function stripCodexChildCooldownFields(psd: JsonRecord): JsonRecord {
|
||||
if (!connectionHasCodexChildCooldown(psd)) return psd;
|
||||
const next = { ...psd };
|
||||
for (const key of CODEX_CHILD_COOLDOWN_KEYS) delete next[key];
|
||||
return next;
|
||||
}
|
||||
|
||||
/** PUT/CAS payload that clears the parent column must also drop nested maps. */
|
||||
export function applyCodexChildCooldownClearOnUpdate<T extends JsonRecord | undefined>(
|
||||
data: JsonRecord,
|
||||
psd: T
|
||||
): T {
|
||||
if (psd == null) return psd;
|
||||
if (!Object.hasOwn(data, "rateLimitedUntil")) return psd;
|
||||
if (data.rateLimitedUntil != null && data.rateLimitedUntil !== "") return psd;
|
||||
return stripCodexChildCooldownFields(psd) as T;
|
||||
}
|
||||
|
||||
function connectionHasCodexChildCooldown(psd: JsonRecord): boolean {
|
||||
return "codexScopeRateLimitedUntil" in psd || "codexScopeRateLimitSource" in psd;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist a full-parent cooldown lift into the nested Codex child maps.
|
||||
* When `alsoClearTopLevel` is set, the parent `rate_limited_until` column is
|
||||
* nulled in the same transaction so a crash between the two writes cannot
|
||||
* leave a nested child map behind a cleared parent column.
|
||||
*/
|
||||
export function stripCodexChildCooldownsFromConnection(
|
||||
id: string,
|
||||
options?: { alsoClearTopLevel?: boolean }
|
||||
): void {
|
||||
if (typeof id !== "string" || id.length === 0) return;
|
||||
const db = getDbInstance() as unknown as DbLike;
|
||||
const alsoClearTopLevel = options?.alsoClearTopLevel === true;
|
||||
const candidate = db
|
||||
.prepare("SELECT provider FROM provider_connections WHERE id = ?")
|
||||
.get(id);
|
||||
const isCodex = toRecord(candidate).provider === "codex";
|
||||
if (!alsoClearTopLevel && !isCodex) return;
|
||||
|
||||
backupDbFile("pre-write");
|
||||
const wrote = db.transaction(() => {
|
||||
const existing = db
|
||||
.prepare(
|
||||
"SELECT provider, provider_specific_data FROM provider_connections WHERE id = ?"
|
||||
)
|
||||
.get(id);
|
||||
if (!existing) return false;
|
||||
const existingRecord = toRecord(rowToCamel(existing));
|
||||
const providerSpecificData = toRecord(existingRecord.providerSpecificData);
|
||||
const stripNested =
|
||||
existingRecord.provider === "codex" &&
|
||||
connectionHasCodexChildCooldown(providerSpecificData);
|
||||
if (!alsoClearTopLevel && !stripNested) return false;
|
||||
|
||||
const now = new Date().toISOString();
|
||||
if (alsoClearTopLevel && stripNested) {
|
||||
db.prepare(
|
||||
`UPDATE provider_connections
|
||||
SET rate_limited_until = NULL,
|
||||
provider_specific_data = ?,
|
||||
updated_at = ?
|
||||
WHERE id = ?`
|
||||
).run(JSON.stringify(stripCodexChildCooldownFields(providerSpecificData)), now, id);
|
||||
return true;
|
||||
}
|
||||
if (alsoClearTopLevel) {
|
||||
db.prepare(
|
||||
`UPDATE provider_connections
|
||||
SET rate_limited_until = NULL, updated_at = ?
|
||||
WHERE id = ?`
|
||||
).run(now, id);
|
||||
return true;
|
||||
}
|
||||
db.prepare(
|
||||
`UPDATE provider_connections
|
||||
SET provider_specific_data = ?, updated_at = ?
|
||||
WHERE id = ?`
|
||||
).run(JSON.stringify(stripCodexChildCooldownFields(providerSpecificData)), now, id);
|
||||
return true;
|
||||
})();
|
||||
if (wrote) invalidateDbCache("connections");
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomically merge one virtual Codex child's quota evidence into its persisted parent.
|
||||
* The transaction reads the latest row so sibling child state cannot be lost.
|
||||
@@ -70,28 +165,35 @@ export async function updateCodexScopedQuotaState(
|
||||
}
|
||||
}
|
||||
|
||||
if (patch.rateLimitedUntil) {
|
||||
const scopeCooldowns = toRecord(providerSpecificData.codexScopeRateLimitedUntil);
|
||||
const sourceByScope = toRecord(providerSpecificData.codexScopeRateLimitSource);
|
||||
const existingCooldownMs =
|
||||
typeof scopeCooldowns[scope] === "string"
|
||||
? new Date(scopeCooldowns[scope] as string).getTime()
|
||||
: NaN;
|
||||
const existingIsAuthoritative =
|
||||
sourceByScope[scope] === "quota_reset" &&
|
||||
patch.rateLimitSource !== "quota_reset" &&
|
||||
Number.isFinite(existingCooldownMs) &&
|
||||
existingCooldownMs > Date.now();
|
||||
nextProviderSpecificData.codexScopeRateLimitedUntil = {
|
||||
...scopeCooldowns,
|
||||
[scope]: existingIsAuthoritative ? scopeCooldowns[scope] : patch.rateLimitedUntil,
|
||||
};
|
||||
nextProviderSpecificData.codexScopeRateLimitSource = {
|
||||
...sourceByScope,
|
||||
[scope]: existingIsAuthoritative
|
||||
if (patch.rateLimitedUntil !== undefined) {
|
||||
const scopeCooldowns = { ...toRecord(providerSpecificData.codexScopeRateLimitedUntil) };
|
||||
const sourceByScope = { ...toRecord(providerSpecificData.codexScopeRateLimitSource) };
|
||||
if (patch.rateLimitedUntil) {
|
||||
const existingCooldownMs =
|
||||
typeof scopeCooldowns[scope] === "string"
|
||||
? new Date(scopeCooldowns[scope] as string).getTime()
|
||||
: NaN;
|
||||
const existingIsAuthoritative =
|
||||
sourceByScope[scope] === "quota_reset" &&
|
||||
patch.rateLimitSource !== "quota_reset" &&
|
||||
Number.isFinite(existingCooldownMs) &&
|
||||
existingCooldownMs > Date.now();
|
||||
scopeCooldowns[scope] = existingIsAuthoritative
|
||||
? scopeCooldowns[scope]
|
||||
: patch.rateLimitedUntil;
|
||||
sourceByScope[scope] = existingIsAuthoritative
|
||||
? sourceByScope[scope]
|
||||
: (patch.rateLimitSource ?? "fallback"),
|
||||
};
|
||||
: (patch.rateLimitSource ?? "fallback");
|
||||
} else {
|
||||
delete scopeCooldowns[scope];
|
||||
delete sourceByScope[scope];
|
||||
}
|
||||
const nextCooldowns = omitEmptyRecord(scopeCooldowns);
|
||||
const nextSources = omitEmptyRecord(sourceByScope);
|
||||
if (nextCooldowns) nextProviderSpecificData.codexScopeRateLimitedUntil = nextCooldowns;
|
||||
else delete nextProviderSpecificData.codexScopeRateLimitedUntil;
|
||||
if (nextSources) nextProviderSpecificData.codexScopeRateLimitSource = nextSources;
|
||||
else delete nextProviderSpecificData.codexScopeRateLimitSource;
|
||||
}
|
||||
|
||||
db.prepare(
|
||||
@@ -106,6 +208,107 @@ export async function updateCodexScopedQuotaState(
|
||||
return persisted;
|
||||
}
|
||||
|
||||
/** Grace window absorbing clock skew against the upstream quota server. */
|
||||
const QUOTA_RESET_CLOCK_SKEW_GRACE_MS = 30_000;
|
||||
|
||||
/** Cheap probe: does this connection+scope currently carry a child cooldown? */
|
||||
export function hasCodexScopeCooldown(id: string, scope: "codex" | "spark"): boolean {
|
||||
if (typeof id !== "string" || id.length === 0) return false;
|
||||
const db = getDbInstance() as unknown as DbLike;
|
||||
const row = db
|
||||
.prepare("SELECT provider, provider_specific_data FROM provider_connections WHERE id = ?")
|
||||
.get(id);
|
||||
if (!row) return false;
|
||||
const record = toRecord(rowToCamel(row));
|
||||
if (record.provider !== "codex") return false;
|
||||
const psd = toRecord(record.providerSpecificData);
|
||||
return Boolean(toRecord(psd.codexScopeRateLimitedUntil)[scope]);
|
||||
}
|
||||
|
||||
/**
|
||||
* #12860: When fresh quota snapshot data demonstrates headroom on a scope,
|
||||
* lift any fallback-sourced cooldown (e.g. parked by quota preflight).
|
||||
* Cooldowns sourced from upstream 429 quota_reset retain their authority
|
||||
* until their reset timestamp has elapsed.
|
||||
*/
|
||||
export function liftCodexScopeCooldownOnHeadroom(
|
||||
id: string,
|
||||
scope: "codex" | "spark"
|
||||
): boolean {
|
||||
if (typeof id !== "string" || id.length === 0) return false;
|
||||
const db = getDbInstance() as unknown as DbLike;
|
||||
|
||||
// Cheap eligibility probe so the backup only runs when a write is plausible.
|
||||
// The transaction below re-reads under lock and remains the authority.
|
||||
if (!hasCodexScopeCooldown(id, scope)) return false;
|
||||
|
||||
backupDbFile("pre-write");
|
||||
const wrote = db.transaction(() => {
|
||||
const existing = db
|
||||
.prepare(
|
||||
"SELECT provider, provider_specific_data FROM provider_connections WHERE id = ?"
|
||||
)
|
||||
.get(id);
|
||||
if (!existing) return false;
|
||||
const existingRecord = toRecord(rowToCamel(existing));
|
||||
if (existingRecord.provider !== "codex") return false;
|
||||
const currentPsd = toRecord(existingRecord.providerSpecificData);
|
||||
const currentCooldowns = { ...toRecord(currentPsd.codexScopeRateLimitedUntil) };
|
||||
if (!currentCooldowns[scope]) return false;
|
||||
|
||||
const currentSources = { ...toRecord(currentPsd.codexScopeRateLimitSource) };
|
||||
const curSource = currentSources[scope];
|
||||
const curUntilMs =
|
||||
typeof currentCooldowns[scope] === "string"
|
||||
? new Date(currentCooldowns[scope] as string).getTime()
|
||||
: NaN;
|
||||
// An upstream-authoritative `quota_reset` deadline outranks a local snapshot:
|
||||
// the grace window absorbs clock skew against the quota server, so a fast
|
||||
// local clock cannot lift a cooldown upstream still considers active.
|
||||
if (
|
||||
curSource === "quota_reset" &&
|
||||
Number.isFinite(curUntilMs) &&
|
||||
curUntilMs > Date.now() - QUOTA_RESET_CLOCK_SKEW_GRACE_MS
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
delete currentCooldowns[scope];
|
||||
delete currentSources[scope];
|
||||
|
||||
const nextPsd: JsonRecord = { ...currentPsd };
|
||||
const nextCooldowns = omitEmptyRecord(currentCooldowns);
|
||||
const nextSources = omitEmptyRecord(currentSources);
|
||||
if (nextCooldowns) nextPsd.codexScopeRateLimitedUntil = nextCooldowns;
|
||||
else delete nextPsd.codexScopeRateLimitedUntil;
|
||||
if (nextSources) nextPsd.codexScopeRateLimitSource = nextSources;
|
||||
else delete nextPsd.codexScopeRateLimitSource;
|
||||
|
||||
const exhaustedByScope = { ...toRecord(currentPsd.codexExhaustedWindowByScope) };
|
||||
if (exhaustedByScope[scope]) {
|
||||
delete exhaustedByScope[scope];
|
||||
const nextExhausted = omitEmptyRecord(exhaustedByScope);
|
||||
if (nextExhausted) {
|
||||
nextPsd.codexExhaustedWindowByScope = nextExhausted;
|
||||
} else {
|
||||
delete nextPsd.codexExhaustedWindowByScope;
|
||||
delete nextPsd.codexExhaustedWindow;
|
||||
}
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
db.prepare(
|
||||
`UPDATE provider_connections
|
||||
SET provider_specific_data = ?, updated_at = ?
|
||||
WHERE id = ?`
|
||||
).run(JSON.stringify(nextPsd), now, id);
|
||||
return true;
|
||||
})();
|
||||
|
||||
if (wrote) invalidateDbCache("connections");
|
||||
return wrote;
|
||||
}
|
||||
|
||||
/** Persist one child cooldown through the shared scoped quota-state transaction. */
|
||||
export async function updateCodexScopeCooldown(
|
||||
id: string,
|
||||
|
||||
@@ -4,6 +4,9 @@
|
||||
|
||||
import { getDbInstance } from "../core";
|
||||
import { invalidateDbCache } from "../readCache";
|
||||
import { backupDbFile } from "../backup";
|
||||
import { bumpProxyConfigGeneration } from "../settings";
|
||||
import { stripCodexChildCooldownsFromConnection } from "./codexAccountState";
|
||||
|
||||
interface StatementLike<TRow = unknown> {
|
||||
all: (...params: unknown[]) => TRow[];
|
||||
@@ -33,6 +36,10 @@ export function setConnectionRateLimitUntil(connectionId: string, until: number
|
||||
// is the only clear path (via clearConnectionRateLimit); past/zero
|
||||
// timestamps are noops so an expired write cannot overwrite a live row.
|
||||
if (until !== null && (!Number.isFinite(until) || until <= Date.now())) return;
|
||||
if (until == null) {
|
||||
stripCodexChildCooldownsFromConnection(connectionId, { alsoClearTopLevel: true });
|
||||
return;
|
||||
}
|
||||
const db = getDbInstance() as unknown as DbLike;
|
||||
db.prepare(
|
||||
"UPDATE provider_connections SET rate_limited_until = ?, updated_at = ? WHERE id = ?"
|
||||
@@ -234,6 +241,72 @@ export function clearStaleCrashCooldowns(): { cleared: number } {
|
||||
return { cleared: toReset.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomic conditional clear of recoverable error state on a connection row.
|
||||
*
|
||||
* Returns true when the row was cleared, false when a concurrent writer
|
||||
* (markAccountUnavailable, connectionRecovery tick, test, etc.) changed the
|
||||
* row between the caller's snapshot read and this UPDATE — in which case the
|
||||
* clear is skipped to preserve the freshest error state. Closes the TOCTOU
|
||||
* window in the quota-recovery path.
|
||||
*
|
||||
* CAS token = (test_status, last_error_at, rate_limited_until).
|
||||
* Nested Codex child cooldown maps are stripped in the same UPDATE so a
|
||||
* concurrent writer cannot re-persist them between two statements.
|
||||
*/
|
||||
export async function clearConnectionErrorIfUnchanged(
|
||||
id: string,
|
||||
expected: {
|
||||
testStatus: string | null | undefined;
|
||||
lastErrorAt: string | null | undefined;
|
||||
rateLimitedUntil: string | null | undefined;
|
||||
}
|
||||
): Promise<boolean> {
|
||||
const db = getDbInstance() as unknown as DbLike;
|
||||
backupDbFile("pre-write");
|
||||
const result = db
|
||||
.prepare(
|
||||
`
|
||||
UPDATE provider_connections SET
|
||||
test_status = 'active',
|
||||
last_error = NULL,
|
||||
last_error_at = NULL,
|
||||
last_error_type = NULL,
|
||||
last_error_source = NULL,
|
||||
error_code = NULL,
|
||||
rate_limited_until = NULL,
|
||||
backoff_level = 0,
|
||||
provider_specific_data = CASE
|
||||
WHEN provider = 'codex' AND json_valid(provider_specific_data)
|
||||
THEN json_remove(
|
||||
provider_specific_data,
|
||||
'$.codexScopeRateLimitedUntil',
|
||||
'$.codexScopeRateLimitSource'
|
||||
)
|
||||
ELSE provider_specific_data
|
||||
END,
|
||||
updated_at = ?
|
||||
WHERE id = ?
|
||||
AND IFNULL(test_status, '') = ?
|
||||
AND IFNULL(last_error_at, '') = ?
|
||||
AND IFNULL(rate_limited_until, '') = ?
|
||||
`
|
||||
)
|
||||
.run(
|
||||
new Date().toISOString(),
|
||||
id,
|
||||
expected.testStatus ?? "",
|
||||
expected.lastErrorAt ?? "",
|
||||
expected.rateLimitedUntil ?? ""
|
||||
);
|
||||
const applied = (result.changes ?? 0) > 0;
|
||||
if (applied) {
|
||||
invalidateDbCache("connections");
|
||||
bumpProxyConfigGeneration();
|
||||
}
|
||||
return applied;
|
||||
}
|
||||
|
||||
// T13: Format a reset countdown as a human-readable string ("2h 35m" / "4m 30s").
|
||||
// The implementation lives in the client-safe formatting utils so client
|
||||
// components (e.g. CoolingConnectionsPanel) can import it without pulling this
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import { getDbInstance, rowToCamel } from "./core";
|
||||
import type { QuotaSnapshotRow, ProviderUtilizationPoint } from "@/shared/types/utilization";
|
||||
import {
|
||||
hasCodexScopeCooldown,
|
||||
liftCodexScopeCooldownOnHeadroom,
|
||||
} from "./providers/codexAccountState";
|
||||
import { isCodexSparkQuotaKey } from "@omniroute/open-sse/config/codexQuotaScopes";
|
||||
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
|
||||
@@ -45,6 +50,61 @@ export function saveQuotaSnapshot(snapshot: Omit<QuotaSnapshotRow, "id" | "creat
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
// #12860: When a new snapshot demonstrates headroom on a Codex connection,
|
||||
// lift any fallback-sourced scope cooldown parked by quota preflight.
|
||||
maybeLiftCodexCooldownOnHeadroom(snapshot);
|
||||
}
|
||||
|
||||
/**
|
||||
* `rowToCamel` leaves snake_case keys intact on some driver paths, so accept
|
||||
* either casing rather than trusting one shape.
|
||||
*/
|
||||
type SnapshotShape = Partial<QuotaSnapshotRow> & {
|
||||
windowKey?: string;
|
||||
remainingPercentage?: number;
|
||||
isExhausted?: number;
|
||||
};
|
||||
|
||||
function snapshotHasHeadroom(s: SnapshotShape): boolean {
|
||||
const pct = s.remainingPercentage ?? s.remaining_percentage ?? 0;
|
||||
const exhausted = s.isExhausted ?? s.is_exhausted ?? 0;
|
||||
return pct > 0 && exhausted !== 1;
|
||||
}
|
||||
|
||||
function maybeLiftCodexCooldownOnHeadroom(
|
||||
snapshot: Omit<QuotaSnapshotRow, "id" | "created_at">
|
||||
): void {
|
||||
if (
|
||||
snapshot.provider?.toLowerCase() !== "codex" ||
|
||||
typeof snapshot.connection_id !== "string" ||
|
||||
snapshot.connection_id.length === 0 ||
|
||||
(snapshot.remaining_percentage ?? 0) <= 0 ||
|
||||
snapshot.is_exhausted === 1
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const scope = isCodexSparkQuotaKey(snapshot.window_key) ? "spark" : "codex";
|
||||
// The scope-wide read is only worth paying for when a cooldown is
|
||||
// actually parked on this connection+scope; the common case is clean.
|
||||
if (!hasCodexScopeCooldown(snapshot.connection_id, scope)) return;
|
||||
|
||||
const scopeWindows = getLatestQuotaSnapshotsForConnection(snapshot.connection_id).filter(
|
||||
(s: SnapshotShape) => {
|
||||
const key = s.windowKey ?? s.window_key;
|
||||
return scope === "spark" ? isCodexSparkQuotaKey(key) : !isCodexSparkQuotaKey(key);
|
||||
}
|
||||
);
|
||||
// Every window in the scope must be healthy: one exhausted window still
|
||||
// justifies the cooldown even when a sibling reports full headroom.
|
||||
if (scopeWindows.length > 0 && scopeWindows.every(snapshotHasHeadroom)) {
|
||||
liftCodexScopeCooldownOnHeadroom(snapshot.connection_id, scope);
|
||||
}
|
||||
} catch (err) {
|
||||
console.debug("[QuotaSnapshots] Headroom evaluation skipped:", err);
|
||||
}
|
||||
}
|
||||
|
||||
export function getQuotaSnapshots(opts: {
|
||||
|
||||
@@ -11,6 +11,7 @@ process.env.API_KEY_SECRET = "test-codex-reset-credits-secret";
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const providersDb = await import("../../src/lib/db/providers.ts");
|
||||
const resetCredits = await import("../../src/lib/usage/codexResetCredits.ts");
|
||||
const codexAccount = await import("../../open-sse/services/codexAccount/index.ts");
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
type QuotaUsageRecord = Record<string, { used?: unknown } | undefined>;
|
||||
@@ -45,6 +46,69 @@ test.after(async () => {
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
||||
});
|
||||
|
||||
async function persistBothChildCooldowns(id: string) {
|
||||
const codexUntil = new Date(Date.now() + 60_000).toISOString();
|
||||
const sparkUntil = new Date(Date.now() + 120_000).toISOString();
|
||||
await codexAccount.persistCodexChildCooldown({
|
||||
connectionId: id,
|
||||
model: "gpt-5.5",
|
||||
rateLimitedUntil: codexUntil,
|
||||
});
|
||||
await codexAccount.persistCodexChildCooldown({
|
||||
connectionId: id,
|
||||
model: "gpt-5.3-codex-spark",
|
||||
rateLimitedUntil: sparkUntil,
|
||||
});
|
||||
return { codexUntil, sparkUntil };
|
||||
}
|
||||
|
||||
async function readConnection(id: string) {
|
||||
const connection = await providersDb.getProviderConnectionById(id);
|
||||
assert.ok(connection);
|
||||
return connection as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function mockResetThenUsage(opts: {
|
||||
consumeBody: unknown;
|
||||
consumeStatus?: number;
|
||||
usageBody?: unknown;
|
||||
usageStatus?: number;
|
||||
}) {
|
||||
globalThis.fetch = (async (url) => {
|
||||
const href = String(url);
|
||||
if (href.endsWith("/rate-limit-reset-credits")) {
|
||||
return new Response(
|
||||
JSON.stringify({ credits: [{ id: "credit-123", status: "available" }] }),
|
||||
{ status: 200, headers: { "content-type": "application/json" } }
|
||||
);
|
||||
}
|
||||
if (href.includes("/rate-limit-reset-credits/consume")) {
|
||||
return new Response(JSON.stringify(opts.consumeBody), {
|
||||
status: opts.consumeStatus ?? 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
if (href.includes("/backend-api/wham/usage")) {
|
||||
if (opts.usageStatus && opts.usageStatus >= 500) {
|
||||
return new Response("upstream down", { status: opts.usageStatus });
|
||||
}
|
||||
return new Response(
|
||||
JSON.stringify(
|
||||
opts.usageBody ?? {
|
||||
plan_type: "plus",
|
||||
rate_limit: {
|
||||
primary_window: { used_percent: 0 },
|
||||
secondary_window: { used_percent: 0 },
|
||||
},
|
||||
}
|
||||
),
|
||||
{ status: 200, headers: { "content-type": "application/json" } }
|
||||
);
|
||||
}
|
||||
return new Response("unexpected", { status: 500 });
|
||||
}) as typeof fetch;
|
||||
}
|
||||
|
||||
test("consumeCodexResetCredit fetches a credit id, posts it, then refreshes usage", async () => {
|
||||
const connection = (await createCodexConnection()) as { id: string };
|
||||
const calls: Array<{ url: string; init: RequestInit }> = [];
|
||||
@@ -355,3 +419,92 @@ test("consumeCodexResetCredit rejects non-Codex and missing connections", async
|
||||
error.code === "codex_provider_required"
|
||||
);
|
||||
});
|
||||
|
||||
test("#12951 consumeCodexResetCredit lifts fallback Codex child cooldown after healthy usage refresh", async () => {
|
||||
const connection = (await createCodexConnection()) as { id: string };
|
||||
const parked = await persistBothChildCooldowns(connection.id);
|
||||
const before = await readConnection(connection.id);
|
||||
assert.equal(codexAccount.getCodexChildCooldown(before as never, "gpt-5.5"), parked.codexUntil);
|
||||
assert.equal(
|
||||
codexAccount.getCodexChildCooldown(before as never, "gpt-5.3-codex-spark"),
|
||||
parked.sparkUntil
|
||||
);
|
||||
|
||||
mockResetThenUsage({ consumeBody: { code: "reset" } });
|
||||
|
||||
const result = await resetCredits.consumeCodexResetCredit(connection.id, "redeem-button-path");
|
||||
assert.equal(result.outcome, "reset");
|
||||
|
||||
const after = await readConnection(connection.id);
|
||||
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), null);
|
||||
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
|
||||
assert.equal(
|
||||
codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"),
|
||||
parked.sparkUntil
|
||||
);
|
||||
});
|
||||
|
||||
test("#12951 consumeCodexResetCredit alreadyRedeemed still needs healthy usage to lift fallback cooldown", async () => {
|
||||
const connection = (await createCodexConnection()) as { id: string };
|
||||
await persistBothChildCooldowns(connection.id);
|
||||
|
||||
mockResetThenUsage({
|
||||
consumeBody: { code: "alreadyRedeemed" },
|
||||
usageBody: {
|
||||
plan_type: "plus",
|
||||
rate_limit: {
|
||||
primary_window: { used_percent: 100 },
|
||||
secondary_window: { used_percent: 100 },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await resetCredits.consumeCodexResetCredit(connection.id, "redeem-already");
|
||||
assert.equal(result.outcome, "alreadyRedeemed");
|
||||
|
||||
const after = await readConnection(connection.id);
|
||||
assert.ok(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"));
|
||||
assert.ok(codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"));
|
||||
});
|
||||
|
||||
test("#12951 consumeCodexResetCredit failed redeem keeps nested child cooldowns", async () => {
|
||||
const connection = (await createCodexConnection()) as { id: string };
|
||||
const parked = await persistBothChildCooldowns(connection.id);
|
||||
|
||||
mockResetThenUsage({
|
||||
consumeBody: { code: "noCredit" },
|
||||
consumeStatus: 409,
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
() => resetCredits.consumeCodexResetCredit(connection.id, "redeem-fail"),
|
||||
(error: unknown) =>
|
||||
error instanceof resetCredits.CodexResetCreditError &&
|
||||
error.status === 409 &&
|
||||
error.code === "no_credit"
|
||||
);
|
||||
|
||||
const after = await readConnection(connection.id);
|
||||
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), parked.codexUntil);
|
||||
assert.equal(
|
||||
codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"),
|
||||
parked.sparkUntil
|
||||
);
|
||||
});
|
||||
|
||||
test("#12951 consumeCodexResetCredit incomplete usage refresh keeps nested child cooldowns", async () => {
|
||||
const connection = (await createCodexConnection()) as { id: string };
|
||||
const parked = await persistBothChildCooldowns(connection.id);
|
||||
|
||||
mockResetThenUsage({ consumeBody: { code: "reset" }, usageStatus: 500 });
|
||||
|
||||
const result = await resetCredits.consumeCodexResetCredit(connection.id, "redeem-incomplete");
|
||||
assert.equal(result.outcome, "reset");
|
||||
|
||||
const after = await readConnection(connection.id);
|
||||
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), parked.codexUntil);
|
||||
assert.equal(
|
||||
codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"),
|
||||
parked.sparkUntil
|
||||
);
|
||||
});
|
||||
|
||||
449
tests/unit/codex-scope-cooldown-clear-12817.test.ts
Normal file
449
tests/unit/codex-scope-cooldown-clear-12817.test.ts
Normal file
@@ -0,0 +1,449 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-codex-clear-12817-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "codex-clear-12817-secret";
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const providersDb = await import("../../src/lib/db/providers.ts");
|
||||
const rateLimit = await import("../../src/lib/db/providers/rateLimit.ts");
|
||||
const codexAccount = await import("../../open-sse/services/codexAccount/index.ts");
|
||||
const quotaSnapshots = await import("../../src/lib/db/quotaSnapshots.ts");
|
||||
const quotaCache = await import("../../src/domain/quotaCache.ts");
|
||||
|
||||
async function resetStorage(): Promise<void> {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
interface SeededConnection {
|
||||
id: string;
|
||||
providerSpecificData: Record<string, unknown>;
|
||||
}
|
||||
|
||||
async function seedCodexConnection(): Promise<SeededConnection> {
|
||||
return providersDb.createProviderConnection({
|
||||
provider: "codex",
|
||||
authType: "oauth",
|
||||
name: "codex-clear-12817",
|
||||
email: "codex-clear-12817@example.com",
|
||||
apiKey: "codex-clear-12817-key",
|
||||
accessToken: "codex-clear-12817-access",
|
||||
refreshToken: "codex-clear-12817-refresh",
|
||||
providerSpecificData: {
|
||||
unrelated: { retained: true },
|
||||
},
|
||||
}) as unknown as Promise<SeededConnection>;
|
||||
}
|
||||
|
||||
async function seedGlmConnection(): Promise<SeededConnection> {
|
||||
return providersDb.createProviderConnection({
|
||||
provider: "glm",
|
||||
authType: "apikey",
|
||||
name: "glm-clear-12817",
|
||||
apiKey: "glm-clear-12817-key",
|
||||
providerSpecificData: {
|
||||
leftover: "keep-me",
|
||||
},
|
||||
}) as unknown as Promise<SeededConnection>;
|
||||
}
|
||||
|
||||
async function readConnection(id: string): Promise<Record<string, unknown>> {
|
||||
const connection = await providersDb.getProviderConnectionById(id);
|
||||
assert.ok(connection);
|
||||
return connection as unknown as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function psd(connection: Record<string, unknown>): Record<string, unknown> {
|
||||
return (connection.providerSpecificData ?? {}) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function futureIso(ms: number): string {
|
||||
return new Date(Date.now() + ms).toISOString();
|
||||
}
|
||||
|
||||
function quotaHeaders(resetAt5h: string, resetAt7d: string, usage5h = "10") {
|
||||
return {
|
||||
"x-codex-5h-usage": usage5h,
|
||||
"x-codex-5h-limit": "100",
|
||||
"x-codex-5h-reset-at": resetAt5h,
|
||||
"x-codex-7d-usage": "10",
|
||||
"x-codex-7d-limit": "100",
|
||||
"x-codex-7d-reset-at": resetAt7d,
|
||||
};
|
||||
}
|
||||
|
||||
async function persistBothChildCooldowns(id: string): Promise<{
|
||||
codexUntil: string;
|
||||
sparkUntil: string;
|
||||
}> {
|
||||
const codexUntil = futureIso(60_000);
|
||||
const sparkUntil = futureIso(120_000);
|
||||
await codexAccount.persistCodexChildCooldown({
|
||||
connectionId: id,
|
||||
model: "gpt-5.5",
|
||||
rateLimitedUntil: codexUntil,
|
||||
});
|
||||
await codexAccount.persistCodexChildCooldown({
|
||||
connectionId: id,
|
||||
model: "gpt-5.3-codex-spark",
|
||||
rateLimitedUntil: sparkUntil,
|
||||
});
|
||||
return { codexUntil, sparkUntil };
|
||||
}
|
||||
|
||||
test.beforeEach(resetStorage);
|
||||
|
||||
test.after(() => {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
||||
});
|
||||
|
||||
test("#12817 PUT rateLimitedUntil:null also drops nested Codex child cooldowns", async () => {
|
||||
const connection = await seedCodexConnection();
|
||||
const { sparkUntil } = await persistBothChildCooldowns(connection.id);
|
||||
await providersDb.updateProviderConnection(connection.id, {
|
||||
rateLimitedUntil: futureIso(90_000),
|
||||
});
|
||||
|
||||
const before = await readConnection(connection.id);
|
||||
assert.equal(
|
||||
codexAccount.getCodexChildCooldown(before as never, "gpt-5.3-codex-spark"),
|
||||
sparkUntil
|
||||
);
|
||||
|
||||
await providersDb.updateProviderConnection(connection.id, { rateLimitedUntil: null });
|
||||
|
||||
const after = await readConnection(connection.id);
|
||||
const data = psd(after);
|
||||
assert.equal(after.rateLimitedUntil, undefined);
|
||||
assert.equal(data.codexScopeRateLimitedUntil, undefined);
|
||||
assert.equal(data.codexScopeRateLimitSource, undefined);
|
||||
assert.deepEqual(data.unrelated, { retained: true });
|
||||
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), null);
|
||||
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"), null);
|
||||
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
|
||||
});
|
||||
|
||||
test("#12817 PUT rateLimitedUntil:\"\" also drops nested Codex child cooldowns", async () => {
|
||||
const connection = await seedCodexConnection();
|
||||
await persistBothChildCooldowns(connection.id);
|
||||
await providersDb.updateProviderConnection(connection.id, {
|
||||
rateLimitedUntil: futureIso(90_000),
|
||||
});
|
||||
|
||||
await providersDb.updateProviderConnection(connection.id, { rateLimitedUntil: "" });
|
||||
|
||||
const after = await readConnection(connection.id);
|
||||
const data = psd(after);
|
||||
assert.equal(after.rateLimitedUntil, undefined);
|
||||
assert.equal(data.codexScopeRateLimitedUntil, undefined);
|
||||
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
|
||||
});
|
||||
|
||||
test("#12817 clearConnectionRateLimit strips nested Codex child cooldowns", async () => {
|
||||
const connection = await seedCodexConnection();
|
||||
await persistBothChildCooldowns(connection.id);
|
||||
rateLimit.setConnectionRateLimitUntil(connection.id, Date.now() + 90_000);
|
||||
|
||||
rateLimit.clearConnectionRateLimit(connection.id);
|
||||
|
||||
const after = await readConnection(connection.id);
|
||||
const data = psd(after);
|
||||
assert.equal(after.rateLimitedUntil, undefined);
|
||||
assert.equal(data.codexScopeRateLimitedUntil, undefined);
|
||||
assert.equal(data.codexScopeRateLimitSource, undefined);
|
||||
assert.deepEqual(data.unrelated, { retained: true });
|
||||
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
|
||||
});
|
||||
|
||||
test("#12817 CAS error-clear also strips nested Codex child cooldowns", async () => {
|
||||
const connection = await seedCodexConnection();
|
||||
await persistBothChildCooldowns(connection.id);
|
||||
const until = futureIso(90_000);
|
||||
await providersDb.updateProviderConnection(connection.id, {
|
||||
testStatus: "unavailable",
|
||||
lastError: "429",
|
||||
lastErrorAt: new Date().toISOString(),
|
||||
lastErrorType: "rate_limit_exceeded",
|
||||
rateLimitedUntil: until,
|
||||
});
|
||||
const before = await readConnection(connection.id);
|
||||
|
||||
const applied = await providersDb.clearConnectionErrorIfUnchanged(connection.id, {
|
||||
testStatus: (before.testStatus as string) ?? null,
|
||||
lastErrorAt: (before.lastErrorAt as string) ?? null,
|
||||
rateLimitedUntil: (before.rateLimitedUntil as string) ?? null,
|
||||
});
|
||||
assert.equal(applied, true);
|
||||
|
||||
const after = await readConnection(connection.id);
|
||||
assert.equal(after.testStatus, "active");
|
||||
assert.equal(psd(after).codexScopeRateLimitedUntil, undefined);
|
||||
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
|
||||
});
|
||||
|
||||
test("#12817 a successful quota observation clears that child's leftover cooldown", async () => {
|
||||
const connection = await seedCodexConnection();
|
||||
const reset5h = futureIso(60_000);
|
||||
const reset7d = futureIso(600_000);
|
||||
|
||||
await persistBothChildCooldowns(connection.id);
|
||||
await codexAccount.persistCodexChildQuotaResponse({
|
||||
connectionId: connection.id,
|
||||
model: "gpt-5.5",
|
||||
headers: quotaHeaders(reset5h, reset7d, "95"),
|
||||
status: 429,
|
||||
});
|
||||
await codexAccount.persistCodexChildQuotaResponse({
|
||||
connectionId: connection.id,
|
||||
model: "gpt-5.5",
|
||||
headers: quotaHeaders(reset5h, reset7d, "10"),
|
||||
status: 200,
|
||||
});
|
||||
|
||||
const after = await readConnection(connection.id);
|
||||
const data = psd(after);
|
||||
const until = data.codexScopeRateLimitedUntil as Record<string, unknown> | undefined;
|
||||
const exhausted = data.codexExhaustedWindowByScope as Record<string, unknown> | undefined;
|
||||
assert.equal(until?.codex, undefined);
|
||||
assert.equal(typeof until?.spark, "string");
|
||||
assert.equal(exhausted?.codex, undefined);
|
||||
assert.deepEqual(data.unrelated, { retained: true });
|
||||
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
|
||||
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.3-codex-spark"), true);
|
||||
});
|
||||
|
||||
test("#12817 clearing a non-Codex cooldown leaves providerSpecificData alone", async () => {
|
||||
const connection = await seedGlmConnection();
|
||||
await providersDb.updateProviderConnection(connection.id, {
|
||||
rateLimitedUntil: futureIso(90_000),
|
||||
});
|
||||
await providersDb.updateProviderConnection(connection.id, { rateLimitedUntil: null });
|
||||
const after = await readConnection(connection.id);
|
||||
assert.equal(after.rateLimitedUntil, undefined);
|
||||
assert.equal(psd(after).leftover, "keep-me");
|
||||
});
|
||||
|
||||
test("#12860 saveQuotaSnapshot with headroom lifts fallback-sourced scope cooldown", async () => {
|
||||
const connection = await seedCodexConnection();
|
||||
await persistBothChildCooldowns(connection.id);
|
||||
|
||||
const before = await readConnection(connection.id);
|
||||
assert.ok(codexAccount.getCodexChildCooldown(before as never, "gpt-5.5"));
|
||||
|
||||
quotaSnapshots.saveQuotaSnapshot({
|
||||
provider: "codex",
|
||||
connection_id: connection.id,
|
||||
window_key: "primary",
|
||||
remaining_percentage: 100,
|
||||
is_exhausted: 0,
|
||||
next_reset_at: futureIso(3600_000),
|
||||
window_duration_ms: 18_000_000,
|
||||
raw_data: null,
|
||||
});
|
||||
|
||||
const after = await readConnection(connection.id);
|
||||
const data = psd(after);
|
||||
const until = data.codexScopeRateLimitedUntil as Record<string, unknown> | undefined;
|
||||
assert.equal(until?.codex, undefined);
|
||||
assert.equal(typeof until?.spark, "string");
|
||||
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), null);
|
||||
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
|
||||
});
|
||||
|
||||
test("#12860 saveQuotaSnapshot with headroom does NOT lift authoritative quota_reset cooldown", async () => {
|
||||
const connection = await seedCodexConnection();
|
||||
const reset5h = futureIso(120_000);
|
||||
const reset7d = futureIso(600_000);
|
||||
|
||||
await codexAccount.persistCodexChildQuotaResponse({
|
||||
connectionId: connection.id,
|
||||
model: "gpt-5.5",
|
||||
headers: quotaHeaders(reset5h, reset7d, "95"),
|
||||
status: 429,
|
||||
});
|
||||
|
||||
const before = await readConnection(connection.id);
|
||||
assert.ok(codexAccount.getCodexChildCooldown(before as never, "gpt-5.5"));
|
||||
|
||||
quotaSnapshots.saveQuotaSnapshot({
|
||||
provider: "codex",
|
||||
connection_id: connection.id,
|
||||
window_key: "session",
|
||||
remaining_percentage: 100,
|
||||
is_exhausted: 0,
|
||||
next_reset_at: futureIso(3600_000),
|
||||
window_duration_ms: 18_000_000,
|
||||
raw_data: null,
|
||||
});
|
||||
|
||||
const after = await readConnection(connection.id);
|
||||
const data = psd(after);
|
||||
const until = data.codexScopeRateLimitedUntil as Record<string, unknown> | undefined;
|
||||
assert.equal(typeof until?.codex, "string");
|
||||
assert.ok(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"));
|
||||
});
|
||||
|
||||
test("#12860 saveQuotaSnapshot does NOT lift scope cooldown if another window for that scope is still exhausted", async () => {
|
||||
const connection = await seedCodexConnection();
|
||||
await codexAccount.persistCodexChildCooldown({
|
||||
connectionId: connection.id,
|
||||
model: "gpt-5.5",
|
||||
rateLimitedUntil: futureIso(120_000),
|
||||
});
|
||||
|
||||
// Weekly window is currently exhausted (0% remaining)
|
||||
quotaSnapshots.saveQuotaSnapshot({
|
||||
provider: "codex",
|
||||
connection_id: connection.id,
|
||||
window_key: "weekly",
|
||||
remaining_percentage: 0,
|
||||
is_exhausted: 1,
|
||||
next_reset_at: futureIso(7200_000),
|
||||
window_duration_ms: 604_800_000,
|
||||
raw_data: null,
|
||||
});
|
||||
|
||||
// Session window reports 100% headroom, but weekly is still exhausted
|
||||
quotaSnapshots.saveQuotaSnapshot({
|
||||
provider: "codex",
|
||||
connection_id: connection.id,
|
||||
window_key: "session",
|
||||
remaining_percentage: 100,
|
||||
is_exhausted: 0,
|
||||
next_reset_at: futureIso(3600_000),
|
||||
window_duration_ms: 18_000_000,
|
||||
raw_data: null,
|
||||
});
|
||||
|
||||
let mid = await readConnection(connection.id);
|
||||
assert.ok(codexAccount.getCodexChildCooldown(mid as never, "gpt-5.5"));
|
||||
|
||||
// Now weekly also recovers to 100% headroom
|
||||
quotaSnapshots.saveQuotaSnapshot({
|
||||
provider: "codex",
|
||||
connection_id: connection.id,
|
||||
window_key: "weekly",
|
||||
remaining_percentage: 100,
|
||||
is_exhausted: 0,
|
||||
next_reset_at: futureIso(7200_000),
|
||||
window_duration_ms: 604_800_000,
|
||||
raw_data: null,
|
||||
});
|
||||
|
||||
let after = await readConnection(connection.id);
|
||||
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), null);
|
||||
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
|
||||
});
|
||||
|
||||
test("#12860 saveQuotaSnapshot for spark scope with headroom lifts only spark cooldown", async () => {
|
||||
const connection = await seedCodexConnection();
|
||||
await persistBothChildCooldowns(connection.id);
|
||||
|
||||
quotaSnapshots.saveQuotaSnapshot({
|
||||
provider: "codex",
|
||||
connection_id: connection.id,
|
||||
window_key: "gpt_5_3_codex_spark_session",
|
||||
remaining_percentage: 100,
|
||||
is_exhausted: 0,
|
||||
next_reset_at: futureIso(3600_000),
|
||||
window_duration_ms: 18_000_000,
|
||||
raw_data: null,
|
||||
});
|
||||
|
||||
const after = await readConnection(connection.id);
|
||||
const data = psd(after);
|
||||
const until = data.codexScopeRateLimitedUntil as Record<string, unknown> | undefined;
|
||||
assert.equal(typeof until?.codex, "string");
|
||||
assert.equal(until?.spark, undefined);
|
||||
assert.ok(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"));
|
||||
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.3-codex-spark"), null);
|
||||
});
|
||||
|
||||
test("#12860 setQuotaCache with fresh usage headroom lifts fallback-sourced scope cooldown", async () => {
|
||||
const connection = await seedCodexConnection();
|
||||
await persistBothChildCooldowns(connection.id);
|
||||
|
||||
const before = await readConnection(connection.id);
|
||||
assert.ok(codexAccount.getCodexChildCooldown(before as never, "gpt-5.5"));
|
||||
|
||||
quotaCache.setQuotaCache(connection.id, "codex", {
|
||||
session: {
|
||||
used: 0,
|
||||
total: 100,
|
||||
remainingPercentage: 100,
|
||||
resetAt: futureIso(3600_000),
|
||||
},
|
||||
});
|
||||
|
||||
const after = await readConnection(connection.id);
|
||||
const data = psd(after);
|
||||
const until = data.codexScopeRateLimitedUntil as Record<string, unknown> | undefined;
|
||||
assert.equal(until?.codex, undefined);
|
||||
assert.equal(typeof until?.spark, "string");
|
||||
assert.equal(codexAccount.getCodexChildCooldown(after as never, "gpt-5.5"), null);
|
||||
assert.equal(codexAccount.isCodexChildUnavailable(after as never, "gpt-5.5"), false);
|
||||
});
|
||||
|
||||
test("#12860 quota_reset cooldown that just elapsed is still held by the skew grace window", async () => {
|
||||
const connection = await seedCodexConnection();
|
||||
// Reset deadline sits 5s in the past — inside the 30s clock-skew grace, so a
|
||||
// fast local clock must not lift an upstream-authoritative cooldown early.
|
||||
const justElapsed = new Date(Date.now() - 5_000).toISOString();
|
||||
|
||||
await providersDb.updateProviderConnection(connection.id, {
|
||||
providerSpecificData: {
|
||||
codexScopeRateLimitedUntil: { codex: justElapsed },
|
||||
codexScopeRateLimitSource: { codex: "quota_reset" },
|
||||
},
|
||||
});
|
||||
|
||||
const lifted = providersDb.liftCodexScopeCooldownOnHeadroom(connection.id, "codex");
|
||||
assert.equal(lifted, false);
|
||||
|
||||
const after = await readConnection(connection.id);
|
||||
const until = psd(after).codexScopeRateLimitedUntil as Record<string, unknown> | undefined;
|
||||
assert.equal(until?.codex, justElapsed);
|
||||
});
|
||||
|
||||
test("#12860 quota_reset cooldown past the skew grace window is lifted", async () => {
|
||||
const connection = await seedCodexConnection();
|
||||
const wellElapsed = new Date(Date.now() - 120_000).toISOString();
|
||||
|
||||
await providersDb.updateProviderConnection(connection.id, {
|
||||
providerSpecificData: {
|
||||
codexScopeRateLimitedUntil: { codex: wellElapsed },
|
||||
codexScopeRateLimitSource: { codex: "quota_reset" },
|
||||
},
|
||||
});
|
||||
|
||||
const lifted = providersDb.liftCodexScopeCooldownOnHeadroom(connection.id, "codex");
|
||||
assert.equal(lifted, true);
|
||||
|
||||
const after = await readConnection(connection.id);
|
||||
assert.equal(psd(after).codexScopeRateLimitedUntil, undefined);
|
||||
});
|
||||
|
||||
test("#12860 hasCodexScopeCooldown short-circuits the snapshot scan when nothing is parked", async () => {
|
||||
const connection = await seedCodexConnection();
|
||||
assert.equal(providersDb.hasCodexScopeCooldown(connection.id, "codex"), false);
|
||||
|
||||
await codexAccount.persistCodexChildCooldown({
|
||||
connectionId: connection.id,
|
||||
model: "gpt-5.5",
|
||||
rateLimitedUntil: futureIso(120_000),
|
||||
});
|
||||
|
||||
assert.equal(providersDb.hasCodexScopeCooldown(connection.id, "codex"), true);
|
||||
assert.equal(providersDb.hasCodexScopeCooldown(connection.id, "spark"), false);
|
||||
|
||||
const glm = await seedGlmConnection();
|
||||
assert.equal(providersDb.hasCodexScopeCooldown(glm.id, "codex"), false);
|
||||
assert.equal(providersDb.hasCodexScopeCooldown("does-not-exist", "codex"), false);
|
||||
});
|
||||
Reference in New Issue
Block a user