fix(codex): lift nested child cooldowns on parent clear and on snapshot headroom (#12951)

Validado numa worktree combinada com a onda de persistência desta leva sobre `release/v3.8.51`: check-file-size e check-changelog-integrity OK, typecheck:core limpo, check-api-typecheck OK (289). Revalidei o head atual mergeado com o tip: typecheck:core limpo e **36/36** entre `db-rate-limit-guard` e as suítes desta PR. Levantar o cooldown do escopo pai sem deixar os filhos aninhados presos é o miolo — um cooldown órfão em filho é invisível no dashboard e mantém a conexão fora de rota sem explicação. **Nota de coordenação:** o `setConnectionRateLimitUntil` colidiu com o #12788 (guard contra timestamp não-finito ou já expirado), que mergeei nesta mesma onda. Eu tinha resolvido a integração na minha worktree, mas ao empurrar o push foi rejeitado — você já tinha empurrado `441fd44`, `f853ba5` e `af2ed01` com a integração feita, e a sua ordenação é equivalente à minha. Descartei a minha e mantive a sua; o crédito é seu inteiro. Fica o registro de que push rejeitado não é erro leve: se eu tivesse mergeado sem reler, teria levado a branch errada.
2026-09-18 21:02:50 +03:00 · 2026-09-10 07:15:57 -04:00
parent 9492357360
commit bfbd090a96
8 changed files with 979 additions and 85 deletions
--- a/src/lib/db/providers.ts
+++ b/src/lib/db/providers.ts
@@ -39,6 +39,7 @@ import { pickCodexConnectionForUser } from "@/lib/oauth/utils/codexConnectionSel
 import { isMicrosoftDesignerWebRetiredProviderId } from "@/shared/constants/designerWebRetirement";
 import { reconcileCodexUsageHistory } from "./providers/usageIdentityReconciliation";
 import { isRuntimeRetiredProviderId } from "@/shared/constants/providerRetirement";
+import { applyCodexChildCooldownClearOnUpdate } from "./providers/codexAccountState";

 /**
 * normalizeProviderSpecificData + the Codex fingerprint-seed invariant: Codex
@@ -951,11 +952,14 @@ export async function updateProviderConnection(id: string, data: JsonRecord) {
    ...data,
    updatedAt: new Date().toISOString(),
  };
-  merged.providerSpecificData = normalizeConnectionProviderSpecificData(
-    toStringOrNull(merged.provider),
-    merged.providerSpecificData,
-    merged,
-    existingCamel.providerSpecificData
+  merged.providerSpecificData = applyCodexChildCooldownClearOnUpdate(
+    data,
+    normalizeConnectionProviderSpecificData(
+      toStringOrNull(merged.provider),
+      merged.providerSpecificData,
+      merged,
+      existingCamel.providerSpecificData
+    )
  );
  // Mirror the sanitization the create path applies — keep the returned
  // object in lockstep with what we persist.
@@ -1024,65 +1028,13 @@ export async function updateProviderConnection(id: string, data: JsonRecord) {
 export {
  updateCodexScopedQuotaState,
  updateCodexScopeCooldown,
+  applyCodexChildCooldownClearOnUpdate,
+  stripCodexChildCooldownFields,
+  stripCodexChildCooldownsFromConnection,
+  hasCodexScopeCooldown,
+  liftCodexScopeCooldownOnHeadroom,
 } from "./providers/codexAccountState";

-/**
- * Atomic conditional clear of recoverable error state on a connection row.
- *
- * Returns true when the row was cleared, false when a concurrent writer
- * (markAccountUnavailable, connectionRecovery tick, test, etc.) changed the
- * row between the caller's snapshot read and this UPDATE — in which case the
- * clear is skipped to preserve the freshest error state. Closes the TOCTOU
- * window in the quota-recovery path.
- *
- * CAS token = (test_status, last_error_at, rate_limited_until).
- * markAccountUnavailable always bumps last_error_at on every cooldown/error
- * write, so an unchanged last_error_at reliably indicates no concurrent write.
- */
-export async function clearConnectionErrorIfUnchanged(
-  id: string,
-  expected: {
-    testStatus: string | null | undefined;
-    lastErrorAt: string | null | undefined;
-    rateLimitedUntil: string | null | undefined;
-  }
-): Promise<boolean> {
-  const db = getDbInstance() as unknown as DbLike;
-  const result = db
-    .prepare(
-      `
-    UPDATE provider_connections SET
-      test_status = 'active',
-      last_error = NULL,
-      last_error_at = NULL,
-      last_error_type = NULL,
-      last_error_source = NULL,
-      error_code = NULL,
-      rate_limited_until = NULL,
-      backoff_level = 0,
-      updated_at = ?
-    WHERE id = ?
-      AND IFNULL(test_status, '') = ?
-      AND IFNULL(last_error_at, '') = ?
-      AND IFNULL(rate_limited_until, '') = ?
-    `
-    )
-    .run(
-      new Date().toISOString(),
-      id,
-      expected.testStatus ?? "",
-      expected.lastErrorAt ?? "",
-      expected.rateLimitedUntil ?? ""
-    );
-  const applied = (result.changes ?? 0) > 0;
-  if (applied) {
-    backupDbFile("pre-write");
-    invalidateDbCache("connections");
-    bumpProxyConfigGeneration();
-  }
-  return applied;
-}
-
 /**
 * Lightweight stat bump — updates lastUsedAt and consecutiveUseCount without
 * SELECT, re-encrypt, cache invalidation, or file backup.
@@ -1183,4 +1135,5 @@ export {
  formatResetCountdown,
  isConnectionRateLimited,
  getRateLimitedConnections,
+  clearConnectionErrorIfUnchanged,
 } from "./providers/rateLimit";
--- a/src/lib/db/providers/codexAccountState.ts
+++ b/src/lib/db/providers/codexAccountState.ts
@@ -18,10 +18,105 @@ interface DbLike {
 type CodexScopedQuotaPatch = {
  quotaState?: JsonRecord;
  exhaustedWindow?: "5h" | "7d" | null;
-  rateLimitedUntil?: string;
+  rateLimitedUntil?: string | null;
  rateLimitSource?: "fallback" | "quota_reset";
 };

+const CODEX_CHILD_COOLDOWN_KEYS = [
+  "codexScopeRateLimitedUntil",
+  "codexScopeRateLimitSource",
+] as const;
+
+function omitEmptyRecord(record: JsonRecord): JsonRecord | undefined {
+  return Object.keys(record).length > 0 ? record : undefined;
+}
+
+/** Drop nested Codex child cooldowns; keep quota snapshots and unrelated keys. */
+export function stripCodexChildCooldownFields(psd: JsonRecord): JsonRecord {
+  if (!connectionHasCodexChildCooldown(psd)) return psd;
+  const next = { ...psd };
+  for (const key of CODEX_CHILD_COOLDOWN_KEYS) delete next[key];
+  return next;
+}
+
+/** PUT/CAS payload that clears the parent column must also drop nested maps. */
+export function applyCodexChildCooldownClearOnUpdate<T extends JsonRecord | undefined>(
+  data: JsonRecord,
+  psd: T
+): T {
+  if (psd == null) return psd;
+  if (!Object.hasOwn(data, "rateLimitedUntil")) return psd;
+  if (data.rateLimitedUntil != null && data.rateLimitedUntil !== "") return psd;
+  return stripCodexChildCooldownFields(psd) as T;
+}
+
+function connectionHasCodexChildCooldown(psd: JsonRecord): boolean {
+  return "codexScopeRateLimitedUntil" in psd || "codexScopeRateLimitSource" in psd;
+}
+
+/**
+ * Persist a full-parent cooldown lift into the nested Codex child maps.
+ * When `alsoClearTopLevel` is set, the parent `rate_limited_until` column is
+ * nulled in the same transaction so a crash between the two writes cannot
+ * leave a nested child map behind a cleared parent column.
+ */
+export function stripCodexChildCooldownsFromConnection(
+  id: string,
+  options?: { alsoClearTopLevel?: boolean }
+): void {
+  if (typeof id !== "string" || id.length === 0) return;
+  const db = getDbInstance() as unknown as DbLike;
+  const alsoClearTopLevel = options?.alsoClearTopLevel === true;
+  const candidate = db
+    .prepare("SELECT provider FROM provider_connections WHERE id = ?")
+    .get(id);
+  const isCodex = toRecord(candidate).provider === "codex";
+  if (!alsoClearTopLevel && !isCodex) return;
+
+  backupDbFile("pre-write");
+  const wrote = db.transaction(() => {
+    const existing = db
+      .prepare(
+        "SELECT provider, provider_specific_data FROM provider_connections WHERE id = ?"
+      )
+      .get(id);
+    if (!existing) return false;
+    const existingRecord = toRecord(rowToCamel(existing));
+    const providerSpecificData = toRecord(existingRecord.providerSpecificData);
+    const stripNested =
+      existingRecord.provider === "codex" &&
+      connectionHasCodexChildCooldown(providerSpecificData);
+    if (!alsoClearTopLevel && !stripNested) return false;
+
+    const now = new Date().toISOString();
+    if (alsoClearTopLevel && stripNested) {
+      db.prepare(
+        `UPDATE provider_connections
+         SET rate_limited_until = NULL,
+             provider_specific_data = ?,
+             updated_at = ?
+         WHERE id = ?`
+      ).run(JSON.stringify(stripCodexChildCooldownFields(providerSpecificData)), now, id);
+      return true;
+    }
+    if (alsoClearTopLevel) {
+      db.prepare(
+        `UPDATE provider_connections
+         SET rate_limited_until = NULL, updated_at = ?
+         WHERE id = ?`
+      ).run(now, id);
+      return true;
+    }
+    db.prepare(
+      `UPDATE provider_connections
+       SET provider_specific_data = ?, updated_at = ?
+       WHERE id = ?`
+    ).run(JSON.stringify(stripCodexChildCooldownFields(providerSpecificData)), now, id);
+    return true;
+  })();
+  if (wrote) invalidateDbCache("connections");
+}
+
 /**
 * Atomically merge one virtual Codex child's quota evidence into its persisted parent.
 * The transaction reads the latest row so sibling child state cannot be lost.
@@ -70,28 +165,35 @@ export async function updateCodexScopedQuotaState(
      }
    }

-    if (patch.rateLimitedUntil) {
-      const scopeCooldowns = toRecord(providerSpecificData.codexScopeRateLimitedUntil);
-      const sourceByScope = toRecord(providerSpecificData.codexScopeRateLimitSource);
-      const existingCooldownMs =
-        typeof scopeCooldowns[scope] === "string"
-          ? new Date(scopeCooldowns[scope] as string).getTime()
-          : NaN;
-      const existingIsAuthoritative =
-        sourceByScope[scope] === "quota_reset" &&
-        patch.rateLimitSource !== "quota_reset" &&
-        Number.isFinite(existingCooldownMs) &&
-        existingCooldownMs > Date.now();
-      nextProviderSpecificData.codexScopeRateLimitedUntil = {
-        ...scopeCooldowns,
-        [scope]: existingIsAuthoritative ? scopeCooldowns[scope] : patch.rateLimitedUntil,
-      };
-      nextProviderSpecificData.codexScopeRateLimitSource = {
-        ...sourceByScope,
-        [scope]: existingIsAuthoritative
+    if (patch.rateLimitedUntil !== undefined) {
+      const scopeCooldowns = { ...toRecord(providerSpecificData.codexScopeRateLimitedUntil) };
+      const sourceByScope = { ...toRecord(providerSpecificData.codexScopeRateLimitSource) };
+      if (patch.rateLimitedUntil) {
+        const existingCooldownMs =
+          typeof scopeCooldowns[scope] === "string"
+            ? new Date(scopeCooldowns[scope] as string).getTime()
+            : NaN;
+        const existingIsAuthoritative =
+          sourceByScope[scope] === "quota_reset" &&
+          patch.rateLimitSource !== "quota_reset" &&
+          Number.isFinite(existingCooldownMs) &&
+          existingCooldownMs > Date.now();
+        scopeCooldowns[scope] = existingIsAuthoritative
+          ? scopeCooldowns[scope]
+          : patch.rateLimitedUntil;
+        sourceByScope[scope] = existingIsAuthoritative
          ? sourceByScope[scope]
-          : (patch.rateLimitSource ?? "fallback"),
-      };
+          : (patch.rateLimitSource ?? "fallback");
+      } else {
+        delete scopeCooldowns[scope];
+        delete sourceByScope[scope];
+      }
+      const nextCooldowns = omitEmptyRecord(scopeCooldowns);
+      const nextSources = omitEmptyRecord(sourceByScope);
+      if (nextCooldowns) nextProviderSpecificData.codexScopeRateLimitedUntil = nextCooldowns;
+      else delete nextProviderSpecificData.codexScopeRateLimitedUntil;
+      if (nextSources) nextProviderSpecificData.codexScopeRateLimitSource = nextSources;
+      else delete nextProviderSpecificData.codexScopeRateLimitSource;
    }

    db.prepare(
@@ -106,6 +208,107 @@ export async function updateCodexScopedQuotaState(
  return persisted;
 }

+/** Grace window absorbing clock skew against the upstream quota server. */
+const QUOTA_RESET_CLOCK_SKEW_GRACE_MS = 30_000;
+
+/** Cheap probe: does this connection+scope currently carry a child cooldown? */
+export function hasCodexScopeCooldown(id: string, scope: "codex" | "spark"): boolean {
+  if (typeof id !== "string" || id.length === 0) return false;
+  const db = getDbInstance() as unknown as DbLike;
+  const row = db
+    .prepare("SELECT provider, provider_specific_data FROM provider_connections WHERE id = ?")
+    .get(id);
+  if (!row) return false;
+  const record = toRecord(rowToCamel(row));
+  if (record.provider !== "codex") return false;
+  const psd = toRecord(record.providerSpecificData);
+  return Boolean(toRecord(psd.codexScopeRateLimitedUntil)[scope]);
+}
+
+/**
+ * #12860: When fresh quota snapshot data demonstrates headroom on a scope,
+ * lift any fallback-sourced cooldown (e.g. parked by quota preflight).
+ * Cooldowns sourced from upstream 429 quota_reset retain their authority
+ * until their reset timestamp has elapsed.
+ */
+export function liftCodexScopeCooldownOnHeadroom(
+  id: string,
+  scope: "codex" | "spark"
+): boolean {
+  if (typeof id !== "string" || id.length === 0) return false;
+  const db = getDbInstance() as unknown as DbLike;
+
+  // Cheap eligibility probe so the backup only runs when a write is plausible.
+  // The transaction below re-reads under lock and remains the authority.
+  if (!hasCodexScopeCooldown(id, scope)) return false;
+
+  backupDbFile("pre-write");
+  const wrote = db.transaction(() => {
+    const existing = db
+      .prepare(
+        "SELECT provider, provider_specific_data FROM provider_connections WHERE id = ?"
+      )
+      .get(id);
+    if (!existing) return false;
+    const existingRecord = toRecord(rowToCamel(existing));
+    if (existingRecord.provider !== "codex") return false;
+    const currentPsd = toRecord(existingRecord.providerSpecificData);
+    const currentCooldowns = { ...toRecord(currentPsd.codexScopeRateLimitedUntil) };
+    if (!currentCooldowns[scope]) return false;
+
+    const currentSources = { ...toRecord(currentPsd.codexScopeRateLimitSource) };
+    const curSource = currentSources[scope];
+    const curUntilMs =
+      typeof currentCooldowns[scope] === "string"
+        ? new Date(currentCooldowns[scope] as string).getTime()
+        : NaN;
+    // An upstream-authoritative `quota_reset` deadline outranks a local snapshot:
+    // the grace window absorbs clock skew against the quota server, so a fast
+    // local clock cannot lift a cooldown upstream still considers active.
+    if (
+      curSource === "quota_reset" &&
+      Number.isFinite(curUntilMs) &&
+      curUntilMs > Date.now() - QUOTA_RESET_CLOCK_SKEW_GRACE_MS
+    ) {
+      return false;
+    }
+
+    delete currentCooldowns[scope];
+    delete currentSources[scope];
+
+    const nextPsd: JsonRecord = { ...currentPsd };
+    const nextCooldowns = omitEmptyRecord(currentCooldowns);
+    const nextSources = omitEmptyRecord(currentSources);
+    if (nextCooldowns) nextPsd.codexScopeRateLimitedUntil = nextCooldowns;
+    else delete nextPsd.codexScopeRateLimitedUntil;
+    if (nextSources) nextPsd.codexScopeRateLimitSource = nextSources;
+    else delete nextPsd.codexScopeRateLimitSource;
+
+    const exhaustedByScope = { ...toRecord(currentPsd.codexExhaustedWindowByScope) };
+    if (exhaustedByScope[scope]) {
+      delete exhaustedByScope[scope];
+      const nextExhausted = omitEmptyRecord(exhaustedByScope);
+      if (nextExhausted) {
+        nextPsd.codexExhaustedWindowByScope = nextExhausted;
+      } else {
+        delete nextPsd.codexExhaustedWindowByScope;
+        delete nextPsd.codexExhaustedWindow;
+      }
+    }
+
+    const now = new Date().toISOString();
+    db.prepare(
+      `UPDATE provider_connections
+       SET provider_specific_data = ?, updated_at = ?
+       WHERE id = ?`
+    ).run(JSON.stringify(nextPsd), now, id);
+    return true;
+  })();
+
+  if (wrote) invalidateDbCache("connections");
+  return wrote;
+}
+
 /** Persist one child cooldown through the shared scoped quota-state transaction. */
 export async function updateCodexScopeCooldown(
  id: string,
--- a/src/lib/db/providers/rateLimit.ts
+++ b/src/lib/db/providers/rateLimit.ts
@@ -4,6 +4,9 @@

 import { getDbInstance } from "../core";
 import { invalidateDbCache } from "../readCache";
+import { backupDbFile } from "../backup";
+import { bumpProxyConfigGeneration } from "../settings";
+import { stripCodexChildCooldownsFromConnection } from "./codexAccountState";

 interface StatementLike<TRow = unknown> {
  all: (...params: unknown[]) => TRow[];
@@ -33,6 +36,10 @@ export function setConnectionRateLimitUntil(connectionId: string, until: number
  // is the only clear path (via clearConnectionRateLimit); past/zero
  // timestamps are noops so an expired write cannot overwrite a live row.
  if (until !== null && (!Number.isFinite(until) || until <= Date.now())) return;
+  if (until == null) {
+    stripCodexChildCooldownsFromConnection(connectionId, { alsoClearTopLevel: true });
+    return;
+  }
  const db = getDbInstance() as unknown as DbLike;
  db.prepare(
    "UPDATE provider_connections SET rate_limited_until = ?, updated_at = ? WHERE id = ?"
@@ -234,6 +241,72 @@ export function clearStaleCrashCooldowns(): { cleared: number } {
  return { cleared: toReset.length };
 }

+/**
+ * Atomic conditional clear of recoverable error state on a connection row.
+ *
+ * Returns true when the row was cleared, false when a concurrent writer
+ * (markAccountUnavailable, connectionRecovery tick, test, etc.) changed the
+ * row between the caller's snapshot read and this UPDATE — in which case the
+ * clear is skipped to preserve the freshest error state. Closes the TOCTOU
+ * window in the quota-recovery path.
+ *
+ * CAS token = (test_status, last_error_at, rate_limited_until).
+ * Nested Codex child cooldown maps are stripped in the same UPDATE so a
+ * concurrent writer cannot re-persist them between two statements.
+ */
+export async function clearConnectionErrorIfUnchanged(
+  id: string,
+  expected: {
+    testStatus: string | null | undefined;
+    lastErrorAt: string | null | undefined;
+    rateLimitedUntil: string | null | undefined;
+  }
+): Promise<boolean> {
+  const db = getDbInstance() as unknown as DbLike;
+  backupDbFile("pre-write");
+  const result = db
+    .prepare(
+      `
+    UPDATE provider_connections SET
+      test_status = 'active',
+      last_error = NULL,
+      last_error_at = NULL,
+      last_error_type = NULL,
+      last_error_source = NULL,
+      error_code = NULL,
+      rate_limited_until = NULL,
+      backoff_level = 0,
+      provider_specific_data = CASE
+        WHEN provider = 'codex' AND json_valid(provider_specific_data)
+          THEN json_remove(
+            provider_specific_data,
+            '$.codexScopeRateLimitedUntil',
+            '$.codexScopeRateLimitSource'
+          )
+        ELSE provider_specific_data
+      END,
+      updated_at = ?
+    WHERE id = ?
+      AND IFNULL(test_status, '') = ?
+      AND IFNULL(last_error_at, '') = ?
+      AND IFNULL(rate_limited_until, '') = ?
+    `
+    )
+    .run(
+      new Date().toISOString(),
+      id,
+      expected.testStatus ?? "",
+      expected.lastErrorAt ?? "",
+      expected.rateLimitedUntil ?? ""
+    );
+  const applied = (result.changes ?? 0) > 0;
+  if (applied) {
+    invalidateDbCache("connections");
+    bumpProxyConfigGeneration();
+  }
+  return applied;
+}
+
 // T13: Format a reset countdown as a human-readable string ("2h 35m" / "4m 30s").
 // The implementation lives in the client-safe formatting utils so client
 // components (e.g. CoolingConnectionsPanel) can import it without pulling this
--- a/src/lib/db/quotaSnapshots.ts
+++ b/src/lib/db/quotaSnapshots.ts
@@ -1,5 +1,10 @@
 import { getDbInstance, rowToCamel } from "./core";
 import type { QuotaSnapshotRow, ProviderUtilizationPoint } from "@/shared/types/utilization";
+import {
+  hasCodexScopeCooldown,
+  liftCodexScopeCooldownOnHeadroom,
+} from "./providers/codexAccountState";
+import { isCodexSparkQuotaKey } from "@omniroute/open-sse/config/codexQuotaScopes";

 type JsonRecord = Record<string, unknown>;

@@ -45,6 +50,61 @@ export function saveQuotaSnapshot(snapshot: Omit<QuotaSnapshotRow, "id" | "creat
    }
    throw err;
  }
+
+  // #12860: When a new snapshot demonstrates headroom on a Codex connection,
+  // lift any fallback-sourced scope cooldown parked by quota preflight.
+  maybeLiftCodexCooldownOnHeadroom(snapshot);
+}
+
+/**
+ * `rowToCamel` leaves snake_case keys intact on some driver paths, so accept
+ * either casing rather than trusting one shape.
+ */
+type SnapshotShape = Partial<QuotaSnapshotRow> & {
+  windowKey?: string;
+  remainingPercentage?: number;
+  isExhausted?: number;
+};
+
+function snapshotHasHeadroom(s: SnapshotShape): boolean {
+  const pct = s.remainingPercentage ?? s.remaining_percentage ?? 0;
+  const exhausted = s.isExhausted ?? s.is_exhausted ?? 0;
+  return pct > 0 && exhausted !== 1;
+}
+
+function maybeLiftCodexCooldownOnHeadroom(
+  snapshot: Omit<QuotaSnapshotRow, "id" | "created_at">
+): void {
+  if (
+    snapshot.provider?.toLowerCase() !== "codex" ||
+    typeof snapshot.connection_id !== "string" ||
+    snapshot.connection_id.length === 0 ||
+    (snapshot.remaining_percentage ?? 0) <= 0 ||
+    snapshot.is_exhausted === 1
+  ) {
+    return;
+  }
+
+  try {
+    const scope = isCodexSparkQuotaKey(snapshot.window_key) ? "spark" : "codex";
+    // The scope-wide read is only worth paying for when a cooldown is
+    // actually parked on this connection+scope; the common case is clean.
+    if (!hasCodexScopeCooldown(snapshot.connection_id, scope)) return;
+
+    const scopeWindows = getLatestQuotaSnapshotsForConnection(snapshot.connection_id).filter(
+      (s: SnapshotShape) => {
+        const key = s.windowKey ?? s.window_key;
+        return scope === "spark" ? isCodexSparkQuotaKey(key) : !isCodexSparkQuotaKey(key);
+      }
+    );
+    // Every window in the scope must be healthy: one exhausted window still
+    // justifies the cooldown even when a sibling reports full headroom.
+    if (scopeWindows.length > 0 && scopeWindows.every(snapshotHasHeadroom)) {
+      liftCodexScopeCooldownOnHeadroom(snapshot.connection_id, scope);
+    }
+  } catch (err) {
+    console.debug("[QuotaSnapshots] Headroom evaluation skipped:", err);
+  }
 }

 export function getQuotaSnapshots(opts: {