fix(sse): stop Codex multi-account family-revocation cascade on quota-sync

The Quota / Providers dashboard (POST /api/usage/provider-limits -> syncAllProviderLimits, GET /api/usage/[connectionId]) calls refreshAndUpdateCredentials() per connection, in concurrent chunks. For rotating-refresh providers (Codex/OpenAI share one Auth0 client_id) the single-use refresh_token is rotated on every refresh; refreshing siblings concurrently makes Auth0 revoke the whole token family (openai/codex#9648), killing every account but the last with [403] <!DOCTYPE html>. On the affected VM expires_at was persisted as ~0 so needsRefresh() was effectively always true -> every codex account refreshed on every page visit -> guaranteed cascade. Fix #1: refreshAndUpdateCredentials skips proactive refresh for rotating providers (rotationGroupFor) and reuses the current access_token for the quota fetch; genuine expiry is handled by the reactive, serialized 401 path. Fix #2 (defense in depth): serializeRefresh inserts a settle gap between two QUEUED sibling refreshes (default 2000ms, CODEX_REFRESH_SPACING_MS, '0' to opt out) but releases a lone refresh immediately, adding no latency to the reactive request path. Tests: codex-quota-sync-no-proactive-refresh (skip + non-rotating guard), refresh-serializer-spacing (default/opt-out + lone-vs-queued behavior).
2026-07-26 09:52:11 +03:00 · 2026-05-31 19:13:41 -03:00
parent 7cb77a9083
commit 6d2e695882
6 changed files with 240 additions and 6 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -63,6 +63,19 @@

 ### Fixed

+- **codex/quota:** opening the Quota / Providers dashboard no longer disconnects
+  Codex multi-account setups. The quota-sync path
+  (`refreshAndUpdateCredentials`) proactively refreshed every connection — for
+  rotating-refresh providers (Codex/OpenAI share one Auth0 `client_id`) it
+  refreshed siblings concurrently, so Auth0 revoked the whole token family
+  (`openai/codex#9648`) and every account but the last died with
+  `[403] <!DOCTYPE html>`. The quota path now skips proactive refresh for
+  rotating providers (`rotationGroupFor`) and reuses the current access_token,
+  deferring genuine expiry to the reactive, serialized 401 path. Defense in
+  depth: `serializeRefresh` now leaves a settle gap between two *queued* sibling
+  refreshes (default 2000 ms, tunable via `CODEX_REFRESH_SPACING_MS`, `"0"` to
+  opt out) while releasing a lone refresh immediately, so the reactive path adds
+  no latency.
 - **payload-rules:** saved payload rules now survive a server restart. When no
  in-memory override is set (fresh process before the boot hook ran, or a
  separate module instance in the standalone build), `getPayloadRulesConfig`
--- a/open-sse/services/refreshSerializer.ts
+++ b/open-sse/services/refreshSerializer.ts
@@ -30,9 +30,27 @@ const ROTATION_LOCK_GROUP: Record<string, string> = {
  qwen: "qwen",
 };

-function readSpacingMs(): number {
-  const raw = Number(process.env.CODEX_REFRESH_SPACING_MS);
-  return Number.isFinite(raw) && raw >= 0 ? raw : 0;
+// Protective settle gap (ms) between two consecutive sibling refreshes when the
+// env var is unset. Conservative by default; bursts are rare and correctness
+// (not revoking the family) outweighs the extra wall-clock on a queued refresh.
+const DEFAULT_REFRESH_SPACING_MS = 2000;
+
+/**
+ * Gap (ms) inserted between two consecutive refreshes in the same rotation group.
+ * It is only paid when a sibling is already queued behind the current refresh —
+ * a lone refresh is released immediately so the reactive request path pays no
+ * extra latency. The gap gives Auth0 time to settle a rotation before the next
+ * sibling presents its (now superseded) refresh_token, closing the
+ * family-revocation race window (openai/codex#9648).
+ *
+ * Tunable via `CODEX_REFRESH_SPACING_MS`; set it to `"0"` to opt out entirely.
+ */
+export function getRefreshSpacingMs(): number {
+  const rawEnv = process.env.CODEX_REFRESH_SPACING_MS;
+  if (rawEnv === undefined || rawEnv === "") return DEFAULT_REFRESH_SPACING_MS;
+  const raw = Number(rawEnv);
+  // Explicit "0" opts out; anything unparseable falls back to the safe default.
+  return Number.isFinite(raw) && raw >= 0 ? raw : DEFAULT_REFRESH_SPACING_MS;
 }

 // Tail promise per group — each new refresh chains after the previous one.
@@ -69,8 +87,14 @@ export async function serializeRefresh<T>(provider: string, fn: () => Promise<T>
  try {
    return await fn();
  } finally {
-    const spacing = readSpacingMs();
-    if (spacing > 0) await delay(spacing);
+    // Only pay the settle gap when a sibling is already queued behind us — a
+    // lone refresh has nobody to collide with, so it must be released
+    // immediately (zero added latency on the reactive request path).
+    const hasSuccessor = groupTail.get(group) !== myTail;
+    if (hasSuccessor) {
+      const spacing = getRefreshSpacingMs();
+      if (spacing > 0) await delay(spacing);
+    }
    releaseMine();
    // Garbage-collect the lane when nobody chained after us.
    if (groupTail.get(group) === myTail) groupTail.delete(group);
--- a/src/lib/usage/providerLimits.ts
+++ b/src/lib/usage/providerLimits.ts
@@ -18,6 +18,7 @@ import { getMachineId } from "@/shared/utils/machine";
 import { USAGE_SUPPORTED_PROVIDERS } from "@/shared/constants/providers";
 import { getExecutor } from "@omniroute/open-sse/executors/index.ts";
 import { getUsageForProvider } from "@omniroute/open-sse/services/usage.ts";
+import { rotationGroupFor } from "@omniroute/open-sse/services/refreshSerializer.ts";
 import {
  extractCodeAssistOnboardTierId,
  extractCodeAssistSubscriptionTier,
@@ -110,7 +111,17 @@ async function syncToCloudIfEnabled() {
  }
 }

-async function refreshAndUpdateCredentials(connection: ProviderConnectionLike) {
+export async function refreshAndUpdateCredentials(connection: ProviderConnectionLike) {
+  // Rotating-refresh providers (Codex/OpenAI share one Auth0 client_id, etc.)
+  // mint a single-use refresh_token on every refresh. The quota-sync path runs
+  // many connections concurrently; refreshing sibling accounts in parallel makes
+  // Auth0 revoke the whole token family (openai/codex#9648) and kills every
+  // account but the last. Never proactively refresh them here — reuse the current
+  // access_token for the quota fetch and let the reactive, serialized 401 path
+  // handle genuine expiry during real requests.
+  if (rotationGroupFor(connection.provider) !== null) {
+    return { connection, refreshed: false };
+  }
  const executor = getExecutor(connection.provider);
  const credentials = {
    connectionId: connection.id,
--- a/tests/unit/codex-quota-sync-no-proactive-refresh.test.ts
+++ b/tests/unit/codex-quota-sync-no-proactive-refresh.test.ts
@@ -0,0 +1,102 @@
+/**
+ * Codex multi-account family-revocation cascade — quota-sync trigger.
+ *
+ * The Quota / Providers dashboard (POST /api/usage/provider-limits ->
+ * syncAllProviderLimits, and GET /api/usage/[connectionId]) calls
+ * refreshAndUpdateCredentials() for each connection, in chunks of N CONCURRENT.
+ * For rotating-refresh providers (Codex/OpenAI share one Auth0 client_id) a
+ * single-use refresh_token is rotated on every refresh; refreshing several
+ * sibling accounts concurrently makes Auth0 revoke the WHOLE token family
+ * (openai/codex#9648) -> every account but the last dies with `[403] <!DOCTYPE`.
+ *
+ * On the affected VM expires_at is persisted as ~0, so needsRefresh() is
+ * effectively always true -> every codex account gets refreshed on every page
+ * visit -> guaranteed cascade.
+ *
+ * Fix: refreshAndUpdateCredentials must NEVER proactively rotate the
+ * refresh_token of a rotating-refresh provider. The quota fetch reuses the
+ * current access_token (multi-day) and genuine expiry is handled by the
+ * reactive, serialized 401 path during real requests. Non-rotating providers
+ * must keep refreshing proactively (no over-broadening).
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+process.env.DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-codex-quota-"));
+
+const { getExecutor } = await import("../../open-sse/executors/index.ts");
+const { refreshAndUpdateCredentials } = await import("../../src/lib/usage/providerLimits.ts");
+
+test("codex: quota-sync must NOT proactively rotate the refresh_token (Auth0 family-revocation cascade guard)", async () => {
+  const exec = getExecutor("codex");
+  const origNeeds = exec.needsRefresh;
+  const origRefresh = exec.refreshCredentials;
+  let refreshCalls = 0;
+  // Simulate the VM state: expires_at ~0 makes needsRefresh always-true.
+  exec.needsRefresh = () => true;
+  exec.refreshCredentials = async () => {
+    refreshCalls++;
+    return null;
+  };
+  try {
+    const result = await refreshAndUpdateCredentials({
+      id: "codex-1",
+      provider: "codex",
+      accessToken: "existing-access-token",
+      refreshToken: "rotating-refresh-token",
+      tokenExpiresAt: new Date(2000).toISOString(),
+      providerSpecificData: {},
+    });
+    assert.equal(
+      refreshCalls,
+      0,
+      "the rotating refresh_token must NOT be exercised from the quota-sync path"
+    );
+    assert.equal(result.refreshed, false, "no proactive refresh happened");
+    assert.equal(
+      result.connection.accessToken,
+      "existing-access-token",
+      "the current access_token is reused for the quota fetch"
+    );
+  } finally {
+    exec.needsRefresh = origNeeds;
+    exec.refreshCredentials = origRefresh;
+  }
+});
+
+test("non-rotating OAuth provider is still refreshed proactively from quota-sync (gate is not over-broad)", async () => {
+  const exec = getExecutor("cursor");
+  const origNeeds = exec.needsRefresh;
+  const origRefresh = exec.refreshCredentials;
+  let refreshCalls = 0;
+  exec.needsRefresh = () => true;
+  // null -> non-github -> surfaces a 401; proves the refresh was actually attempted.
+  exec.refreshCredentials = async () => {
+    refreshCalls++;
+    return null;
+  };
+  try {
+    await assert.rejects(
+      refreshAndUpdateCredentials({
+        id: "cursor-1",
+        provider: "cursor",
+        accessToken: "a",
+        refreshToken: "r",
+        tokenExpiresAt: new Date(2000).toISOString(),
+        providerSpecificData: {},
+      }),
+      "a non-rotating provider with a failed refresh should surface the 401"
+    );
+    assert.equal(
+      refreshCalls,
+      1,
+      "non-rotating provider must still attempt the proactive refresh"
+    );
+  } finally {
+    exec.needsRefresh = origNeeds;
+    exec.refreshCredentials = origRefresh;
+  }
+});
--- a/tests/unit/refresh-serializer-spacing.test.ts
+++ b/tests/unit/refresh-serializer-spacing.test.ts
@@ -0,0 +1,79 @@
+/**
+ * Codex multi-account cascade — defense-in-depth refresh spacing (Fix #2).
+ *
+ * Serialization (concurrency=1 per Auth0 family) already stops OVERLAPPING
+ * refreshes. Spacing adds a settle gap BETWEEN two consecutive sibling refreshes
+ * so the second account never presents a refresh_token that Auth0 is still
+ * rotating for the family (the `refresh_token_reused` race, openai/codex#9648).
+ *
+ * Two invariants:
+ *  - The gap is only paid when a sibling is already queued behind us. A lone
+ *    refresh — the common reactive case during a real request — is released
+ *    immediately, so we add ZERO latency to user traffic.
+ *  - The gap has a protective non-zero default, is tunable via
+ *    CODEX_REFRESH_SPACING_MS, and can be disabled with "0".
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+import {
+  serializeRefresh,
+  getRefreshSpacingMs,
+  __resetRefreshSerializerForTest,
+} from "../../open-sse/services/refreshSerializer.ts";
+
+const ENV = "CODEX_REFRESH_SPACING_MS";
+
+function withEnv(value: string | undefined, fn: () => void | Promise<void>) {
+  const prev = process.env[ENV];
+  if (value === undefined) delete process.env[ENV];
+  else process.env[ENV] = value;
+  __resetRefreshSerializerForTest();
+  const restore = () => {
+    if (prev === undefined) delete process.env[ENV];
+    else process.env[ENV] = prev;
+    __resetRefreshSerializerForTest();
+  };
+  return Promise.resolve()
+    .then(fn)
+    .finally(restore);
+}
+
+test("getRefreshSpacingMs defaults to a protective non-zero gap, honors overrides and explicit opt-out", () => {
+  const prev = process.env[ENV];
+  try {
+    delete process.env[ENV];
+    assert.equal(getRefreshSpacingMs(), 2000, "unset -> protective default");
+    process.env[ENV] = "0";
+    assert.equal(getRefreshSpacingMs(), 0, "explicit 0 -> opt out");
+    process.env[ENV] = "750";
+    assert.equal(getRefreshSpacingMs(), 750, "explicit value honored");
+    process.env[ENV] = "not-a-number";
+    assert.equal(getRefreshSpacingMs(), 2000, "garbage -> protective default");
+  } finally {
+    if (prev === undefined) delete process.env[ENV];
+    else process.env[ENV] = prev;
+  }
+});
+
+test("a lone rotating refresh is released immediately (no added latency on the reactive request path)", async () => {
+  await withEnv("1000", async () => {
+    const start = Date.now();
+    await serializeRefresh("codex", async () => "ok");
+    const elapsed = Date.now() - start;
+    assert.ok(elapsed < 400, `a lone refresh must not pay the spacing gap; took ${elapsed}ms`);
+  });
+});
+
+test("queued sibling refreshes in the same group are spaced apart by the configured gap", async () => {
+  await withEnv("120", async () => {
+    const starts: number[] = [];
+    const run = () =>
+      serializeRefresh("codex", async () => {
+        starts.push(Date.now());
+      });
+    await Promise.all([run(), run()]);
+    assert.equal(starts.length, 2, "both refreshes ran");
+    const gap = starts[1] - starts[0];
+    assert.ok(gap >= 100, `the second sibling must start after the gap; gap was ${gap}ms`);
+  });
+});
--- a/tests/unit/refresh-serializer.test.ts
+++ b/tests/unit/refresh-serializer.test.ts
@@ -20,6 +20,11 @@ import {
  __resetRefreshSerializerForTest,
 } from "../../open-sse/services/refreshSerializer.ts";

+// These tests assert serialization ORDERING (concurrency=1 per group); the
+// inter-refresh settle gap is covered by refresh-serializer-spacing.test.ts.
+// Opt out of the gap here so the ordering checks stay fast and deterministic.
+process.env.CODEX_REFRESH_SPACING_MS = "0";
+
 const delay = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));

 test("codex and openai share one group and never refresh concurrently", async () => {