fix(resilience): evict sticky affinity on pinned-account failover (#6219)

CHANGELOG.md

@@ -12,6 +12,10 @@
 - **ci:** heavy-pipeline dedup ([#6215](https://github.com/diegosouzapw/OmniRoute/pull/6215)) — the release-PR pipeline ran the unit suite 4× per sync (95 jobs, 208 machine-min; the v3.8.44 cycle fired 123 such runs, 88 cancelled). Now: Node 24/26 compat matrices move to a daily `nightly-compat.yml` (−28%/run; resolves the active release branch, opens a tracking issue on failure), coverage is collected inside the unit shards themselves via c8/`NODE_V8_COVERAGE` (−18%/run; the Coverage Shard ×8 matrix is gone — nodejs/node's own CI pattern), the ~40-job per-language i18n matrix becomes 1 job (the account has 20 concurrent-job slots total), and heavy jobs skip **draft** PRs — paired with `/generate-release` now opening the living release PR as draft (flipped ready at the new Phase 0a.0a), killing the per-merge churn for the whole cycle. Validated by a full `workflow_dispatch` of the new pipeline: 35 jobs, 0 failures, 23 min, merged coverage 80.16% (> ratchet baseline).
 - **feat(quality):** no-new-warnings per PR ([#6218](https://github.com/diegosouzapw/OmniRoute/pull/6218)) — native ESLint bulk suppressions (≥9.24) freeze the pre-existing debt (476 files / 4,273 violations in `config/quality/eslint-suppressions.json`); `npm run lint`, lint-staged (pre-commit) and a new fork-aware `lint-guard` job in quality.yml all run suppressions-aware, so a NEW warning goes red in the PR that introduces it instead of accruing invisibly (+41/+88 per cycle) and being blind-rebaselined at release. 3 warn rules promoted to error in `src/**` (`react-hooks/exhaustive-deps`, `@next/next/no-img-element`, `import/no-anonymous-default-export`); `collect-metrics` measures under the frozen baseline (ratchet metric = net-NEW debt; baseline tightened 4,279→0 in-PR per require-tighten); fork PRs run report-only (contributors are never blocked — the maintainer campaigns fix via co-authorship). Baseline stock shrinks via `--prune-suppressions` at release reconciliation.
 
+### 🔧 Bug Fixes
+
+- fix(resilience): sticky session affinity now evicts and fails over to another account when the pinned account is exhausted/unavailable (#6219)
+
 ---
 
 ## [3.8.44] — TBD

src/lib/db/sessionAccountAffinity.ts

@@ -128,6 +128,39 @@ export function deleteSessionAccountAffinity(sessionKey: string, provider: strin
   deleteAffinityKey(affinityKey(sessionKey, provider));
 }
 
+/**
+ * #6219 — Evict a session's pin ONLY when it currently points at `connectionId`.
+ *
+ * Used by the account-failover paths in chat.ts: when the pinned connection is
+ * marked unavailable/exhausted, the sticky pin must be dropped so the next
+ * request fails over to another account instead of re-pinning the dead one
+ * (previously the session stayed pinned until process restart).
+ *
+ * Unlike `getSessionAccountAffinity`, this reads the stored record INDEPENDENT
+ * of the TTL gate — a 2-arg `getSessionAccountAffinity(key, provider)` read
+ * (ttl defaulting to 0) always returns null, which silently defeated the
+ * connection-match guard on the earlier failover branches. The connection-match
+ * guard is preserved here so a pin pointing at a different (still-healthy)
+ * connection is never nuked. Returns true when a pin was evicted.
+ */
+export function evictSessionAccountAffinityForConnection(
+  sessionKey: string,
+  provider: string,
+  connectionId: string
+): boolean {
+  if (!sessionKey || !provider || !connectionId) return false;
+
+  const key = affinityKey(sessionKey, provider);
+  const row = getDbInstance()
+    .prepare("SELECT value FROM key_value WHERE namespace = ? AND key = ?")
+    .get(NAMESPACE, key) as { value?: unknown } | undefined;
+  const record = parseRecord(row?.value);
+  if (!record || record.connectionId !== connectionId) return false;
+
+  deleteAffinityKey(key);
+  return true;
+}
+
 export function cleanupStaleSessionAccountAffinities(
   _ttlMs: number = 30 * 60 * 1000,
   now: number = Date.now()

src/lib/localDb.ts

@@ -440,6 +440,7 @@ export {
   upsertSessionAccountAffinity,
   touchSessionAccountAffinity,
   deleteSessionAccountAffinity,
+  evictSessionAccountAffinityForConnection,
   cleanupStaleSessionAccountAffinities,
   startSessionAccountAffinityCleanup,
   stopSessionAccountAffinityCleanupForTests,

src/sse/handlers/chat.ts

@@ -56,6 +56,7 @@ import { updateCombo } from "@/lib/db/combos";
 import { promoteSuccessfulComboModel } from "@/lib/combos/autoPromote";
 import {
   deleteSessionAccountAffinity,
+  evictSessionAccountAffinityForConnection,
   getCachedSettings,
   getCombos,
   getCombosCacheVersion,
@@ -1639,6 +1640,21 @@ async function handleSingleModelChat(
           requestRetryLastCooldownMs = cooldownMs;
         }
         log.warn("AUTH", `Account ${accountId}... unavailable (${result.status}), trying fallback`);
+        // #6219: evict the sticky session pin when the pinned account fails over,
+        // otherwise the next request re-pins the same throttled account until
+        // restart. Guarded by connection match so a pin for a different (healthy)
+        // account is left intact.
+        if (runtimeOptions.sessionAffinityKey) {
+          try {
+            evictSessionAccountAffinityForConnection(
+              runtimeOptions.sessionAffinityKey,
+              provider,
+              credentials.connectionId
+            );
+          } catch {
+            // best-effort: selection also excludes this connection for the current retry.
+          }
+        }
         excludedConnectionIds.add(credentials.connectionId);
         lastError = result.error;
         lastStatus = result.status;

tests/unit/sticky-affinity-failover-6219.test.ts

@@ -0,0 +1,97 @@
+// #6219 — Sticky session affinity must fail over when the pinned account is
+// exhausted/unavailable. Before the fix, the generic account-fallback path in
+// src/sse/handlers/chat.ts marked the account unavailable + excluded it for the
+// current retry, but never evicted the persisted session-affinity pin — so the
+// next request re-pinned the same throttled account until process restart.
+//
+// The fix adds evictSessionAccountAffinityForConnection() (src/lib/db/
+// sessionAccountAffinity.ts) and calls it on that generic failover path. The
+// helper reads the stored pin INDEPENDENT of the TTL gate — the pre-existing
+// guarded reads via getSessionAccountAffinity(key, provider) (2-arg, ttl=0)
+// always returned null, making that guard a silent no-op.
+//
+// These tests drive the extracted eviction seam directly (running the full
+// chat.ts handler is too heavy) plus a source-level guard that the generic
+// fallback path wires the eviction in.
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-sticky-failover-6219-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "sticky-failover-6219-test-secret";
+
+const core = await import("../../src/lib/db/core.ts");
+const affinityDb = await import("../../src/lib/db/sessionAccountAffinity.ts");
+
+const PROVIDER = "codex";
+const SESSION = "session-6219";
+const CONN_A = "conn-A-exhausted";
+const CONN_B = "conn-B-healthy";
+const TTL = 60_000;
+
+test.beforeEach(() => {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
+});
+
+test.after(() => {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+});
+
+test("evicts the sticky pin when the pinned connection fails over (#6219)", () => {
+  affinityDb.upsertSessionAccountAffinity(SESSION, PROVIDER, CONN_A, Date.now(), TTL);
+  assert.equal(
+    affinityDb.getSessionAccountAffinity(SESSION, PROVIDER, TTL)?.connectionId,
+    CONN_A,
+    "precondition: session pinned to the (soon-exhausted) connection A"
+  );
+
+  const evicted = affinityDb.evictSessionAccountAffinityForConnection(SESSION, PROVIDER, CONN_A);
+
+  assert.equal(evicted, true, "failover eviction should report it removed the pin");
+  assert.equal(
+    affinityDb.getSessionAccountAffinity(SESSION, PROVIDER, TTL),
+    null,
+    "after failover the sticky pin to the exhausted connection must be gone (re-pins next request)"
+  );
+});
+
+test("does NOT evict a pin that points at a different (healthy) connection (#6219)", () => {
+  affinityDb.upsertSessionAccountAffinity(SESSION, PROVIDER, CONN_B, Date.now(), TTL);
+
+  const evicted = affinityDb.evictSessionAccountAffinityForConnection(SESSION, PROVIDER, CONN_A);
+
+  assert.equal(evicted, false, "must not evict when the pin is for another connection");
+  assert.equal(
+    affinityDb.getSessionAccountAffinity(SESSION, PROVIDER, TTL)?.connectionId,
+    CONN_B,
+    "a healthy pin to B must survive a failover on the unrelated connection A"
+  );
+});
+
+test("eviction is not TTL-gated — clears the pin even for a stale stored record (#6219)", () => {
+  // The pre-fix guard read via getSessionAccountAffinity(key, provider) (ttl=0)
+  // returned null and never deleted. The helper reads raw so eviction fires
+  // regardless of the TTL gate.
+  const past = Date.now() - 120_000;
+  affinityDb.upsertSessionAccountAffinity(SESSION, PROVIDER, CONN_A, past, 60_000); // already expired
+
+  const evicted = affinityDb.evictSessionAccountAffinityForConnection(SESSION, PROVIDER, CONN_A);
+
+  assert.equal(evicted, true, "raw connection-matched eviction still removes the stale stored pin");
+});
+
+test("chat.ts generic account-failover path wires in the sticky eviction (#6219)", () => {
+  const src = fs.readFileSync(new URL("../../src/sse/handlers/chat.ts", import.meta.url), "utf8");
+  assert.match(
+    src,
+    /evictSessionAccountAffinityForConnection\(/,
+    "chat.ts must call evictSessionAccountAffinityForConnection on the failover path"
+  );
+});