diff --git a/CHANGELOG.md b/CHANGELOG.md index fbd429a96d..7a3ff5f138 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ - **ci:** heavy-pipeline dedup ([#6215](https://github.com/diegosouzapw/OmniRoute/pull/6215)) — the release-PR pipeline ran the unit suite 4× per sync (95 jobs, 208 machine-min; the v3.8.44 cycle fired 123 such runs, 88 cancelled). Now: Node 24/26 compat matrices move to a daily `nightly-compat.yml` (−28%/run; resolves the active release branch, opens a tracking issue on failure), coverage is collected inside the unit shards themselves via c8/`NODE_V8_COVERAGE` (−18%/run; the Coverage Shard ×8 matrix is gone — nodejs/node's own CI pattern), the ~40-job per-language i18n matrix becomes 1 job (the account has 20 concurrent-job slots total), and heavy jobs skip **draft** PRs — paired with `/generate-release` now opening the living release PR as draft (flipped ready at the new Phase 0a.0a), killing the per-merge churn for the whole cycle. Validated by a full `workflow_dispatch` of the new pipeline: 35 jobs, 0 failures, 23 min, merged coverage 80.16% (> ratchet baseline). - **feat(quality):** no-new-warnings per PR ([#6218](https://github.com/diegosouzapw/OmniRoute/pull/6218)) — native ESLint bulk suppressions (≥9.24) freeze the pre-existing debt (476 files / 4,273 violations in `config/quality/eslint-suppressions.json`); `npm run lint`, lint-staged (pre-commit) and a new fork-aware `lint-guard` job in quality.yml all run suppressions-aware, so a NEW warning goes red in the PR that introduces it instead of accruing invisibly (+41/+88 per cycle) and being blind-rebaselined at release. 3 warn rules promoted to error in `src/**` (`react-hooks/exhaustive-deps`, `@next/next/no-img-element`, `import/no-anonymous-default-export`); `collect-metrics` measures under the frozen baseline (ratchet metric = net-NEW debt; baseline tightened 4,279→0 in-PR per require-tighten); fork PRs run report-only (contributors are never blocked — the maintainer campaigns fix via co-authorship). Baseline stock shrinks via `--prune-suppressions` at release reconciliation. +### 🔧 Bug Fixes + +- fix(resilience): sticky session affinity now evicts and fails over to another account when the pinned account is exhausted/unavailable (#6219) + --- ## [3.8.44] — TBD diff --git a/src/lib/db/sessionAccountAffinity.ts b/src/lib/db/sessionAccountAffinity.ts index ddb81301e2..9a284d1265 100644 --- a/src/lib/db/sessionAccountAffinity.ts +++ b/src/lib/db/sessionAccountAffinity.ts @@ -128,6 +128,39 @@ export function deleteSessionAccountAffinity(sessionKey: string, provider: strin deleteAffinityKey(affinityKey(sessionKey, provider)); } +/** + * #6219 — Evict a session's pin ONLY when it currently points at `connectionId`. + * + * Used by the account-failover paths in chat.ts: when the pinned connection is + * marked unavailable/exhausted, the sticky pin must be dropped so the next + * request fails over to another account instead of re-pinning the dead one + * (previously the session stayed pinned until process restart). + * + * Unlike `getSessionAccountAffinity`, this reads the stored record INDEPENDENT + * of the TTL gate — a 2-arg `getSessionAccountAffinity(key, provider)` read + * (ttl defaulting to 0) always returns null, which silently defeated the + * connection-match guard on the earlier failover branches. The connection-match + * guard is preserved here so a pin pointing at a different (still-healthy) + * connection is never nuked. Returns true when a pin was evicted. + */ +export function evictSessionAccountAffinityForConnection( + sessionKey: string, + provider: string, + connectionId: string +): boolean { + if (!sessionKey || !provider || !connectionId) return false; + + const key = affinityKey(sessionKey, provider); + const row = getDbInstance() + .prepare("SELECT value FROM key_value WHERE namespace = ? AND key = ?") + .get(NAMESPACE, key) as { value?: unknown } | undefined; + const record = parseRecord(row?.value); + if (!record || record.connectionId !== connectionId) return false; + + deleteAffinityKey(key); + return true; +} + export function cleanupStaleSessionAccountAffinities( _ttlMs: number = 30 * 60 * 1000, now: number = Date.now() diff --git a/src/lib/localDb.ts b/src/lib/localDb.ts index d911599723..fcf46560e2 100755 --- a/src/lib/localDb.ts +++ b/src/lib/localDb.ts @@ -440,6 +440,7 @@ export { upsertSessionAccountAffinity, touchSessionAccountAffinity, deleteSessionAccountAffinity, + evictSessionAccountAffinityForConnection, cleanupStaleSessionAccountAffinities, startSessionAccountAffinityCleanup, stopSessionAccountAffinityCleanupForTests, diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 9598061d07..1305fd4271 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -56,6 +56,7 @@ import { updateCombo } from "@/lib/db/combos"; import { promoteSuccessfulComboModel } from "@/lib/combos/autoPromote"; import { deleteSessionAccountAffinity, + evictSessionAccountAffinityForConnection, getCachedSettings, getCombos, getCombosCacheVersion, @@ -1639,6 +1640,21 @@ async function handleSingleModelChat( requestRetryLastCooldownMs = cooldownMs; } log.warn("AUTH", `Account ${accountId}... unavailable (${result.status}), trying fallback`); + // #6219: evict the sticky session pin when the pinned account fails over, + // otherwise the next request re-pins the same throttled account until + // restart. Guarded by connection match so a pin for a different (healthy) + // account is left intact. + if (runtimeOptions.sessionAffinityKey) { + try { + evictSessionAccountAffinityForConnection( + runtimeOptions.sessionAffinityKey, + provider, + credentials.connectionId + ); + } catch { + // best-effort: selection also excludes this connection for the current retry. + } + } excludedConnectionIds.add(credentials.connectionId); lastError = result.error; lastStatus = result.status; diff --git a/tests/unit/sticky-affinity-failover-6219.test.ts b/tests/unit/sticky-affinity-failover-6219.test.ts new file mode 100644 index 0000000000..44b8709b9f --- /dev/null +++ b/tests/unit/sticky-affinity-failover-6219.test.ts @@ -0,0 +1,97 @@ +// #6219 — Sticky session affinity must fail over when the pinned account is +// exhausted/unavailable. Before the fix, the generic account-fallback path in +// src/sse/handlers/chat.ts marked the account unavailable + excluded it for the +// current retry, but never evicted the persisted session-affinity pin — so the +// next request re-pinned the same throttled account until process restart. +// +// The fix adds evictSessionAccountAffinityForConnection() (src/lib/db/ +// sessionAccountAffinity.ts) and calls it on that generic failover path. The +// helper reads the stored pin INDEPENDENT of the TTL gate — the pre-existing +// guarded reads via getSessionAccountAffinity(key, provider) (2-arg, ttl=0) +// always returned null, making that guard a silent no-op. +// +// These tests drive the extracted eviction seam directly (running the full +// chat.ts handler is too heavy) plus a source-level guard that the generic +// fallback path wires the eviction in. + +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-sticky-failover-6219-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "sticky-failover-6219-test-secret"; + +const core = await import("../../src/lib/db/core.ts"); +const affinityDb = await import("../../src/lib/db/sessionAccountAffinity.ts"); + +const PROVIDER = "codex"; +const SESSION = "session-6219"; +const CONN_A = "conn-A-exhausted"; +const CONN_B = "conn-B-healthy"; +const TTL = 60_000; + +test.beforeEach(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("evicts the sticky pin when the pinned connection fails over (#6219)", () => { + affinityDb.upsertSessionAccountAffinity(SESSION, PROVIDER, CONN_A, Date.now(), TTL); + assert.equal( + affinityDb.getSessionAccountAffinity(SESSION, PROVIDER, TTL)?.connectionId, + CONN_A, + "precondition: session pinned to the (soon-exhausted) connection A" + ); + + const evicted = affinityDb.evictSessionAccountAffinityForConnection(SESSION, PROVIDER, CONN_A); + + assert.equal(evicted, true, "failover eviction should report it removed the pin"); + assert.equal( + affinityDb.getSessionAccountAffinity(SESSION, PROVIDER, TTL), + null, + "after failover the sticky pin to the exhausted connection must be gone (re-pins next request)" + ); +}); + +test("does NOT evict a pin that points at a different (healthy) connection (#6219)", () => { + affinityDb.upsertSessionAccountAffinity(SESSION, PROVIDER, CONN_B, Date.now(), TTL); + + const evicted = affinityDb.evictSessionAccountAffinityForConnection(SESSION, PROVIDER, CONN_A); + + assert.equal(evicted, false, "must not evict when the pin is for another connection"); + assert.equal( + affinityDb.getSessionAccountAffinity(SESSION, PROVIDER, TTL)?.connectionId, + CONN_B, + "a healthy pin to B must survive a failover on the unrelated connection A" + ); +}); + +test("eviction is not TTL-gated — clears the pin even for a stale stored record (#6219)", () => { + // The pre-fix guard read via getSessionAccountAffinity(key, provider) (ttl=0) + // returned null and never deleted. The helper reads raw so eviction fires + // regardless of the TTL gate. + const past = Date.now() - 120_000; + affinityDb.upsertSessionAccountAffinity(SESSION, PROVIDER, CONN_A, past, 60_000); // already expired + + const evicted = affinityDb.evictSessionAccountAffinityForConnection(SESSION, PROVIDER, CONN_A); + + assert.equal(evicted, true, "raw connection-matched eviction still removes the stale stored pin"); +}); + +test("chat.ts generic account-failover path wires in the sticky eviction (#6219)", () => { + const src = fs.readFileSync(new URL("../../src/sse/handlers/chat.ts", import.meta.url), "utf8"); + assert.match( + src, + /evictSessionAccountAffinityForConnection\(/, + "chat.ts must call evictSessionAccountAffinityForConnection on the failover path" + ); +});