From ddd546483f1b58f9cdcefa218a555e9ec4b649dd Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Sun, 5 Jul 2026 17:33:51 -0300 Subject: [PATCH] fix(resilience): evict sticky affinity on pinned-account failover (#6219) (#6231) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merged into release/v3.8.45. Sticky affinity failover (#6219). Sincronizado com tip; CHANGELOG restaurado (base bullets preservados, net +1). Reds pré-existentes: dast-smoke + file-size drift. --- CHANGELOG.md | 2 + src/lib/db/sessionAccountAffinity.ts | 33 +++++++ src/lib/localDb.ts | 1 + src/sse/handlers/chat.ts | 16 +++ .../sticky-affinity-failover-6219.test.ts | 97 +++++++++++++++++++ 5 files changed, 149 insertions(+) create mode 100644 tests/unit/sticky-affinity-failover-6219.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 396ae33f45..579bd9db05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ ### 🐛 Bug Fixes +- fix(resilience): sticky session affinity now evicts and fails over to another account when the pinned account is exhausted/unavailable (#6219) + - **fix(proxy):** stop the v3.8.44 proxy regression that leaked the real IP and disabled healthy proxies ([#6246](https://github.com/diegosouzapw/OmniRoute/issues/6246)). Two coupled defects from the new health scheduler: (1) **IP leak** — when a proxy assigned to a connection was marked `inactive`, resolution fell through to a **direct** egress instead of blocking, exposing the operator's real IP; (2) **over-deactivation** — the sweep flipped a proxy to `inactive` on the **first** failed probe and counted our own 5s timeout / a probe-target `5xx` as the proxy's fault, so healthy paid proxies vanished from egress selection ("my proxies are not being used anymore"). Fix: the sweep decision is extracted into a pure, network-free `decideProxyHealthAction` (`src/lib/proxyHealth/decision.ts`) — by default the health check now **only counts/logs and never downgrades status** (a proxy is downgraded/removed only with `PROXY_AUTO_REMOVE=true`, after `PROXY_AUTO_REMOVE_AFTER` **consecutive** conclusive failures); probes are classified tri-state so an inconclusive result (our timeout, or a `5xx` from the probe target) never penalizes the proxy, and the probe timeout is raised 5s→15s. Separately, `safeResolveProxy` now **fails closed** via the existing policy: a connection whose assigned proxy is dead is blocked instead of leaking direct (`hasBlockingProxyAssignment`), honoring the explicit `proxy off` toggles and the `PROXY_FAIL_OPEN=true` opt-out. Existing proxies stuck `inactive` by the old behavior need a one-time manual re-activate (the operator owns proxy status). Regression guards: `tests/unit/proxy-health-decide-action-6246.test.ts`, `tests/unit/proxy-assigned-unavailable-6246.test.ts`. - **fix(proxy):** make "Test All" read-only and add bulk enable/disable ([#6246](https://github.com/diegosouzapw/OmniRoute/issues/6246)). Complements the core fail-closed / scheduler fix (#6296) with the two remaining reporter asks. (1) The **"Test All" button** (`POST /api/settings/proxies/auto-test`) used to flip a proxy to `inactive` on a failed reachability probe; since the egress selector excludes `inactive` proxies, a flaky probe (an unreachable `httpbin.org`, a proxy that blocks `HEAD`, or a slow paid proxy) silently disabled every proxy that failed — "Test All" is now **read-only by default** (only the operator sets a proxy active/inactive; opt back into the legacy test-and-set with `PROXY_HEALTH_AUTO_DEACTIVATE=true`). (2) Adds a **bulk enable/disable** proxies endpoint + toolbar action (`POST /api/settings/proxies/batch-activate`) so an operator can re-activate proxies in one click. Regression guard: `tests/unit/proxy-health-6246.test.ts`. (thanks @tenshiak) diff --git a/src/lib/db/sessionAccountAffinity.ts b/src/lib/db/sessionAccountAffinity.ts index ddb81301e2..9a284d1265 100644 --- a/src/lib/db/sessionAccountAffinity.ts +++ b/src/lib/db/sessionAccountAffinity.ts @@ -128,6 +128,39 @@ export function deleteSessionAccountAffinity(sessionKey: string, provider: strin deleteAffinityKey(affinityKey(sessionKey, provider)); } +/** + * #6219 — Evict a session's pin ONLY when it currently points at `connectionId`. + * + * Used by the account-failover paths in chat.ts: when the pinned connection is + * marked unavailable/exhausted, the sticky pin must be dropped so the next + * request fails over to another account instead of re-pinning the dead one + * (previously the session stayed pinned until process restart). + * + * Unlike `getSessionAccountAffinity`, this reads the stored record INDEPENDENT + * of the TTL gate — a 2-arg `getSessionAccountAffinity(key, provider)` read + * (ttl defaulting to 0) always returns null, which silently defeated the + * connection-match guard on the earlier failover branches. The connection-match + * guard is preserved here so a pin pointing at a different (still-healthy) + * connection is never nuked. Returns true when a pin was evicted. + */ +export function evictSessionAccountAffinityForConnection( + sessionKey: string, + provider: string, + connectionId: string +): boolean { + if (!sessionKey || !provider || !connectionId) return false; + + const key = affinityKey(sessionKey, provider); + const row = getDbInstance() + .prepare("SELECT value FROM key_value WHERE namespace = ? AND key = ?") + .get(NAMESPACE, key) as { value?: unknown } | undefined; + const record = parseRecord(row?.value); + if (!record || record.connectionId !== connectionId) return false; + + deleteAffinityKey(key); + return true; +} + export function cleanupStaleSessionAccountAffinities( _ttlMs: number = 30 * 60 * 1000, now: number = Date.now() diff --git a/src/lib/localDb.ts b/src/lib/localDb.ts index d911599723..fcf46560e2 100755 --- a/src/lib/localDb.ts +++ b/src/lib/localDb.ts @@ -440,6 +440,7 @@ export { upsertSessionAccountAffinity, touchSessionAccountAffinity, deleteSessionAccountAffinity, + evictSessionAccountAffinityForConnection, cleanupStaleSessionAccountAffinities, startSessionAccountAffinityCleanup, stopSessionAccountAffinityCleanupForTests, diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 177804ff75..af00b793eb 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -56,6 +56,7 @@ import { updateCombo } from "@/lib/db/combos"; import { promoteSuccessfulComboModel } from "@/lib/combos/autoPromote"; import { deleteSessionAccountAffinity, + evictSessionAccountAffinityForConnection, getCachedSettings, getCombos, getCombosCacheVersion, @@ -1642,6 +1643,21 @@ async function handleSingleModelChat( requestRetryLastCooldownMs = cooldownMs; } log.warn("AUTH", `Account ${accountId}... unavailable (${result.status}), trying fallback`); + // #6219: evict the sticky session pin when the pinned account fails over, + // otherwise the next request re-pins the same throttled account until + // restart. Guarded by connection match so a pin for a different (healthy) + // account is left intact. + if (runtimeOptions.sessionAffinityKey) { + try { + evictSessionAccountAffinityForConnection( + runtimeOptions.sessionAffinityKey, + provider, + credentials.connectionId + ); + } catch { + // best-effort: selection also excludes this connection for the current retry. + } + } excludedConnectionIds.add(credentials.connectionId); lastError = result.error; lastStatus = result.status; diff --git a/tests/unit/sticky-affinity-failover-6219.test.ts b/tests/unit/sticky-affinity-failover-6219.test.ts new file mode 100644 index 0000000000..44b8709b9f --- /dev/null +++ b/tests/unit/sticky-affinity-failover-6219.test.ts @@ -0,0 +1,97 @@ +// #6219 — Sticky session affinity must fail over when the pinned account is +// exhausted/unavailable. Before the fix, the generic account-fallback path in +// src/sse/handlers/chat.ts marked the account unavailable + excluded it for the +// current retry, but never evicted the persisted session-affinity pin — so the +// next request re-pinned the same throttled account until process restart. +// +// The fix adds evictSessionAccountAffinityForConnection() (src/lib/db/ +// sessionAccountAffinity.ts) and calls it on that generic failover path. The +// helper reads the stored pin INDEPENDENT of the TTL gate — the pre-existing +// guarded reads via getSessionAccountAffinity(key, provider) (2-arg, ttl=0) +// always returned null, making that guard a silent no-op. +// +// These tests drive the extracted eviction seam directly (running the full +// chat.ts handler is too heavy) plus a source-level guard that the generic +// fallback path wires the eviction in. + +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-sticky-failover-6219-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "sticky-failover-6219-test-secret"; + +const core = await import("../../src/lib/db/core.ts"); +const affinityDb = await import("../../src/lib/db/sessionAccountAffinity.ts"); + +const PROVIDER = "codex"; +const SESSION = "session-6219"; +const CONN_A = "conn-A-exhausted"; +const CONN_B = "conn-B-healthy"; +const TTL = 60_000; + +test.beforeEach(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("evicts the sticky pin when the pinned connection fails over (#6219)", () => { + affinityDb.upsertSessionAccountAffinity(SESSION, PROVIDER, CONN_A, Date.now(), TTL); + assert.equal( + affinityDb.getSessionAccountAffinity(SESSION, PROVIDER, TTL)?.connectionId, + CONN_A, + "precondition: session pinned to the (soon-exhausted) connection A" + ); + + const evicted = affinityDb.evictSessionAccountAffinityForConnection(SESSION, PROVIDER, CONN_A); + + assert.equal(evicted, true, "failover eviction should report it removed the pin"); + assert.equal( + affinityDb.getSessionAccountAffinity(SESSION, PROVIDER, TTL), + null, + "after failover the sticky pin to the exhausted connection must be gone (re-pins next request)" + ); +}); + +test("does NOT evict a pin that points at a different (healthy) connection (#6219)", () => { + affinityDb.upsertSessionAccountAffinity(SESSION, PROVIDER, CONN_B, Date.now(), TTL); + + const evicted = affinityDb.evictSessionAccountAffinityForConnection(SESSION, PROVIDER, CONN_A); + + assert.equal(evicted, false, "must not evict when the pin is for another connection"); + assert.equal( + affinityDb.getSessionAccountAffinity(SESSION, PROVIDER, TTL)?.connectionId, + CONN_B, + "a healthy pin to B must survive a failover on the unrelated connection A" + ); +}); + +test("eviction is not TTL-gated — clears the pin even for a stale stored record (#6219)", () => { + // The pre-fix guard read via getSessionAccountAffinity(key, provider) (ttl=0) + // returned null and never deleted. The helper reads raw so eviction fires + // regardless of the TTL gate. + const past = Date.now() - 120_000; + affinityDb.upsertSessionAccountAffinity(SESSION, PROVIDER, CONN_A, past, 60_000); // already expired + + const evicted = affinityDb.evictSessionAccountAffinityForConnection(SESSION, PROVIDER, CONN_A); + + assert.equal(evicted, true, "raw connection-matched eviction still removes the stale stored pin"); +}); + +test("chat.ts generic account-failover path wires in the sticky eviction (#6219)", () => { + const src = fs.readFileSync(new URL("../../src/sse/handlers/chat.ts", import.meta.url), "utf8"); + assert.match( + src, + /evictSessionAccountAffinityForConnection\(/, + "chat.ts must call evictSessionAccountAffinityForConnection on the failover path" + ); +});