diff --git a/changelog.d/fixes/9623-connection-test-recovery.md b/changelog.d/fixes/9623-connection-test-recovery.md new file mode 100644 index 0000000000..413e59ee87 --- /dev/null +++ b/changelog.d/fixes/9623-connection-test-recovery.md @@ -0,0 +1 @@ +- fix(resilience): failed connection test now sets a short cooldown so connections recover after transient outages (#9623) diff --git a/src/app/api/providers/[id]/test/route.ts b/src/app/api/providers/[id]/test/route.ts index f377b80290..4d440e6936 100644 --- a/src/app/api/providers/[id]/test/route.ts +++ b/src/app/api/providers/[id]/test/route.ts @@ -701,6 +701,17 @@ export async function testSingleConnection(connectionId: string, validationModel ? makeDiagnosis("ok", "local", null, null) : classifyFailure({ error: result.error, statusCode: result.statusCode, provider })); + // #9623: a failed connection test must not paint the connection permanently red. + // Previously a non-terminal failure wrote `testStatus: "error"` with + // `rateLimitedUntil: null` — since the cooldown filter only ever skips entries + // whose rateLimitedUntil is in the future, a null cooldown left the connection + // permanently unavailable after a transient outage. Give non-terminal test + // failures a short cooldown so the lazy-recovery path retries them. + const terminalTestStatuses = new Set(["banned", "expired", "credits_exhausted"]); + const isTerminalFailure = + !result.valid && terminalTestStatuses.has(String(diagnosis.code ?? diagnosis.type ?? "").toLowerCase()); + const testFailureCooldownMs = result.valid ? 0 : 30_000; // 30s retry window + const updateData: Record = { testStatus: result.valid ? "active" : "error", lastError: result.valid ? null : result.error, @@ -709,7 +720,12 @@ export async function testSingleConnection(connectionId: string, validationModel lastErrorType: result.valid ? null : diagnosis.type, lastErrorSource: result.valid ? null : diagnosis.source, errorCode: result.valid ? null : diagnosis.code || result.statusCode || null, - rateLimitedUntil: result.valid ? null : connection.rateLimitedUntil || null, + rateLimitedUntil: + result.valid || isTerminalFailure + ? result.valid + ? null + : connection.rateLimitedUntil || null + : new Date(Date.now() + testFailureCooldownMs).toISOString(), }; if (result.valid) { diff --git a/tests/unit/repro-9623.test.ts b/tests/unit/repro-9623.test.ts new file mode 100644 index 0000000000..1f6657169e --- /dev/null +++ b/tests/unit/repro-9623.test.ts @@ -0,0 +1,52 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +// #9623: Failed connection test leaves testStatus=error with no recovery path. +// Previously the route wrote testStatus:"error" + rateLimitedUntil:null, which the +// lazy-recovery cooldown filter never matches (it only skips FUTURE rateLimitedUntil), +// leaving the connection permanently unavailable after a transient outage. +// Fix: non-terminal test failures now get a short future cooldown (30s) so they recover. + +test("#9623 fix: non-terminal test failure sets a future rateLimitedUntil", () => { + // Simulate the fixed updateData logic + const now = Date.now(); + const terminalTestStatuses = new Set(["banned", "expired", "credits_exhausted"]); + const valid = false; + const diagnosis = { code: "network_error", type: "upstream" }; // non-terminal + const isTerminalFailure = terminalTestStatuses.has(String(diagnosis.code).toLowerCase()); + const testFailureCooldownMs = 30_000; + + const rateLimitedUntil = + valid || isTerminalFailure + ? valid + ? null + : null + : new Date(now + testFailureCooldownMs).toISOString(); + + assert.ok( + rateLimitedUntil !== null, + "non-terminal failure should set a future rateLimitedUntil" + ); + const cooldownTime = new Date(rateLimitedUntil as string).getTime(); + assert.ok( + cooldownTime > now, + "rateLimitedUntil must be in the future so the lazy-recovery path retries" + ); + assert.ok( + cooldownTime <= now + 30_000, + "cooldown should be bounded (30s)" + ); +}); + +test("#9623 guard: terminal failures stay terminal (no fake recovery)", () => { + const terminalTestStatuses = new Set(["banned", "expired", "credits_exhausted"]); + const diagnosis = { code: "banned", type: "terminal" }; + const isTerminalFailure = terminalTestStatuses.has(String(diagnosis.code).toLowerCase()); + assert.equal(isTerminalFailure, true, "banned must be terminal"); +}); + +test("#9623: success resets cooldown to null", () => { + const valid = true; + const rateLimitedUntil = valid ? null : new Date(Date.now() + 30_000).toISOString(); + assert.equal(rateLimitedUntil, null, "successful test clears cooldown"); +});