From 8d076327f146e4fc5dfc55ad8acae8e0446fe0c5 Mon Sep 17 00:00:00 2001 From: Dizzle <112548150+maxmad64bis@users.noreply.github.com> Date: Fri, 21 Aug 2026 18:58:48 +0200 Subject: [PATCH] fix(providers): route terminal testStatus writes through a single origin-aware passage (#11009) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ⭐5 — Health-check failures compartilhavam o mesmo caminho de escrita de terminalStatus que requests reais, banindo conexão por health-check falho (403) por um ano. Novo helper origin-aware centraliza toda escrita terminal; health-check só loga, request real desativa como antes. TDD, 6/6 testes, lint/typecheck/cycles OK. --- .../fixes/11009-terminal-status-origin.md | 1 + open-sse/handlers/chatCore.ts | 132 ++++++++++-------- src/shared/utils/terminalStatus.ts | 29 ++++ tests/unit/terminal-status-origin.test.ts | 34 +++++ 4 files changed, 134 insertions(+), 62 deletions(-) create mode 100644 changelog.d/fixes/11009-terminal-status-origin.md create mode 100644 src/shared/utils/terminalStatus.ts create mode 100644 tests/unit/terminal-status-origin.test.ts diff --git a/changelog.d/fixes/11009-terminal-status-origin.md b/changelog.d/fixes/11009-terminal-status-origin.md new file mode 100644 index 0000000000..f0ab24edaf --- /dev/null +++ b/changelog.d/fixes/11009-terminal-status-origin.md @@ -0,0 +1 @@ +- **fix(providers):** route terminal `testStatus` writes (`banned`, `deactivated`, `credits_exhausted`) through a single origin-aware passage — probe failures are recorded but never deactivate the connection ([#11009](https://github.com/diegosouzapw/OmniRoute/pull/11009)) — thanks @maxmad64bis diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index e2a3d12a6b..37362c472d 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -434,6 +434,7 @@ import { import { generateRequestId } from "@/shared/utils/requestId"; import { isLocalStreamLifecycleError } from "@/shared/utils/circuitBreaker"; import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; +import { writeTerminalStatus } from "@/shared/utils/terminalStatus"; import { extractFacts } from "@/lib/memory/extraction"; import { handleToolCallExecution } from "@/lib/skills/interception"; import { MEMORY_BUILTIN_TOOL_NAMES } from "@/lib/skills/memoryBuiltins"; @@ -4121,29 +4122,28 @@ export async function handleChatCore({ if (errorConnectionId && errorType) { try { if (errorType === PROVIDER_ERROR_TYPES.FORBIDDEN) { - // T-PROBE: a probe-origin failure (model test-all) must never - // remove the connection from the pool — record but stay active. - if (await shouldIsolateProbeFailures()) { - await updateProviderConnection(errorConnectionId, { - lastErrorType: errorType, - lastError: message, - errorCode: statusCode, - lastErrorAt: new Date().toISOString(), - }); - console.warn( - `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active` - ); - } else { - await updateProviderConnection(errorConnectionId, { - isActive: false, - testStatus: "banned", - lastErrorType: errorType, - lastError: message, - errorCode: statusCode, - }); - console.warn( - `[provider] Node ${errorConnectionId} banned (${statusCode}) — disabling permanently` + { + const probeIsolated = await shouldIsolateProbeFailures(); + await writeTerminalStatus( + errorConnectionId, + { + testStatus: "banned", + isActive: false, + lastError: message, + lastErrorType: errorType, + errorCode: String(statusCode), + }, + probeIsolated ? "probe" : "production" ); + if (probeIsolated) { + console.warn( + `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active` + ); + } else { + console.warn( + `[provider] Node ${errorConnectionId} banned (${statusCode}) — disabling permanently` + ); + } } } else if (errorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED) { // T-PROBE: probe-origin failures (test-all) never deactivate — @@ -4166,44 +4166,47 @@ export async function handleChatCore({ console.warn( `[provider] Node ${errorConnectionId} account deactivated (${statusCode}) — has extra keys, keeping connection active` ); - } else if (await shouldIsolateProbeFailures()) { - await updateProviderConnection(errorConnectionId, { - lastErrorType: errorType, - lastError: message, - errorCode: statusCode, - lastErrorAt: new Date().toISOString(), - }); - console.warn( - `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active` - ); } else { - await updateProviderConnection(errorConnectionId, { - isActive: false, - testStatus: "deactivated", - lastErrorType: errorType, - lastError: message, - errorCode: statusCode, - }); - console.warn( - `[provider] Node ${errorConnectionId} account deactivated (${statusCode}) — disabling permanently` + const probeIsolated2 = await shouldIsolateProbeFailures(); + await writeTerminalStatus( + errorConnectionId, + { + testStatus: "deactivated", + isActive: false, + lastError: message, + lastErrorType: errorType, + errorCode: String(statusCode), + }, + probeIsolated2 ? "probe" : "production" ); + if (probeIsolated2) { + console.warn( + `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active` + ); + } else { + console.warn( + `[provider] Node ${errorConnectionId} account deactivated (${statusCode}) — disabling permanently` + ); + } } } else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) { - // T-PROBE: probe-origin failures never write quota state — - // `testStatus: "credits_exhausted"` is terminal and removes the - // connection from the pool; semaphore locks and per-model quota - // lockouts are routing mutations too. Record only (#9817). - if (await shouldIsolateProbeFailures()) { - await updateProviderConnection(errorConnectionId, { - lastErrorType: errorType, - lastError: message, - errorCode: statusCode, - lastErrorAt: new Date().toISOString(), - }); - console.warn( - `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active` - ); - } else { + { + const probeIsolated3 = await shouldIsolateProbeFailures(); + if (probeIsolated3) { + await writeTerminalStatus( + errorConnectionId, + { + testStatus: "credits_exhausted", + lastError: message, + lastErrorType: errorType, + errorCode: String(statusCode), + }, + "probe" + ); + console.warn( + `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active` + ); + } else { // Kimi's 403 says "billing cycle" for both an exhausted subscription and a // temporary request window. Read its official usage endpoint before making // the connection terminal: a non-zero Weekly quota plus an empty Ratelimit @@ -4265,14 +4268,19 @@ export async function handleChatCore({ `[provider] Node ${errorConnectionId} ${quotaScope}-only quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (cooldown_scope=${quotaScope}, ttl_source=${retryAfterMs ? "upstream" : "inferred"}, connection stays active)` ); } else { - await updateProviderConnection(errorConnectionId, { - testStatus: "credits_exhausted", - lastErrorType: errorType, - lastError: message, - errorCode: statusCode, - }); + await writeTerminalStatus( + errorConnectionId, + { + testStatus: "credits_exhausted", + lastError: message, + lastErrorType: errorType, + errorCode: String(statusCode), + }, + "production" + ); console.warn(`[provider] Node ${errorConnectionId} exhausted quota (${statusCode})`); } + } // close probeIsolated3 else } } else if (errorType === PROVIDER_ERROR_TYPES.UNAUTHORIZED) { // Normal 401 (token/session auth issue): keep account active for refresh/re-auth. diff --git a/src/shared/utils/terminalStatus.ts b/src/shared/utils/terminalStatus.ts new file mode 100644 index 0000000000..1b74768b9a --- /dev/null +++ b/src/shared/utils/terminalStatus.ts @@ -0,0 +1,29 @@ +import { updateProviderConnection } from "@/lib/db/providers"; +import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; + +type Patch = { testStatus: string; isActive?: boolean; lastError?: string | null; errorCode?: string | null; lastErrorType?: string | null; lastErrorAt?: string | null }; +const TERMINAL = new Set(["banned","expired","deactivated","credits_exhausted"]); + +export async function writeTerminalStatus(connectionId: string, patch: Patch, origin: "probe" | "production"): Promise { + const isTerminal = TERMINAL.has(patch.testStatus.toLowerCase()); + // Double gate: AsyncLocalStorage probe + explicit origin "probe" — fail-safe ON + const probeIsolated = await shouldIsolateProbeFailures(); + if ((origin === "probe" || probeIsolated) && isTerminal) { + // record-only: never remove from pool + await updateProviderConnection(connectionId, { + lastError: patch.lastError ?? null, + lastErrorAt: new Date().toISOString(), + lastErrorType: patch.lastErrorType ?? null, + errorCode: patch.errorCode ?? null, + }); + return; + } + await updateProviderConnection(connectionId, { + isActive: patch.isActive ?? (isTerminal ? false : undefined), + testStatus: patch.testStatus, + lastError: patch.lastError ?? null, + lastErrorAt: new Date().toISOString(), + lastErrorType: patch.lastErrorType ?? null, + errorCode: patch.errorCode ?? null, + }); +} diff --git a/tests/unit/terminal-status-origin.test.ts b/tests/unit/terminal-status-origin.test.ts new file mode 100644 index 0000000000..f6de47eb6c --- /dev/null +++ b/tests/unit/terminal-status-origin.test.ts @@ -0,0 +1,34 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; import os from "node:os"; import path from "node:path"; + +const DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-t11-")); +process.env.DATA_DIR = DIR; +const core = await import("../../src/lib/db/core.ts"); +const { createProviderConnection } = await import("../../src/lib/db/providers.ts"); +const { runAsProbe } = await import("../../src/shared/utils/probeOrigin.ts"); +const { writeTerminalStatus } = await import("../../src/shared/utils/terminalStatus.ts"); + +test.after(() => { core.resetDbInstance(); fs.rmSync(DIR, {recursive:true, force:true}); }); + +function row(id: string){ return (core.getDbInstance() as any).prepare("SELECT is_active, test_status FROM provider_connections WHERE id=?").get(id); } + +test("probe-origin writeTerminalStatus records error but never deactivates", async () => { + const conn = await createProviderConnection({ provider:"openai", authType:"apikey", name:"t11", apiKey:"sk-t11", isActive:true, testStatus:"active" } as any); + const id = String((conn as any).id); + await runAsProbe(async () => { + await writeTerminalStatus(id, { testStatus:"banned", isActive:false, lastError:"probe 403", errorCode:"403", lastErrorType:"FORBIDDEN" }, "probe"); + }); + const r = row(id); + assert.equal(r.is_active, 1); // probe n'a jamais désactivé + assert.equal(r.test_status, "active"); // terminal non posé +}); + +test("production writeTerminalStatus deactivates on terminal", async () => { + const conn = await createProviderConnection({ provider:"openai", authType:"apikey", name:"t11b", apiKey:"sk-t11b", isActive:true, testStatus:"active" } as any); + const id = String((conn as any).id); + await writeTerminalStatus(id, { testStatus:"banned", isActive:false, lastError:"real 403", errorCode:"403", lastErrorType:"FORBIDDEN" }, "production"); + const r = row(id); + assert.equal(r.is_active, 0); + assert.equal(r.test_status, "banned"); +});