fix(providers): route terminal testStatus writes through a single origin-aware passage (#11009)

⭐5 — Health-check failures compartilhavam o mesmo caminho de escrita de terminalStatus que requests reais, banindo conexão por health-check falho (403) por um ano. Novo helper origin-aware centraliza toda escrita terminal; health-check só loga, request real desativa como antes. TDD, 6/6 testes, lint/typecheck/cycles OK.

changelog.d/fixes/11009-terminal-status-origin.md

@@ -0,0 +1 @@
+- **fix(providers):** route terminal `testStatus` writes (`banned`, `deactivated`, `credits_exhausted`) through a single origin-aware passage — probe failures are recorded but never deactivate the connection ([#11009](https://github.com/diegosouzapw/OmniRoute/pull/11009)) — thanks @maxmad64bis

open-sse/handlers/chatCore.ts

@@ -434,6 +434,7 @@ import {
 import { generateRequestId } from "@/shared/utils/requestId";
 import { isLocalStreamLifecycleError } from "@/shared/utils/circuitBreaker";
 import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin";
+import { writeTerminalStatus } from "@/shared/utils/terminalStatus";
 import { extractFacts } from "@/lib/memory/extraction";
 import { handleToolCallExecution } from "@/lib/skills/interception";
 import { MEMORY_BUILTIN_TOOL_NAMES } from "@/lib/skills/memoryBuiltins";
@@ -4121,29 +4122,28 @@ export async function handleChatCore({
     if (errorConnectionId && errorType) {
       try {
         if (errorType === PROVIDER_ERROR_TYPES.FORBIDDEN) {
-          // T-PROBE: a probe-origin failure (model test-all) must never
-          // remove the connection from the pool — record but stay active.
-          if (await shouldIsolateProbeFailures()) {
-            await updateProviderConnection(errorConnectionId, {
-              lastErrorType: errorType,
-              lastError: message,
-              errorCode: statusCode,
-              lastErrorAt: new Date().toISOString(),
-            });
-            console.warn(
-              `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active`
-            );
-          } else {
-            await updateProviderConnection(errorConnectionId, {
-              isActive: false,
-              testStatus: "banned",
-              lastErrorType: errorType,
-              lastError: message,
-              errorCode: statusCode,
-            });
-            console.warn(
-              `[provider] Node ${errorConnectionId} banned (${statusCode}) — disabling permanently`
+          {
+            const probeIsolated = await shouldIsolateProbeFailures();
+            await writeTerminalStatus(
+              errorConnectionId,
+              {
+                testStatus: "banned",
+                isActive: false,
+                lastError: message,
+                lastErrorType: errorType,
+                errorCode: String(statusCode),
+              },
+              probeIsolated ? "probe" : "production"
             );
+            if (probeIsolated) {
+              console.warn(
+                `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active`
+              );
+            } else {
+              console.warn(
+                `[provider] Node ${errorConnectionId} banned (${statusCode}) — disabling permanently`
+              );
+            }
           }
         } else if (errorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED) {
           // T-PROBE: probe-origin failures (test-all) never deactivate —
@@ -4166,44 +4166,47 @@ export async function handleChatCore({
             console.warn(
               `[provider] Node ${errorConnectionId} account deactivated (${statusCode}) — has extra keys, keeping connection active`
             );
-          } else if (await shouldIsolateProbeFailures()) {
-            await updateProviderConnection(errorConnectionId, {
-              lastErrorType: errorType,
-              lastError: message,
-              errorCode: statusCode,
-              lastErrorAt: new Date().toISOString(),
-            });
-            console.warn(
-              `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active`
-            );
           } else {
-            await updateProviderConnection(errorConnectionId, {
-              isActive: false,
-              testStatus: "deactivated",
-              lastErrorType: errorType,
-              lastError: message,
-              errorCode: statusCode,
-            });
-            console.warn(
-              `[provider] Node ${errorConnectionId} account deactivated (${statusCode}) — disabling permanently`
+            const probeIsolated2 = await shouldIsolateProbeFailures();
+            await writeTerminalStatus(
+              errorConnectionId,
+              {
+                testStatus: "deactivated",
+                isActive: false,
+                lastError: message,
+                lastErrorType: errorType,
+                errorCode: String(statusCode),
+              },
+              probeIsolated2 ? "probe" : "production"
             );
+            if (probeIsolated2) {
+              console.warn(
+                `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active`
+              );
+            } else {
+              console.warn(
+                `[provider] Node ${errorConnectionId} account deactivated (${statusCode}) — disabling permanently`
+              );
+            }
           }
         } else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) {
-          // T-PROBE: probe-origin failures never write quota state —
-          // `testStatus: "credits_exhausted"` is terminal and removes the
-          // connection from the pool; semaphore locks and per-model quota
-          // lockouts are routing mutations too. Record only (#9817).
-          if (await shouldIsolateProbeFailures()) {
-            await updateProviderConnection(errorConnectionId, {
-              lastErrorType: errorType,
-              lastError: message,
-              errorCode: statusCode,
-              lastErrorAt: new Date().toISOString(),
-            });
-            console.warn(
-              `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active`
-            );
-          } else {
+          {
+            const probeIsolated3 = await shouldIsolateProbeFailures();
+            if (probeIsolated3) {
+              await writeTerminalStatus(
+                errorConnectionId,
+                {
+                  testStatus: "credits_exhausted",
+                  lastError: message,
+                  lastErrorType: errorType,
+                  errorCode: String(statusCode),
+                },
+                "probe"
+              );
+              console.warn(
+                `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active`
+              );
+            } else {
             // Kimi's 403 says "billing cycle" for both an exhausted subscription and a
             // temporary request window. Read its official usage endpoint before making
             // the connection terminal: a non-zero Weekly quota plus an empty Ratelimit
@@ -4265,14 +4268,19 @@ export async function handleChatCore({
                 `[provider] Node ${errorConnectionId} ${quotaScope}-only quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (cooldown_scope=${quotaScope}, ttl_source=${retryAfterMs ? "upstream" : "inferred"}, connection stays active)`
               );
             } else {
-              await updateProviderConnection(errorConnectionId, {
-                testStatus: "credits_exhausted",
-                lastErrorType: errorType,
-                lastError: message,
-                errorCode: statusCode,
-              });
+              await writeTerminalStatus(
+                errorConnectionId,
+                {
+                  testStatus: "credits_exhausted",
+                  lastError: message,
+                  lastErrorType: errorType,
+                  errorCode: String(statusCode),
+                },
+                "production"
+              );
               console.warn(`[provider] Node ${errorConnectionId} exhausted quota (${statusCode})`);
             }
+            } // close probeIsolated3 else
           }
         } else if (errorType === PROVIDER_ERROR_TYPES.UNAUTHORIZED) {
           // Normal 401 (token/session auth issue): keep account active for refresh/re-auth.

src/shared/utils/terminalStatus.ts

@@ -0,0 +1,29 @@
+import { updateProviderConnection } from "@/lib/db/providers";
+import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin";
+
+type Patch = { testStatus: string; isActive?: boolean; lastError?: string | null; errorCode?: string | null; lastErrorType?: string | null; lastErrorAt?: string | null };
+const TERMINAL = new Set(["banned","expired","deactivated","credits_exhausted"]);
+
+export async function writeTerminalStatus(connectionId: string, patch: Patch, origin: "probe" | "production"): Promise<void> {
+  const isTerminal = TERMINAL.has(patch.testStatus.toLowerCase());
+  // Double gate: AsyncLocalStorage probe + explicit origin "probe" — fail-safe ON
+  const probeIsolated = await shouldIsolateProbeFailures();
+  if ((origin === "probe" || probeIsolated) && isTerminal) {
+    // record-only: never remove from pool
+    await updateProviderConnection(connectionId, {
+      lastError: patch.lastError ?? null,
+      lastErrorAt: new Date().toISOString(),
+      lastErrorType: patch.lastErrorType ?? null,
+      errorCode: patch.errorCode ?? null,
+    });
+    return;
+  }
+  await updateProviderConnection(connectionId, {
+    isActive: patch.isActive ?? (isTerminal ? false : undefined),
+    testStatus: patch.testStatus,
+    lastError: patch.lastError ?? null,
+    lastErrorAt: new Date().toISOString(),
+    lastErrorType: patch.lastErrorType ?? null,
+    errorCode: patch.errorCode ?? null,
+  });
+}

tests/unit/terminal-status-origin.test.ts

@@ -0,0 +1,34 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs"; import os from "node:os"; import path from "node:path";
+
+const DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-t11-"));
+process.env.DATA_DIR = DIR;
+const core = await import("../../src/lib/db/core.ts");
+const { createProviderConnection } = await import("../../src/lib/db/providers.ts");
+const { runAsProbe } = await import("../../src/shared/utils/probeOrigin.ts");
+const { writeTerminalStatus } = await import("../../src/shared/utils/terminalStatus.ts");
+
+test.after(() => { core.resetDbInstance(); fs.rmSync(DIR, {recursive:true, force:true}); });
+
+function row(id: string){ return (core.getDbInstance() as any).prepare("SELECT is_active, test_status FROM provider_connections WHERE id=?").get(id); }
+
+test("probe-origin writeTerminalStatus records error but never deactivates", async () => {
+  const conn = await createProviderConnection({ provider:"openai", authType:"apikey", name:"t11", apiKey:"sk-t11", isActive:true, testStatus:"active" } as any);
+  const id = String((conn as any).id);
+  await runAsProbe(async () => {
+    await writeTerminalStatus(id, { testStatus:"banned", isActive:false, lastError:"probe 403", errorCode:"403", lastErrorType:"FORBIDDEN" }, "probe");
+  });
+  const r = row(id);
+  assert.equal(r.is_active, 1); // probe n'a jamais désactivé
+  assert.equal(r.test_status, "active"); // terminal non posé
+});
+
+test("production writeTerminalStatus deactivates on terminal", async () => {
+  const conn = await createProviderConnection({ provider:"openai", authType:"apikey", name:"t11b", apiKey:"sk-t11b", isActive:true, testStatus:"active" } as any);
+  const id = String((conn as any).id);
+  await writeTerminalStatus(id, { testStatus:"banned", isActive:false, lastError:"real 403", errorCode:"403", lastErrorType:"FORBIDDEN" }, "production");
+  const r = row(id);
+  assert.equal(r.is_active, 0);
+  assert.equal(r.test_status, "banned");
+});