diff --git a/config/quality/eslint-suppressions.json b/config/quality/eslint-suppressions.json index 8f68e5f07c..81775983bd 100644 --- a/config/quality/eslint-suppressions.json +++ b/config/quality/eslint-suppressions.json @@ -1018,11 +1018,6 @@ "count": 1 } }, - "src/lib/api/modelTestRunner.ts": { - "no-restricted-imports": { - "count": 1 - } - }, "src/lib/api/proxyRegistryRouteHandlers.ts": { "no-restricted-imports": { "count": 1 diff --git a/docs/security/BAN_DETECTION.md b/docs/security/BAN_DETECTION.md index 4f72fe7708..faf585267e 100644 --- a/docs/security/BAN_DETECTION.md +++ b/docs/security/BAN_DETECTION.md @@ -38,7 +38,7 @@ this service has been disabled in this account (Antigravity) > copy is `ACCOUNT_DEACTIVATED_SIGNALS` in `open-sse/services/accountFallback.ts`; > treat the block above as a snapshot. -Two adjacent, **separate** signal tables live in the same file and are *not* part +Two adjacent, **separate** signal tables live in the same file and are _not_ part of banned-keyword detection: - `CREDITS_EXHAUSTED_SIGNALS` — billing/quota depleted (`insufficient_quota`, @@ -70,7 +70,7 @@ upstream error response narrower **`deactivated`** label (`isActive=false` when the connection has no spare API keys) is written by the inline `chatCore.ts` path on **HTTP 401 / 403** (classified via `classifyProviderError` → `ACCOUNT_DEACTIVATED`). Note the - `markAccountUnavailable()` path writes a *different* terminal status — + `markAccountUnavailable()` path writes a _different_ terminal status — **`expired`** — for the same `ACCOUNT_DEACTIVATED` signal (via `resolveTerminalConnectionStatus`), so the same ban can surface as either `deactivated` or `expired` depending on which path handled the response. (The @@ -86,7 +86,7 @@ every failed upstream request flows through — it is **not** gated to OAuth/subscription scrapers. The resulting terminal state is per **connection**, not per provider. -That said, the built-in *strings* are oriented toward subscription/OAuth +That said, the built-in _strings_ are oriented toward subscription/OAuth providers with real ban risk (ChatGPT Web, Claude Web, Codex, Muse Spark, Antigravity). An API-key provider will only trip the detector if its error body literally contains one of the substrings. @@ -136,14 +136,62 @@ There is no separate "clear ban flag" button — recovery is re-test, re-auth, o re-enable, matching the general terminal-state rule in [RESILIENCE_GUIDE](../architecture/RESILIENCE_GUIDE.md). +## Probe isolation (model test-all) + +A **probe-origin failure** (model test-all / health-check dispatches executed +inside `runAsProbe`) never removes a connection from the pool (#9817): it is +**recorded for visibility** (`last_error`, `last_error_type`, `error_code`, +`last_error_at`) but skips **every** routing mutation — cooldowns, terminal +status (`banned` / `deactivated` / `credits_exhausted`), per-model lockouts, +the provider circuit breaker, the 5-minute quota cache, OAuth token refresh +and auto-disable. Only a real request-path failure deactivates. The recorded +error is what makes a flagged account visible in the dashboard while it stays +serving traffic. + +The single decision point is `shouldIsolateProbeFailures()` +(`src/shared/utils/probeOrigin.ts`), consulted by **every** site that could +mutate routing state from a probe-origin failure: + +- `markAccountUnavailable` (`auth.ts`) — record-only (`lastError` raw text, + `lastErrorType`, `errorCode`, `lastErrorAt`; deliberately **no** + `backoffLevel`, which would trigger the selection-time auto-decay and wipe + the record) +- `maybeAutoDisableBannedAccount` — no auto-disable +- `chatCore` — FORBIDDEN, ACCOUNT_DEACTIVATED, QUOTA_EXHAUSTED (record-only, + no terminal `credits_exhausted`), GEO_BLOCKED (no 24h exclusion), + MODEL_NOT_FOUND (no `lockModel`), the codex 429 account-rotation failover + (no `markCodexScopeRateLimited`, no persisted `rate_limited_until`, no + session-affinity clear), `persistCodexQuotaState` (no quota-state write, + no cache invalidation), `recordKeyHealthStatus` (key-health rotator + untouched) +- OAuth refresh — both the proactive refresh in the executor base + (`base.ts` `execute()`, no refresh-token rotation consumed) and the + reactive 401/403 path in `chatCore` (no `expired` deactivation) +- `chat.ts` — provider circuit breaker and the 5-minute quota cache + (`markAccountExhaustedFrom429`) never degraded + +The recorded error is what makes a flagged account visible in the dashboard +while it stays serving traffic. Note: the probe record stores the **raw** +(unsliced) error text, unlike the real path's `slice(0,100)` truncation. + +Operators who use test-all as a maintenance tool can restore the historical +behavior (probe counts as a real generation) via either: + +- the `probeCanDisable` setting (`POST /api/settings` with + `{"probeCanDisable": true}`, or a direct `key_value` DB edit), or +- feature flag **`PROBE_CAN_DISABLE=true`** (env or DB override; wins over the + setting). + +Fail-safe: if the flag or settings lookup throws, isolation stays ON. + ## Source files -| Concern | File | -| --- | --- | -| Signal tables + match | `open-sse/services/accountFallback.ts` | -| Terminalization / persistence | `src/sse/services/auth.ts` (`markAccountUnavailable`, `resolveTerminalConnectionStatus`, `clearAccountError`) | -| Auto-disable scope | `src/shared/utils/autoDisableBanned.ts`, `src/sse/services/autoDisableBannedAccount.ts` | -| Inline classification | `open-sse/handlers/chatCore.ts`, `open-sse/services/errorClassifier.ts` | -| Terminal-state recovery exclusion | `src/lib/quota/connectionRecovery.ts` | -| Custom-keyword runtime load | `src/lib/config/runtimeSettings.ts` (`setCustomBannedSignals`) | -| Settings UI | `src/app/(dashboard)/dashboard/settings/components/SecurityTab.tsx` | +| Concern | File | +| --------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| Signal tables + match | `open-sse/services/accountFallback.ts` | +| Terminalization / persistence | `src/sse/services/auth.ts` (`markAccountUnavailable`, `resolveTerminalConnectionStatus`, `clearAccountError`) | +| Auto-disable scope | `src/shared/utils/autoDisableBanned.ts`, `src/sse/services/autoDisableBannedAccount.ts` | +| Inline classification | `open-sse/handlers/chatCore.ts`, `open-sse/services/errorClassifier.ts` | +| Terminal-state recovery exclusion | `src/lib/quota/connectionRecovery.ts` | +| Custom-keyword runtime load | `src/lib/config/runtimeSettings.ts` (`setCustomBannedSignals`) | +| Settings UI | `src/app/(dashboard)/dashboard/settings/components/SecurityTab.tsx` | diff --git a/open-sse/executors/base.ts b/open-sse/executors/base.ts index c91cb5cbb7..877431b68e 100644 --- a/open-sse/executors/base.ts +++ b/open-sse/executors/base.ts @@ -103,6 +103,7 @@ import { } from "./base/headers.ts"; import { applyPeerTraceHeader } from "@/shared/resilience/peerRouting"; import { applyClineProtocolHeaders } from "@/shared/utils/clineAuth"; +import { isProbeContext } from "@/shared/utils/probeOrigin"; // Header helpers extracted to a pure leaf; re-exported for external importers // (executors + tests) that import them from "./base.ts". export { @@ -689,7 +690,10 @@ export class BaseExecutor { // Track per-URL intra-retry attempts to avoid infinite loops const retryAttemptsByUrl: Record = {}; - if (this.needsRefresh(credentials)) { + // Probe-origin dispatches must not consume a refresh-token rotation — + // routing state untouched; the reactive 401/403 path is probe-guarded + // in chatCore (#9817). + if (!isProbeContext() && this.needsRefresh(credentials)) { try { // Fix A: wire onCredentialsRefreshed through runWithOnPersist so it runs // INSIDE the per-connection mutex inside getAccessToken. Not every @@ -800,7 +804,14 @@ export class BaseExecutor { activeCredentials ); const url = this.buildUrl(model, stream, urlIndex, requestCredentials); - const headers = this.buildHeaders(requestCredentials, stream, clientHeaders, model, undefined, body); + const headers = this.buildHeaders( + requestCredentials, + stream, + clientHeaders, + model, + undefined, + body + ); applyConfiguredUserAgent(headers, requestCredentials?.providerSpecificData); // Strip OpenAI SDK (X-Stainless-*) metadata + normalize SDK-derived User-Agent diff --git a/open-sse/executors/gitlab.ts b/open-sse/executors/gitlab.ts index fa0b22c5c1..b82d5c47aa 100644 --- a/open-sse/executors/gitlab.ts +++ b/open-sse/executors/gitlab.ts @@ -10,6 +10,7 @@ import { } from "./base.ts"; import { FETCH_TIMEOUT_MS } from "../config/constants.ts"; import { getAccessToken } from "../services/tokenRefresh.ts"; +import { isProbeContext } from "@/shared/utils/probeOrigin"; import { prepareToolMessages, buildToolAwareResult } from "../translator/webTools.ts"; import { buildStreamingResponse, @@ -208,9 +209,7 @@ function buildToolExchangePrompt(messages: OpenAIMessage[]): string { const line = renderConversationTurn(message, role, text); if (line) convo.push(line); } - const header = systemParts.length - ? `System instructions:\n${systemParts.join("\n\n")}\n\n` - : ""; + const header = systemParts.length ? `System instructions:\n${systemParts.join("\n\n")}\n\n` : ""; const body = `${header}${convo.join( "\n\n" )}\n\nContinue the response using the tool result above; do not repeat the tool call.`.trim(); @@ -672,7 +671,9 @@ export class GitlabExecutor extends BaseExecutor { } let activeCredentials = input.credentials; - if (this.needsRefresh(activeCredentials)) { + // Probe-origin dispatches must not consume a refresh-token rotation — + // routing state untouched; mirrors the base.ts guard (#9817). + if (!isProbeContext() && this.needsRefresh(activeCredentials)) { const refreshed = await this.refreshCredentials(activeCredentials, input.log || null); if (refreshed) { activeCredentials = mergeCredentials(activeCredentials, refreshed); diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index d93d3d1482..3555467a40 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -397,6 +397,7 @@ import { } from "../utils/aiSdkCompat.ts"; import { generateRequestId } from "@/shared/utils/requestId"; import { isLocalStreamLifecycleError } from "@/shared/utils/circuitBreaker"; +import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; import { extractFacts } from "@/lib/memory/extraction"; import { handleToolCallExecution } from "@/lib/skills/interception"; import { OMNIROUTE_RESPONSE_HEADERS } from "@/shared/constants/headers"; @@ -3022,7 +3023,11 @@ export async function handleChatCore({ const res = normalizeExecutorResult(rawExecutorResult); trace("post_executor", { status: res?.response?.status }); - if (provider === "codex" && attemptConnectionId) { + if ( + provider === "codex" && + attemptConnectionId && + !(await shouldIsolateProbeFailures()) + ) { try { const persistedQuota = await persistCodexChildQuotaResponse({ connectionId: String(attemptConnectionId), @@ -3054,7 +3059,11 @@ export async function handleChatCore({ stage: "provider_response_started", }); - if (res.response.status === 401 && executionConnectionId) { + if ( + res.response.status === 401 && + executionConnectionId && + !(await shouldIsolateProbeFailures()) + ) { recordKeyHealthStatus(401, execCreds); } @@ -3084,7 +3093,10 @@ export async function handleChatCore({ !managedLease && comboStrategy !== "context-relay" && res.response.status === 429 && - attempts < maxAttempts - 1 + attempts < maxAttempts - 1 && + // Probe-origin (test-all) 429 must not rotate accounts or persist + // cooldowns — routing state untouched (#9817). + !(await shouldIsolateProbeFailures()) ) { const failedConnectionId = executionConnectionId || credentials?.connectionId || connectionId; @@ -3689,10 +3701,15 @@ export async function handleChatCore({ } // Handle 401/403 - try token refresh using executor + // T-PROBE: probe-origin failures never attempt the refresh — a probe must + // not consume a rotating refresh token nor persist an "expired" + // deactivation on refresh failure (#9817). The 401/403 then flows into + // the normal providerFailure classification (record-only in probe mode). if ( (providerResponse.status === HTTP_STATUS.UNAUTHORIZED || providerResponse.status === HTTP_STATUS.FORBIDDEN) && - !hadStreamOptions // Skip refresh if failure may be from stream_options removal, not auth + !hadStreamOptions && // Skip refresh if failure may be from stream_options removal, not auth + !(await shouldIsolateProbeFailures()) ) { // Fix A: wrap refreshCredentials in runWithOnPersist so the persist callback // executes INSIDE the per-connection mutex held by getAccessToken. This makes @@ -3972,17 +3989,34 @@ export async function handleChatCore({ if (errorConnectionId && errorType) { try { if (errorType === PROVIDER_ERROR_TYPES.FORBIDDEN) { - await updateProviderConnection(errorConnectionId, { - isActive: false, - testStatus: "banned", - lastErrorType: errorType, - lastError: message, - errorCode: statusCode, - }); - console.warn( - `[provider] Node ${errorConnectionId} banned (${statusCode}) — disabling permanently` - ); + // T-PROBE: a probe-origin failure (model test-all) must never + // remove the connection from the pool — record but stay active. + if (await shouldIsolateProbeFailures()) { + await updateProviderConnection(errorConnectionId, { + lastErrorType: errorType, + lastError: message, + errorCode: statusCode, + lastErrorAt: new Date().toISOString(), + }); + console.warn( + `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active` + ); + } else { + await updateProviderConnection(errorConnectionId, { + isActive: false, + testStatus: "banned", + lastErrorType: errorType, + lastError: message, + errorCode: statusCode, + }); + console.warn( + `[provider] Node ${errorConnectionId} banned (${statusCode}) — disabling permanently` + ); + } } else if (errorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED) { + // T-PROBE: probe-origin failures (test-all) never deactivate — + // record but stay active; Plan A (extra keys) stays first so the + // real path keeps its existing priority (#9817). // Plan A: if connection has extra API keys, don't disable — only the failing key is affected. // Single-key connections still get disabled as before. if ( @@ -4000,6 +4034,16 @@ export async function handleChatCore({ console.warn( `[provider] Node ${errorConnectionId} account deactivated (${statusCode}) — has extra keys, keeping connection active` ); + } else if (await shouldIsolateProbeFailures()) { + await updateProviderConnection(errorConnectionId, { + lastErrorType: errorType, + lastError: message, + errorCode: statusCode, + lastErrorAt: new Date().toISOString(), + }); + console.warn( + `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active` + ); } else { await updateProviderConnection(errorConnectionId, { isActive: false, @@ -4013,73 +4057,90 @@ export async function handleChatCore({ ); } } else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) { - // Kimi's 403 says "billing cycle" for both an exhausted subscription and a - // temporary request window. Read its official usage endpoint before making - // the connection terminal: a non-zero Weekly quota plus an empty Ratelimit - // window must recover automatically at the reported reset time. - let kimiRateLimitResetAt: string | null = null; - if (provider === "kimi-coding") { - try { - const { fetchAndPersistProviderLimits } = await import("@/lib/usage/providerLimits"); - const { usage } = await fetchAndPersistProviderLimits(errorConnectionId, "manual"); - kimiRateLimitResetAt = getKimiTemporaryRateLimitResetAt(usage); - } catch { - // Preserve the existing quota handling when Kimi's usage endpoint is unavailable. - } - } - - // Providers with per-model quotas — lock the model only, not the connection - const quotaCooldownMs = kimiRateLimitResetAt - ? Math.max(new Date(kimiRateLimitResetAt).getTime() - Date.now(), 0) - : retryAfterMs || COOLDOWN_MS.rateLimit; - const accountSemaphoreKey = resolveAccountSemaphoreKey({ - provider, - model: currentModel, - connectionId: errorConnectionId, - credentials, - }); - if (accountSemaphoreKey) { - markAccountSemaphoreBlocked(accountSemaphoreKey, quotaCooldownMs); - } - if (kimiRateLimitResetAt) { + // T-PROBE: probe-origin failures never write quota state — + // `testStatus: "credits_exhausted"` is terminal and removes the + // connection from the pool; semaphore locks and per-model quota + // lockouts are routing mutations too. Record only (#9817). + if (await shouldIsolateProbeFailures()) { await updateProviderConnection(errorConnectionId, { - testStatus: "unavailable", - rateLimitedUntil: kimiRateLimitResetAt, - backoffLevel: 0, - lastErrorType: PROVIDER_ERROR_TYPES.RATE_LIMITED, - lastError: message, - errorCode: statusCode, - }); - console.warn( - `[provider] Node ${errorConnectionId} Kimi request window exhausted (${statusCode}) — retrying after ${kimiRateLimitResetAt}` - ); - } else if (isModelScope() && errorConnectionId) { - const lockFn = provider === "antigravity" ? lockExactModel : lockModel; - lockFn(provider, errorConnectionId, model, "quota_exhausted", quotaCooldownMs); - console.warn( - `[provider] Node ${errorConnectionId} ModelScope model quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)` - ); - } else if ( - lockModelIfPerModelQuota( - provider, - errorConnectionId, - model, - "quota_exhausted", - quotaCooldownMs - ) - ) { - const quotaScope = getQuotaScopeLabelForProvider(provider, model); - console.warn( - `[provider] Node ${errorConnectionId} ${quotaScope}-only quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (cooldown_scope=${quotaScope}, ttl_source=${retryAfterMs ? "upstream" : "inferred"}, connection stays active)` - ); - } else { - await updateProviderConnection(errorConnectionId, { - testStatus: "credits_exhausted", lastErrorType: errorType, lastError: message, errorCode: statusCode, + lastErrorAt: new Date().toISOString(), }); - console.warn(`[provider] Node ${errorConnectionId} exhausted quota (${statusCode})`); + console.warn( + `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active` + ); + } else { + // Kimi's 403 says "billing cycle" for both an exhausted subscription and a + // temporary request window. Read its official usage endpoint before making + // the connection terminal: a non-zero Weekly quota plus an empty Ratelimit + // window must recover automatically at the reported reset time. + let kimiRateLimitResetAt: string | null = null; + if (provider === "kimi-coding") { + try { + const { fetchAndPersistProviderLimits } = + await import("@/lib/usage/providerLimits"); + const { usage } = await fetchAndPersistProviderLimits(errorConnectionId, "manual"); + kimiRateLimitResetAt = getKimiTemporaryRateLimitResetAt(usage); + } catch { + // Preserve the existing quota handling when Kimi's usage endpoint is unavailable. + } + } + + // Providers with per-model quotas — lock the model only, not the connection + const quotaCooldownMs = kimiRateLimitResetAt + ? Math.max(new Date(kimiRateLimitResetAt).getTime() - Date.now(), 0) + : retryAfterMs || COOLDOWN_MS.rateLimit; + const accountSemaphoreKey = resolveAccountSemaphoreKey({ + provider, + model: currentModel, + connectionId: errorConnectionId, + credentials, + }); + if (accountSemaphoreKey) { + markAccountSemaphoreBlocked(accountSemaphoreKey, quotaCooldownMs); + } + if (kimiRateLimitResetAt) { + await updateProviderConnection(errorConnectionId, { + testStatus: "unavailable", + rateLimitedUntil: kimiRateLimitResetAt, + backoffLevel: 0, + lastErrorType: PROVIDER_ERROR_TYPES.RATE_LIMITED, + lastError: message, + errorCode: statusCode, + }); + console.warn( + `[provider] Node ${errorConnectionId} Kimi request window exhausted (${statusCode}) — retrying after ${kimiRateLimitResetAt}` + ); + } else if (isModelScope() && errorConnectionId) { + const lockFn = provider === "antigravity" ? lockExactModel : lockModel; + lockFn(provider, errorConnectionId, model, "quota_exhausted", quotaCooldownMs); + console.warn( + `[provider] Node ${errorConnectionId} ModelScope model quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)` + ); + } else if ( + lockModelIfPerModelQuota( + provider, + errorConnectionId, + model, + "quota_exhausted", + quotaCooldownMs + ) + ) { + const quotaScope = getQuotaScopeLabelForProvider(provider, model); + console.warn( + `[provider] Node ${errorConnectionId} ${quotaScope}-only quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (cooldown_scope=${quotaScope}, ttl_source=${retryAfterMs ? "upstream" : "inferred"}, connection stays active)` + ); + } else { + await updateProviderConnection(errorConnectionId, { + testStatus: "credits_exhausted", + lastErrorType: errorType, + lastError: message, + errorCode: statusCode, + }); + console.warn(`[provider] Node ${errorConnectionId} exhausted quota (${statusCode})`); + } } } else if (errorType === PROVIDER_ERROR_TYPES.UNAUTHORIZED) { // Normal 401 (token/session auth issue): keep account active for refresh/re-auth. @@ -4121,11 +4182,15 @@ export async function handleChatCore({ lastError: message, errorCode: statusCode, }); - try { - const { setConnectionRateLimitUntil } = await import("@/lib/db/providers"); - setConnectionRateLimitUntil(errorConnectionId, Date.now() + geoCooldownMs); - } catch { - // DB write failure must never break the fallback loop + // T-PROBE: the 24h exclusion is a routing mutation — a probe must + // not push a connection into a day-long cooldown (#9817). + if (!(await shouldIsolateProbeFailures())) { + try { + const { setConnectionRateLimitUntil } = await import("@/lib/db/providers"); + setConnectionRateLimitUntil(errorConnectionId, Date.now() + geoCooldownMs); + } catch { + // DB write failure must never break the fallback loop + } } console.warn( `[provider] Node ${errorConnectionId} geo-blocked (${statusCode}) — excluded for ${Math.ceil(geoCooldownMs / 1000)}s, trying other accounts` @@ -4136,16 +4201,20 @@ export async function handleChatCore({ // otherwise degenerate into a 429 rate-limit storm). Connection stays // active since only the specific model is unavailable. (#6827) const notFoundCooldownMs = COOLDOWN_MS.notFound; - lockModel( - provider, - errorConnectionId, - currentModel, - "model_not_found", - notFoundCooldownMs - ); - console.warn( - `[provider] Node ${errorConnectionId} model not found (${statusCode}) for ${currentModel} - locking model for ${Math.ceil(notFoundCooldownMs / 1000)}s (connection stays active)` - ); + // T-PROBE: the model lockout is a routing mutation — a probe must + // not lock a model for the cooldown window (#9817). + if (!(await shouldIsolateProbeFailures())) { + lockModel( + provider, + errorConnectionId, + currentModel, + "model_not_found", + notFoundCooldownMs + ); + console.warn( + `[provider] Node ${errorConnectionId} model not found (${statusCode}) for ${currentModel} - locking model for ${Math.ceil(notFoundCooldownMs / 1000)}s (connection stays active)` + ); + } } } catch { // Best-effort state update; request flow should continue with fallback handling. diff --git a/src/lib/api/modelTestRunner.ts b/src/lib/api/modelTestRunner.ts index f68454f6f5..c72248f55a 100644 --- a/src/lib/api/modelTestRunner.ts +++ b/src/lib/api/modelTestRunner.ts @@ -8,7 +8,7 @@ import { extractComboTestResponseText, extractComboTestStreamResult, } from "@/lib/combos/testHealth"; -import { getCustomModels } from "@/lib/localDb"; +import { getCustomModels } from "@/lib/db/models"; import { getProviderNodeById } from "@/lib/db/providers"; import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error"; import { withRateLimit } from "@omniroute/open-sse/services/rateLimitManager"; @@ -18,6 +18,7 @@ import { } from "@omniroute/open-sse/services/accountFallback"; import { looksLikeQuotaExhausted } from "@/shared/utils/classify429"; import { getTrustedLocalRateLimitError } from "@omniroute/open-sse/services/rateLimitManager/errors"; +import { runAsProbe } from "@/shared/utils/probeOrigin"; import { isConnectionUnavailableToAuxiliaryActivity } from "@/lib/exclusiveLeaseIsolation"; const INTERNAL_ORIGIN = "http://omniroute.internal"; @@ -482,11 +483,14 @@ export async function runSingleModelTest( providerId, connectionId, fullModelStr, - (signal) => runInner(signal), + // T-PROBE: wrap the scheduled fn, not the withRateLimit call — a + // queued Bottleneck job executes from its own async resource and + // would otherwise run outside the probe context below. + (signal) => runAsProbe(() => runInner(signal)), controller.signal ); } else { - res = await runInner(controller.signal); + res = await runAsProbe(() => runInner(controller.signal)); } } catch (error: unknown) { clearTimeout(timeoutHandle); @@ -566,9 +570,14 @@ export async function runSingleModelTest( let responseText = ""; let streamError: ModelTestResponseText["error"]; try { - const parsedResponse = await extractModelTestResponseText( - res, - !isEmbedding && !isRerank && streamChat + // T-PROBE: consume the stream inside the probe context too — the SSE + // body is transformed by chatCore/chatHelpers generator code that + // resumes in the CONSUMER's async context. Without this wrapper, an + // error frame inside a 200 stream (Sentinel blocks, "account + // deactivated") would run outside runAsProbe and could still reach + // markAccountUnavailable (#9817). + const parsedResponse = await runAsProbe(() => + extractModelTestResponseText(res, !isEmbedding && !isRerank && streamChat) ); responseText = parsedResponse.text; streamError = parsedResponse.error; diff --git a/src/shared/utils/probeOrigin.ts b/src/shared/utils/probeOrigin.ts new file mode 100644 index 0000000000..1c426e117c --- /dev/null +++ b/src/shared/utils/probeOrigin.ts @@ -0,0 +1,63 @@ +/** + * Probe-origin tracking via AsyncLocalStorage. + * + * Convention: ANY probe flow (model test-all, future batch tests, + * credential-health if it ever routes through the chat path) MUST execute + * inside runAsProbe() so deactivation guards can refuse probe-origin + * failures (invariant #9817: only a real request-path failure deactivates + * a connection). Pinned by tests/unit/probe-testall-isolation.test.ts. + * + * NOTE: when a probe dispatches through a scheduler with a queue + * (Bottleneck via withRateLimit), runAsProbe must wrap the scheduled fn + * itself — a queued job otherwise executes outside this context + * (pinned by the queued-scheduler test below). + * + * EXCEPTIONS (deliberate, documented in the PR): tokenHealthCheck refresh + * failures keep deactivating (re-auth semantics — a dead refresh token is + * a real death, not a probe artifact), and circuit-breaker HALF_OPEN + * probes are real generations by design. Those flows stay outside + * runAsProbe. + */ +import { AsyncLocalStorage } from "node:async_hooks"; + +const probeContext = new AsyncLocalStorage<{ probe: true }>(); + +export function runAsProbe(fn: () => Promise): Promise { + return probeContext.run({ probe: true }, fn); +} + +export function isProbeContext(): boolean { + return probeContext.getStore() !== undefined; +} + +/** + * Central probe-isolation decision used by every deactivation site. + * + * True when the current execution is probe-origin AND the opt-in setting + * `probeCanDisable` is OFF (default): the probe failure is recorded but + * must never remove the connection from the pool (cooldowns, terminal + * status, per-model lockouts, auto-disable, circuit breaker). Operators + * who use test-all as a maintenance tool set `probeCanDisable: true` to + * restore the historical behavior where a probe counts as a real + * generation. + */ +export async function shouldIsolateProbeFailures(): Promise { + if (!isProbeContext()) return false; + // Feature-flag kill-switch (env/DB override; fail-safe false like the + // AUTH_LOG_INCLUDE_ACCOUNT_ID usage): PROBE_CAN_DISABLE restores the + // historical behavior where a probe counts as a real generation. + try { + const { isFeatureFlagEnabled } = await import("@/shared/utils/featureFlags"); + if (isFeatureFlagEnabled("PROBE_CAN_DISABLE")) return false; + } catch { + // Fail-safe: on lookup failure the isolation stays ON. + } + try { + const { getCachedSettings } = await import("@/lib/db/readCache"); + const settings = await getCachedSettings(); + return !settings.probeCanDisable; + } catch { + // Fail-safe: on settings-lookup failure the isolation stays ON. + return true; + } +} diff --git a/src/shared/validation/settingsSchemas.ts b/src/shared/validation/settingsSchemas.ts index c4cb849573..83d87893f9 100644 --- a/src/shared/validation/settingsSchemas.ts +++ b/src/shared/validation/settingsSchemas.ts @@ -174,6 +174,10 @@ export const updateSettingsSchema = z.object({ ) .optional(), customBannedSignals: z.array(z.string().max(200)).optional(), + // #9817: opt-in (default off) — lets a probe-origin (model test-all) + // failure deactivate a connection like real traffic. Off by default: + // probe failures are recorded but never mutate routing state. + probeCanDisable: z.boolean().optional(), debugMode: z.boolean().optional(), logToolSources: z.boolean().optional(), hiddenSidebarItems: z.array(z.enum(HIDEABLE_SIDEBAR_ITEM_IDS)).optional(), diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 4e5ab7b6ef..b200674d32 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -126,6 +126,7 @@ import { classify429FromError, type FailureKind } from "@/shared/utils/classify4 import { isSubscriptionQuotaText } from "@omniroute/open-sse/services/quotaTextCooldowns.ts"; import { resolveUseUpstream429BreakerHints } from "@/shared/utils/providerHints"; import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags"; +import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; import { getCircuitBreaker, isLocalStreamLifecycleError } from "../../shared/utils/circuitBreaker"; import { markAccountExhaustedFrom429 } from "../../domain/quotaCache"; import { resolveForcedConnectionForCredentialPool } from "../services/sessionAffinityPin.ts"; @@ -1664,7 +1665,10 @@ async function handleSingleModelChat( credentials?.allRateLimited && isProviderBreakerFailureStatus(breakerFailureStatus) && !isNetworkError && - !isQueueTimeout + !isQueueTimeout && + // Probe-origin dispatches must not degrade the provider breaker — + // routing state untouched (#9817). + !(await shouldIsolateProbeFailures()) ) { breaker._onFailure(); } @@ -2206,7 +2210,10 @@ async function handleSingleModelChat( const passthroughModels = credentials.providerSpecificData?.passthroughModels; if ( result.status === 429 && - shouldMarkAccountExhaustedFrom429(provider, model, passthroughModels, failureKind) + shouldMarkAccountExhaustedFrom429(provider, model, passthroughModels, failureKind) && + // T-PROBE: a probe must not poison the 5min quotaCache for real + // traffic (#9817). + !(await shouldIsolateProbeFailures()) ) { markAccountExhaustedFrom429(credentials.connectionId, provider); } @@ -2308,7 +2315,12 @@ async function handleSingleModelChat( continue; } - if (shouldTripProviderBreakerForResult(result, isCombo, forceLiveComboTest)) { + // T-PROBE: a probe failure must not degrade the provider-wide circuit + // breaker for real traffic (#9817). + if ( + !(await shouldIsolateProbeFailures()) && + shouldTripProviderBreakerForResult(result, isCombo, forceLiveComboTest) + ) { breaker._onFailure(); } diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index 96a0de2761..7f831b81c5 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -126,6 +126,7 @@ import { getNextFromDeckSync, planNextFromDeckSync, } from "@/shared/utils/shuffleDeck"; +import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; import { applyExclusiveConnectionLeasePolicy, invalidateManagedConnectionLease, @@ -2446,6 +2447,33 @@ export async function markAccountUnavailable( effectiveProviderProfile ); + // T-PROBE: probe-origin failures (model test-all) must never remove the + // connection from the pool. Record the failure for visibility but leave + // ALL routing state untouched — cooldowns, terminal status, per-model + // lockouts (T09 codex-scope, per-model quota, agentrouter #10334) and + // auto-disable. Only a real request-path failure deactivates (#9817); + // the opt-in setting probeCanDisable restores the historical behavior. + if (await shouldIsolateProbeFailures()) { + await updateProviderConnection(connectionId, { + // lastError kept RAW (full text) — maximal probe visibility; the + // divergence vs the normal path's slice(0,100) is intentional. + // backoffLevel is deliberately NOT written: a positive backoff + // triggers the selection-time auto-decay (resetConnectionBackoff, + // auth.ts getProviderCredentials) which wipes lastError back to + // NULL on the next attempt — silently destroying the probe record. + // The backoff is also routing state a probe must not touch (#9817). + lastError: errorText, + lastErrorType: fallbackResult.reason || null, + errorCode: status, + lastErrorAt: new Date().toISOString(), + }); + log.warn( + "AUTH", + `[T-PROBE] ${connectionId.slice(0, 8)} ${provider ?? ""} failure ${status} recorded — connection stays in the pool` + ); + return { shouldFallback: true, cooldownMs: 0 }; + } + // Read passthroughModels from connection config (user-configured per-model quota) const connProviderSpecificData = (conn?.providerSpecificData as Record) || {}; if (provider && conn) { diff --git a/src/sse/services/autoDisableBannedAccount.ts b/src/sse/services/autoDisableBannedAccount.ts index 2d1d8d33d1..b55a8b8417 100644 --- a/src/sse/services/autoDisableBannedAccount.ts +++ b/src/sse/services/autoDisableBannedAccount.ts @@ -9,6 +9,7 @@ import { getCachedSettings } from "@/lib/db/readCache"; import { updateProviderConnection } from "@/lib/db/providers"; import { resolveProviderId, WEB_COOKIE_PROVIDERS } from "@/shared/constants/providers"; import { shouldAutoDisableBannedConnection } from "@/shared/utils/autoDisableBanned"; +import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; import * as log from "../utils/logger"; /** Deactivate a connection after a permanent ban when settings and scope allow it. */ @@ -20,6 +21,15 @@ export async function maybeAutoDisableBannedAccount(input: { permanent?: boolean; }): Promise { if (!input.permanent) return; + // T-PROBE: a probe-origin failure (model test-all) must never disable a + // connection — only a real request-path failure deactivates (#9817). + if (await shouldIsolateProbeFailures()) { + log.info( + "AUTH", + `Skipped auto-disable for ${input.connectionId.slice(0, 8)} — probe origin (permanent failure, connection stays active)` + ); + return; + } try { const settings = await getCachedSettings(); const scope = settings.autoDisableBannedScope; diff --git a/tests/unit/probe-autodisable-isolation.test.ts b/tests/unit/probe-autodisable-isolation.test.ts new file mode 100644 index 0000000000..5cb4479b67 --- /dev/null +++ b/tests/unit/probe-autodisable-isolation.test.ts @@ -0,0 +1,60 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-probe-autodisable-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const settingsDb = await import("../../src/lib/db/settings.ts"); +const { createProviderConnection } = await import("../../src/lib/db/providers.ts"); +const { maybeAutoDisableBannedAccount } = + await import("../../src/sse/services/autoDisableBannedAccount.ts"); +const { runAsProbe } = await import("../../src/shared/utils/probeOrigin.ts"); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +function readIsActive(connId: string): unknown { + const db = core.getDbInstance() as unknown as { + prepare: (sql: string) => { get: (id: string) => { is_active: unknown } | undefined }; + }; + return db.prepare("SELECT is_active FROM provider_connections WHERE id = ?").get(connId) + ?.is_active; +} + +async function setupConnection(): Promise { + await settingsDb.updateSettings({ autoDisableBannedAccounts: true }); + const conn = await createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "probe-autodisable", + apiKey: "sk-probe-autodisable", // pragma: allowlist secret + isActive: true, + testStatus: "active", + }); + return String((conn as { id: string }).id); +} + +const PROBE_INPUT = { + provider: "openai", + authType: "apikey", + connectionProvider: "openai", + permanent: true, +}; + +test("probe-origin permanent failure never disables the connection", async () => { + const connId = await setupConnection(); + await runAsProbe(() => maybeAutoDisableBannedAccount({ connectionId: connId, ...PROBE_INPUT })); + assert.equal(readIsActive(connId), 1); +}); + +test("real-path permanent failure still disables (setting ON)", async () => { + const connId = await setupConnection(); + await maybeAutoDisableBannedAccount({ connectionId: connId, ...PROBE_INPUT }); + assert.equal(readIsActive(connId), 0); +}); diff --git a/tests/unit/probe-gate-autodisable.test.ts b/tests/unit/probe-gate-autodisable.test.ts new file mode 100644 index 0000000000..a4cb5755fb --- /dev/null +++ b/tests/unit/probe-gate-autodisable.test.ts @@ -0,0 +1,294 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-probe-gate-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const { createProviderConnection, deleteProviderConnection } = + await import("../../src/lib/db/providers.ts"); +const { runSingleModelTest, buildInternalChatRequest } = + await import("../../src/lib/api/modelTestRunner.ts"); +const chatRouteModule = await import("../../src/app/api/v1/chat/completions/route.ts"); +const postChatCompletion = chatRouteModule.POST; +const { resetAllCircuitBreakers, getCircuitBreaker } = + await import("../../src/shared/utils/circuitBreaker.ts"); +const { invalidateDbCache } = await import("../../src/lib/db/readCache.ts"); + +const originalFetch = globalThis.fetch; + +test.beforeEach(() => { + resetAllCircuitBreakers(); + invalidateDbCache("connections"); +}); + +test.after(() => { + globalThis.fetch = originalFetch; + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +function readConnectionRow(connId: string) { + const db = core.getDbInstance() as unknown as { + prepare: (sql: string) => { + get: (id: string) => Record | undefined; + }; + }; + return db + .prepare( + "SELECT is_active, test_status, rate_limited_until, last_error, refresh_token, access_token FROM provider_connections WHERE id = ?" + ) + .get(connId); +} + +async function createConnection(provider = "openai", extra: Record = {}) { + const conn = await createProviderConnection({ + provider, + authType: "apikey", + name: "probe-gate", + apiKey: "sk-probe-gate", // pragma: allowlist secret + isActive: true, + testStatus: "active", + ...extra, + }); + return String((conn as { id: string }).id); +} + +async function warmUp(connId: string): Promise { + globalThis.fetch = async () => + new Response(JSON.stringify({ choices: [{ message: { role: "assistant", content: "OK" } }] }), { + headers: { "content-type": "application/json" }, + }); + await runSingleModelTest({ + providerId: "openai", + modelId: "gpt-4o", + connectionId: connId, + timeoutMs: 10_000, + }); +} + +function mockUpstream(status: number, message: string): void { + globalThis.fetch = async () => + new Response(JSON.stringify({ error: { message } }), { + status, + headers: { "content-type": "application/json" }, + }); +} + +const ASSERT_NO_COOLDOWN = (connId: string, label: string) => { + const row = readConnectionRow(connId); + assert.equal(row?.is_active, 1, `${label}: connection stays active`); + assert.notEqual(row?.test_status, "banned", `${label}: no terminal banned status`); + assert.notEqual(row?.test_status, "deactivated", `${label}: no deactivated status`); + assert.notEqual(row?.test_status, "credits_exhausted", `${label}: no credits_exhausted`); + assert.equal(row?.rate_limited_until, null, `${label}: no persisted cooldown`); + assert.ok(row?.last_error, `${label}: probe failure is recorded for visibility`); +}; + +test("GEO_BLOCKED (403 region) probe records but never persists the 24h cooldown", async () => { + const connId = await createConnection("gemini"); + await warmUp(connId); + mockUpstream(403, "user location is not supported"); + const result = await runSingleModelTest({ + providerId: "gemini", + modelId: "gemini-2.5-flash", + connectionId: connId, + timeoutMs: 10_000, + }); + assert.equal(result.status, "error"); + ASSERT_NO_COOLDOWN(connId, "GEO probe"); +}); + +test("QUOTA_EXHAUSTED (402) probe records but never writes the terminal credits state", async () => { + const connId = await createConnection(); + await warmUp(connId); + mockUpstream(402, "billing cycle exhausted"); + const result = await runSingleModelTest({ + providerId: "openai", + modelId: "gpt-4o", + connectionId: connId, + timeoutMs: 10_000, + }); + assert.equal(result.status, "error"); + ASSERT_NO_COOLDOWN(connId, "QUOTA probe"); +}); + +test("MODEL_NOT_FOUND (404) probe does not lock the model for real traffic", async () => { + const connId = await createConnection(); + await warmUp(connId); + mockUpstream(404, "Model gpt-4o not found"); + const failed = await runSingleModelTest({ + providerId: "openai", + modelId: "gpt-4o", + connectionId: connId, + timeoutMs: 10_000, + }); + assert.equal(failed.status, "error"); + ASSERT_NO_COOLDOWN(connId, "404 probe"); + + globalThis.fetch = async () => + new Response(JSON.stringify({ choices: [{ message: { role: "assistant", content: "OK" } }] }), { + headers: { "content-type": "application/json" }, + }); + const after = await runSingleModelTest({ + providerId: "openai", + modelId: "gpt-4o", + connectionId: connId, + timeoutMs: 10_000, + }); + assert.equal(after.status, "ok", "model lockout was not persisted by the probe"); +}); + +test("401 probe never consumes the OAuth refresh token (no executor refresh call)", async () => { + let fetchCalls = 0; + const countingFetch = async (): Promise => { + fetchCalls += 1; + return new Response(JSON.stringify({ error: { message: "invalid token" } }), { + status: 401, + headers: { "content-type": "application/json" }, + }); + }; + globalThis.fetch = countingFetch as typeof fetch; + const connId = await createConnection("github", { + authType: "oauth", + accessToken: "gh-probe-access", + refreshToken: "gh-probe-refresh", + providerSpecificData: { copilotToken: "gh-probe-copilot" }, + }); + const result = await runSingleModelTest({ + providerId: "github", + modelId: "gpt-4o", + connectionId: connId, + timeoutMs: 10_000, + }); + assert.equal(result.status, "error"); + assert.equal( + fetchCalls, + 1, + "exactly one upstream call — the refresh executor must never run for a probe" + ); + const row = readConnectionRow(connId); + assert.equal(row?.refresh_token, "gh-probe-refresh", "refresh token untouched"); + assert.equal(row?.access_token, "gh-probe-access", "access token untouched"); + ASSERT_NO_COOLDOWN(connId, "401 probe"); + // Remove this connection so the stale-token test below runs with exactly + // one github connection (the chatCore fallback would otherwise try the + // second github account, inflating its upstream fetch count). + await deleteProviderConnection(connId); +}); + +test("probe with a stale token never runs the PROACTIVE refresh (base.ts execute)", async () => { + let fetchCalls = 0; + globalThis.fetch = (async () => { + fetchCalls += 1; + return new Response(JSON.stringify({ error: { message: "invalid token" } }), { + status: 401, + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; + const connId = await createConnection("github", { + authType: "oauth", + accessToken: "gh-probe-access", + refreshToken: "gh-probe-refresh", + providerSpecificData: { + copilotToken: "gh-probe-copilot", + copilotTokenExpiresAt: new Date(Date.now() + 60_000).toISOString(), + }, + }); + const result = await runSingleModelTest({ + providerId: "github", + modelId: "gpt-4o", + connectionId: connId, + timeoutMs: 10_000, + }); + assert.equal(result.status, "error"); + assert.equal( + fetchCalls, + 1, + "stale-token probe must skip the proactive refresh rotation (no extra fetch)" + ); + ASSERT_NO_COOLDOWN(connId, "stale-token probe"); +}); + +test("gitlab stale-token probe never runs its own execute() refresh override", async () => { + let fetchCalls = 0; + globalThis.fetch = (async () => { + fetchCalls += 1; + return new Response(JSON.stringify({ error: { message: "invalid token" } }), { + status: 401, + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; + const connId = await createConnection("gitlab", { + authType: "oauth", + accessToken: "gl-probe-access", + refreshToken: "gl-probe-refresh", + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }); + const result = await runSingleModelTest({ + providerId: "gitlab", + modelId: "gpt-4o", + connectionId: connId, + timeoutMs: 10_000, + }); + assert.equal(result.status, "error"); + assert.equal( + fetchCalls, + 1, + "GitLabExecutor.execute must not consume a refresh rotation under a probe" + ); + ASSERT_NO_COOLDOWN(connId, "gitlab stale-token probe"); +}); + +test("codex 429 probe never runs the account-rotation failover (no persisted cooldown)", async () => { + const connId = await createConnection("codex"); + await warmUp(connId); + mockUpstream(429, "rate limited"); + const result = await runSingleModelTest({ + providerId: "codex", + modelId: "gpt-5-codex", + connectionId: connId, + timeoutMs: 10_000, + }); + assert.equal(result.status, "rate_limited"); + ASSERT_NO_COOLDOWN(connId, "codex 429 probe"); +}); + +test("upstream-timeout probe keeps the breaker and the connection intact", async () => { + const connId = await createConnection(); + await warmUp(connId); + + const timeoutError = new Error("upstream deadline exceeded"); + timeoutError.name = "TimeoutError"; + globalThis.fetch = async () => { + throw timeoutError; + }; + + const realRes = await postChatCompletion( + buildInternalChatRequest( + { model: "openai/gpt-4o", messages: [{ role: "user", content: "hi" }], stream: false }, + new AbortController().signal, + connId + ) + ); + assert.notEqual(realRes.status, 200, "real timeout path exercised"); + const breaker = getCircuitBreaker("openai"); + assert.equal(breaker.failureCount, 0, "locally-tagged timeout never trips the breaker (design)"); + + const probe = await runSingleModelTest({ + providerId: "openai", + modelId: "gpt-4o", + connectionId: connId, + timeoutMs: 10_000, + }); + assert.equal(probe.status, "error"); + assert.equal(breaker.failureCount, 0, "probe failure must not degrade the breaker"); + assert.equal( + readConnectionRow(connId)?.is_active, + 1, + "probe timeout keeps the connection active" + ); +}); diff --git a/tests/unit/probe-origin.test.ts b/tests/unit/probe-origin.test.ts new file mode 100644 index 0000000000..6a5ee83a04 --- /dev/null +++ b/tests/unit/probe-origin.test.ts @@ -0,0 +1,38 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import Bottleneck from "bottleneck"; + +const { runAsProbe, isProbeContext } = await import("../../src/shared/utils/probeOrigin.ts"); + +test("runAsProbe propagates through nested async/await; false outside", async () => { + assert.equal(isProbeContext(), false); + await runAsProbe(async () => { + assert.equal(isProbeContext(), true); + await Promise.resolve(); + const inner = async () => { + await Promise.resolve(); + return isProbeContext(); + }; + assert.equal(await inner(), true); + await new Promise((r) => setTimeout(r, 5)); + assert.equal(isProbeContext(), true); + }); + assert.equal(isProbeContext(), false); +}); + +test("queued scheduler job wrapped in runAsProbe keeps the probe context", async () => { + const limiter = new Bottleneck({ maxConcurrent: 1, minTime: 50 }); + let insideQueuedJob: boolean | null = null; + // Schedule job 1 WITHOUT awaiting it so job 2 really queues behind it + // (maxConcurrent 1 + minTime 50) — the queued job must execute with the + // probe context established at scheduling time. + const job1 = limiter.schedule(() => new Promise((r) => setTimeout(r, 30))); + const job2 = limiter.schedule(() => + runAsProbe(() => { + insideQueuedJob = isProbeContext(); + return Promise.resolve(); + }) + ); + await Promise.all([job1, job2]); + assert.equal(insideQueuedJob, true); +}); diff --git a/tests/unit/probe-policy.test.ts b/tests/unit/probe-policy.test.ts new file mode 100644 index 0000000000..bed1fc6f55 --- /dev/null +++ b/tests/unit/probe-policy.test.ts @@ -0,0 +1,61 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-probe-policy-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const settingsDb = await import("../../src/lib/db/settings.ts"); +const { runAsProbe, shouldIsolateProbeFailures, isProbeContext } = + await import("../../src/shared/utils/probeOrigin.ts"); + +test.after(() => { + delete process.env.PROBE_CAN_DISABLE; + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("outside a probe context the decision is always false (real path)", async () => { + await settingsDb.updateSettings({ probeCanDisable: true }); + assert.equal(await shouldIsolateProbeFailures(), false); +}); + +test("probe + default settings => isolation ON (probe never disables)", async () => { + await settingsDb.updateSettings({ probeCanDisable: false }); + await runAsProbe(async () => { + assert.equal(await shouldIsolateProbeFailures(), true); + }); +}); + +test("probe + opt-in setting probeCanDisable => isolation OFF (historical behavior)", async () => { + await settingsDb.updateSettings({ probeCanDisable: true }); + await runAsProbe(async () => { + assert.equal(await shouldIsolateProbeFailures(), false); + }); +}); + +test("probe + env feature flag PROBE_CAN_DISABLE=true => isolation OFF", async () => { + process.env.PROBE_CAN_DISABLE = "true"; + await settingsDb.updateSettings({ probeCanDisable: false }); + await runAsProbe(async () => { + assert.equal(await shouldIsolateProbeFailures(), false); + }); +}); + +test("probe + flag false + setting true => setting wins (flag must not re-enable isolation)", async () => { + process.env.PROBE_CAN_DISABLE = "false"; + await settingsDb.updateSettings({ probeCanDisable: true }); + await runAsProbe(async () => { + assert.equal(await shouldIsolateProbeFailures(), false); + }); +}); + +test("probe context propagation is still pinned (policy uses the same gate)", async () => { + assert.equal(isProbeContext(), false); + await runAsProbe(async () => { + assert.equal(isProbeContext(), true); + }); +}); diff --git a/tests/unit/probe-production-path.test.ts b/tests/unit/probe-production-path.test.ts new file mode 100644 index 0000000000..3b099b99dc --- /dev/null +++ b/tests/unit/probe-production-path.test.ts @@ -0,0 +1,58 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-probe-prodpath-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const settingsDb = await import("../../src/lib/db/settings.ts"); +const { createProviderConnection } = await import("../../src/lib/db/providers.ts"); +const { markAccountUnavailable } = await import("../../src/sse/services/auth.ts"); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +function readConnectionRow(connId: string) { + const db = core.getDbInstance() as unknown as { + prepare: (sql: string) => { + get: (id: string) => + | { + is_active: unknown; + test_status: unknown; + rate_limited_until: unknown; + } + | undefined; + }; + }; + return db + .prepare( + "SELECT is_active, test_status, rate_limited_until FROM provider_connections WHERE id = ?" + ) + .get(connId); +} + +test("real-path permanent failure keeps the full production behavior", async () => { + await settingsDb.updateSettings({ autoDisableBannedAccounts: true }); + const conn = await createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "probe-prodpath", + apiKey: "sk-probe-prodpath", // pragma: allowlist secret + isActive: true, + testStatus: "active", + }); + const connId = String((conn as { id: string }).id); + + const res = await markAccountUnavailable(connId, 403, "this account is deactivated", "openai"); + assert.equal(res.shouldFallback, true); + + const row = readConnectionRow(connId); + assert.equal(row?.is_active, 0, "is_active flipped off (auto-disable, setting ON)"); + assert.equal(row?.test_status, "banned", "terminal status written"); + assert.equal(row?.rate_limited_until, null, "no persisted cooldown on the terminal path"); +}); diff --git a/tests/unit/probe-testall-isolation.test.ts b/tests/unit/probe-testall-isolation.test.ts new file mode 100644 index 0000000000..40016fec82 --- /dev/null +++ b/tests/unit/probe-testall-isolation.test.ts @@ -0,0 +1,176 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-probe-testall-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const { createProviderConnection } = await import("../../src/lib/db/providers.ts"); +const { runSingleModelTest } = await import("../../src/lib/api/modelTestRunner.ts"); +const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts"); +const { invalidateDbCache } = await import("../../src/lib/db/readCache.ts"); +const { refreshConnectionRateLimits, enableRateLimitProtection } = + await import("@omniroute/open-sse/services/rateLimitManager.ts"); + +const originalFetch = globalThis.fetch; + +// A test-all 403 can also open the provider circuit breaker and stale the +// 5s connections read cache (rawConnectionsCache) — either would +// short-circuit the NEXT tests before chatCore, a false positive for the +// isolation asserts. Reset both before every test. +test.beforeEach(() => { + resetAllCircuitBreakers(); + invalidateDbCache("connections"); +}); + +test.after(() => { + globalThis.fetch = originalFetch; + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +function readConnectionRow(connId: string) { + const db = core.getDbInstance() as unknown as { + prepare: (sql: string) => { + get: (id: string) => + | { + is_active: unknown; + test_status: unknown; + rate_limited_until: unknown; + last_error: unknown; + } + | undefined; + }; + }; + return db + .prepare( + "SELECT is_active, test_status, rate_limited_until, last_error FROM provider_connections WHERE id = ?" + ) + .get(connId); +} + +async function createConnection(): Promise { + const conn = await createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "probe-testall", + apiKey: "sk-probe-testall", // pragma: allowlist secret + isActive: true, + testStatus: "active", + }); + return String((conn as { id: string }).id); +} + +// Warm up the chat-completions pipeline (SSE translators, compression +// settings, etc. lazy-init on the first real request in a process) with a +// fast success mock, mirroring model-test-runner.test.ts. +async function warmUp(connId: string): Promise { + globalThis.fetch = async () => + new Response(JSON.stringify({ choices: [{ message: { role: "assistant", content: "OK" } }] }), { + headers: { "content-type": "application/json" }, + }); + await runSingleModelTest({ + providerId: "openai", + modelId: "gpt-4o", + connectionId: connId, + timeoutMs: 10_000, + }); +} + +async function mockUpstream(status: number, message: string): Promise { + globalThis.fetch = async () => + new Response(JSON.stringify({ error: { message } }), { + status, + headers: { "content-type": "application/json" }, + }); +} + +const ASSERT_ISOLATED = (connId: string) => { + const row = readConnectionRow(connId); + assert.equal(row?.is_active, 1, "connection stays active after a probe failure"); + assert.notEqual(row?.test_status, "banned", "no terminal banned status from a probe"); + assert.notEqual(row?.test_status, "deactivated", "no deactivated status from a probe"); + assert.equal(row?.rate_limited_until, null, "no cooldown persisted by a probe"); + assert.ok(row?.last_error, "probe failure is recorded for visibility"); +}; + +test("test-all FORBIDDEN failure (Sentinel) does not deactivate the connection", async () => { + const connId = await createConnection(); + await warmUp(connId); + await mockUpstream(403, "SENTINEL_BLOCKED"); + const result = await runSingleModelTest({ + providerId: "openai", + modelId: "gpt-4o", + connectionId: connId, + timeoutMs: 10_000, + }); + assert.equal(result.status, "error"); + ASSERT_ISOLATED(connId); +}); + +test("test-all ACCOUNT_DEACTIVATED failure does not deactivate the connection", async () => { + const connId = await createConnection(); + await warmUp(connId); + await mockUpstream(403, "this account is deactivated"); + const result = await runSingleModelTest({ + providerId: "openai", + modelId: "gpt-4o", + connectionId: connId, + timeoutMs: 10_000, + }); + assert.equal(result.status, "error"); + ASSERT_ISOLATED(connId); +}); + +test("queued (rate-limited) test-all failure stays isolated", async () => { + const connId = await createConnection(); + await warmUp(connId); + // Rate-limit protection is OFF by default for test connections (empty + // enabledConnections — withRateLimit:537-540 would short-circuit directly, + // a false positive on the inner wrapper). Enable it so the calls really + // go through the Bottleneck limiter. + enableRateLimitProtection(connId); + // minTime 200 forces the 2nd job to wait behind the 1st — the queued job + // must run inside the probe context (inner wrapper in withRateLimit); + // without it, ASSERT_ISOLATED goes red. + refreshConnectionRateLimits(connId, { minTime: 200 }); + await mockUpstream(403, "SENTINEL_BLOCKED"); + await runSingleModelTest({ + providerId: "openai", + modelId: "gpt-4o", + connectionId: connId, + timeoutMs: 10_000, + }); + await runSingleModelTest({ + providerId: "openai", + modelId: "gpt-4o", + connectionId: connId, + timeoutMs: 10_000, + }); + ASSERT_ISOLATED(connId); +}); + +test("RESTORE: probe with opt-in probeCanDisable=true deactivates like real traffic", async () => { + const settingsDb = await import("../../src/lib/db/settings.ts"); + await settingsDb.updateSettings({ probeCanDisable: true }); + try { + const connId = await createConnection(); + await warmUp(connId); + await mockUpstream(403, "SENTINEL_BLOCKED"); + const result = await runSingleModelTest({ + providerId: "openai", + modelId: "gpt-4o", + connectionId: connId, + timeoutMs: 10_000, + }); + assert.equal(result.status, "error"); + const row = readConnectionRow(connId); + assert.equal(row?.is_active, 0, "opt-in restores historical behavior: probe deactivates"); + assert.equal(row?.test_status, "banned", "terminal banned status restored for probe"); + } finally { + await settingsDb.updateSettings({ probeCanDisable: false }); + } +});