diff --git a/changelog.d/fixes/12864-anthropic-request-not-allowed-not-a-ban.md b/changelog.d/fixes/12864-anthropic-request-not-allowed-not-a-ban.md new file mode 100644 index 0000000000..41f9af91db --- /dev/null +++ b/changelog.d/fixes/12864-anthropic-request-not-allowed-not-a-ban.md @@ -0,0 +1 @@ +- **fix(sse):** An Anthropic OAuth `403 "Request not allowed"` no longer bans the Claude connection on the first response — it is a per-request refusal on an otherwise healthy token, so it is now classified as the non-terminal `request_rejected` type, the connection is excluded for a growing cooldown (5 min, then 15 min) and only three consecutive refusals with no success in between escalate to `banned`; previously a single such response flipped the only Claude connection to `banned` and every later request was short-circuited with "All 1 connection(s) banned by upstream" until an operator reconnected ([#12859](https://github.com/diegosouzapw/OmniRoute/issues/12859), [#12864](https://github.com/diegosouzapw/OmniRoute/pull/12864) — thanks @insoln) diff --git a/docs/architecture/CODEBASE_DOCUMENTATION.md b/docs/architecture/CODEBASE_DOCUMENTATION.md index 0f60d2fdcc..a3df7cc0e1 100644 --- a/docs/architecture/CODEBASE_DOCUMENTATION.md +++ b/docs/architecture/CODEBASE_DOCUMENTATION.md @@ -523,7 +523,7 @@ Highlights (full list under `open-sse/services/`): | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Combo routing | `combo.ts` (19 strategies), `comboConfig.ts`, `comboMetrics.ts`, `comboManifestMetrics.ts`, `comboAgentMiddleware.ts` | | Auto Combo engine | `autoCombo/` — `engine.ts`, `scoring.ts`, `taskFitness.ts`, `virtualFactory.ts`, `modePacks.ts`, `autoPrefix.ts`, `persistence.ts`, `providerDiversity.ts`, `providerRegistryAccessor.ts`, `routerStrategy.ts`, `selfHealing.ts`, `index.ts` | -| Resilience | `accountFallback.ts` (cooldown + lockout), `errorClassifier.ts`, `emergencyFallback.ts`, `rateLimitManager.ts`, `rateLimitSemaphore.ts`, `accountSemaphore.ts`, `accountSelector.ts` | +| Resilience | `accountFallback.ts` (cooldown + lockout), `errorClassifier.ts`, `requestRejectedStreak.ts`, `emergencyFallback.ts`, `rateLimitManager.ts`, `rateLimitSemaphore.ts`, `accountSemaphore.ts`, `accountSelector.ts` | | Quotas | `quotaMonitor.ts`, `quotaPreflight.ts`, `bailianQuotaFetcher.ts`, `codexQuotaFetcher.ts`, `deepseekQuotaFetcher.ts`, `openrouterQuotaFetcher.ts`, `openrouterFreeWindow.ts`, `crofUsageFetcher.ts`, `antigravityCredits.ts` | | Caching | `reasoningCache.ts`, `searchCache.ts`, `signatureCache.ts`, `requestDedup.ts` | | Routing intelligence | `intentClassifier.ts`, `taskAwareRouter.ts`, `backgroundTaskDetector.ts`, `volumeDetector.ts`, `wildcardRouter.ts`, `workflowFSM.ts`, `specificityDetector.ts`, `specificityRules.ts`, `specificityTypes.ts` | diff --git a/docs/architecture/RESILIENCE_GUIDE.md b/docs/architecture/RESILIENCE_GUIDE.md index cc49c443b2..7ce95a00e2 100644 --- a/docs/architecture/RESILIENCE_GUIDE.md +++ b/docs/architecture/RESILIENCE_GUIDE.md @@ -100,7 +100,7 @@ Regression guard: `tests/unit/provider-cooldown-window-gate.test.ts`. **Terminal states (NOT cooldowns):** -- `banned` — set by banned-keyword / account-ban detection (see [BAN_DETECTION](../security/BAN_DETECTION.md)) +- `banned` — set by banned-keyword / account-ban detection (see [BAN_DETECTION](../security/BAN_DETECTION.md)), and by three consecutive upstream per-request refusals (`request_rejected`, e.g. Anthropic OAuth 403 "Request not allowed" — `open-sse/services/requestRejectedStreak.ts`); a single refusal only cools the connection down - `expired` (transitions to terminal after bounded retries — `EXPIRED_RETRY_MAX = 3` with exponential backoff — so transient OAuth errors can self-heal before the account is permanently deactivated) - `credits_exhausted` diff --git a/open-sse/config/errorConfig.ts b/open-sse/config/errorConfig.ts index 7e7355948f..6464a6eab1 100644 --- a/open-sse/config/errorConfig.ts +++ b/open-sse/config/errorConfig.ts @@ -74,6 +74,12 @@ export const COOLDOWN_MS = { transientMax: 60 * 1000, transient: TRANSIENT_COOLDOWN_MS, requestNotAllowed: 5 * 1000, + // Anthropic OAuth 403 "Request not allowed" (#12859): a per-request refusal + // on a healthy token. chatCore excludes the connection for requestRejected + // after the first refusal, requestRejectedRepeat after the second, and bans + // it on the third consecutive one (services/requestRejectedStreak.ts). + requestRejected: 5 * 60 * 1000, + requestRejectedRepeat: 15 * 60 * 1000, rateLimit: 2 * 60 * 1000, serviceUnavailable: 2 * 1000, authExpired: 2 * 60 * 1000, @@ -99,6 +105,10 @@ export const ERROR_RULES: ErrorRule[] = [ reason: "auth_error", }, { + // For provider `claude` this text is classified REQUEST_REJECTED and the + // connection-level cooldown is written by chatCore before the fallback + // layer runs (#12859); markAccountUnavailable then keeps the longer + // cooldown. This 5 s rule still serves every other provider. id: "request_not_allowed", text: "request not allowed", cooldownMs: COOLDOWN_MS.requestNotAllowed, diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 506f14f58e..e1570743f0 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -275,6 +275,8 @@ import { stageTrace } from "./chatCore/stageTrace.ts"; import { attachCompressionUsageReceiptAfterAnalytics as attachCompressionUsageReceiptAfterAnalyticsFor } from "./chatCore/compressionUsageReceipt.ts"; import { prepareUpstreamBody } from "./chatCore/upstreamBody.ts"; import { getQuotaScopeLabelForProvider } from "../services/antigravityQuotaFamily.ts"; +import { excludeConnectionForCooldown } from "./chatCore/connectionCooldown.ts"; +import { handleRequestRejectedFailure } from "./chatCore/requestRejectedFailure.ts"; import { getKimiTemporaryRateLimitResetAt } from "./chatCore/kimiQuotaRecovery.ts"; import { getCallLogPipelineCaptureStreamChunks, @@ -4049,35 +4051,37 @@ export async function handleChatCore({ `[provider] Node ${errorConnectionId} project routing error (${statusCode}) -- not banning` ); } else if (errorType === PROVIDER_ERROR_TYPES.GEO_BLOCKED) { - const geoCooldownMs = COOLDOWN_MS.geoBlocked ?? 24 * 60 * 60 * 1000; - await updateProviderConnection(errorConnectionId, { - lastErrorType: errorType, - lastError: persistentMessage, - errorCode: statusCode, + // Google regional refusal: account-independent, non-terminal; park the connection + // until egress uses a supported region; probes skip the day-long cooldown (#9817). + await excludeConnectionForCooldown({ + connectionId: errorConnectionId, + errorType, + message: persistentMessage, + statusCode, + cooldownMs: COOLDOWN_MS.geoBlocked ?? 24 * 60 * 60 * 1000, + skipCooldownForProbe: true, + label: "geo-blocked", + suffix: "trying other accounts", + }); + } else if (errorType === PROVIDER_ERROR_TYPES.REQUEST_REJECTED) { + // Per-request refusal (#12859): growing cooldown, streak → banned. + await handleRequestRejectedFailure({ + connectionId: errorConnectionId, + statusCode, + message: persistentMessage, }); - if (!(await shouldIsolateProbeFailures())) { - try { - const { setConnectionRateLimitUntil } = await import("@/lib/db/providers"); - setConnectionRateLimitUntil(errorConnectionId, Date.now() + geoCooldownMs); - } catch {} - } - console.warn( - `[provider] Node ${errorConnectionId} geo-blocked (${statusCode}) -- excluded for ${Math.ceil(geoCooldownMs / 1000)}s, trying other accounts` - ); } else if (errorType === PROVIDER_ERROR_TYPES.GCP_PROJECT_REQUIRED) { - const byopCooldownMs = COOLDOWN_MS.gcpProjectRequired ?? 24 * 60 * 60 * 1000; - await updateProviderConnection(errorConnectionId, { - lastErrorType: errorType, - lastError: persistentMessage, - errorCode: statusCode, + // Antigravity BYOP: fixable via a Project ID; never a lockout/ban. Park the connection. + await excludeConnectionForCooldown({ + connectionId: errorConnectionId, + errorType, + message: persistentMessage, + statusCode, + cooldownMs: COOLDOWN_MS.gcpProjectRequired ?? 24 * 60 * 60 * 1000, + skipCooldownForProbe: false, + label: "GCP project required", + suffix: "routing to other accounts (enter a Project ID to restore)", }); - try { - const { setConnectionRateLimitUntil } = await import("@/lib/db/providers"); - setConnectionRateLimitUntil(errorConnectionId, Date.now() + byopCooldownMs); - } catch {} - console.warn( - `[provider] Node ${errorConnectionId} GCP project required (${statusCode}) -- excluded for ${Math.ceil(byopCooldownMs / 1000)}s, routing to other accounts (enter a Project ID to restore)` - ); } else if (errorType === PROVIDER_ERROR_TYPES.MODEL_NOT_FOUND) { const notFoundCooldownMs = COOLDOWN_MS.notFound; if (!(await shouldIsolateProbeFailures())) { diff --git a/open-sse/handlers/chatCore/connectionCooldown.ts b/open-sse/handlers/chatCore/connectionCooldown.ts new file mode 100644 index 0000000000..fe007ed7e4 --- /dev/null +++ b/open-sse/handlers/chatCore/connectionCooldown.ts @@ -0,0 +1,42 @@ +import { updateProviderConnection } from "@/lib/db/providers"; +import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; + +/** + * chatCore leaf for the non-terminal "exclude this connection for a while" + * outcomes (GEO_BLOCKED, GCP_PROJECT_REQUIRED): record the error on the + * connection and park it via rateLimitedUntil so selection prefers sibling + * accounts, without ever marking it banned/expired. + */ +export async function excludeConnectionForCooldown(params: { + connectionId: string; + errorType: string; + message: string; + statusCode: number; + cooldownMs: number; + /** + * T-PROBE (#9817): when true, a probe-origin failure records the error but + * does not push the connection into the cooldown (routing state untouched). + */ + skipCooldownForProbe: boolean; + /** Log wording: `[provider] Node