diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index a53bdffef6..5204c7410d 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -57,7 +57,7 @@ import { parseDayGranularityResetMs, shouldPreserveQuotaSignals } from "./quotaR import { evictLockoutOverflow } from "./accountFallback/lockoutEviction.ts"; export { MODEL_LOCKOUT_EVICTION_CAP } from "./accountFallback/lockoutEviction.ts"; import { capScaledCooldownMs } from "./accountFallback/cooldownCap.ts"; - +import { resolveApiKeyForbiddenFallback } from "./accountFallback/nonRetryableUpstream.ts"; export type ProviderProfile = { baseCooldownMs: number; useUpstreamRetryHints: boolean; @@ -1625,7 +1625,7 @@ export function checkFallbackError( !errorStr.toLowerCase().includes("hour quota") && !errorStr.toLowerCase().includes("quota has been exceeded") ) { - return buildRetryableFallback(RateLimitReason.AUTH_ERROR); + return resolveApiKeyForbiddenFallback(errorStr, buildRetryableFallback, RateLimitReason.AUTH_ERROR); } } diff --git a/open-sse/services/accountFallback/nonRetryableUpstream.ts b/open-sse/services/accountFallback/nonRetryableUpstream.ts new file mode 100644 index 0000000000..df4084b49c --- /dev/null +++ b/open-sse/services/accountFallback/nonRetryableUpstream.ts @@ -0,0 +1,100 @@ +/** + * #8775 — Detect Cloudflare (and similar edge) errors that explicitly forbid + * retries. When these bodies are misclassified as short AUTH_ERROR cooldowns, + * OmniRoute waits 3× for a permanent client-signature ban and burns 21–33s + * before falling through to the next combo tier. + * + * Kept as a pure helper so checkFallbackError stays within its file-size freeze + * and every branch is unit-testable without DB/auth wiring. + */ + +function tryParseJsonObject(text: string): Record | null { + const trimmed = text.trim(); + if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) return null; + try { + const parsed: unknown = JSON.parse(trimmed); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + return parsed as Record; + } + } catch { + // Body may be truncated HTML/JSON hybrids — fall through to string probes. + } + return null; +} + +function hasCloudflareBanSignal(value: unknown): boolean { + if (typeof value === "number") return value === 1010; + if (typeof value !== "string") return false; + const lower = value.toLowerCase(); + return ( + lower.includes("browser_signature_banned") || + lower.includes("error 1010") || + lower === "1010" || + lower.includes("cloudflare") + ); +} + +/** + * Returns true when upstream error text says the failure must not be retried — + * Cloudflare error 1010 / browser_signature_banned, or an explicit + * `retryable: false` paired with Cloudflare / owner_action_required signals. + */ +export function isNonRetryableCloudflareError(errorText: string | null | undefined): boolean { + if (typeof errorText !== "string" || errorText.length === 0) return false; + + const obj = tryParseJsonObject(errorText); + if (obj) { + const retryable = obj.retryable; + const errorCode = obj.error_code ?? obj.errorCode; + const errorName = typeof obj.error_name === "string" ? obj.error_name : ""; + const cloudflareError = obj.cloudflare_error === true || obj.cloudflareError === true; + const ownerActionRequired = + obj.owner_action_required === true || obj.ownerActionRequired === true; + + if (errorCode === 1010 || errorCode === "1010") return true; + if (errorName.toLowerCase() === "browser_signature_banned") return true; + if (retryable === false && (cloudflareError || ownerActionRequired)) return true; + if (retryable === false && hasCloudflareBanSignal(obj.title ?? obj.type ?? obj.detail)) { + return true; + } + } + + // Non-JSON / nested-string fallbacks (logs often stringify the body). + if (/browser_signature_banned/i.test(errorText)) return true; + if (/"error_code"\s*:\s*1010\b/.test(errorText)) return true; + if ( + /"retryable"\s*:\s*false/.test(errorText) && + (/"cloudflare_error"\s*:\s*true/.test(errorText) || + /"owner_action_required"\s*:\s*true/.test(errorText) || + /cloudflare/i.test(errorText)) + ) { + return true; + } + + return false; +} + +type ApiKeyForbiddenFallback = { + shouldFallback: boolean; + cooldownMs: number; + reason: string; + baseCooldownMs?: number; + newBackoffLevel?: number; + usedUpstreamRetryHint?: boolean; +}; + +/** + * Api-key 403 branch helper (#8775): CF 1010 / retryable:false gets cooldown 0 + * so COOLDOWN_RETRY does not wait; other 403s keep the short retryable cooldown. + */ +export function resolveApiKeyForbiddenFallback( + errorStr: string, + buildRetryableFallback: (reason: string) => ApiKeyForbiddenFallback, + authErrorReason: string +): ApiKeyForbiddenFallback { + if (isNonRetryableCloudflareError(errorStr)) { + return { shouldFallback: true, cooldownMs: 0, reason: authErrorReason }; + } + return buildRetryableFallback(authErrorReason); +} + diff --git a/tests/unit/account-fallback-cf1010-no-retry-8775.test.ts b/tests/unit/account-fallback-cf1010-no-retry-8775.test.ts new file mode 100644 index 0000000000..cf20707761 --- /dev/null +++ b/tests/unit/account-fallback-cf1010-no-retry-8775.test.ts @@ -0,0 +1,81 @@ +/** + * Regression #8775: Cloudflare error 1010 (`browser_signature_banned`) carries + * `"retryable": false` / "Do not retry", but checkFallbackError classified the + * apikey 403 as a short AUTH_ERROR cooldown (3s). That made the chat loop enter + * COOLDOWN_RETRY 3× and burn 21–33s before falling through to the next tier. + * + * Expected: shouldFallback=true with cooldownMs=0 so the connection is excluded + * for this request without arming rateLimitedUntil / cooldown-aware waits. + * Must NOT set permanent=true (TLS/fingerprint ban is not an account ban). + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { RateLimitReason } from "../../open-sse/config/constants.ts"; +import { isNonRetryableCloudflareError } from "../../open-sse/services/accountFallback/nonRetryableUpstream.ts"; + +const { checkFallbackError } = await import("../../open-sse/services/accountFallback.ts"); + +const CF_1010_BODY = JSON.stringify({ + type: "https://developers.cloudflare.com/support/troubleshooting/http-status-codes/cloudflare-1xxx-errors/error-1010/", + title: "Error 1010: Access denied", + status: 403, + detail: "The site owner has blocked access based on your browser's signature.", + error_code: 1010, + error_name: "browser_signature_banned", + error_category: "access_denied", + zone: "api.groq.com", + cloudflare_error: true, + retryable: false, + owner_action_required: true, + what_you_should_do: "**Do not retry.** Your user-agent has been banned by the site owner.", +}); + +test("isNonRetryableCloudflareError: detects verbatim CF 1010 JSON", () => { + assert.equal(isNonRetryableCloudflareError(CF_1010_BODY), true); +}); + +test("isNonRetryableCloudflareError: ignores ordinary api-key 403 text", () => { + assert.equal(isNonRetryableCloudflareError("invalid api key"), false); + assert.equal(isNonRetryableCloudflareError('{"error":"forbidden"}'), false); +}); + +test("#8775: CF 1010 on groq returns fallback with zero cooldown (no COOLDOWN_RETRY fuel)", () => { + const result = checkFallbackError(403, CF_1010_BODY, 0, null, "groq"); + assert.equal(result.shouldFallback, true); + assert.equal( + result.cooldownMs, + 0, + "must not arm a short connection cooldown that triggers COOLDOWN_RETRY waits" + ); + assert.notEqual( + (result as { permanent?: boolean }).permanent, + true, + "fingerprint ban must not permanently ban the account" + ); + assert.equal(result.reason, RateLimitReason.AUTH_ERROR); +}); + +test("#8775: same for cerebras and together (Cloudflare-fronted apikey providers)", () => { + for (const provider of ["cerebras", "together"] as const) { + const result = checkFallbackError(403, CF_1010_BODY, 0, null, provider); + assert.equal(result.cooldownMs, 0, `${provider} must get zero cooldown`); + assert.equal(result.shouldFallback, true, `${provider} must still fall through`); + } +}); + +test("#8775: ordinary apikey 403 still gets a short retryable cooldown (no over-broadening)", () => { + const result = checkFallbackError(403, "invalid api key", 0, null, "groq"); + assert.equal(result.shouldFallback, true); + assert.ok( + result.cooldownMs > 0, + "genuine auth 403 must keep the short cooldown path so bad keys back off" + ); +}); + +test("#8775: retryable:false + cloudflare_error string body (non-pretty JSON) is detected", () => { + const compact = + '{"cloudflare_error":true,"retryable":false,"error_code":1010,"error_name":"browser_signature_banned"}'; + assert.equal(isNonRetryableCloudflareError(compact), true); + const result = checkFallbackError(403, compact, 0, null, "groq"); + assert.equal(result.cooldownMs, 0); +});