fix(resilience): honor Cloudflare 1010 retryable:false — skip COOLDOWN_RETRY (#8775) (#8800)

Api-key 403 bodies with Cloudflare error 1010 / browser_signature_banned /
retryable:false were treated as short AUTH_ERROR cooldowns, so the chat loop
waited ~21–33s before falling through. Return cooldownMs:0 so the tier fails
fast without permanently banning the account.

open-sse/services/accountFallback.ts

@@ -57,7 +57,7 @@ import { parseDayGranularityResetMs, shouldPreserveQuotaSignals } from "./quotaR
 import { evictLockoutOverflow } from "./accountFallback/lockoutEviction.ts";
 export { MODEL_LOCKOUT_EVICTION_CAP } from "./accountFallback/lockoutEviction.ts";
 import { capScaledCooldownMs } from "./accountFallback/cooldownCap.ts";
-
+import { resolveApiKeyForbiddenFallback } from "./accountFallback/nonRetryableUpstream.ts";
 export type ProviderProfile = {
   baseCooldownMs: number;
   useUpstreamRetryHints: boolean;
@@ -1625,7 +1625,7 @@ export function checkFallbackError(
       !errorStr.toLowerCase().includes("hour quota") &&
       !errorStr.toLowerCase().includes("quota has been exceeded")
     ) {
-      return buildRetryableFallback(RateLimitReason.AUTH_ERROR);
+      return resolveApiKeyForbiddenFallback(errorStr, buildRetryableFallback, RateLimitReason.AUTH_ERROR);
     }
   }
 

open-sse/services/accountFallback/nonRetryableUpstream.ts

@@ -0,0 +1,100 @@
+/**
+ * #8775 — Detect Cloudflare (and similar edge) errors that explicitly forbid
+ * retries. When these bodies are misclassified as short AUTH_ERROR cooldowns,
+ * OmniRoute waits 3× for a permanent client-signature ban and burns 21–33s
+ * before falling through to the next combo tier.
+ *
+ * Kept as a pure helper so checkFallbackError stays within its file-size freeze
+ * and every branch is unit-testable without DB/auth wiring.
+ */
+
+function tryParseJsonObject(text: string): Record<string, unknown> | null {
+  const trimmed = text.trim();
+  if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) return null;
+  try {
+    const parsed: unknown = JSON.parse(trimmed);
+    if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+      return parsed as Record<string, unknown>;
+    }
+  } catch {
+    // Body may be truncated HTML/JSON hybrids — fall through to string probes.
+  }
+  return null;
+}
+
+function hasCloudflareBanSignal(value: unknown): boolean {
+  if (typeof value === "number") return value === 1010;
+  if (typeof value !== "string") return false;
+  const lower = value.toLowerCase();
+  return (
+    lower.includes("browser_signature_banned") ||
+    lower.includes("error 1010") ||
+    lower === "1010" ||
+    lower.includes("cloudflare")
+  );
+}
+
+/**
+ * Returns true when upstream error text says the failure must not be retried —
+ * Cloudflare error 1010 / browser_signature_banned, or an explicit
+ * `retryable: false` paired with Cloudflare / owner_action_required signals.
+ */
+export function isNonRetryableCloudflareError(errorText: string | null | undefined): boolean {
+  if (typeof errorText !== "string" || errorText.length === 0) return false;
+
+  const obj = tryParseJsonObject(errorText);
+  if (obj) {
+    const retryable = obj.retryable;
+    const errorCode = obj.error_code ?? obj.errorCode;
+    const errorName = typeof obj.error_name === "string" ? obj.error_name : "";
+    const cloudflareError = obj.cloudflare_error === true || obj.cloudflareError === true;
+    const ownerActionRequired =
+      obj.owner_action_required === true || obj.ownerActionRequired === true;
+
+    if (errorCode === 1010 || errorCode === "1010") return true;
+    if (errorName.toLowerCase() === "browser_signature_banned") return true;
+    if (retryable === false && (cloudflareError || ownerActionRequired)) return true;
+    if (retryable === false && hasCloudflareBanSignal(obj.title ?? obj.type ?? obj.detail)) {
+      return true;
+    }
+  }
+
+  // Non-JSON / nested-string fallbacks (logs often stringify the body).
+  if (/browser_signature_banned/i.test(errorText)) return true;
+  if (/"error_code"\s*:\s*1010\b/.test(errorText)) return true;
+  if (
+    /"retryable"\s*:\s*false/.test(errorText) &&
+    (/"cloudflare_error"\s*:\s*true/.test(errorText) ||
+      /"owner_action_required"\s*:\s*true/.test(errorText) ||
+      /cloudflare/i.test(errorText))
+  ) {
+    return true;
+  }
+
+  return false;
+}
+
+type ApiKeyForbiddenFallback = {
+  shouldFallback: boolean;
+  cooldownMs: number;
+  reason: string;
+  baseCooldownMs?: number;
+  newBackoffLevel?: number;
+  usedUpstreamRetryHint?: boolean;
+};
+
+/**
+ * Api-key 403 branch helper (#8775): CF 1010 / retryable:false gets cooldown 0
+ * so COOLDOWN_RETRY does not wait; other 403s keep the short retryable cooldown.
+ */
+export function resolveApiKeyForbiddenFallback(
+  errorStr: string,
+  buildRetryableFallback: (reason: string) => ApiKeyForbiddenFallback,
+  authErrorReason: string
+): ApiKeyForbiddenFallback {
+  if (isNonRetryableCloudflareError(errorStr)) {
+    return { shouldFallback: true, cooldownMs: 0, reason: authErrorReason };
+  }
+  return buildRetryableFallback(authErrorReason);
+}
+

tests/unit/account-fallback-cf1010-no-retry-8775.test.ts

@@ -0,0 +1,81 @@
+/**
+ * Regression #8775: Cloudflare error 1010 (`browser_signature_banned`) carries
+ * `"retryable": false` / "Do not retry", but checkFallbackError classified the
+ * apikey 403 as a short AUTH_ERROR cooldown (3s). That made the chat loop enter
+ * COOLDOWN_RETRY 3× and burn 21–33s before falling through to the next tier.
+ *
+ * Expected: shouldFallback=true with cooldownMs=0 so the connection is excluded
+ * for this request without arming rateLimitedUntil / cooldown-aware waits.
+ * Must NOT set permanent=true (TLS/fingerprint ban is not an account ban).
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+import { RateLimitReason } from "../../open-sse/config/constants.ts";
+import { isNonRetryableCloudflareError } from "../../open-sse/services/accountFallback/nonRetryableUpstream.ts";
+
+const { checkFallbackError } = await import("../../open-sse/services/accountFallback.ts");
+
+const CF_1010_BODY = JSON.stringify({
+  type: "https://developers.cloudflare.com/support/troubleshooting/http-status-codes/cloudflare-1xxx-errors/error-1010/",
+  title: "Error 1010: Access denied",
+  status: 403,
+  detail: "The site owner has blocked access based on your browser's signature.",
+  error_code: 1010,
+  error_name: "browser_signature_banned",
+  error_category: "access_denied",
+  zone: "api.groq.com",
+  cloudflare_error: true,
+  retryable: false,
+  owner_action_required: true,
+  what_you_should_do: "**Do not retry.** Your user-agent has been banned by the site owner.",
+});
+
+test("isNonRetryableCloudflareError: detects verbatim CF 1010 JSON", () => {
+  assert.equal(isNonRetryableCloudflareError(CF_1010_BODY), true);
+});
+
+test("isNonRetryableCloudflareError: ignores ordinary api-key 403 text", () => {
+  assert.equal(isNonRetryableCloudflareError("invalid api key"), false);
+  assert.equal(isNonRetryableCloudflareError('{"error":"forbidden"}'), false);
+});
+
+test("#8775: CF 1010 on groq returns fallback with zero cooldown (no COOLDOWN_RETRY fuel)", () => {
+  const result = checkFallbackError(403, CF_1010_BODY, 0, null, "groq");
+  assert.equal(result.shouldFallback, true);
+  assert.equal(
+    result.cooldownMs,
+    0,
+    "must not arm a short connection cooldown that triggers COOLDOWN_RETRY waits"
+  );
+  assert.notEqual(
+    (result as { permanent?: boolean }).permanent,
+    true,
+    "fingerprint ban must not permanently ban the account"
+  );
+  assert.equal(result.reason, RateLimitReason.AUTH_ERROR);
+});
+
+test("#8775: same for cerebras and together (Cloudflare-fronted apikey providers)", () => {
+  for (const provider of ["cerebras", "together"] as const) {
+    const result = checkFallbackError(403, CF_1010_BODY, 0, null, provider);
+    assert.equal(result.cooldownMs, 0, `${provider} must get zero cooldown`);
+    assert.equal(result.shouldFallback, true, `${provider} must still fall through`);
+  }
+});
+
+test("#8775: ordinary apikey 403 still gets a short retryable cooldown (no over-broadening)", () => {
+  const result = checkFallbackError(403, "invalid api key", 0, null, "groq");
+  assert.equal(result.shouldFallback, true);
+  assert.ok(
+    result.cooldownMs > 0,
+    "genuine auth 403 must keep the short cooldown path so bad keys back off"
+  );
+});
+
+test("#8775: retryable:false + cloudflare_error string body (non-pretty JSON) is detected", () => {
+  const compact =
+    '{"cloudflare_error":true,"retryable":false,"error_code":1010,"error_name":"browser_signature_banned"}';
+  assert.equal(isNonRetryableCloudflareError(compact), true);
+  const result = checkFallbackError(403, compact, 0, null, "groq");
+  assert.equal(result.cooldownMs, 0);
+});