From 994f1c78a09f4542c2f88d3ba007103a8fa2531c Mon Sep 17 00:00:00 2001 From: Rafael Dias Zendron Date: Thu, 16 Jul 2026 14:12:18 -0300 Subject: [PATCH] fix(6980): classify Cloudflare AI neuron exhaustion as quota_exhausted (#6983) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloudflare Workers AI free tier (10k Neurons/day, account-wide) returns 429 with body 'you have used up your daily free allocation of 10,000 neurons' which matched no QUOTA_PATTERNS keyword — falling through to rate_limit (~60s cooldown) instead of quota_exhausted. Two layers: 1. Provider-specific rule for 'cloudflare-ai' in providerRuleRegistry (scope: connection — budget is account-wide, not per-model) 2. Defense-in-depth: /daily free allocation/i in classify429 QUOTA_PATTERNS Tests: 11/11 pass (provider rule + classify429 paths covered). Closes #6980 Co-authored-by: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> --- open-sse/config/providerErrorRules.ts | 30 ++++- src/shared/utils/classify429.ts | 8 ++ ...oudflare-ai-neuron-exhaustion-6980.test.ts | 108 ++++++++++++++++++ 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 tests/unit/cloudflare-ai-neuron-exhaustion-6980.test.ts diff --git a/open-sse/config/providerErrorRules.ts b/open-sse/config/providerErrorRules.ts index d9eb14535b..dde56a53d2 100644 --- a/open-sse/config/providerErrorRules.ts +++ b/open-sse/config/providerErrorRules.ts @@ -130,6 +130,31 @@ function buildMinimaxRules(): ProviderErrorRule[] { ]; } +// ─── Cloudflare Workers AI ───────────────────────────────────────────────────── +// Free tier = 10,000 Neurons/day, shared across the WHOLE account +// (docs/reference/FREE_TIERS.md; official: developers.cloudflare.com/ +// workers-ai/platform/errors/). The exhaustion body doesn't match any +// QUOTA_PATTERNS keyword so it falls through to rate_limit and gets +// retried every ~60s against a budget that only resets at UTC midnight. +// Issue #6980. +function buildCloudflareAiRules(): ProviderErrorRule[] { + return [ + { + id: "cloudflare-ai-daily-neuron-allocation", + match: ({ status, body }) => { + if (status !== 429) return null; + const text = JSON.stringify(body ?? "").toLowerCase(); + // Body: "you have used up your daily free allocation of 10,000 neurons, + // please upgrade to Cloudflare's Workers Paid plan..." + if (!text.includes("daily free allocation")) return null; + // No cooldownMs: recordModelLockoutFailure already sets + // quota_exhausted without one to "next UTC midnight". + return { reason: "quota_exhausted", scope: "connection" }; + }, + }, + ]; +} + /** * Global registry. Provider name → ordered list of rules (first match wins). * Add new providers here; the matcher in classifyError will pick them up @@ -141,6 +166,7 @@ export const providerRuleRegistry = new Map([ ["opencode-cli", buildOpencodeRules()], ["minimax", buildMinimaxRules()], ["minimax-passthrough", buildMinimaxRules()], + ["cloudflare-ai", buildCloudflareAiRules()], ]); /** @@ -194,7 +220,9 @@ export function getProviderErrorRuleMatch( */ export function parseResetCountdownMs(text: string): number | null { if (typeof text !== "string" || text.length === 0) return null; - const match = text.match(/resets?\s+in\s+(\d+)\s+(day|days|hour|hours|minute|minutes|second|seconds)\b/); + const match = text.match( + /resets?\s+in\s+(\d+)\s+(day|days|hour|hours|minute|minutes|second|seconds)\b/ + ); if (!match) return null; const n = Number(match[1]); if (!Number.isFinite(n) || n <= 0) return null; diff --git a/src/shared/utils/classify429.ts b/src/shared/utils/classify429.ts index ff6f21ab31..a824679d3b 100644 --- a/src/shared/utils/classify429.ts +++ b/src/shared/utils/classify429.ts @@ -53,6 +53,14 @@ const QUOTA_PATTERNS: ReadonlyArray = [ /individual quota reached/i, /enable overages/i, /INSUFFICIENT_G1_CREDITS_BALANCE/i, + + // Cloudflare Workers AI daily neuron exhaustion (Issue #6980). + // Body: "you have used up your daily free allocation of 10,000 neurons, + // please upgrade to Cloudflare's Workers Paid plan..." + // No existing pattern matches "daily free allocation" — without this, + // the 429 is misclassified as transient rate_limit and retried every + // ~60s against a budget that only resets at UTC midnight. + /daily free allocation/i, ]; /** diff --git a/tests/unit/cloudflare-ai-neuron-exhaustion-6980.test.ts b/tests/unit/cloudflare-ai-neuron-exhaustion-6980.test.ts new file mode 100644 index 0000000000..4df5826c59 --- /dev/null +++ b/tests/unit/cloudflare-ai-neuron-exhaustion-6980.test.ts @@ -0,0 +1,108 @@ +/** + * Issue #6980 — Cloudflare Workers AI daily neuron exhaustion 429 must be + * classified as quota_exhausted (not transient rate_limit). + * + * Two layers of defense: + * 1. Provider-specific rule in providerErrorRules.ts → getProviderErrorRuleMatch + * 2. Global QUOTA_PATTERNS in classify429.ts → looksLikeQuotaExhausted + * + * Without these, the 429 body "you have used up your daily free allocation of + * 10,000 neurons" matches no keyword, falls through to rate_limit (~60s cooldown), + * and the combo router keeps cycling through every cloudflare model on retry + * against a budget that only resets at UTC midnight. + */ + +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; + +import { + getProviderErrorRuleMatch, + providerRuleRegistry, +} from "../../open-sse/config/providerErrorRules.ts"; +import { classify429, looksLikeQuotaExhausted } from "../../src/shared/utils/classify429.ts"; + +// ─── Fixtures ──────────────────────────────────────────────────────────────── + +const CF_NEURON_BODY = + "you have used up your daily free allocation of 10,000 neurons, please upgrade to Cloudflare's Workers Paid plan"; + +const CF_NEURON_BODY_JSON = { + errors: [ + { + code: 4006, + message: + "you have used up your daily free allocation of 10,000 neurons, please upgrade to Cloudflare's Workers Paid plan", + }, + ], +}; + +// ─── Tests: provider-specific rule (primary path) ─────────────────────────── + +describe("#6980 provider rule: cloudflare-ai neuron exhaustion", () => { + test("cloudflare-ai is registered in providerRuleRegistry", () => { + assert.ok(providerRuleRegistry.has("cloudflare-ai")); + }); + + test("429 with plain-string neuron body → quota_exhausted, scope connection", () => { + const result = getProviderErrorRuleMatch("cloudflare-ai", 429, {}, CF_NEURON_BODY); + assert.ok(result, "expected a match"); + assert.equal(result!.reason, "quota_exhausted"); + assert.equal(result!.scope, "connection"); + // No explicit cooldownMs — recordModelLockoutFailure resolves to next UTC midnight. + assert.equal(result!.cooldownMs, undefined); + }); + + test("429 with JSON-structured neuron body → quota_exhausted", () => { + const result = getProviderErrorRuleMatch("cloudflare-ai", 429, {}, CF_NEURON_BODY_JSON); + assert.ok(result); + assert.equal(result!.reason, "quota_exhausted"); + assert.equal(result!.scope, "connection"); + }); + + test("non-429 status does not match even with neuron body", () => { + const result = getProviderErrorRuleMatch("cloudflare-ai", 500, {}, CF_NEURON_BODY); + assert.equal(result, null); + }); + + test("429 with unrelated body does not match", () => { + const result = getProviderErrorRuleMatch( + "cloudflare-ai", + 429, + {}, + { + error: "rate limited, try again later", + } + ); + assert.equal(result, null); + }); + + test("provider name matching is case-insensitive", () => { + const result = getProviderErrorRuleMatch("Cloudflare-AI", 429, {}, CF_NEURON_BODY); + assert.ok(result); + assert.equal(result!.reason, "quota_exhausted"); + }); +}); + +// ─── Tests: classify429 defense-in-depth (fallback path) ──────────────────── + +describe("#6980 classify429: daily free allocation pattern", () => { + test("looksLikeQuotaExhausted matches neuron body string", () => { + assert.ok(looksLikeQuotaExhausted(CF_NEURON_BODY)); + }); + + test("looksLikeQuotaExhausted matches neuron body JSON-stringified", () => { + assert.ok(looksLikeQuotaExhausted(CF_NEURON_BODY_JSON)); + }); + + test("classify429 returns quota_exhausted for neuron body", () => { + assert.equal(classify429({ status: 429, body: CF_NEURON_BODY }), "quota_exhausted"); + }); + + test("classify429 returns quota_exhausted for neuron JSON body", () => { + assert.equal(classify429({ status: 429, body: CF_NEURON_BODY_JSON }), "quota_exhausted"); + }); + + test("classify429 returns rate_limit for generic 429 without quota keywords", () => { + assert.equal(classify429({ status: 429, body: "Too many requests" }), "rate_limit"); + }); +});