fix(6980): classify Cloudflare AI neuron exhaustion as quota_exhausted (#6983)

Cloudflare Workers AI free tier (10k Neurons/day, account-wide) returns
429 with body 'you have used up your daily free allocation of 10,000
neurons' which matched no QUOTA_PATTERNS keyword — falling through to
rate_limit (~60s cooldown) instead of quota_exhausted.

Two layers:
1. Provider-specific rule for 'cloudflare-ai' in providerRuleRegistry
   (scope: connection — budget is account-wide, not per-model)
2. Defense-in-depth: /daily free allocation/i in classify429 QUOTA_PATTERNS

Tests: 11/11 pass (provider rule + classify429 paths covered).

Closes #6980

Co-authored-by: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com>
This commit is contained in:
Rafael Dias Zendron
2026-07-16 14:12:18 -03:00
committed by GitHub
parent df9808c0e4
commit 994f1c78a0
3 changed files with 145 additions and 1 deletions

View File

@@ -130,6 +130,31 @@ function buildMinimaxRules(): ProviderErrorRule[] {
];
}
// ─── Cloudflare Workers AI ─────────────────────────────────────────────────────
// Free tier = 10,000 Neurons/day, shared across the WHOLE account
// (docs/reference/FREE_TIERS.md; official: developers.cloudflare.com/
// workers-ai/platform/errors/). The exhaustion body doesn't match any
// QUOTA_PATTERNS keyword so it falls through to rate_limit and gets
// retried every ~60s against a budget that only resets at UTC midnight.
// Issue #6980.
function buildCloudflareAiRules(): ProviderErrorRule[] {
return [
{
id: "cloudflare-ai-daily-neuron-allocation",
match: ({ status, body }) => {
if (status !== 429) return null;
const text = JSON.stringify(body ?? "").toLowerCase();
// Body: "you have used up your daily free allocation of 10,000 neurons,
// please upgrade to Cloudflare's Workers Paid plan..."
if (!text.includes("daily free allocation")) return null;
// No cooldownMs: recordModelLockoutFailure already sets
// quota_exhausted without one to "next UTC midnight".
return { reason: "quota_exhausted", scope: "connection" };
},
},
];
}
/**
* Global registry. Provider name → ordered list of rules (first match wins).
* Add new providers here; the matcher in classifyError will pick them up
@@ -141,6 +166,7 @@ export const providerRuleRegistry = new Map<string, ProviderErrorRule[]>([
["opencode-cli", buildOpencodeRules()],
["minimax", buildMinimaxRules()],
["minimax-passthrough", buildMinimaxRules()],
["cloudflare-ai", buildCloudflareAiRules()],
]);
/**
@@ -194,7 +220,9 @@ export function getProviderErrorRuleMatch(
*/
export function parseResetCountdownMs(text: string): number | null {
if (typeof text !== "string" || text.length === 0) return null;
const match = text.match(/resets?\s+in\s+(\d+)\s+(day|days|hour|hours|minute|minutes|second|seconds)\b/);
const match = text.match(
/resets?\s+in\s+(\d+)\s+(day|days|hour|hours|minute|minutes|second|seconds)\b/
);
if (!match) return null;
const n = Number(match[1]);
if (!Number.isFinite(n) || n <= 0) return null;

View File

@@ -53,6 +53,14 @@ const QUOTA_PATTERNS: ReadonlyArray<RegExp> = [
/individual quota reached/i,
/enable overages/i,
/INSUFFICIENT_G1_CREDITS_BALANCE/i,
// Cloudflare Workers AI daily neuron exhaustion (Issue #6980).
// Body: "you have used up your daily free allocation of 10,000 neurons,
// please upgrade to Cloudflare's Workers Paid plan..."
// No existing pattern matches "daily free allocation" — without this,
// the 429 is misclassified as transient rate_limit and retried every
// ~60s against a budget that only resets at UTC midnight.
/daily free allocation/i,
];
/**

View File

@@ -0,0 +1,108 @@
/**
* Issue #6980 — Cloudflare Workers AI daily neuron exhaustion 429 must be
* classified as quota_exhausted (not transient rate_limit).
*
* Two layers of defense:
* 1. Provider-specific rule in providerErrorRules.ts → getProviderErrorRuleMatch
* 2. Global QUOTA_PATTERNS in classify429.ts → looksLikeQuotaExhausted
*
* Without these, the 429 body "you have used up your daily free allocation of
* 10,000 neurons" matches no keyword, falls through to rate_limit (~60s cooldown),
* and the combo router keeps cycling through every cloudflare model on retry
* against a budget that only resets at UTC midnight.
*/
import { describe, test } from "node:test";
import assert from "node:assert/strict";
import {
getProviderErrorRuleMatch,
providerRuleRegistry,
} from "../../open-sse/config/providerErrorRules.ts";
import { classify429, looksLikeQuotaExhausted } from "../../src/shared/utils/classify429.ts";
// ─── Fixtures ────────────────────────────────────────────────────────────────
const CF_NEURON_BODY =
"you have used up your daily free allocation of 10,000 neurons, please upgrade to Cloudflare's Workers Paid plan";
const CF_NEURON_BODY_JSON = {
errors: [
{
code: 4006,
message:
"you have used up your daily free allocation of 10,000 neurons, please upgrade to Cloudflare's Workers Paid plan",
},
],
};
// ─── Tests: provider-specific rule (primary path) ───────────────────────────
describe("#6980 provider rule: cloudflare-ai neuron exhaustion", () => {
test("cloudflare-ai is registered in providerRuleRegistry", () => {
assert.ok(providerRuleRegistry.has("cloudflare-ai"));
});
test("429 with plain-string neuron body → quota_exhausted, scope connection", () => {
const result = getProviderErrorRuleMatch("cloudflare-ai", 429, {}, CF_NEURON_BODY);
assert.ok(result, "expected a match");
assert.equal(result!.reason, "quota_exhausted");
assert.equal(result!.scope, "connection");
// No explicit cooldownMs — recordModelLockoutFailure resolves to next UTC midnight.
assert.equal(result!.cooldownMs, undefined);
});
test("429 with JSON-structured neuron body → quota_exhausted", () => {
const result = getProviderErrorRuleMatch("cloudflare-ai", 429, {}, CF_NEURON_BODY_JSON);
assert.ok(result);
assert.equal(result!.reason, "quota_exhausted");
assert.equal(result!.scope, "connection");
});
test("non-429 status does not match even with neuron body", () => {
const result = getProviderErrorRuleMatch("cloudflare-ai", 500, {}, CF_NEURON_BODY);
assert.equal(result, null);
});
test("429 with unrelated body does not match", () => {
const result = getProviderErrorRuleMatch(
"cloudflare-ai",
429,
{},
{
error: "rate limited, try again later",
}
);
assert.equal(result, null);
});
test("provider name matching is case-insensitive", () => {
const result = getProviderErrorRuleMatch("Cloudflare-AI", 429, {}, CF_NEURON_BODY);
assert.ok(result);
assert.equal(result!.reason, "quota_exhausted");
});
});
// ─── Tests: classify429 defense-in-depth (fallback path) ────────────────────
describe("#6980 classify429: daily free allocation pattern", () => {
test("looksLikeQuotaExhausted matches neuron body string", () => {
assert.ok(looksLikeQuotaExhausted(CF_NEURON_BODY));
});
test("looksLikeQuotaExhausted matches neuron body JSON-stringified", () => {
assert.ok(looksLikeQuotaExhausted(CF_NEURON_BODY_JSON));
});
test("classify429 returns quota_exhausted for neuron body", () => {
assert.equal(classify429({ status: 429, body: CF_NEURON_BODY }), "quota_exhausted");
});
test("classify429 returns quota_exhausted for neuron JSON body", () => {
assert.equal(classify429({ status: 429, body: CF_NEURON_BODY_JSON }), "quota_exhausted");
});
test("classify429 returns rate_limit for generic 429 without quota keywords", () => {
assert.equal(classify429({ status: 429, body: "Too many requests" }), "rate_limit");
});
});