mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
fix(6980): classify Cloudflare AI neuron exhaustion as quota_exhausted (#6983)
Cloudflare Workers AI free tier (10k Neurons/day, account-wide) returns 429 with body 'you have used up your daily free allocation of 10,000 neurons' which matched no QUOTA_PATTERNS keyword — falling through to rate_limit (~60s cooldown) instead of quota_exhausted. Two layers: 1. Provider-specific rule for 'cloudflare-ai' in providerRuleRegistry (scope: connection — budget is account-wide, not per-model) 2. Defense-in-depth: /daily free allocation/i in classify429 QUOTA_PATTERNS Tests: 11/11 pass (provider rule + classify429 paths covered). Closes #6980 Co-authored-by: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com>
This commit is contained in:
committed by
GitHub
parent
df9808c0e4
commit
994f1c78a0
@@ -130,6 +130,31 @@ function buildMinimaxRules(): ProviderErrorRule[] {
|
||||
];
|
||||
}
|
||||
|
||||
// ─── Cloudflare Workers AI ─────────────────────────────────────────────────────
|
||||
// Free tier = 10,000 Neurons/day, shared across the WHOLE account
|
||||
// (docs/reference/FREE_TIERS.md; official: developers.cloudflare.com/
|
||||
// workers-ai/platform/errors/). The exhaustion body doesn't match any
|
||||
// QUOTA_PATTERNS keyword so it falls through to rate_limit and gets
|
||||
// retried every ~60s against a budget that only resets at UTC midnight.
|
||||
// Issue #6980.
|
||||
function buildCloudflareAiRules(): ProviderErrorRule[] {
|
||||
return [
|
||||
{
|
||||
id: "cloudflare-ai-daily-neuron-allocation",
|
||||
match: ({ status, body }) => {
|
||||
if (status !== 429) return null;
|
||||
const text = JSON.stringify(body ?? "").toLowerCase();
|
||||
// Body: "you have used up your daily free allocation of 10,000 neurons,
|
||||
// please upgrade to Cloudflare's Workers Paid plan..."
|
||||
if (!text.includes("daily free allocation")) return null;
|
||||
// No cooldownMs: recordModelLockoutFailure already sets
|
||||
// quota_exhausted without one to "next UTC midnight".
|
||||
return { reason: "quota_exhausted", scope: "connection" };
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Global registry. Provider name → ordered list of rules (first match wins).
|
||||
* Add new providers here; the matcher in classifyError will pick them up
|
||||
@@ -141,6 +166,7 @@ export const providerRuleRegistry = new Map<string, ProviderErrorRule[]>([
|
||||
["opencode-cli", buildOpencodeRules()],
|
||||
["minimax", buildMinimaxRules()],
|
||||
["minimax-passthrough", buildMinimaxRules()],
|
||||
["cloudflare-ai", buildCloudflareAiRules()],
|
||||
]);
|
||||
|
||||
/**
|
||||
@@ -194,7 +220,9 @@ export function getProviderErrorRuleMatch(
|
||||
*/
|
||||
export function parseResetCountdownMs(text: string): number | null {
|
||||
if (typeof text !== "string" || text.length === 0) return null;
|
||||
const match = text.match(/resets?\s+in\s+(\d+)\s+(day|days|hour|hours|minute|minutes|second|seconds)\b/);
|
||||
const match = text.match(
|
||||
/resets?\s+in\s+(\d+)\s+(day|days|hour|hours|minute|minutes|second|seconds)\b/
|
||||
);
|
||||
if (!match) return null;
|
||||
const n = Number(match[1]);
|
||||
if (!Number.isFinite(n) || n <= 0) return null;
|
||||
|
||||
@@ -53,6 +53,14 @@ const QUOTA_PATTERNS: ReadonlyArray<RegExp> = [
|
||||
/individual quota reached/i,
|
||||
/enable overages/i,
|
||||
/INSUFFICIENT_G1_CREDITS_BALANCE/i,
|
||||
|
||||
// Cloudflare Workers AI daily neuron exhaustion (Issue #6980).
|
||||
// Body: "you have used up your daily free allocation of 10,000 neurons,
|
||||
// please upgrade to Cloudflare's Workers Paid plan..."
|
||||
// No existing pattern matches "daily free allocation" — without this,
|
||||
// the 429 is misclassified as transient rate_limit and retried every
|
||||
// ~60s against a budget that only resets at UTC midnight.
|
||||
/daily free allocation/i,
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
108
tests/unit/cloudflare-ai-neuron-exhaustion-6980.test.ts
Normal file
108
tests/unit/cloudflare-ai-neuron-exhaustion-6980.test.ts
Normal file
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* Issue #6980 — Cloudflare Workers AI daily neuron exhaustion 429 must be
|
||||
* classified as quota_exhausted (not transient rate_limit).
|
||||
*
|
||||
* Two layers of defense:
|
||||
* 1. Provider-specific rule in providerErrorRules.ts → getProviderErrorRuleMatch
|
||||
* 2. Global QUOTA_PATTERNS in classify429.ts → looksLikeQuotaExhausted
|
||||
*
|
||||
* Without these, the 429 body "you have used up your daily free allocation of
|
||||
* 10,000 neurons" matches no keyword, falls through to rate_limit (~60s cooldown),
|
||||
* and the combo router keeps cycling through every cloudflare model on retry
|
||||
* against a budget that only resets at UTC midnight.
|
||||
*/
|
||||
|
||||
import { describe, test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
getProviderErrorRuleMatch,
|
||||
providerRuleRegistry,
|
||||
} from "../../open-sse/config/providerErrorRules.ts";
|
||||
import { classify429, looksLikeQuotaExhausted } from "../../src/shared/utils/classify429.ts";
|
||||
|
||||
// ─── Fixtures ────────────────────────────────────────────────────────────────
|
||||
|
||||
const CF_NEURON_BODY =
|
||||
"you have used up your daily free allocation of 10,000 neurons, please upgrade to Cloudflare's Workers Paid plan";
|
||||
|
||||
const CF_NEURON_BODY_JSON = {
|
||||
errors: [
|
||||
{
|
||||
code: 4006,
|
||||
message:
|
||||
"you have used up your daily free allocation of 10,000 neurons, please upgrade to Cloudflare's Workers Paid plan",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// ─── Tests: provider-specific rule (primary path) ───────────────────────────
|
||||
|
||||
describe("#6980 provider rule: cloudflare-ai neuron exhaustion", () => {
|
||||
test("cloudflare-ai is registered in providerRuleRegistry", () => {
|
||||
assert.ok(providerRuleRegistry.has("cloudflare-ai"));
|
||||
});
|
||||
|
||||
test("429 with plain-string neuron body → quota_exhausted, scope connection", () => {
|
||||
const result = getProviderErrorRuleMatch("cloudflare-ai", 429, {}, CF_NEURON_BODY);
|
||||
assert.ok(result, "expected a match");
|
||||
assert.equal(result!.reason, "quota_exhausted");
|
||||
assert.equal(result!.scope, "connection");
|
||||
// No explicit cooldownMs — recordModelLockoutFailure resolves to next UTC midnight.
|
||||
assert.equal(result!.cooldownMs, undefined);
|
||||
});
|
||||
|
||||
test("429 with JSON-structured neuron body → quota_exhausted", () => {
|
||||
const result = getProviderErrorRuleMatch("cloudflare-ai", 429, {}, CF_NEURON_BODY_JSON);
|
||||
assert.ok(result);
|
||||
assert.equal(result!.reason, "quota_exhausted");
|
||||
assert.equal(result!.scope, "connection");
|
||||
});
|
||||
|
||||
test("non-429 status does not match even with neuron body", () => {
|
||||
const result = getProviderErrorRuleMatch("cloudflare-ai", 500, {}, CF_NEURON_BODY);
|
||||
assert.equal(result, null);
|
||||
});
|
||||
|
||||
test("429 with unrelated body does not match", () => {
|
||||
const result = getProviderErrorRuleMatch(
|
||||
"cloudflare-ai",
|
||||
429,
|
||||
{},
|
||||
{
|
||||
error: "rate limited, try again later",
|
||||
}
|
||||
);
|
||||
assert.equal(result, null);
|
||||
});
|
||||
|
||||
test("provider name matching is case-insensitive", () => {
|
||||
const result = getProviderErrorRuleMatch("Cloudflare-AI", 429, {}, CF_NEURON_BODY);
|
||||
assert.ok(result);
|
||||
assert.equal(result!.reason, "quota_exhausted");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Tests: classify429 defense-in-depth (fallback path) ────────────────────
|
||||
|
||||
describe("#6980 classify429: daily free allocation pattern", () => {
|
||||
test("looksLikeQuotaExhausted matches neuron body string", () => {
|
||||
assert.ok(looksLikeQuotaExhausted(CF_NEURON_BODY));
|
||||
});
|
||||
|
||||
test("looksLikeQuotaExhausted matches neuron body JSON-stringified", () => {
|
||||
assert.ok(looksLikeQuotaExhausted(CF_NEURON_BODY_JSON));
|
||||
});
|
||||
|
||||
test("classify429 returns quota_exhausted for neuron body", () => {
|
||||
assert.equal(classify429({ status: 429, body: CF_NEURON_BODY }), "quota_exhausted");
|
||||
});
|
||||
|
||||
test("classify429 returns quota_exhausted for neuron JSON body", () => {
|
||||
assert.equal(classify429({ status: 429, body: CF_NEURON_BODY_JSON }), "quota_exhausted");
|
||||
});
|
||||
|
||||
test("classify429 returns rate_limit for generic 429 without quota keywords", () => {
|
||||
assert.equal(classify429({ status: 429, body: "Too many requests" }), "rate_limit");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user