From 33ca6caef3b085ce67ba21ec90a05eb3a89c32ed Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Fri, 10 Jul 2026 19:47:05 -0300 Subject: [PATCH] fix(resilience): apikey-provider 429s honor explicit quota-exhausted text (#6638) (#6731) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(resilience): apikey-provider 429s honor explicit quota-exhausted text (#6638) Ollama Cloud (and any other apikey-category provider) 429s skipped body-text quota classification entirely; a genuine multi-day quota exhaustion was misclassified as a plain rate_limit_exceeded with a few seconds of cooldown, so combo routing retried the account immediately. shouldPreserveQuotaSignals() now lets an explicit quota-exhausted signal (looksLikeQuotaExhausted) override the apikey-category default, and parseDayGranularityResetMs() adds day- granularity reset-hint parsing ("...reset in 3 days.") alongside the existing Xh/Ym/Zs parsing. Regression guard: tests/unit/issue-6638-ollama-quota.test.ts (RED before the fix, GREEN after). Aligned two tests/unit/account-fallback-service.test.ts cases that had codified the old buggy behavior for apikey-provider quota text. * chore(6731): re-sync onto release tip; CHANGELOG entry → changelog.d fragment (fragments-first) --- .../fixes/6731-apikey-429-quota-exhausted.md | 1 + open-sse/services/accountFallback.ts | 12 ++--- open-sse/services/quotaResetParsing.ts | 44 +++++++++++++++++++ tests/unit/account-fallback-service.test.ts | 10 ++--- tests/unit/issue-6638-ollama-quota.test.ts | 33 ++++++++++++++ 5 files changed, 87 insertions(+), 13 deletions(-) create mode 100644 changelog.d/fixes/6731-apikey-429-quota-exhausted.md create mode 100644 open-sse/services/quotaResetParsing.ts create mode 100644 tests/unit/issue-6638-ollama-quota.test.ts diff --git a/changelog.d/fixes/6731-apikey-429-quota-exhausted.md b/changelog.d/fixes/6731-apikey-429-quota-exhausted.md new file mode 100644 index 0000000000..bdfabc45ef --- /dev/null +++ b/changelog.d/fixes/6731-apikey-429-quota-exhausted.md @@ -0,0 +1 @@ +- **fix(resilience):** OmniRoute didn't respect an exhausted Ollama Cloud (or any other apikey-category provider) quota — it retried the account seconds later instead of waiting out the real reset window ([#6638](https://github.com/diegosouzapw/OmniRoute/issues/6638)) — `shouldPreserveQuotaSignalsFor429()`/`checkFallbackError()` (`open-sse/services/accountFallback.ts`) only applied body-text quota classification (daily/monthly/weekly quota-exhausted detection) to OAuth-category providers; apikey-category 429s (Ollama Cloud, OpenAI, etc.) always fell through to the generic short rate-limit cooldown regardless of what the error body said, and `parseRetryFromErrorText()` also had no support for day-granularity reset hints ("Your quota will reset in 3 days.") — only Xh/Ym/Zs combos. An explicit quota-exhausted signal in the body (`looksLikeQuotaExhausted()`) now overrides the apikey-category default via the new `shouldPreserveQuotaSignals()` (`open-sse/services/quotaResetParsing.ts`), and `parseDayGranularityResetMs()` parses whole-day reset countdowns so the real multi-day window is honored instead of a few seconds of backoff. Regression guard: `tests/unit/issue-6638-ollama-quota.test.ts` + 2 aligned `tests/unit/account-fallback-service.test.ts` cases that previously asserted the buggy rate_limit_exceeded/undefined-dailyQuotaExhausted behavior for apikey-provider quota text. diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index d37acc6752..f0289340cb 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -35,6 +35,7 @@ import { getQuotaScopedModelForProvider } from "./antigravityQuotaFamily.ts"; import { isRpdExhausted, isRpmExhausted } from "./geminiRateLimitTracker.ts"; import { setConnectionRateLimitUntil } from "@/lib/db/providers"; import { parseRetryHintFromJsonBody } from "./retryAfterJson.ts"; +import { parseDayGranularityResetMs, shouldPreserveQuotaSignals } from "./quotaResetParsing.ts"; export type ProviderProfile = { baseCooldownMs: number; @@ -364,11 +365,6 @@ export function getProviderProfile(provider: string): ProviderProfile { return buildProviderProfile(category); } -function shouldPreserveQuotaSignalsFor429(provider: string | null | undefined): boolean { - if (!provider) return true; - return getProviderCategory(provider) === "oauth"; -} - export async function getRuntimeProviderProfile(provider: string | null | undefined) { try { const { getCachedSettings } = await import("@/lib/db/readCache"); @@ -676,7 +672,7 @@ export function shouldMarkAccountExhaustedFrom429( // without making this one look quota-depleted for 5 minutes. if (failureKind === "rate_limit" || failureKind === "transient") return false; return ( - shouldPreserveQuotaSignalsFor429(provider) && + shouldPreserveQuotaSignals(provider) && !hasPerModelQuota(provider, model, connectionPassthroughModels) ); } @@ -1071,7 +1067,7 @@ export function parseRetryFromErrorText(errorText: unknown): number | null { return computeDurationMs(resetsInMatch); } - return null; + return parseDayGranularityResetMs(msg, MAX_PROVIDER_COOLDOWN_MS); } /** @@ -1417,7 +1413,7 @@ export function checkFallbackError( } const isRateLimitStatus = status === HTTP_STATUS.RATE_LIMITED; - const preserveQuota429 = shouldPreserveQuotaSignalsFor429(provider); + const preserveQuota429 = shouldPreserveQuotaSignals(provider, errorText); const shouldUseQuotaSignal = !isRateLimitStatus || preserveQuota429; // Check error message FIRST - specific patterns take priority over status codes diff --git a/open-sse/services/quotaResetParsing.ts b/open-sse/services/quotaResetParsing.ts new file mode 100644 index 0000000000..b3606b02a2 --- /dev/null +++ b/open-sse/services/quotaResetParsing.ts @@ -0,0 +1,44 @@ +import { looksLikeQuotaExhausted } from "../../src/shared/utils/classify429"; +import { getProviderCategory } from "../config/providerRegistry.ts"; + +/** + * Issue #6638 — Ollama Cloud (and any other apikey-category provider) 429s + * skip body-text quota classification by default: a bare 429 usually just + * means "too many requests/min" for these providers, so a short exponential + * backoff applies instead of the long cooldown reserved for genuine + * daily/monthly/weekly quota exhaustion. + * + * That default is correct for plain rate limiting, but it must not swallow + * an EXPLICIT quota-exhausted signal in the body (see `looksLikeQuotaExhausted` + * / QUOTA_PATTERNS) — otherwise the account looks "available" again seconds + * after a multi-day quota was exhausted, and combo routing retries it right + * away (the reported symptom). OAuth-category providers always preserve + * quota signals; apikey-category providers only do when the body explicitly + * says a long-period cap was hit. + */ +export function shouldPreserveQuotaSignals( + provider: string | null | undefined, + errorText?: string | null +): boolean { + if (!provider) return true; + if (getProviderCategory(provider) === "oauth") return true; + return Boolean(errorText) && looksLikeQuotaExhausted(errorText); +} + +/** + * Parse a day-granularity quota reset countdown ("Your quota will reset in + * 3 days.", "Resets in 13 days") out of an upstream 429 body. + * + * Companion to the Xh/Ym/Zs countdown parsing already handled inline by + * `parseRetryFromErrorText` — none of those patterns match when the upstream + * expresses the reset window in whole days rather than hours/minutes/seconds, + * so a multi-day quota reset previously parsed to `null` and fell back to the + * engine's ~seconds-scale default cooldown. + */ +export function parseDayGranularityResetMs(msg: string, maxMs: number): number | null { + const dayMatch = /reset(?:s)?\s+in\s+(\d+)\s*day(?:s)?/i.exec(msg); + if (!dayMatch) return null; + const days = Number.parseInt(dayMatch[1], 10); + if (!Number.isFinite(days) || days <= 0) return null; + return Math.min(days * 24 * 3600 * 1000, maxMs); +} diff --git a/tests/unit/account-fallback-service.test.ts b/tests/unit/account-fallback-service.test.ts index 02fd6ed006..768f21f231 100644 --- a/tests/unit/account-fallback-service.test.ts +++ b/tests/unit/account-fallback-service.test.ts @@ -202,11 +202,11 @@ test("checkFallbackError preserves OAuth 429 exhausted-credit semantics", () => assert.equal(result.cooldownMs, COOLDOWN_MS.paymentRequired ?? 3600 * 1000); }); -test("checkFallbackError keeps API-key 429 quota text on the status-based resilience path", () => { +test("#6638: checkFallbackError classifies API-key 429 explicit quota text as quota_exhausted", () => { const result = checkFallbackError(429, "quota exceeded", 0, null, "openai", null, makeProfile()); assert.equal(result.shouldFallback, true); - assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED); + assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); assert.equal(result.cooldownMs, 125); }); @@ -844,7 +844,7 @@ test("checkFallbackError routes API-key 429 'try again tomorrow' through resilie assert.equal(result.cooldownMs, 125); }); -test("checkFallbackError routes API-key 429 'daily quota' text through resilience cooldown", () => { +test("#6638: checkFallbackError routes API-key 429 'daily quota' text as quota_exhausted", () => { const result = checkFallbackError( 429, "You have exceeded your daily quota", @@ -855,8 +855,8 @@ test("checkFallbackError routes API-key 429 'daily quota' text through resilienc makeProfile() ); assert.equal(result.shouldFallback, true); - assert.equal(result.dailyQuotaExhausted, undefined); - assert.equal(result.cooldownMs, 125); + assert.equal(result.dailyQuotaExhausted, true); + assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); }); test("checkFallbackError preserves OAuth 429 daily quota semantics", () => { diff --git a/tests/unit/issue-6638-ollama-quota.test.ts b/tests/unit/issue-6638-ollama-quota.test.ts new file mode 100644 index 0000000000..79eb8fc4ca --- /dev/null +++ b/tests/unit/issue-6638-ollama-quota.test.ts @@ -0,0 +1,33 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { checkFallbackError } from "../../open-sse/services/accountFallback.ts"; + +// Repro for GitHub issue #6638: "OmniRoute doesn't respect exhausted quotas" +test("#6638: Ollama Cloud weekly-quota-exhausted 429 must NOT get a short generic rate-limit cooldown", () => { + const errorText = JSON.stringify({ + error: "You have exceeded your weekly usage quota. Your quota will reset in 3 days.", + }); + + const result = checkFallbackError( + 429, + errorText, + 0, + "deepseek-v4-pro", + "ollama-cloud", + null, + null, + undefined + ); + + console.log("checkFallbackError result:", result); + + assert.equal( + result.reason, + "quota_exhausted", + `expected reason "quota_exhausted" but got "${result.reason}" — quota text is being ignored for apikey-category 429s` + ); + assert.ok( + result.cooldownMs > 60 * 60 * 1000, + `expected a long (>1h) cooldown reflecting the weekly quota reset, got ${result.cooldownMs}ms` + ); +});