fix(resilience): apikey-provider 429s honor explicit quota-exhausted text (#6638) (#6731)

* fix(resilience): apikey-provider 429s honor explicit quota-exhausted text (#6638)

Ollama Cloud (and any other apikey-category provider) 429s skipped body-text
quota classification entirely; a genuine multi-day quota exhaustion was
misclassified as a plain rate_limit_exceeded with a few seconds of cooldown,
so combo routing retried the account immediately. shouldPreserveQuotaSignals()
now lets an explicit quota-exhausted signal (looksLikeQuotaExhausted) override
the apikey-category default, and parseDayGranularityResetMs() adds day-
granularity reset-hint parsing ("...reset in 3 days.") alongside the existing
Xh/Ym/Zs parsing.

Regression guard: tests/unit/issue-6638-ollama-quota.test.ts (RED before the
fix, GREEN after). Aligned two tests/unit/account-fallback-service.test.ts
cases that had codified the old buggy behavior for apikey-provider quota
text.

* chore(6731): re-sync onto release tip; CHANGELOG entry → changelog.d fragment (fragments-first)

changelog.d/fixes/6731-apikey-429-quota-exhausted.md

@@ -0,0 +1 @@
+- **fix(resilience):** OmniRoute didn't respect an exhausted Ollama Cloud (or any other apikey-category provider) quota — it retried the account seconds later instead of waiting out the real reset window ([#6638](https://github.com/diegosouzapw/OmniRoute/issues/6638)) — `shouldPreserveQuotaSignalsFor429()`/`checkFallbackError()` (`open-sse/services/accountFallback.ts`) only applied body-text quota classification (daily/monthly/weekly quota-exhausted detection) to OAuth-category providers; apikey-category 429s (Ollama Cloud, OpenAI, etc.) always fell through to the generic short rate-limit cooldown regardless of what the error body said, and `parseRetryFromErrorText()` also had no support for day-granularity reset hints ("Your quota will reset in 3 days.") — only Xh/Ym/Zs combos. An explicit quota-exhausted signal in the body (`looksLikeQuotaExhausted()`) now overrides the apikey-category default via the new `shouldPreserveQuotaSignals()` (`open-sse/services/quotaResetParsing.ts`), and `parseDayGranularityResetMs()` parses whole-day reset countdowns so the real multi-day window is honored instead of a few seconds of backoff. Regression guard: `tests/unit/issue-6638-ollama-quota.test.ts` + 2 aligned `tests/unit/account-fallback-service.test.ts` cases that previously asserted the buggy rate_limit_exceeded/undefined-dailyQuotaExhausted behavior for apikey-provider quota text.

open-sse/services/accountFallback.ts

@@ -35,6 +35,7 @@ import { getQuotaScopedModelForProvider } from "./antigravityQuotaFamily.ts";
 import { isRpdExhausted, isRpmExhausted } from "./geminiRateLimitTracker.ts";
 import { setConnectionRateLimitUntil } from "@/lib/db/providers";
 import { parseRetryHintFromJsonBody } from "./retryAfterJson.ts";
+import { parseDayGranularityResetMs, shouldPreserveQuotaSignals } from "./quotaResetParsing.ts";
 
 export type ProviderProfile = {
   baseCooldownMs: number;
@@ -364,11 +365,6 @@ export function getProviderProfile(provider: string): ProviderProfile {
   return buildProviderProfile(category);
 }
 
-function shouldPreserveQuotaSignalsFor429(provider: string | null | undefined): boolean {
-  if (!provider) return true;
-  return getProviderCategory(provider) === "oauth";
-}
-
 export async function getRuntimeProviderProfile(provider: string | null | undefined) {
   try {
     const { getCachedSettings } = await import("@/lib/db/readCache");
@@ -676,7 +672,7 @@ export function shouldMarkAccountExhaustedFrom429(
   // without making this one look quota-depleted for 5 minutes.
   if (failureKind === "rate_limit" || failureKind === "transient") return false;
   return (
-    shouldPreserveQuotaSignalsFor429(provider) &&
+    shouldPreserveQuotaSignals(provider) &&
     !hasPerModelQuota(provider, model, connectionPassthroughModels)
   );
 }
@@ -1071,7 +1067,7 @@ export function parseRetryFromErrorText(errorText: unknown): number | null {
     return computeDurationMs(resetsInMatch);
   }
 
-  return null;
+  return parseDayGranularityResetMs(msg, MAX_PROVIDER_COOLDOWN_MS);
 }
 
 /**
@@ -1417,7 +1413,7 @@ export function checkFallbackError(
   }
 
   const isRateLimitStatus = status === HTTP_STATUS.RATE_LIMITED;
-  const preserveQuota429 = shouldPreserveQuotaSignalsFor429(provider);
+  const preserveQuota429 = shouldPreserveQuotaSignals(provider, errorText);
   const shouldUseQuotaSignal = !isRateLimitStatus || preserveQuota429;
 
   // Check error message FIRST - specific patterns take priority over status codes

open-sse/services/quotaResetParsing.ts

@@ -0,0 +1,44 @@
+import { looksLikeQuotaExhausted } from "../../src/shared/utils/classify429";
+import { getProviderCategory } from "../config/providerRegistry.ts";
+
+/**
+ * Issue #6638 — Ollama Cloud (and any other apikey-category provider) 429s
+ * skip body-text quota classification by default: a bare 429 usually just
+ * means "too many requests/min" for these providers, so a short exponential
+ * backoff applies instead of the long cooldown reserved for genuine
+ * daily/monthly/weekly quota exhaustion.
+ *
+ * That default is correct for plain rate limiting, but it must not swallow
+ * an EXPLICIT quota-exhausted signal in the body (see `looksLikeQuotaExhausted`
+ * / QUOTA_PATTERNS) — otherwise the account looks "available" again seconds
+ * after a multi-day quota was exhausted, and combo routing retries it right
+ * away (the reported symptom). OAuth-category providers always preserve
+ * quota signals; apikey-category providers only do when the body explicitly
+ * says a long-period cap was hit.
+ */
+export function shouldPreserveQuotaSignals(
+  provider: string | null | undefined,
+  errorText?: string | null
+): boolean {
+  if (!provider) return true;
+  if (getProviderCategory(provider) === "oauth") return true;
+  return Boolean(errorText) && looksLikeQuotaExhausted(errorText);
+}
+
+/**
+ * Parse a day-granularity quota reset countdown ("Your quota will reset in
+ * 3 days.", "Resets in 13 days") out of an upstream 429 body.
+ *
+ * Companion to the Xh/Ym/Zs countdown parsing already handled inline by
+ * `parseRetryFromErrorText` — none of those patterns match when the upstream
+ * expresses the reset window in whole days rather than hours/minutes/seconds,
+ * so a multi-day quota reset previously parsed to `null` and fell back to the
+ * engine's ~seconds-scale default cooldown.
+ */
+export function parseDayGranularityResetMs(msg: string, maxMs: number): number | null {
+  const dayMatch = /reset(?:s)?\s+in\s+(\d+)\s*day(?:s)?/i.exec(msg);
+  if (!dayMatch) return null;
+  const days = Number.parseInt(dayMatch[1], 10);
+  if (!Number.isFinite(days) || days <= 0) return null;
+  return Math.min(days * 24 * 3600 * 1000, maxMs);
+}

tests/unit/account-fallback-service.test.ts

@@ -202,11 +202,11 @@ test("checkFallbackError preserves OAuth 429 exhausted-credit semantics", () =>
   assert.equal(result.cooldownMs, COOLDOWN_MS.paymentRequired ?? 3600 * 1000);
 });
 
-test("checkFallbackError keeps API-key 429 quota text on the status-based resilience path", () => {
+test("#6638: checkFallbackError classifies API-key 429 explicit quota text as quota_exhausted", () => {
   const result = checkFallbackError(429, "quota exceeded", 0, null, "openai", null, makeProfile());
 
   assert.equal(result.shouldFallback, true);
-  assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED);
+  assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
   assert.equal(result.cooldownMs, 125);
 });
 
@@ -844,7 +844,7 @@ test("checkFallbackError routes API-key 429 'try again tomorrow' through resilie
   assert.equal(result.cooldownMs, 125);
 });
 
-test("checkFallbackError routes API-key 429 'daily quota' text through resilience cooldown", () => {
+test("#6638: checkFallbackError routes API-key 429 'daily quota' text as quota_exhausted", () => {
   const result = checkFallbackError(
     429,
     "You have exceeded your daily quota",
@@ -855,8 +855,8 @@ test("checkFallbackError routes API-key 429 'daily quota' text through resilienc
     makeProfile()
   );
   assert.equal(result.shouldFallback, true);
-  assert.equal(result.dailyQuotaExhausted, undefined);
-  assert.equal(result.cooldownMs, 125);
+  assert.equal(result.dailyQuotaExhausted, true);
+  assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
 });
 
 test("checkFallbackError preserves OAuth 429 daily quota semantics", () => {

tests/unit/issue-6638-ollama-quota.test.ts

@@ -0,0 +1,33 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { checkFallbackError } from "../../open-sse/services/accountFallback.ts";
+
+// Repro for GitHub issue #6638: "OmniRoute doesn't respect exhausted quotas"
+test("#6638: Ollama Cloud weekly-quota-exhausted 429 must NOT get a short generic rate-limit cooldown", () => {
+  const errorText = JSON.stringify({
+    error: "You have exceeded your weekly usage quota. Your quota will reset in 3 days.",
+  });
+
+  const result = checkFallbackError(
+    429,
+    errorText,
+    0,
+    "deepseek-v4-pro",
+    "ollama-cloud",
+    null,
+    null,
+    undefined
+  );
+
+  console.log("checkFallbackError result:", result);
+
+  assert.equal(
+    result.reason,
+    "quota_exhausted",
+    `expected reason "quota_exhausted" but got "${result.reason}" — quota text is being ignored for apikey-category 429s`
+  );
+  assert.ok(
+    result.cooldownMs > 60 * 60 * 1000,
+    `expected a long (>1h) cooldown reflecting the weekly quota reset, got ${result.cooldownMs}ms`
+  );
+});