From fce2bb67adda3af8a28df53e085141a2aff7e7fc Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Tue, 14 Jul 2026 21:45:20 -0300 Subject: [PATCH] fix: recognize Ollama Cloud session usage-limit 429 as quota-exhausted (#7071) (#7181) * fix: recognize Ollama Cloud session usage-limit 429 as quota-exhausted (#7071) Ollama Cloud's 5-hour "session" usage-limit 429 body ("you (NAME) have reached your session usage limit...") was never recognized as quota-exhausted -- only the sibling "weekly usage limit" wording was fixed (#6638/#3709). Neither the generic QUOTA_PATTERNS list nor the dedicated weekly-quota classifier matched the session wording, so checkFallbackError() fell through to the generic ~3s rate-limit backoff instead of a long QUOTA_EXHAUSTED cooldown -- combo/LKGP routing cycled back to the "exhausted" account almost immediately instead of advancing to the next one. Adds isSessionUsageLimitText()/buildSessionQuotaFallback() to quotaTextCooldowns.ts, mirroring the weekly-quota pair, with a 5h cooldown matching Ollama Cloud's documented session window. Wired unconditionally into checkFallbackError() next to the weekly check so apikey-category providers like ollama-cloud are covered. * chore(test): register issue-7071-ollama-session-quota.test.ts in stryker tap.testFiles (#7071) --- .../fixes/7071-ollama-cloud-session-quota.md | 1 + open-sse/services/accountFallback.ts | 7 ++ open-sse/services/quotaTextCooldowns.ts | 34 ++++++ stryker.conf.json | 1 + .../issue-7071-ollama-session-quota.test.ts | 103 ++++++++++++++++++ 5 files changed, 146 insertions(+) create mode 100644 changelog.d/fixes/7071-ollama-cloud-session-quota.md create mode 100644 tests/unit/issue-7071-ollama-session-quota.test.ts diff --git a/changelog.d/fixes/7071-ollama-cloud-session-quota.md b/changelog.d/fixes/7071-ollama-cloud-session-quota.md new file mode 100644 index 0000000000..81807e30ad --- /dev/null +++ b/changelog.d/fixes/7071-ollama-cloud-session-quota.md @@ -0,0 +1 @@ +- fix(resilience): recognize Ollama Cloud's 5-hour session usage-limit 429 as quota-exhausted instead of a generic rate limit (#7071) diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 275ecbab41..32562fc526 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -41,6 +41,7 @@ import { isSubscriptionQuotaText, buildSubscriptionQuotaFallback, buildWeeklyQuotaFallback, + buildSessionQuotaFallback, } from "./quotaTextCooldowns.ts"; import { parseDayGranularityResetMs, shouldPreserveQuotaSignals } from "./quotaResetParsing.ts"; @@ -1454,6 +1455,12 @@ export function checkFallbackError( } const weeklyResult = buildWeeklyQuotaFallback(errorStr); if (weeklyResult) return weeklyResult; + // Issue #7071 (session usage cap) is the same sibling gap as #3709 above — + // runs UNCONDITIONALLY for the same reason: apikey-category providers + // like ollama-cloud are excluded from the oauth-only shouldUseQuotaSignal + // gate. + const sessionResult = buildSessionQuotaFallback(errorStr); + if (sessionResult) return sessionResult; const quotaResetHintMs = parseRetryFromErrorText(errorStr); if ( diff --git a/open-sse/services/quotaTextCooldowns.ts b/open-sse/services/quotaTextCooldowns.ts index 0146d72ea5..4f481b822d 100644 --- a/open-sse/services/quotaTextCooldowns.ts +++ b/open-sse/services/quotaTextCooldowns.ts @@ -103,3 +103,37 @@ export function buildWeeklyQuotaFallback(errorStr: string): QuotaTextFallback | reason: RateLimitReason.QUOTA_EXHAUSTED, }; } + +// ─── Issue #7071 — Ollama Cloud 5-hour SESSION usage cap ─────────────────── +// +// Ollama Cloud also enforces a rolling 5-hour "session" usage cap, sibling to +// the weekly cap above (#3709/#6638). On cap the upstream returns 429 with a +// body like "you () have reached your session usage limit". Same +// root cause as the weekly gap: neither the generic subscription-quota-text +// classifier nor the weekly one recognize "session" wording, so the account +// fell through to the generic 429 backoff and got retried within the same +// 5-hour window instead of cooling down for it — combo/LKGP routing cycled +// back to the "exhausted" account instead of advancing to the next one. +// +// Patterns are scoped to "session ... usage limit" / "session limit reached" +// / "reached your session ... usage limit" phrasing (not a bare "session" +// match) so unrelated "session expired"/"session token invalid" auth errors +// from other providers are not misclassified as quota-exhausted. +const SESSION_QUOTA_COOLDOWN_MS = 5 * 60 * 60 * 1000; // 5 hours + +export function isSessionUsageLimitText(lower: string): boolean { + return ( + lower.includes("session usage limit") || + lower.includes("session limit reached") || + (lower.includes("reached your session") && lower.includes("usage limit")) + ); +} + +export function buildSessionQuotaFallback(errorStr: string): QuotaTextFallback | null { + if (!isSessionUsageLimitText(errorStr.toLowerCase())) return null; + return { + shouldFallback: true, + cooldownMs: SESSION_QUOTA_COOLDOWN_MS, + reason: RateLimitReason.QUOTA_EXHAUSTED, + }; +} diff --git a/stryker.conf.json b/stryker.conf.json index f944f3c27d..6aac6d8ee0 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -182,6 +182,7 @@ "tests/unit/issue-6343-v0-web-alias-collision.test.ts", "tests/unit/issue-6638-ollama-quota.test.ts", "tests/unit/issue-6686-quota-preflight-coverage.test.ts", + "tests/unit/issue-7071-ollama-session-quota.test.ts", "tests/unit/livews-forward-backoff-4604.test.ts", "tests/unit/management-auth-hardening.test.ts", "tests/unit/mark-account-unavailable-numeric-epoch-guard.test.ts", diff --git a/tests/unit/issue-7071-ollama-session-quota.test.ts b/tests/unit/issue-7071-ollama-session-quota.test.ts new file mode 100644 index 0000000000..25b297c0ad --- /dev/null +++ b/tests/unit/issue-7071-ollama-session-quota.test.ts @@ -0,0 +1,103 @@ +/** + * Issue #7071 — Ollama Cloud's 5-hour "session" usage-limit 429 is never + * recognized as quota-exhausted. The upstream returns a body like: + * "you () have reached your session usage limit" + * + * This exactly mirrors the already-fixed "weekly usage limit" gap (#3709, + * #6638): ollama-cloud is an apikey-category provider (not oauth), so the + * oauth-only `shouldUseQuotaSignal` gate in checkFallbackError skips the + * generic subscription-quota-text branch (#2321) for its 429s. Without a + * dedicated, ungated session check the account fell through to the generic + * 429 backoff (~3s, capped low) and got retried within the same 5-hour + * session window instead of cooling down for the session's duration — + * combo/LKGP routing cycled back to the "exhausted" account instead of + * advancing to the next one. + * + * This test proves: (1) the session-usage-limit text is classified as + * QUOTA_EXHAUSTED with a cooldown far longer than the generic backoff cap, + * for BOTH apikey and oauth provider categories, and (2) unrelated + * session-expired/auth wording and the sibling weekly-quota text are + * unaffected. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { checkFallbackError } = await import("../../open-sse/services/accountFallback.ts"); +const { isSessionUsageLimitText, buildSessionQuotaFallback, isWeeklyUsageLimitText } = + await import("../../open-sse/services/quotaTextCooldowns.ts"); +const { RateLimitReason, BACKOFF_CONFIG } = await import("../../open-sse/config/constants.ts"); +const { BACKOFF_CONFIG: ERROR_BACKOFF_CONFIG } = await import("../../open-sse/config/errorConfig.ts"); + +const SESSION_BODY = "you (acme-corp) have reached your session usage limit"; +const SESSION_COOLDOWN_MS = 5 * 60 * 60 * 1000; // 5 hours + +test("#7071 sanity: weekly text IS recognized (already fixed by #3709/#6638)", () => { + assert.equal(isWeeklyUsageLimitText("you (acme-corp) have reached your weekly usage limit"), true); +}); + +test("#7071 isSessionUsageLimitText matches the ollama-cloud 429 body", () => { + assert.equal(isSessionUsageLimitText(SESSION_BODY.toLowerCase()), true); + assert.equal(isSessionUsageLimitText("session limit reached, try later"), true); + assert.equal(isSessionUsageLimitText("rate_limit_exceeded: too many requests"), false); + // Must not false-positive on unrelated "session expired" auth errors. + assert.equal(isSessionUsageLimitText("your session has expired, please log in again"), false); + assert.equal(isSessionUsageLimitText("session token invalid"), false); +}); + +test("#7071 buildSessionQuotaFallback returns a 5h QUOTA_EXHAUSTED cooldown, far above the generic backoff cap", () => { + const result = buildSessionQuotaFallback(SESSION_BODY); + assert.ok(result, "expected a non-null fallback for session-usage-limit text"); + assert.equal(result!.reason, RateLimitReason.QUOTA_EXHAUSTED); + assert.equal(result!.cooldownMs, SESSION_COOLDOWN_MS); + assert.ok(result!.cooldownMs > (ERROR_BACKOFF_CONFIG.max ?? BACKOFF_CONFIG.max)); +}); + +test("#7071 buildSessionQuotaFallback returns null for unrelated error text", () => { + assert.equal(buildSessionQuotaFallback("rate_limit_exceeded: too many requests"), null); + assert.equal(buildSessionQuotaFallback("your session has expired, please log in again"), null); +}); + +test("#7071 BUG: checkFallbackError misclassifies ollama-cloud session-quota 429 as generic RATE_LIMIT_EXCEEDED instead of QUOTA_EXHAUSTED", () => { + const out = checkFallbackError( + 429, + SESSION_BODY, + 0, // backoffLevel + null, // model + "ollama-cloud", // provider (apikey category) + null, // headers + null, // profileOverride + null // structuredError + ); + + assert.equal(out.shouldFallback, true); + assert.equal( + out.reason, + RateLimitReason.QUOTA_EXHAUSTED, + `expected QUOTA_EXHAUSTED for session-usage-limit text, got reason=${out.reason} cooldownMs=${out.cooldownMs}` + ); + assert.equal(out.cooldownMs, SESSION_COOLDOWN_MS); +}); + +test("#7071 checkFallbackError: oauth-category provider with session-limit text also gets the long cooldown", () => { + const out = checkFallbackError(429, SESSION_BODY, 0, null, "claude", null, null, null); + assert.equal(out.reason, RateLimitReason.QUOTA_EXHAUSTED); + assert.equal(out.cooldownMs, SESSION_COOLDOWN_MS); +}); + +test("#7071 checkFallbackError: ollama-cloud generic rate-limit body is unaffected (no false positive)", () => { + const out = checkFallbackError( + 429, + "rate_limit_exceeded: too many requests", + 0, + null, + "ollama-cloud", + null, + null, + null + ); + assert.equal(out.reason, RateLimitReason.RATE_LIMIT_EXCEEDED); + assert.ok( + out.cooldownMs <= 2 * 60 * 1000, + "generic rate limit text must keep the normal short backoff, not the 5h session cooldown" + ); +});