From 7042d562c49d482b0b93d99af8ee8cd0c72c7621 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Wed, 3 Jun 2026 07:25:50 -0300 Subject: [PATCH] fix(sse): scope ollama-cloud per-model 403 to model lockout, not connection cooldown (#3027) (#3096) A per-model subscription/permission 403 from a passthrough provider (hasPerModelQuota) now locks only the failing model instead of cooling the whole connection, so free models on the same key keep serving and repeated paid-model 403s don't escalate a connection-wide backoff. Generalizes the grok-web 403 precedent; terminal/credential 403s still deactivate the connection (guarded by resolveTerminalConnectionStatus). TDD: 3 tests in sse-auth.test.ts (2 reproductions fail pre-fix, regression guard passes both ways). --- CHANGELOG.md | 1 + src/sse/services/auth.ts | 40 +++++++++++++++++ tests/unit/sse-auth.test.ts | 86 +++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f6cf8816e..f0aeed3e97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ _Development cycle in progress — entries are added as work merges into `releas - **providers:** fix claude-web persistent 403 — `execute()` was calling the synchronous `normalizeClaudeSessionCookie()` which never injects `cf_clearance`; changed to async `normalizeClaudeSessionCookieWithAutoRefresh()` with `allowAutoSolve:true`. Also removes dead executor `claude-web-auto-refresh.ts` and correctly reclassifies `duckduckgo-web` and `veoaifree-web` as `NOAUTH_PROVIDERS`. (#3090 — thanks @oyi77) - **autoCombo:** rotate across all provider connections, never waste capacity — `buildAutoCandidates` now expands each provider into one candidate per active connection (e.g. 43 Cerebras keys → 43 candidates). Adds `ScoreTierRotator` with per-combo round-robin state, combo-name-aware tier preferences (smart/fast/cheap/coding), `connectionDensity` factor (weight 0.05), and budget-cap degradation using the rotator. (#3078 — thanks @oyi77) - **providers:** fix SiliconFlow model sync from configured endpoint — routes model discovery through `providerSpecificData.baseUrl` so CN (`api.siliconflow.cn`) vs Global endpoint selection is respected, and prevents `/sync-models` from treating `source: "local_catalog"` fallback responses as successful remote syncs. (#3094 — thanks @xz-dev) +- **resilience:** a per-model subscription/permission `403` from a passthrough provider (e.g. Ollama Cloud `deepseek-v4-pro` → _"this model requires a subscription"_) now locks out **only that model** instead of cooling down the whole connection — the free models on the same key keep serving, and repeated paid-model 403s no longer escalate a connection-wide backoff. Generalizes the grok-web 403 precedent to all `hasPerModelQuota` providers; terminal/credential 403s (banned/deactivated key) still deactivate the connection. ([#3027](https://github.com/diegosouzapw/OmniRoute/issues/3027)) - **cache:** preserve client-side `cache_control` breakpoints for Xiaomi MiMo — added `xiaomi-mimo` to the prompt-caching provider allowlist so Claude Code (via cc-switch) cache hints are no longer stripped by the OpenAI-format translator, restoring cache hits. ([#3088](https://github.com/diegosouzapw/OmniRoute/issues/3088)) ### 📦 Dependencies diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index 2975861c79..b81e1cef46 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -1815,6 +1815,46 @@ export async function markAccountUnavailable( ); const cooldownMs = terminalStatus ? 0 : rawCooldownMs; + // ── #3027: per-model subscription/permission 403 → model-only lockout ── + // Passthrough / per-model-quota providers (e.g. ollama-cloud with + // passthroughModels:true) multiplex many upstream models behind one key. + // A scoped 403 like "this model requires a subscription, upgrade for access" + // is about the paid model, not the key — cooling the whole connection would + // knock out the free models on the same key too and escalate backoff + // (#3001/#3027). This generalizes the grok-web 403 precedent above to every + // hasPerModelQuota provider. Terminal/credential 403s (banned/deactivated + // key, credits exhausted) are excluded here because + // resolveTerminalConnectionStatus() returns a non-null status for them, so + // they keep their existing connection-level cooldown/deactivation path. + if (isPerModelQuotaProvider && status === 403 && provider && model && !terminalStatus) { + const lockout = recordModelLockoutFailure( + provider, + connectionId, + model, + "forbidden", + status, + fallbackResult.baseCooldownMs ?? + effectiveProviderProfile?.baseCooldownMs ?? + COOLDOWN_MS.serviceUnavailable, + effectiveProviderProfile, + { + exactCooldownMs: + fallbackResult.usedUpstreamRetryHint === true ? fallbackResult.cooldownMs : null, + } + ); + updateProviderConnection(connectionId, { + lastErrorType: "forbidden", + lastError: `Model ${model} forbidden (per-model access/subscription)`, + lastErrorAt: new Date().toISOString(), + errorCode: status, + }).catch(() => {}); + log.info( + "AUTH", + `Model-only lockout for ${provider}:${model} — 403 forbidden ${Math.ceil(lockout.cooldownMs / 1000)}s (per-model quota provider, connection stays active)` + ); + return { shouldFallback: true, cooldownMs: lockout.cooldownMs }; + } + // ── 404 model-only lockout: connection stays active ── // For local providers (detected by URL), a 404 means the specific model // doesn't exist or isn't available for this account — it should NOT lock diff --git a/tests/unit/sse-auth.test.ts b/tests/unit/sse-auth.test.ts index 0109b87e01..b89dbba18d 100644 --- a/tests/unit/sse-auth.test.ts +++ b/tests/unit/sse-auth.test.ts @@ -14,6 +14,7 @@ const settingsDb = await import("../../src/lib/db/settings.ts"); const apiKeysDb = await import("../../src/lib/db/apiKeys.ts"); const auth = await import("../../src/sse/services/auth.ts"); const quotaCache = await import("../../src/domain/quotaCache.ts"); +const fallback = await import("../../open-sse/services/accountFallback.ts"); async function resetStorage() { core.resetDbInstance(); @@ -1122,6 +1123,91 @@ test("markAccountUnavailable applies a model-only lockout for compatible provide assert.equal(Number(updated.errorCode), 429); }); +// #3027 — a per-model subscription/permission 403 from a passthrough provider +// (ollama-cloud) must lock only the paid model, not the whole connection. +test("markAccountUnavailable: ollama-cloud per-model subscription 403 locks the model, not the connection (#3027)", async () => { + fallback.clearAllModelLockouts(); + const connection = await seedConnection("ollama-cloud", { + name: "ollama-paid-model", + providerSpecificData: { passthroughModels: true }, + }); + + const result = await auth.markAccountUnavailable( + connection.id, + 403, + "this model requires a subscription, upgrade for access: https://ollama.com/upgrade", + "ollama-cloud", + "deepseek-v4-pro" + ); + await flushWrites(); + const updated = await providersDb.getProviderConnectionById(connection.id); + + // Connection stays eligible — only the paid model is cooled down. + assert.equal(result.shouldFallback, true); + assert.ok(result.cooldownMs > 0); + assert.equal(updated.testStatus, "active"); + assert.equal(updated.rateLimitedUntil, undefined); + assert.equal(updated.lastErrorType, "forbidden"); + assert.equal(Number(updated.errorCode), 403); + + assert.equal(fallback.isModelLocked("ollama-cloud", connection.id, "deepseek-v4-pro"), true); + // A free model on the same key is still usable. + assert.equal(fallback.isModelLocked("ollama-cloud", connection.id, "gemma4:31b"), false); +}); + +// #3027 regression — a genuine whole-key 403 (deactivated/banned key) must NOT +// be downgraded to a model lockout; the connection still becomes terminal. +test("markAccountUnavailable: a whole-key 403 still deactivates the ollama-cloud connection (#3027 regression)", async () => { + fallback.clearAllModelLockouts(); + const connection = await seedConnection("ollama-cloud", { + name: "ollama-banned-key", + providerSpecificData: { passthroughModels: true }, + }); + + await auth.markAccountUnavailable( + connection.id, + 403, + "account has been deactivated", + "ollama-cloud", + "deepseek-v4-pro" + ); + await flushWrites(); + const updated = await providersDb.getProviderConnectionById(connection.id); + + assert.equal(fallback.isModelLocked("ollama-cloud", connection.id, "deepseek-v4-pro"), false); + assert.ok( + ["banned", "expired", "credits_exhausted"].includes(updated.testStatus), + `expected a terminal connection status, got ${updated.testStatus}` + ); +}); + +// #3027 — repeated subscription 403s on the paid model must never escalate a +// connection-wide cooldown/backoff (only the model lockout escalates). +test("markAccountUnavailable: repeated ollama-cloud subscription 403s never escalate connection backoff (#3027)", async () => { + fallback.clearAllModelLockouts(); + const connection = await seedConnection("ollama-cloud", { + name: "ollama-repeat-403", + providerSpecificData: { passthroughModels: true }, + }); + + for (let i = 0; i < 3; i++) { + await auth.markAccountUnavailable( + connection.id, + 403, + "this model requires a subscription, upgrade for access", + "ollama-cloud", + "deepseek-v4-pro" + ); + await flushWrites(); + } + const updated = await providersDb.getProviderConnectionById(connection.id); + + assert.equal(updated.rateLimitedUntil, undefined); + assert.notEqual(updated.testStatus, "unavailable"); + assert.ok(!updated.backoffLevel, "connection backoffLevel must not escalate"); + assert.equal(fallback.isModelLocked("ollama-cloud", connection.id, "deepseek-v4-pro"), true); +}); + test("markAccountUnavailable honors configured api-key rate-limit cooldowns", async () => { await settingsDb.updateSettings({ providerProfiles: {