From f9bcc9418bf735c6d9cd234612a1bef54f51355e Mon Sep 17 00:00:00 2001 From: Regis <92858615+Regis-RCR@users.noreply.github.com> Date: Mon, 16 Mar 2026 18:55:41 +0100 Subject: [PATCH 1/2] fix(sse): model-only lockout for local provider 404 (connection stays active) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a local inference backend (oMLX, Ollama, LM Studio) returns 404 for an unknown model, OmniRoute previously locked the entire connection for 2 minutes — blocking all valid models on that connection. This fix introduces local provider detection and changes the 404 behavior for local providers: - Model-only lockout (5s) instead of connection-level lockout (2min) - Connection stays active — other models continue working immediately - Detection via URL heuristic (localhost/127.0.0.1) + apiKey===null fallback - Configurable via LOCAL_HOSTNAMES env var for Docker setups Also fixes a pre-existing bug where the model parameter was not passed to markAccountUnavailable() from chat.ts, preventing per-model lockouts from working at all. Changes: - Add isLocalProvider(baseUrl) helper in providerRegistry.ts - Add COOLDOWN_MS.notFoundLocal (5s) and PROVIDER_PROFILES.local - Add local 404 branch in markAccountUnavailable() in auth.ts - Pass model param to markAccountUnavailable() in chat.ts (bug fix) --- open-sse/config/constants.ts | 8 ++++++++ open-sse/config/providerRegistry.ts | 31 +++++++++++++++++++++++++++++ src/sse/handlers/chat.ts | 3 ++- src/sse/services/auth.ts | 21 +++++++++++++++++++ 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/open-sse/config/constants.ts b/open-sse/config/constants.ts index f310f8ab2c..ab2d154c91 100644 --- a/open-sse/config/constants.ts +++ b/open-sse/config/constants.ts @@ -135,6 +135,7 @@ export const COOLDOWN_MS = { unauthorized: 2 * 60 * 1000, // 401 → 2 min paymentRequired: 2 * 60 * 1000, // 402/403 → 2 min notFound: 2 * 60 * 1000, // 404 → 2 minutes + notFoundLocal: 5 * 1000, // 404 on local provider → 5s model-only lockout (connection stays active) transientInitial: 5 * 1000, // 408/500/502/503/504 first hit → 5s (backoff from here) transientMax: 60 * 1000, // 502/503/504 backoff ceiling → 60s transient: 5 * 1000, // Legacy alias → points to transientInitial @@ -162,6 +163,13 @@ export const PROVIDER_PROFILES = { circuitBreakerThreshold: 5, // More tolerant (occasional 502 is normal) circuitBreakerReset: 30000, // 30s reset }, + local: { + transientCooldown: 2000, // 2s (local — very fast recovery) + rateLimitCooldown: 5000, // 5s (local — no real rate limits) + maxBackoffLevel: 3, // Low ceiling (local either works or doesn't) + circuitBreakerThreshold: 2, // Opens fast (if local is down, it's down) + circuitBreakerReset: 15000, // 15s reset (check again quickly) + }, }; // Default rate limit values for API Key providers (auto-enabled safety net) diff --git a/open-sse/config/providerRegistry.ts b/open-sse/config/providerRegistry.ts index 531c111c4b..579b0b9787 100644 --- a/open-sse/config/providerRegistry.ts +++ b/open-sse/config/providerRegistry.ts @@ -1042,6 +1042,37 @@ export function generateAliasMap(): Record { return map; } +// ── Local Provider Detection ────────────────────────────────────────────── + +const LOCAL_HOSTNAMES = new Set([ + "localhost", + "127.0.0.1", + "::1", + "[::1]", + ...(typeof process !== "undefined" && process.env.LOCAL_HOSTNAMES + ? process.env.LOCAL_HOSTNAMES.split(",") + .map((h) => h.trim()) + .filter(Boolean) + : []), +]); + +/** + * Detect if a base URL points to a local inference backend. + * Used for shorter 404 cooldowns (model-only, not connection) and health check targets. + * + * Operators can extend via LOCAL_HOSTNAMES env var (comma-separated) for Docker + * hostnames (e.g., LOCAL_HOSTNAMES=omlx,mlx-audio). + */ +export function isLocalProvider(baseUrl?: string | null): boolean { + if (!baseUrl) return false; + try { + const url = new URL(baseUrl); + return LOCAL_HOSTNAMES.has(url.hostname); + } catch { + return false; + } +} + // ── Registry Lookup Helpers ─────────────────────────────────────────────── const _byAlias = new Map(); diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index f10c5077d4..bd0b0e59c8 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -382,7 +382,8 @@ async function handleSingleModelChat( credentials.connectionId, result.status, result.error, - provider + provider, + model ); if (shouldFallback) { diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index ac8bd67785..451d145cbb 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -14,6 +14,8 @@ import { isModelLocked, lockModel, } from "@omniroute/open-sse/services/accountFallback.ts"; +import { isLocalProvider } from "@omniroute/open-sse/config/providerRegistry.ts"; +import { COOLDOWN_MS } from "@omniroute/open-sse/config/constants.ts"; import * as log from "../utils/logger"; import { fisherYatesShuffle, getNextFromDeckSync } from "@/shared/utils/shuffleDeck"; @@ -563,6 +565,25 @@ export async function markAccountUnavailable( ); if (!shouldFallback) return { shouldFallback: false, cooldownMs: 0 }; + // ── Local provider 404: model-only lockout, connection stays active ── + const connBaseUrl = (conn?.providerSpecificData as Record)?.baseUrl as + | string + | undefined; + const isLocal = + isLocalProvider(connBaseUrl) || (conn?.apiKey === null && conn?.accessToken === null); + + if (isLocal && status === 404) { + const localCooldown = COOLDOWN_MS.notFoundLocal; + if (provider && model) { + lockModel(provider, connectionId, model, "local_not_found", localCooldown); + } + log.info( + "AUTH", + `Local 404 for ${model} — model-only lockout ${localCooldown / 1000}s (connection stays active)` + ); + return { shouldFallback: true, cooldownMs: localCooldown }; + } + const rateLimitedUntil = getUnavailableUntil(cooldownMs); const errorMsg = typeof errorText === "string" ? errorText.slice(0, 100) : "Provider error"; From 1f9a402dcd783daea168ac44f3611d2fc884d39c Mon Sep 17 00:00:00 2001 From: Regis <92858615+Regis-RCR@users.noreply.github.com> Date: Mon, 16 Mar 2026 19:03:47 +0100 Subject: [PATCH 2/2] =?UTF-8?q?fix(sse):=20address=20bot=20review=20?= =?UTF-8?q?=E2=80=94=20tighten=20local=20detection,=20guard=20null=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove apiKey===null heuristic (too broad — could match cloud providers with non-standard auth). Use URL-based detection only. - Guard local 404 branch with provider && model check — if either is null, fall through to standard connection lockout (safer behavior). - Document LOCAL_HOSTNAMES as module-load-time constant (restart required). - Document PROVIDER_PROFILES.local as intentionally not yet wired. --- open-sse/config/constants.ts | 3 +++ open-sse/config/providerRegistry.ts | 1 + src/sse/services/auth.ts | 10 ++++------ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/open-sse/config/constants.ts b/open-sse/config/constants.ts index ab2d154c91..9b4aeaf720 100644 --- a/open-sse/config/constants.ts +++ b/open-sse/config/constants.ts @@ -163,6 +163,9 @@ export const PROVIDER_PROFILES = { circuitBreakerThreshold: 5, // More tolerant (occasional 502 is normal) circuitBreakerReset: 30000, // 30s reset }, + // Local providers (localhost inference backends like Ollama, LM Studio, oMLX). + // Not yet wired into getProviderProfile() — will be used when local provider_nodes + // are integrated into the resilience layer. Kept here to avoid a second constants change. local: { transientCooldown: 2000, // 2s (local — very fast recovery) rateLimitCooldown: 5000, // 5s (local — no real rate limits) diff --git a/open-sse/config/providerRegistry.ts b/open-sse/config/providerRegistry.ts index 579b0b9787..1ba3995987 100644 --- a/open-sse/config/providerRegistry.ts +++ b/open-sse/config/providerRegistry.ts @@ -1044,6 +1044,7 @@ export function generateAliasMap(): Record { // ── Local Provider Detection ────────────────────────────────────────────── +// Evaluated once at module load time — process restart required for env var changes. const LOCAL_HOSTNAMES = new Set([ "localhost", "127.0.0.1", diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index 451d145cbb..d4f6ead196 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -566,17 +566,15 @@ export async function markAccountUnavailable( if (!shouldFallback) return { shouldFallback: false, cooldownMs: 0 }; // ── Local provider 404: model-only lockout, connection stays active ── + // Detection: URL-based only (apiKey===null heuristic was too broad — could match + // cloud providers with non-standard auth stored in providerSpecificData). const connBaseUrl = (conn?.providerSpecificData as Record)?.baseUrl as | string | undefined; - const isLocal = - isLocalProvider(connBaseUrl) || (conn?.apiKey === null && conn?.accessToken === null); - if (isLocal && status === 404) { + if (isLocalProvider(connBaseUrl) && status === 404 && provider && model) { const localCooldown = COOLDOWN_MS.notFoundLocal; - if (provider && model) { - lockModel(provider, connectionId, model, "local_not_found", localCooldown); - } + lockModel(provider, connectionId, model, "local_not_found", localCooldown); log.info( "AUTH", `Local 404 for ${model} — model-only lockout ${localCooldown / 1000}s (connection stays active)`