fix(resilience): keep Ollama model-not-found failures scoped to connection model lockout (#11071) (#11078)

Cherry-picked the value commit (2c9202e4) onto the current tip, dropping the stale base-red sync commits. Focused tests: ollama-404-model-lockout 2/2 + the five sibling lockout suites (combo-provider-cooldown-sibling, 8247-model-unhealthy, vertex-passthrough, nvidia-410, account-fallback-service) 112/112; mutation coverage no-drift. Fixes #11071 — local/self-hosted 404s now scope to model lockout per the resilience doctrine. Thank you @rqzbeh!

open-sse/services/accountFallback.ts

@@ -21,7 +21,7 @@ import {
   honorsRuleLockScope,
 } from "../config/providerErrorRules.ts";
 import * as rot from "./rotationConfig.ts";
-import { getPassthroughProviders, getProviderCategory } from "../config/providerRegistry.ts";
+import { getPassthroughProviders, getProviderCategory, isLocalProvider } from "../config/providerRegistry.ts";
 import {
   DEFAULT_RESILIENCE_SETTINGS,
   resolveResilienceSettings,
@@ -37,7 +37,7 @@ import {
   type FailureKind,
 } from "../../src/shared/utils/classify429";
 import { recordProviderSuccess as resetCooldownFailureCount } from "./providerCooldownTracker.ts";
-import { resolveProviderId } from "../../src/shared/constants/providers";
+import { resolveProviderId, isLocalProvider as isLocalProviderId, isSelfHostedChatProvider } from "../../src/shared/constants/providers";
 import { resolveUseUpstream429BreakerHints } from "../../src/shared/utils/providerHints";
 import { getCodexModelScope } from "../config/codexQuotaScopes.ts";
 import { getQuotaScopedModelForProvider } from "./antigravityQuotaFamily.ts";
@@ -791,12 +791,14 @@ export function hasPerModelQuota(
     return connectionPassthroughModels;
   }
   if (!provider) return false;
-  if (getCanonicalLockProvider(provider) === "antigravity") return true;
-  if (getCanonicalLockProvider(provider) === "codex") return true;
-  if (provider === "gemini" || provider === "github") return true;
-  if (provider === "antigravity" || provider === "agy") return true;
-  if (getPassthroughProviders().has(provider)) return true;
-  if (isCompatibleProvider(provider)) return true;
+  const canonicalId = resolveProviderId(provider);
+  if (getCanonicalLockProvider(canonicalId) === "antigravity") return true;
+  if (getCanonicalLockProvider(canonicalId) === "codex") return true;
+  if (canonicalId === "gemini" || canonicalId === "github") return true;
+  if (canonicalId === "antigravity" || canonicalId === "agy") return true;
+  if (getPassthroughProviders().has(canonicalId)) return true;
+  if (isCompatibleProvider(canonicalId)) return true;
+  if (isLocalProviderId(canonicalId) || isSelfHostedChatProvider(canonicalId)) return true;
   return false;
 }
 

stryker.conf.json

@@ -272,6 +272,7 @@
       "tests/unit/model-lockout-max-cooldown.test.ts",
       "tests/unit/no-memory-header.test.ts",
       "tests/unit/noauth-autocombo-lockout-7623.test.ts",
+      "tests/unit/ollama-404-model-lockout-11071.test.ts",
       "tests/unit/non-streaming-sse-terminal-typescan-4459.test.ts",
       "tests/unit/nvidia-410-model-scope.test.ts",
       "tests/unit/nvidia-passthrough-models-6773.test.ts",

tests/unit/ollama-404-model-lockout-11071.test.ts

@@ -0,0 +1,66 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-ollama-404-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+
+const core = await import("../../src/lib/db/core.ts");
+const providersDb = await import("../../src/lib/db/providers.ts");
+const auth = await import("../../src/sse/services/auth.ts");
+const { hasPerModelQuota, isModelLocked } = await import("../../open-sse/services/accountFallback.ts");
+
+async function resetStorage() {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
+}
+
+test.after(() => {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+});
+
+test("hasPerModelQuota returns true for ollama-local and ollama providers", () => {
+  assert.equal(hasPerModelQuota("ollama-local"), true);
+  assert.equal(hasPerModelQuota("ollama"), true);
+});
+
+test("markAccountUnavailable locks only the missing model on a 404 from ollama-local", async () => {
+  await resetStorage();
+
+  const connection = await providersDb.createProviderConnection({
+    provider: "ollama-local",
+    authType: "none",
+    baseUrl: "http://127.0.0.1:11434/v1",
+    isActive: true,
+  });
+
+  const result = await auth.markAccountUnavailable(
+    connection.id,
+    404,
+    "model 'model-b' not found",
+    "ollama-local",
+    "model-b"
+  );
+
+  assert.equal(result.shouldFallback, true);
+
+  // The missing model must be locked
+  assert.equal(isModelLocked("ollama-local", connection.id, "model-b"), true);
+
+  // The connection in DB must remain active / not marked unavailable for sibling models
+  const connInDb = await providersDb.getProviderConnectionById(connection.id);
+  assert.notEqual(connInDb?.testStatus, "unavailable", "connection should not be marked unavailable connection-wide on a 404 model-not-found error");
+
+  // getProviderCredentials must still serve sibling models
+  const selectedForSibling = await auth.getProviderCredentials(
+    "ollama-local",
+    null,
+    null,
+    "model-a"
+  );
+  assert.ok(selectedForSibling && !("allExpired" in selectedForSibling), "sibling model-a must still be selected on the same connection");
+});