diff --git a/open-sse/services/autoCombo/resilienceCandidateFilter.ts b/open-sse/services/autoCombo/resilienceCandidateFilter.ts new file mode 100644 index 0000000000..72c2cc9445 --- /dev/null +++ b/open-sse/services/autoCombo/resilienceCandidateFilter.ts @@ -0,0 +1,112 @@ +/** + * #7623 — exclude models/connections the existing resilience runtime already + * marked unavailable from `auto/*` candidate pools. + * + * Uses the same reads as dispatch-time account selection and the #7819 + * candidate inspector (`isModelLocked`, connection cooldown / testStatus). + * Pure filter kept separate from `virtualFactory.ts` for unit testing, mirroring + * `paidModelFilter.ts` and `candidateOverrides.ts` in this directory. + */ +import { isAccountUnavailable, isModelLocked } from "../accountFallback.ts"; + +export const SYNTHETIC_NOAUTH_CONNECTION_ID = "noauth"; + +const TERMINAL_CONNECTION_STATUSES = new Set([ + "banned", + "expired", + "credits_exhausted", + "deactivated", +]); + +interface ResilienceFilterCandidate { + provider: string; + connectionId: string | null; + allowedConnectionIds?: string[]; + model: string; +} + +export interface ConnectionResilienceView { + id: string; + rateLimitedUntil?: string | null; + testStatus?: string | null; +} + +function isConnectionResilienceBlocked(connection: ConnectionResilienceView): boolean { + if (isAccountUnavailable(connection.rateLimitedUntil)) return true; + const status = connection.testStatus; + if (status === "unavailable") return true; + if (typeof status === "string" && TERMINAL_CONNECTION_STATUSES.has(status)) return true; + return false; +} + +function isConnectionEligibleForModel( + provider: string, + connectionId: string, + model: string, + connectionsById: Map +): boolean { + const connection = connectionsById.get(connectionId); + if (connection && isConnectionResilienceBlocked(connection)) return false; + return !isModelLocked(provider, connectionId, model); +} + +/** + * Remove auto-combo candidates whose provider/model pair is model-locked, and + * trim credentialed logical candidates whose allowed connections are all blocked. + * Returns the input reference when nothing changed. + */ +export function filterResilienceBlockedCandidates( + pool: T[], + connectionsById: Map +): T[] { + if (!Array.isArray(pool) || pool.length === 0) return pool; + + let changed = false; + const filtered = pool.flatMap((candidate) => { + if (candidate.connectionId === SYNTHETIC_NOAUTH_CONNECTION_ID) { + if (isModelLocked(candidate.provider, SYNTHETIC_NOAUTH_CONNECTION_ID, candidate.model)) { + changed = true; + return []; + } + return [candidate]; + } + + if (Array.isArray(candidate.allowedConnectionIds)) { + const allowedConnectionIds = candidate.allowedConnectionIds.filter((connectionId) => + isConnectionEligibleForModel( + candidate.provider, + connectionId, + candidate.model, + connectionsById + ) + ); + if (allowedConnectionIds.length === 0) { + changed = true; + return []; + } + if (allowedConnectionIds.length === candidate.allowedConnectionIds.length) { + return [candidate]; + } + changed = true; + return [{ ...candidate, allowedConnectionIds }]; + } + + if (candidate.connectionId) { + if ( + !isConnectionEligibleForModel( + candidate.provider, + candidate.connectionId, + candidate.model, + connectionsById + ) + ) { + changed = true; + return []; + } + } + + return [candidate]; + }); + + return changed ? filtered : pool; +} diff --git a/open-sse/services/autoCombo/virtualFactory.ts b/open-sse/services/autoCombo/virtualFactory.ts index 7d53ee1b65..21e130df8c 100644 --- a/open-sse/services/autoCombo/virtualFactory.ts +++ b/open-sse/services/autoCombo/virtualFactory.ts @@ -23,6 +23,11 @@ import { filterPaidOnlyCandidates } from "./paidModelFilter"; import { isModelExcludedByConnection } from "@/domain/connectionModelRules"; import { filterExcludedCandidates } from "./candidateOverrides"; import { getExcludedConnectionIds } from "@/lib/db/autoCandidateOverrides"; +import { + filterResilienceBlockedCandidates, + SYNTHETIC_NOAUTH_CONNECTION_ID as RESILIENCE_NOAUTH_CONNECTION_ID, + type ConnectionResilienceView, +} from "./resilienceCandidateFilter"; /** #4235 Phase B: optional category/tier overlay for `auto/:` combos. * #6453: optional `family` overlay for `auto/` combos (e.g. `auto/glm`) — @@ -133,7 +138,7 @@ function hasUsableConnectionCredential(conn: VirtualFactoryConn): boolean { return hasApiKey || hasUsableOAuthToken(conn) || hasProviderSpecificSessionData(conn); } -const SYNTHETIC_NOAUTH_CONNECTION_ID = "noauth"; +const SYNTHETIC_NOAUTH_CONNECTION_ID = RESILIENCE_NOAUTH_CONNECTION_ID; // Allowlist of no-auth (keyless) providers permitted to enter the `auto`/`auto-*` // candidate pool. Narrowed to the backends verified to answer without any @@ -409,6 +414,21 @@ export async function createVirtualAutoCombo( ) ); + // #7623: honor existing model lockouts + connection cooldown/terminal state so + // auto/* never advertises models the dispatch path would immediately skip. + const connectionsById = new Map(); + for (const conn of [...connections, ...disabledNoAuthConnections]) { + connectionsById.set(conn.id, conn); + } + const resilienceFilteredPool = filterResilienceBlockedCandidates( + candidatePool, + connectionsById + ); + if (resilienceFilteredPool !== candidatePool) { + candidatePool.length = 0; + candidatePool.push(...resilienceFilteredPool); + } + // #6512 (follow-up to #6328/#6495): when the operator opts into `hidePaidModels`, // exclude paid-only backends from EVERY `auto/*` candidate pool — not just the // `/v1/models` listing — so auto-routing never picks a model that will 402/403. diff --git a/tests/unit/noauth-autocombo-lockout-7623.test.ts b/tests/unit/noauth-autocombo-lockout-7623.test.ts new file mode 100644 index 0000000000..dea6d7b231 --- /dev/null +++ b/tests/unit/noauth-autocombo-lockout-7623.test.ts @@ -0,0 +1,103 @@ +/** + * #7623 — auto-combo candidate pool must honor existing model lockouts so + * repeatedly failing no-auth models (e.g. opencode 401) are not re-advertised. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-7623-noauth-lockout-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; + +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const virtualFactory = await import("../../open-sse/services/autoCombo/virtualFactory.ts"); +const accountFallback = await import("../../open-sse/services/accountFallback.ts"); + +async function resetStorage() { + core.resetDbInstance(); + accountFallback.clearAllModelLockouts(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.beforeEach(async () => { + await resetStorage(); +}); + +test.after(async () => { + await resetStorage(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + + if (ORIGINAL_DATA_DIR === undefined) { + delete process.env.DATA_DIR; + } else { + process.env.DATA_DIR = ORIGINAL_DATA_DIR; + } +}); + +test("#7623: a model-locked no-auth opencode model is ABSENT from the auto-combo candidate pool", async () => { + accountFallback.lockModel("opencode", "noauth", "mimo-v2.5-free", "model_not_found", 60_000); + + const combo = await virtualFactory.createVirtualAutoCombo(undefined); + const modelStrings = combo.models.map((m: { model: string }) => m.model); + + assert.ok( + !modelStrings.some((model: string) => model.endsWith("/mimo-v2.5-free")), + "BUG #7623: locked no-auth model must not appear in auto-combo pool. Pool: " + + JSON.stringify(modelStrings) + ); +}); + +test("#7623: sibling no-auth models stay in the pool when only one model is locked", async () => { + accountFallback.lockModel("opencode", "noauth", "mimo-v2.5-free", "model_not_found", 60_000); + + const combo = await virtualFactory.createVirtualAutoCombo(undefined); + const modelStrings = combo.models.map((m: { model: string }) => m.model); + + assert.ok( + modelStrings.some((model: string) => model.endsWith("/big-pickle")), + `healthy sibling model must remain in pool. Pool: ${JSON.stringify(modelStrings)}` + ); +}); + +test("#7623: credentialed provider drops a connection from allowedConnectionIds when that model is locked", async () => { + const tokenExpiresAt = new Date(Date.now() + 60_000).toISOString(); + const locked = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + email: "locked@example.com", + accessToken: "fake-token-locked", + tokenExpiresAt, + }); + const healthy = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + email: "healthy@example.com", + accessToken: "fake-token-healthy", + tokenExpiresAt, + }); + + const lockedModel = "claude-sonnet-4-6"; + accountFallback.lockModel("antigravity", locked.id, lockedModel, "rate_limited", 60_000); + + const combo = await virtualFactory.createVirtualAutoCombo(undefined); + const candidate = combo.models.find( + (m: { model: string }) => m.model === `antigravity/${lockedModel}` + ) as { allowedConnectionIds?: string[] } | undefined; + + assert.ok(candidate, `expected antigravity/${lockedModel} logical candidate`); + assert.deepEqual( + [...(candidate.allowedConnectionIds ?? [])].sort(), + [healthy.id].sort(), + "locked connection must be removed from allowedConnectionIds" + ); + assert.ok( + !(candidate.allowedConnectionIds ?? []).includes(locked.id), + "locked connection must not remain eligible" + ); +});