diff --git a/changelog.d/fixes/9133-auto-combo-candidates-unfiltered.md b/changelog.d/fixes/9133-auto-combo-candidates-unfiltered.md new file mode 100644 index 0000000000..ddd3f6165f --- /dev/null +++ b/changelog.d/fixes/9133-auto-combo-candidates-unfiltered.md @@ -0,0 +1 @@ +- fix(sse): stop the auto-combo candidates inspector from silently dropping model-locked/cooled-down rows (#9133) diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 20d3c00edc..acce0e90b8 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -477,7 +477,8 @@ "_rebaseline_2026_08_23_tip_drift_post_batch0823": "Tip drift after the 2026-08-23 merge wave: chatBodyAdmission.ts 1009->1118 (+109, gate count incl. +1) and auth.ts 3337->3344 (+7), both grown by merges already on origin/release/v3.8.50 (verified identical on the pristine tip) — not by the codex-appserver-hardening PR that carries this bump. Owner pre-authorized baseline bumps 2026-08-22.", "_rebaseline_2026_08_24_11355_cooldown_recovery_guards": "PR #11355 own growth: test/route.ts 1215->1237, +22 (startup crash-recovery guard: clearStaleCrashCooldowns() now parses the persisted rate_limited_until deadline and skips clearing rows still genuinely in the future, instead of clearing every non-terminal cooldown unconditionally). Cohesive fix at the existing test-route dispatch chokepoint alongside the #11141 probe builder. Covered by tests/unit/startup-stale-cooldown-recovery.test.ts + tests/unit/repro-zai-cooldown-cleared-by-connection-test.test.ts.", "_rebaseline_2026_08_24_video_bridge_fu02_fu07_sampler": "PRs #11344 (FU-02 one-frame scene-aware determinism) + #11381 (FU-07 opt-in segment_aware structural sampling) own growth: videoBridgeRuntime.ts <1000->1009, +9 (sum of both boarded together in the same merge-batch). #11344 adds the deterministic one-frame midpoint fallback + policyEffective=uniform report at the existing scene_aware seam; #11381 adds the bounded local-only FFmpeg structural pre-analysis pass (scene/freeze/blur/exposure/SI-TI) and its budget-reallocation logic. Covered by tests/unit/guardrails/videoBridgeSampler.test.ts, tests/unit/guardrails/videoBridgeFu07StructuralSampling.test.ts, tests/integration/video-bridge-sampler-ffmpeg.test.ts. Owner pre-authorized rebaseline for legitimate PR growth (2026-08-19 directive).", - "open-sse/services/autoCombo/virtualFactory.ts": 1138, + "open-sse/services/autoCombo/virtualFactory.ts": 1139, + "_rebaseline_2026_08_29_9133_candidates_inspector_skip_flag": "#9133 own growth: open-sse/services/autoCombo/virtualFactory.ts 1138->1139 (+1, net of extraction). Fix: prepareVirtualAutoComboInputs gained an opt-in `skip` parameter so the read-only #7819 candidate inspector (open-sse/handlers/autoComboCandidates.ts) can build the FULL, unfiltered pool and decorate a resilience-blocked candidate as reachable:false instead of filterResilienceBlockedCandidates silently dropping the row before the inspector ever sees it (routing is unaffected — it never passes `skip`). The connectionsById map-building loop was extracted to buildConnectionResilienceMap() in resilienceCandidateFilter.ts (net 0 there since Prettier still breaks the call over multiple lines) and the now-unused ConnectionResilienceView import was dropped; the sole remaining growth is the new `skip` default parameter itself, which Prettier always places on its own line once the preceding options object parameter already breaks across lines — not further reducible without splitting prepareVirtualAutoComboInputs's signature away from its own body. Covered by tests/unit/auto-combo-candidates-locked-model-visible.test.ts (TDD repro: red before the fix, green after) plus the existing tests/unit/noauth-autocombo-lockout-7623.test.ts and tests/unit/auto-combo-credentialed-model-pool.test.ts (unaffected routing-path behavior).", "_rebaseline_2026_08_28_mergebatch_v3851_qwen_retirement": "/merge-batch 2026-08-28 (v3.8.51): #11713 (Qwen Web retirement) own growth: open-sse/services/autoCombo/virtualFactory.ts 1132->1135 (+3, combining the Designer + Runtime retirement-guard filter into the single runtimeConnections predicate at the existing candidate-pool chokepoint, now excluding Qwen Web alongside Felo Web). Irreducible per-mechanism wiring, additive not a new branch. Covered by tests/unit/virtual-auto-combo.test.ts.", "_rebaseline_2026_08_28_mergebatch_v3851_chatgptweb_retirement": "/merge-batch 2026-08-28 (v3.8.51): #11754 (common ChatGPT Web retirement) own growth: open-sse/services/autoCombo/virtualFactory.ts 1135->1138 (+3, an early `available` connection filter for the retired chatgpt-web/cgpt-web ids applied to both the active and disabled-noauth connection lists, ahead of the existing Designer+Runtime runtimeConnections filter). Irreducible per-mechanism wiring, additive not a new branch. Covered by tests/unit/virtual-auto-combo.test.ts.", "src/lib/cloudflaredTunnel.ts": 1078, diff --git a/open-sse/handlers/autoComboCandidates.ts b/open-sse/handlers/autoComboCandidates.ts index c133032306..e7a2762ee7 100644 --- a/open-sse/handlers/autoComboCandidates.ts +++ b/open-sse/handlers/autoComboCandidates.ts @@ -2,11 +2,13 @@ * #7819 (Level 1) — read-only candidate pool + reachability listing for an * `auto/*` channel. * - * Builds the SAME candidate pool `virtualFactory.createVirtualAutoCombo` uses - * for routing (via `createBuiltinAutoCombo`, unfiltered by any per-key - * exclusion so the operator can see — and toggle — excluded candidates), then - * decorates each candidate with live reachability derived from the existing - * resilience reads (CLAUDE.md "Resilience Runtime State"): + * Builds its OWN candidate pool via `prepareVirtualAutoComboInputs({}, true)` + * (#9133, the `skip` opt-out) — the full, unfiltered pool that routing would + * otherwise narrow with `filterResilienceBlockedCandidates` before ever + * handing it to this endpoint. Not filtered by any per-key exclusion either, + * so the operator can see — and toggle — excluded candidates. Every + * candidate is then decorated with live reachability derived from the + * existing resilience reads (CLAUDE.md "Resilience Runtime State"): * - provider circuit breaker: `getCircuitBreaker(provider).getStatus()` / * `.canExecute()` — NEVER raw `state`, so an expired breaker (lazy * recovery) doesn't show as permanently open. @@ -14,13 +16,31 @@ * provider_connections row (no-auth synthetic connections have no row — * treated as always reachable on this axis). * - model lockout: `isModelLocked(provider, connectionId, model)`. + * + * A candidate the routing path would currently skip (model-locked, cooled + * down, breaker open) is never dropped from this listing — it is surfaced + * with `reachable:false` and the specific reason field (`modelLocked` / + * `connectionCooldown` / `breakerState`) set, since the panel's whole + * purpose is read-only transparency, not a preview of the dispatch pool. + * The actual routing path (`createVirtualAutoCombo` / + * `createBuiltinAutoCombo` called without a `prepared` override) is + * unchanged and keeps excluding these candidates before dispatch. */ import { buildErrorBody } from "@omniroute/open-sse/utils/error.ts"; import { getCircuitBreaker } from "@/shared/utils/circuitBreaker"; import { isModelLocked } from "@omniroute/open-sse/services/accountFallback.ts"; +import { parseModel } from "@omniroute/open-sse/services/model.ts"; import { getProviderConnectionById } from "@/lib/db/providers"; import { getExcludedConnectionIds } from "@/lib/db/autoCandidateOverrides"; +/** + * One row of the unfiltered, reason-annotated candidate pool (#9133): every + * eligible provider/model/account combination the routing pool would + * otherwise contain, including rows routing itself would currently skip. + * `reachable:false` plus the relevant reason field (`modelLocked` / + * `connectionCooldown` / `breakerState`) is how a blocked candidate is + * represented here — it is never simply absent from `candidates`. + */ export interface AutoComboCandidateView { provider: string; connectionId: string; @@ -69,7 +89,17 @@ async function decorateCandidate(candidate: { } } - const modelLocked = isModelLocked(candidate.provider, candidate.connectionId, candidate.model); + // #9133 (latent alignment): every lock writer (accountFallback.ts) and the + // routing-side filter (resilienceCandidateFilter.ts, combo.ts) key + // `isModelLocked` off the BARE model id, never the "provider/model" + // `modelStr`. `candidate.model` here is the prefixed form (it comes from + // the materialized combo's `models[].model`, not the raw candidate pool), + // so it must be parsed back to bare before the lock lookup — currently + // masked (both spellings map to the same quota-family key for every + // provider exercised so far) but not guaranteed for a provider whose lock + // key isn't family-scoped. + const bareModel = parseModel(candidate.modelStr).model ?? candidate.model; + const modelLocked = isModelLocked(candidate.provider, candidate.connectionId, bareModel); return { provider: candidate.provider, @@ -99,15 +129,26 @@ export async function getAutoComboCandidates( // virtualFactory — createBuiltinAutoCombo() only recognizes `auto/` // ids (matches classifyAutoModel()'s special-casing of the literal "auto" // model string in src/sse/handlers/autoRouting.ts). + // #9133 — this endpoint's stated role is read-only transparency, so it must + // build its OWN unfiltered pool (`prepareVirtualAutoComboInputs({}, true)`, + // the `skip` opt-out) rather than reuse the routing path's pool. Routing + // legitimately drops resilience-blocked candidates before dispatch; this + // inspector must not, or a model-locked/cooled-down row silently vanishes + // instead of showing up as `reachable:false` with a reason. The routing + // path itself is unchanged — it keeps calling + // `prepareVirtualAutoComboInputs()`/`createVirtualAutoCombo()` with the + // default (filtered) behavior. + const { prepareVirtualAutoComboInputs, createVirtualAutoComboFromPrepared } = + await import("@omniroute/open-sse/services/autoCombo/virtualFactory.ts"); + const prepared = await prepareVirtualAutoComboInputs({}, true); + let virtualCombo; if (channel === "auto") { - const { createVirtualAutoCombo } = - await import("@omniroute/open-sse/services/autoCombo/virtualFactory.ts"); - virtualCombo = await createVirtualAutoCombo(undefined); + virtualCombo = await createVirtualAutoComboFromPrepared(prepared, undefined); } else { const { createBuiltinAutoCombo } = await import("@omniroute/open-sse/services/autoCombo/builtinCatalog.ts"); - virtualCombo = await createBuiltinAutoCombo(modelStr, channel); + virtualCombo = await createBuiltinAutoCombo(modelStr, channel, prepared); } const excludedConnectionIds = apiKeyId diff --git a/open-sse/services/autoCombo/resilienceCandidateFilter.ts b/open-sse/services/autoCombo/resilienceCandidateFilter.ts index 72c2cc9445..8b3a4e682b 100644 --- a/open-sse/services/autoCombo/resilienceCandidateFilter.ts +++ b/open-sse/services/autoCombo/resilienceCandidateFilter.ts @@ -31,6 +31,15 @@ export interface ConnectionResilienceView { testStatus?: string | null; } +/** Index connection resilience views by id, for the O(1) lookups this filter needs. */ +export function buildConnectionResilienceMap( + connections: Iterable +): Map { + const byId = new Map(); + for (const conn of connections) byId.set(conn.id, conn); + return byId; +} + function isConnectionResilienceBlocked(connection: ConnectionResilienceView): boolean { if (isAccountUnavailable(connection.rateLimitedUntil)) return true; const status = connection.testStatus; @@ -54,12 +63,19 @@ function isConnectionEligibleForModel( * Remove auto-combo candidates whose provider/model pair is model-locked, and * trim credentialed logical candidates whose allowed connections are all blocked. * Returns the input reference when nothing changed. + * + * `skip` (#9133) lets the read-only candidate inspector + * (`open-sse/handlers/autoComboCandidates.ts`) opt out entirely: it needs the + * FULL pool so it can decorate blocked candidates as `reachable:false` + * instead of dropping them before they are ever surfaced. Routing callers + * never pass it, so dispatch behavior is unchanged. */ export function filterResilienceBlockedCandidates( pool: T[], - connectionsById: Map + connectionsById: Map, + skip = false ): T[] { - if (!Array.isArray(pool) || pool.length === 0) return pool; + if (skip || !Array.isArray(pool) || pool.length === 0) return pool; let changed = false; const filtered = pool.flatMap((candidate) => { diff --git a/open-sse/services/autoCombo/virtualFactory.ts b/open-sse/services/autoCombo/virtualFactory.ts index 96a46dde2a..a4d0338819 100644 --- a/open-sse/services/autoCombo/virtualFactory.ts +++ b/open-sse/services/autoCombo/virtualFactory.ts @@ -44,8 +44,8 @@ import { filterExcludedCandidates } from "./candidateOverrides"; import { getExcludedConnectionIds } from "@/lib/db/autoCandidateOverrides"; import { filterResilienceBlockedCandidates, + buildConnectionResilienceMap, SYNTHETIC_NOAUTH_CONNECTION_ID as RESILIENCE_NOAUTH_CONNECTION_ID, - type ConnectionResilienceView, } from "./resilienceCandidateFilter"; import type { ChaosTuning } from "./chaosEngine"; @@ -584,7 +584,8 @@ export async function prepareVirtualAutoComboInputs( options: { includeResolvedCapabilities?: boolean; resolutionSnapshot?: ModelCapabilityResolutionSnapshot; - } = {} + } = {}, + skip = false // #9133 — inspector opt-out, see filterResilienceBlockedCandidates ): Promise { const [rawConnections, rawDisabledNoAuthConnections, settings] = await Promise.all([ getCachedProviderConnections({ isActive: true }) as Promise, @@ -708,10 +709,10 @@ export async function prepareVirtualAutoComboInputs( // #7623: honor existing model lockouts + connection cooldown/terminal state so // auto/* never advertises models the dispatch path would immediately skip. - const connectionsById = new Map(); - for (const conn of [...runtimeConnections, ...disabledNoAuthConnections]) { - connectionsById.set(conn.id, conn); - } + const connectionsById = buildConnectionResilienceMap([ + ...runtimeConnections, + ...disabledNoAuthConnections, + ]); const connectedProviders = new Set(validConnections.map((conn) => conn.provider)); const buildPreparedPool = (bypassNoAuthAllowlist: boolean) => { @@ -727,7 +728,7 @@ export async function prepareVirtualAutoComboInputs( ), ]; - const resilienceFilteredPool = filterResilienceBlockedCandidates(pool, connectionsById); + const resilienceFilteredPool = filterResilienceBlockedCandidates(pool, connectionsById, skip); if (resilienceFilteredPool !== pool) pool = resilienceFilteredPool; // #6512 (follow-up to #6328/#6495): when the operator opts into `hidePaidModels`, diff --git a/tests/unit/auto-combo-candidates-locked-model-visible.test.ts b/tests/unit/auto-combo-candidates-locked-model-visible.test.ts new file mode 100644 index 0000000000..6a5e18440c --- /dev/null +++ b/tests/unit/auto-combo-candidates-locked-model-visible.test.ts @@ -0,0 +1,110 @@ +/** + * #9133 — a model lockout on one account must not silently drop that + * account's row from the auto-combo candidates inspector endpoint. The endpoint + * is read-only transparency (#7819 Level 1): a locked/cooled-down candidate + * must still be listed, decorated with `reachable:false` and an accurate + * reason (`modelLocked`), not deleted from the pool before the inspector + * ever sees it. + * + * Root cause (measured by community contributor ntdat812, ratified by the + * owner 2026-08-22): `prepareVirtualAutoComboInputs` unconditionally ran + * `filterResilienceBlockedCandidates` before the listing endpoint decorated + * the pool, so `modelLocked`/`reachable:false` were dead fields — the rows + * they would apply to were already gone. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-auto-locked-visible-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; + +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const candidateHandler = await import("../../open-sse/handlers/autoComboCandidates.ts"); +const accountFallback = await import("../../open-sse/services/accountFallback.ts"); + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +async function seedConnections() { + const tokenExpiresAt = new Date(Date.now() + 60_000).toISOString(); + const first = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + email: "antigravity-locked@example.com", + accessToken: "fake-antigravity-access-token-one", + tokenExpiresAt, + }); + const second = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + email: "antigravity-clean@example.com", + accessToken: "fake-antigravity-access-token-two", + tokenExpiresAt, + }); + return { first, second }; +} + +test.beforeEach(async () => { + await resetStorage(); + accountFallback.clearAllModelLockouts(); +}); + +test.after(async () => { + accountFallback.clearAllModelLockouts(); + await resetStorage(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + + if (ORIGINAL_DATA_DIR === undefined) { + delete process.env.DATA_DIR; + } else { + process.env.DATA_DIR = ORIGINAL_DATA_DIR; + } +}); + +test("a model-locked account row stays in the listing with reachable:false + modelLocked:true", async () => { + const { first, second } = await seedConnections(); + + // Lock a single model on a single account — the bare model id, matching + // every real lock writer (accountFallback.ts, resilienceCandidateFilter.ts). + accountFallback.lockModel( + "antigravity", + first.id, + "claude-sonnet-4-6", + "rate_limit", + 60_000 + ); + + const result = await candidateHandler.getAutoComboCandidates("auto", null); + const sonnetRows = result.candidates.filter( + (candidate) => + candidate.provider === "antigravity" && candidate.model === "antigravity/claude-sonnet-4-6" + ); + + // Baseline: both accounts must still be represented — the endpoint's + // stated role is read-only transparency, so a lock must never make a row + // disappear. + assert.deepEqual( + new Set(sonnetRows.map((candidate) => candidate.connectionId)), + new Set([first.id, second.id]), + "the locked account's row must remain listed, not be silently dropped" + ); + + const lockedRow = sonnetRows.find((candidate) => candidate.connectionId === first.id); + assert.ok(lockedRow, "locked account row must be present"); + assert.equal(lockedRow?.modelLocked, true, "locked row must report modelLocked:true"); + assert.equal(lockedRow?.reachable, false, "locked row must report reachable:false"); + + const cleanRow = sonnetRows.find((candidate) => candidate.connectionId === second.id); + assert.ok(cleanRow, "unlocked account row must be present"); + assert.equal(cleanRow?.modelLocked, false, "unlocked row must not report modelLocked"); + assert.equal(cleanRow?.reachable, true, "unlocked row must remain reachable"); +});