diff --git a/open-sse/services/autoCombo/virtualFactory.ts b/open-sse/services/autoCombo/virtualFactory.ts index 558f4b5694..c9b68d4453 100644 --- a/open-sse/services/autoCombo/virtualFactory.ts +++ b/open-sse/services/autoCombo/virtualFactory.ts @@ -685,10 +685,21 @@ export async function prepareVirtualAutoComboInputs( // back to the static catalog only when the user has none. This keeps catalog-only // models (e.g. openrouter/auto) out of every auto/* pool when the operator only // synced a subset (e.g. OpenRouter with importFreeModelsOnly). - const [syncedByConnection, customModels] = await Promise.all([ + const [syncedByConnection, rawCustomModels] = await Promise.all([ getSyncedAvailableModelsByConnection(providerId), getCustomModels(providerId), ]); + // The `customModels` key_value blob is operator-writable and is stored as raw + // parsed JSON, so a row can be `null` or a non-object. The catalog builder + // already filters those out (catalog.ts, "Add custom models"); without the same + // filter here every read below null-derefs and the whole auto/* pool fails to + // materialize ("Could not materialize built-in auto model auto/"). + const customModels: Array<{ id?: string }> = ( + Array.isArray(rawCustomModels) ? rawCustomModels : [] + ).filter( + (model: unknown): model is { id?: string } => + !!model && typeof model === "object" && !Array.isArray(model) + ); const userVisibleIds = new Set(); for (const models of Object.values(syncedByConnection)) { for (const m of models) if (m.id && !hiddenModels?.has(m.id)) userVisibleIds.add(m.id); diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index a69db56e64..9d8e928864 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -408,7 +408,12 @@ const SCHEMA_SQL = ` CREATE INDEX IF NOT EXISTS idx_cl_timestamp ON call_logs(timestamp); CREATE INDEX IF NOT EXISTS idx_cl_status ON call_logs(status); CREATE INDEX IF NOT EXISTS idx_cl_provider_timestamp ON call_logs(provider, timestamp); - CREATE INDEX IF NOT EXISTS idx_cl_request_provider ON call_logs(request_type, provider); + -- idx_cl_request_provider is NOT declared here: SCHEMA_SQL runs before + -- ensureCallLogsColumns() heals a legacy call_logs table, and a lineage that + -- predates the request_type column has none yet — the CREATE INDEX would abort + -- the whole schema exec with "no such column: request_type" and the server would + -- never boot. It is created next to the other request_type/combo indexes in + -- ensureCallLogsColumns() (db/schemaColumns.ts), after the columns exist. CREATE TABLE IF NOT EXISTS proxy_logs ( id TEXT PRIMARY KEY, diff --git a/src/lib/db/models/activeSyncedCatalog.ts b/src/lib/db/models/activeSyncedCatalog.ts index 4e68a17d2b..9c34f685dd 100644 --- a/src/lib/db/models/activeSyncedCatalog.ts +++ b/src/lib/db/models/activeSyncedCatalog.ts @@ -184,10 +184,12 @@ async function unionCustomModels( */ async function loadConnectionCatalog(storedProviderId: string): Promise { const [connections, modelsByConnection] = await Promise.all([ - getRawProviderConnections({ provider: storedProviderId, isActive: true }, undefined, undefined, [ - "id", - "provider", - ]), + getRawProviderConnections( + { provider: storedProviderId, isActive: true }, + undefined, + undefined, + ["id", "provider"] + ), getSyncedAvailableModelsByConnection(storedProviderId), ]); @@ -248,6 +250,18 @@ export async function getActiveSyncedCatalog(providerId: string): Promise`) and demote the + * embedding/rerank registry's canonical `jina-ai/` to a child — the inverse of + * the identity every other specialty model of that provider carries. */ export async function getAllActiveSyncedModels(): Promise> { try { @@ -277,10 +291,7 @@ export async function getAllActiveSyncedModels(): Promise 0) { diff --git a/src/lib/db/schemaColumns.ts b/src/lib/db/schemaColumns.ts index b288072dac..d0d7b42493 100644 --- a/src/lib/db/schemaColumns.ts +++ b/src/lib/db/schemaColumns.ts @@ -265,6 +265,12 @@ export function ensureCallLogsColumns(db: SqliteDatabase) { "CREATE INDEX IF NOT EXISTS idx_call_logs_requested_model ON call_logs(requested_model)" ); db.exec("CREATE INDEX IF NOT EXISTS idx_call_logs_request_type ON call_logs(request_type)"); + // #12832's provider-stats index. It lives here rather than in SCHEMA_SQL because + // SCHEMA_SQL runs before this healing pass: on a legacy call_logs table that + // predates `request_type` the CREATE INDEX aborts the whole schema exec. + db.exec( + "CREATE INDEX IF NOT EXISTS idx_cl_request_provider ON call_logs(request_type, provider)" + ); db.exec( "CREATE INDEX IF NOT EXISTS idx_cl_combo_target ON call_logs(combo_name, combo_execution_key, timestamp)" ); diff --git a/tests/unit/combo-auto-pool-visible-only.test.ts b/tests/unit/combo-auto-pool-visible-only.test.ts index 002140163e..dd70289798 100644 --- a/tests/unit/combo-auto-pool-visible-only.test.ts +++ b/tests/unit/combo-auto-pool-visible-only.test.ts @@ -173,3 +173,40 @@ test("virtual auto-combo pool filters EVERY provider with partial sync, not just "kilocode pool must contain exactly the two synced models" ); }); + +test("virtual auto-combo pool survives a malformed customModels row", async () => { + // The `customModels` key_value blob is operator-writable and is read back as raw + // parsed JSON, so a row can be null / a non-object / carry no id. Before the guard + // in prepareVirtualAutoComboInputs those rows threw + // "Cannot read properties of null (reading 'id')" and EVERY auto/* combo failed to + // materialize ("[catalog] Could not materialize built-in auto model auto/"). + const conn = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "OpenAI", + apiKey: "sk-test-openai", + }); + const connectionId = (conn as { id?: string }).id; + await modelsDb.replaceSyncedAvailableModelsForConnection("openai", connectionId, [ + { id: "gpt-4o-mini", name: "GPT-4o mini", source: "imported" as const }, + ]); + core + .getDbInstance() + .prepare("INSERT OR REPLACE INTO key_value (namespace, key, value) VALUES (?, ?, ?)") + .run( + "customModels", + "openai", + JSON.stringify([null, "not-an-object", { name: "Missing Id" }, { id: "operator-custom" }]) + ); + + const prepared = await virtualFactory.prepareVirtualAutoComboInputs(); + const openaiCandidates = prepared.regularCandidates.filter((c) => c.provider === "openai"); + assert.ok( + openaiCandidates.some((c) => c.model === "gpt-4o-mini"), + "the synced model must still reach the pool despite the malformed custom rows" + ); + assert.ok( + openaiCandidates.some((c) => c.model === "operator-custom"), + "the one well-formed custom row must still reach the pool" + ); +}); diff --git a/tests/unit/combo-provider-cooldown-sibling.test.ts b/tests/unit/combo-provider-cooldown-sibling.test.ts index 562173fccc..6764cb2d81 100644 --- a/tests/unit/combo-provider-cooldown-sibling.test.ts +++ b/tests/unit/combo-provider-cooldown-sibling.test.ts @@ -129,12 +129,33 @@ test("source guard: auth.ts skips model lockout for per-model-quota providers on }); test("source guard: combo.ts skips provider cooldown for per-model-quota on 500", () => { - const src = fs.readFileSync( + // The combo dispatcher was split out of combo.ts into open-sse/services/combo/* + // (#12746 executeTarget → gates/attempt/loop, #12811 round-robin). The invariant + // did not move: EVERY call site that records a provider cooldown after a failed + // combo target must first exclude a 500 on a per-model-quota provider, or one + // model's outage cools down its siblings. Scan the whole combo surface so the + // guard follows the code instead of one file name. + const comboDir = path.join(process.cwd(), "open-sse", "services", "combo"); + const comboFiles = [ path.join(process.cwd(), "open-sse", "services", "combo.ts"), - "utf-8" + ...fs + .readdirSync(comboDir, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".ts")) + .map((entry) => path.join(comboDir, entry.name)), + ].filter((file) => fs.existsSync(file)); + + const recordingFiles = comboFiles.filter((file) => + fs.readFileSync(file, "utf-8").includes("recordProviderCooldown(") ); assert.ok( - src.includes("hasPerModelQuota(provider, rawModel)") && src.includes("recordProviderCooldown"), - "combo.ts must skip provider cooldown recording for per-model-quota providers on 500" + recordingFiles.length > 0, + "no combo module records a provider cooldown — the guarded call site vanished" ); + for (const file of recordingFiles) { + const normalized = fs.readFileSync(file, "utf-8").replace(/\s+/g, " "); + assert.ok( + /result\.status === 500[^;]{0,160}?hasPerModelQuota\(provider,/.test(normalized), + `${path.basename(file)} must skip provider cooldown recording for per-model-quota providers on 500` + ); + } }); diff --git a/tests/unit/custom-models-live-catalog-12597.test.ts b/tests/unit/custom-models-live-catalog-12597.test.ts index 4e6bb35068..b81745aeb9 100644 --- a/tests/unit/custom-models-live-catalog-12597.test.ts +++ b/tests/unit/custom-models-live-catalog-12597.test.ts @@ -14,10 +14,14 @@ process.env.DATA_DIR = TEST_DATA_DIR; process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "custom-live-12597-test-secret"; const core = await import("../../src/lib/db/core.ts"); -const { addCustomModel, replaceSyncedAvailableModelsForConnection, getActiveProvidersWithSyncedModel } = - await import("../../src/lib/db/models.ts"); +const { + addCustomModel, + replaceSyncedAvailableModelsForConnection, + getActiveProvidersWithSyncedModel, +} = await import("../../src/lib/db/models.ts"); const { getActiveSyncedCatalog, + getAllActiveSyncedModels, catalogContainsModel, reconcileProvidersWithActiveSyncedCatalog, } = await import("../../src/lib/db/models/activeSyncedCatalog.ts"); @@ -124,6 +128,21 @@ test("#12597 sparse custom overlay does not wipe synced capability fields", asyn assert.deepEqual(match?.supportedThinkingEfforts, ["low", "high"]); }); +test("#12597 the union is dispatch-only — getAllActiveSyncedModels stays live-sync-shaped", async () => { + await addCustomModel(PROVIDER, PICKER_MODEL, "DeepSeek R1 via picker"); + + const allActive = await getAllActiveSyncedModels(); + const ids = (allActive[PROVIDER] ?? []).map((model) => model.id); + + assert.deepEqual( + ids, + [SYNCED_MODEL], + "getAllActiveSyncedModels reports what the provider's live sync returned; its consumers (/v1/models' synced loop, /api/models' exclusive-listing suppression, getSyncedAutoAliases) each handle custom rows on their own" + ); + // The dispatch-time readers still see it — that is what #12597 fixed. + assert.equal(catalogContainsModel(await getActiveSyncedCatalog(PROVIDER), PICKER_MODEL), true); +}); + test("#12597 without a custom row the picker id is still absent (lock the old contract)", async () => { const catalog = await getActiveSyncedCatalog(PROVIDER); assert.equal(catalogContainsModel(catalog, PICKER_MODEL), false); diff --git a/tests/unit/hard-session-lease-bypass-inventory.test.ts b/tests/unit/hard-session-lease-bypass-inventory.test.ts index 97e713a0c7..5b89f59225 100644 --- a/tests/unit/hard-session-lease-bypass-inventory.test.ts +++ b/tests/unit/hard-session-lease-bypass-inventory.test.ts @@ -13,7 +13,12 @@ type BypassClass = "A" | "B" | "C"; const EXPECTED: Record> = { credential: { - "open-sse/handlers/chatCore.ts": 2, + // #12867 extracted chatCore.ts's streaming provider-execution loop into + // chatCore/providerExecutionPipeline.ts. Its two getProviderCredentials() + // sites (codex 429 rotation, antigravity BYOP rotation) moved with it and are + // now reached through the injected `connection.getProviderCredentials` handle, + // so countCalls() also inventories property-access calls. + "open-sse/handlers/chatCore/providerExecutionPipeline.ts": 2, "open-sse/services/imageCombo.ts": 1, "open-sse/services/speechCombo.ts": 1, "open-sse/services/videoCombo.ts": 2, @@ -88,8 +93,9 @@ const EXPECTED: Record> = { "open-sse/services/antigravityFamilyCooldown.ts": 1, // v3.8.50 back-merge additions (f95b03d7): combo routing infra and the // volcengine-plan binding/auto-sync services query connections the same - // way as their classified siblings. - "open-sse/services/combo.ts": 1, + // way as their classified siblings. #12746 split executeTarget out of + // combo.ts, moving this lookup into combo/executeTargetGates.ts unchanged. + "open-sse/services/combo/executeTargetGates.ts": 1, "open-sse/services/combo/providerWildcard.ts": 1, "open-sse/services/tokenRefresh.ts": 1, "src/lib/providers/volcPlanAutoSyncBackfill.ts": 1, @@ -127,6 +133,11 @@ const EXPECTED: Record> = { "src/app/api/translator/send/route.ts": 1, "src/app/api/translator/translate/route.ts": 1, "src/app/api/usage/call-logs/route.ts": 1, + // #12805: the reset-credit route resolves the connection's PROVIDER to pick + // the codex or grok-cli library; the exclusive-lease fence itself lives in + // those libraries (both listed in auxiliaryIsolationSources below). It never + // selects a connection to serve a request, so it stays class C. + "src/app/api/usage/codex-reset-credit/route.ts": 1, "src/app/api/usage/quota/route.ts": 1, "src/app/api/usage/utilization/route.ts": 1, "src/app/api/v1/vscode/[token]/api/tags/route.ts": 1, @@ -174,6 +185,9 @@ const EXPECTED: Record> = { "src/lib/usage/callLogs.ts": 1, "src/lib/usage/codexResetCredits.ts": 1, "src/lib/usage/comboScoringInspector.ts": 1, + // #12805: Grok Build sibling of codexResetCredits.ts — same auxiliary-activity + // fence in front of the same connection lookup, so same class B. + "src/lib/usage/grokResetCredits.ts": 1, "src/lib/usage/providerLimits.ts": 4, "src/lib/usage/resilienceExplain.ts": 1, "src/lib/usage/usageStats.ts": 1, @@ -212,10 +226,9 @@ const CLASSIFICATION: Record> = { [ "open-sse/handlers/autoComboCandidates.ts", "open-sse/handlers/chatCore.ts", - "open-sse/services/combo.ts", "open-sse/services/alibabaFreeTier.ts", "open-sse/services/alibabaFreeTierQuotaFetcher.ts", - "open-sse/services/combo.ts", + "open-sse/services/combo/executeTargetGates.ts", "open-sse/services/combo/providerWildcard.ts", "open-sse/services/tokenRefresh.ts", "src/app/api/translator/send/route.ts", @@ -224,6 +237,7 @@ const CLASSIFICATION: Record> = { "src/lib/providers/volcenginePlanBinding.ts", "src/lib/services/quotaAutoPing.ts", "src/lib/usage/codexResetCredits.ts", + "src/lib/usage/grokResetCredits.ts", "src/lib/usage/providerLimits.ts", "src/lib/vncSession/service.ts", "src/lib/warmupScheduler.ts", @@ -260,14 +274,13 @@ function countCalls(): Record> { const key = file.split(path.sep).join("/"); actual[kind][key] = (actual[kind][key] ?? 0) + 1; }; + const isCredentialName = (name: string) => + name === "getProviderCredentials" || name === "getProviderCredentialsWithQuotaPreflight"; const visit = (node: ts.Node): void => { if (ts.isCallExpression(node)) { const expression = node.expression; if (ts.isIdentifier(expression)) { - if ( - expression.text === "getProviderCredentials" || - expression.text === "getProviderCredentialsWithQuotaPreflight" - ) { + if (isCredentialName(expression.text)) { increment("credential"); } if ( @@ -276,15 +289,22 @@ function countCalls(): Record> { ) { increment("connection"); } - } else if ( - ts.isPropertyAccessExpression(expression) && - expression.name.text === "execute" && - ts.isIdentifier(expression.expression) && - ["executor", "fallbackExecutor", "providerExecutor", "streamExecutor"].includes( - expression.expression.text - ) - ) { - increment("executor"); + } else if (ts.isPropertyAccessExpression(expression)) { + // #12867: the extracted execution pipeline reaches the resolver through an + // injected handle (`connection.getProviderCredentials(...)`), so a + // bare-identifier scan alone would let those sites leave the inventory. + if (isCredentialName(expression.name.text)) { + increment("credential"); + } + if ( + expression.name.text === "execute" && + ts.isIdentifier(expression.expression) && + ["executor", "fallbackExecutor", "providerExecutor", "streamExecutor"].includes( + expression.expression.text + ) + ) { + increment("executor"); + } } } ts.forEachChild(node, visit); @@ -312,6 +332,10 @@ test("managed request surfaces are fenced centrally or rejected before independe path.join(REPO_ROOT, "src/app/api/internal/codex-responses-ws/route.ts"), "utf8" ); + const executionPipeline = fs.readFileSync( + path.join(REPO_ROOT, "open-sse/handlers/chatCore/providerExecutionPipeline.ts"), + "utf8" + ); const internalKeys = fs.readFileSync(path.join(REPO_ROOT, "src/lib/db/apiKeys.ts"), "utf8"); const auxiliaryIsolationSources = [ "src/app/api/providers/[id]/models/route.ts", @@ -320,6 +344,7 @@ test("managed request surfaces are fenced centrally or rejected before independe "src/lib/api/modelTestRunner.ts", "src/lib/services/quotaAutoPing.ts", "src/lib/usage/codexResetCredits.ts", + "src/lib/usage/grokResetCredits.ts", "src/lib/vncSession/service.ts", "src/lib/warmupScheduler.ts", "src/shared/services/modelSyncScheduler.ts", @@ -336,7 +361,14 @@ test("managed request surfaces are fenced centrally or rejected before independe core, /assertManagedLeaseFence\(getExecutionConnectionId\(getExecutionCredentials\(\)\)\)/ ); - assert.match(core, /provider === "codex" &&\s*!managedLease/); + // #12867 moved the codex 429 account-rotation out of chatCore.ts into + // chatCore/providerExecutionPipeline.ts. The managed-lease fence moved with it: + // the inline `provider === "codex" && !managedLease` became the policy flag + // chatCore computes and the pipeline gates every rotation on. Assert both halves + // so the fence cannot be dropped on either side of that seam. + assert.match(core, /allowAccountRotation:\s*!managedLease\b/); + assert.match(executionPipeline, /canRotateAccount\s*=\s*policy\.allowAccountRotation\b/); + assert.match(executionPipeline, /canRotateAccount &&\s*target\.provider === "codex"/); assert.match(ws, /LEASE_UNSUPPORTED_TRANSPORT/); assert.match(internalKeys, /!k\.scopes\?\.includes\(EXCLUSIVE_LEASE_SCOPE\)/); for (const source of auxiliaryIsolationSources) { diff --git a/tests/unit/models-catalog-route.test.ts b/tests/unit/models-catalog-route.test.ts index cfecc23c96..9fc3584a2d 100644 --- a/tests/unit/models-catalog-route.test.ts +++ b/tests/unit/models-catalog-route.test.ts @@ -7,6 +7,14 @@ import path from "node:path"; const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-model-catalog-")); process.env.DATA_DIR = TEST_DATA_DIR; process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "catalog-test-secret"; +// #12627 bounds a cold catalog build at 8s and, with no last-good response to fall +// back on, surfaces `catalog_build_timeout` as an error body — no `data` array. That +// bound is sized for a warm production process; a tsx-transpiled test runner building +// the full 500+ model catalog from a fresh SQLite file on a loaded CI box crosses it +// (10-13s observed), which turned the assertions below into a load-dependent flake. +// Raise it here so these cases test catalog CONTENT; the timeout behavior itself is +// covered by tests/unit/12627-catalog-inflight-timeout.test.ts. +process.env.CATALOG_BUILD_TIMEOUT_MS = process.env.CATALOG_BUILD_TIMEOUT_MS || "120000"; const core = await import("../../src/lib/db/core.ts"); const providersDb = await import("../../src/lib/db/providers.ts");